agent-regression-lab 0.1.1 → 0.3.0

package/dist/index.js

@@ -1,7 +1,10 @@
 #!/usr/bin/env node
+import packageJson from "../package.json" with { type: "json" };
+import { pathToFileURL } from "node:url";
 import { createAgentFactory } from "./agent/factory.js";
-import { getAgentRegistration } from "./config.js";
-import { getRunErrorDetail } from "./runOutput.js";
+import { getAgentRegistration, getVariantSet } from "./config.js";
+import { createConfigHash, createSuiteBatchId } from "./lib/id.js";
+import { formatCliErrorMessage, formatRunIdentityLines, getFailedEvaluatorSummaries, getRunErrorDetail } from "./runOutput.js";
 async function main() {
     const [, , command, ...args] = process.argv;
     switch (command) {
@@ -9,27 +12,27 @@ async function main() {
         case "--help":
         case "-h":
             printUsage();
-            return;
+            break;
         case "version":
         case "--version":
         case "-v":
             printVersion();
-            return;
+            break;
         case "list":
             await handleList(args);
-            return;
+            break;
         case "run":
             await handleRun(args);
-            return;
+            break;
         case "show":
             await handleShow(args);
-            return;
+            break;
         case "compare":
             await handleCompare(args);
-            return;
+            break;
         case "ui":
             await handleUi();
-            return;
+            break;
         default:
             printUsage();
     }
@@ -37,16 +40,19 @@ async function main() {
 function printUsage() {
     console.log(`Usage:
   agentlab list scenarios
-  agentlab run <scenario-id> [--agent <name>] [--provider mock|openai|external_process] [--model <model>] [--agent-label <label>]
-  agentlab run --suite <suite-id> [--agent <name>] [--provider mock|openai|external_process] [--model <model>] [--agent-label <label>]
+  agentlab run <scenario-id> [--agent <name>] [--provider mock|openai|external_process|http] [--model <model>] [--agent-label <label>]
+  agentlab run --suite <suite-id> [--agent <name>] [--provider mock|openai|external_process|http] [--model <model>] [--agent-label <label>]
+  agentlab run --suite-def <name> [--agent <name>]
+  agentlab run <scenario-id> [--variant-set <name>]
   agentlab show <run-id>
   agentlab compare <baseline-run-id> <candidate-run-id>
+  agentlab compare --suite <baseline-batch-id> <candidate-batch-id>
   agentlab ui
   agentlab help
   agentlab version`);
 }
 function printVersion() {
-    console.log("0.1.0");
+    console.log(packageJson.version);
 }
 async function handleList(args) {
     if (args[0] !== "scenarios") {
@@ -61,38 +67,108 @@ async function handleList(args) {
 async function handleRun(args) {
     const parsed = parseRunArgs(args);
     const runtimeConfig = validateRuntimeConfig(parsed.runtimeConfig);
-    const { loadScenariosBySuite } = await import("./scenarios.js");
+    const { loadScenariosBySuite, loadScenariosBySuiteDefinition } = await import("./scenarios.js");
+    if (parsed.suite && parsed.suiteDefinition) {
+        throw new Error("--suite and --suite-def cannot be used together.");
+    }
+    if (parsed.runtimeConfig.agentName && parsed.variantSetName) {
+        throw new Error("--agent and --variant-set cannot be used together.");
+    }
     if (parsed.suite) {
         const suite = parsed.suite;
-        if (!suite) {
-            throw new Error("Missing suite id.");
-        }
         const scenarios = loadScenariosBySuite(suite);
         if (scenarios.length === 0) {
             throw new Error(`No scenarios found for suite '${suite}'.`);
         }
+        const suiteBatchId = createSuiteBatchId();
         const runs = [];
-        for (const scenario of scenarios) {
-            runs.push(await executeOne(scenario.definition.id, runtimeConfig));
-        }
-        const passed = runs.filter((bundle) => bundle.run.status === "pass").length;
-        const failed = runs.filter((bundle) => bundle.run.status === "fail").length;
-        const errored = runs.filter((bundle) => bundle.run.status === "error").length;
-        const avgScore = Math.round(runs.reduce((sum, bundle) => sum + bundle.run.score, 0) / runs.length);
-        console.log(`Suite: ${suite}`);
-        console.log(`Passed: ${passed}/${runs.length}`);
-        console.log(`Failed: ${failed}/${runs.length}`);
-        console.log(`Errored: ${errored}/${runs.length}`);
-        console.log(`Average score: ${avgScore}`);
+        if (parsed.variantSetName) {
+            console.log(`Variant set: ${parsed.variantSetName}`);
+            for (const scenario of scenarios) {
+                runs.push(...await executeVariantSetScenario(scenario.definition.id, parsed.variantSetName, suiteBatchId));
+            }
+        }
+        else {
+            for (const scenario of scenarios) {
+                runs.push(await executeOne(scenario.definition.id, runtimeConfig, suiteBatchId));
+            }
+        }
+        printSuiteSummary(suite, runs, suiteBatchId);
+        return;
+    }
+    if (parsed.suiteDefinition) {
+        const suiteDefinition = parsed.suiteDefinition;
+        const scenarios = loadScenariosBySuiteDefinition(suiteDefinition);
+        if (scenarios.length === 0) {
+            throw new Error(`No scenarios found for suite definition '${suiteDefinition}'.`);
+        }
+        const suiteBatchId = createSuiteBatchId();
+        const runs = [];
+        console.log(`Suite definition: ${suiteDefinition}`);
+        if (parsed.variantSetName) {
+            console.log(`Variant set: ${parsed.variantSetName}`);
+            for (const scenario of scenarios) {
+                runs.push(...await executeVariantSetScenario(scenario.definition.id, parsed.variantSetName, suiteBatchId, suiteDefinition));
+            }
+        }
+        else {
+            const suiteRuntimeConfig = { ...runtimeConfig, suiteDefinitionName: suiteDefinition };
+            for (const scenario of scenarios) {
+                runs.push(await executeOne(scenario.definition.id, suiteRuntimeConfig, suiteBatchId));
+            }
+        }
+        printSuiteSummary(suiteDefinition, runs, suiteBatchId);
         return;
     }
     const scenarioId = parsed.scenarioId;
     if (!scenarioId) {
         throw new Error("Missing scenario id.");
     }
-    await executeOne(scenarioId, runtimeConfig);
+    if (parsed.variantSetName) {
+        console.log(`Variant set: ${parsed.variantSetName}`);
+        await executeVariantSetScenario(scenarioId, parsed.variantSetName);
+        return;
+    }
+    // Detect scenario type to route to the right runner
+    const { listScenarioFiles } = await import("./scenarios.js");
+    const { parse } = await import("yaml");
+    const { readFileSync } = await import("node:fs");
+    const { resolve } = await import("node:path");
+    let scenarioType = "task";
+    for (const filePath of listScenarioFiles()) {
+        const raw = readFileSync(resolve(filePath), "utf8");
+        const parsedYaml = parse(raw);
+        if (parsedYaml.id === scenarioId) {
+            scenarioType = parsedYaml.type === "conversation" ? "conversation" : "task";
+            break;
+        }
+    }
+    if (scenarioType === "task" && runtimeConfig.provider === "http") {
+        throw new Error(`Scenario '${scenarioId}' is a task scenario. HTTP agents (provider: http) only work with ` +
+            `type: conversation scenarios.\n` +
+            `To test an HTTP agent, create a conversation scenario (type: conversation) — ` +
+            `conversation scenarios do not use a tools: block. See docs/scenarios.md for the format.`);
+    }
+    if (scenarioType === "conversation") {
+        if (runtimeConfig.provider !== "http") {
+            throw new Error(`Scenario '${scenarioId}' is a conversation scenario and requires provider: http. Use --agent <name> with a configured HTTP agent.`);
+        }
+        const httpConfig = {
+            name: runtimeConfig.agentName ?? "http-agent",
+            provider: "http",
+            url: runtimeConfig.url,
+            request_template: runtimeConfig.request_template,
+            response_field: runtimeConfig.response_field,
+            headers: runtimeConfig.headers,
+            timeout_ms: runtimeConfig.timeout_ms,
+        };
+        await executeConversation(scenarioId, httpConfig, runtimeConfig.label);
+    }
+    else {
+        await executeOne(scenarioId, runtimeConfig);
+    }
 }
-async function executeOne(scenarioId, runtimeConfig) {
+async function executeOne(scenarioId, runtimeConfig, suiteBatchId) {
     const [{ Storage }, { loadToolRegistry, loadToolSpecs }, { loadScenarioById }, { runScenario }] = await Promise.all([
         import("./storage.js"),
         import("./tools.js"),
@@ -100,31 +176,167 @@ async function executeOne(scenarioId, runtimeConfig) {
         import("./runner.js"),
     ]);
     const storage = new Storage();
-    const toolSpecs = await loadToolSpecs();
-    const toolRegistry = await loadToolRegistry();
-    const loaded = loadScenarioById(scenarioId);
-    storage.upsertScenario({
-        id: loaded.definition.id,
-        name: loaded.definition.name,
-        suite: loaded.definition.suite,
-        difficulty: loaded.definition.difficulty,
-        description: loaded.definition.description,
-    }, loaded.definition, loaded.filePath, loaded.fileHash);
-    const factory = createAgentFactory(runtimeConfig);
-    const agentVersion = factory.createVersion(runtimeConfig);
-    storage.upsertAgentVersion(agentVersion);
-    const bundle = await runScenario({
-        agentAdapter: factory.createAdapter(),
-        agentVersion,
-        scenario: loaded.definition,
-        scenarioFileHash: loaded.fileHash,
-        toolSpecs,
-        tools: toolRegistry,
+    try {
+        const toolSpecs = await loadToolSpecs();
+        const toolRegistry = await loadToolRegistry();
+        const loaded = loadScenarioById(scenarioId);
+        storage.upsertScenario({
+            id: loaded.definition.id,
+            name: loaded.definition.name,
+            suite: loaded.definition.suite,
+            difficulty: loaded.definition.difficulty,
+            description: loaded.definition.description,
+        }, loaded.definition, loaded.filePath, loaded.fileHash);
+        const factory = createAgentFactory(runtimeConfig);
+        const agentVersion = factory.createVersion(runtimeConfig);
+        storage.upsertAgentVersion(agentVersion);
+        const bundle = await runScenario({
+            agentAdapter: factory.createAdapter(),
+            agentVersion,
+            scenario: loaded.definition,
+            scenarioFileHash: loaded.fileHash,
+            toolSpecs,
+            tools: toolRegistry,
+        });
+        bundle.run.suiteBatchId = suiteBatchId;
+        bundle.run.variantSetName = agentVersion.variantSetName;
+        bundle.run.variantLabel = agentVersion.variantLabel;
+        bundle.run.promptVersion = agentVersion.promptVersion;
+        bundle.run.modelVersion = agentVersion.modelVersion;
+        bundle.run.toolSchemaVersion = agentVersion.toolSchemaVersion;
+        bundle.run.configLabel = agentVersion.configLabel;
+        bundle.run.configHash = agentVersion.configHash;
+        bundle.run.runtimeProfileName = loaded.definition.runtime_profile;
+        bundle.run.suiteDefinitionName = runtimeConfig.suiteDefinitionName;
+        bundle.agentVersion = agentVersion;
+        storage.saveRun(bundle);
+        printRunSummary(bundle);
+        return bundle;
+    }
+    finally {
+        storage.close();
+    }
+}
+export async function executeVariantSetScenario(scenarioId, variantSetName, suiteBatchId, suiteDefinitionName) {
+    const variantSet = getVariantSet(variantSetName);
+    const runs = [];
+    for (const variant of variantSet.variants) {
+        const registration = getAgentRegistration(variant.agent);
+        const runtimeConfig = buildVariantRuntimeConfig(registration, variantSet.name, variant, suiteDefinitionName);
+        runs.push(await executeOne(scenarioId, runtimeConfig, suiteBatchId));
+    }
+    return runs;
+}
+function buildVariantRuntimeConfig(registration, variantSetName, variant, suiteDefinitionName) {
+    const runtimeConfig = {
+        ...registration,
+        agentName: registration.name,
+        label: registration.label ?? variant.label,
+        variantSetName,
+        variantLabel: variant.label,
+        promptVersion: variant.prompt_version,
+        modelVersion: variant.model_version,
+        toolSchemaVersion: variant.tool_schema_version,
+        configLabel: variant.config_label,
+        suiteDefinitionName,
+    };
+    runtimeConfig.configHash = createConfigHash({
+        provider: runtimeConfig.provider,
+        agentName: runtimeConfig.agentName,
+        label: runtimeConfig.label,
+        model: runtimeConfig.model,
+        command: runtimeConfig.command,
+        args: runtimeConfig.args ?? [],
+        variantSetName,
+        variantLabel: variant.label,
+        promptVersion: variant.prompt_version,
+        modelVersion: variant.model_version,
+        toolSchemaVersion: variant.tool_schema_version,
+        configLabel: variant.config_label,
     });
-    bundle.agentVersion = agentVersion;
-    storage.saveRun(bundle);
-    printRunSummary(bundle);
-    return bundle;
+    return runtimeConfig;
+}
+export async function executeConversation(scenarioId, httpConfig, label, suiteBatchId) {
+    const [{ Storage }, { loadConversationScenarioById }, { runConversation }, { createAgentVersionId }] = await Promise.all([
+        import("./storage.js"),
+        import("./scenarios.js"),
+        import("./conversationRunner.js"),
+        import("./lib/id.js"),
+    ]);
+    const storage = new Storage();
+    try {
+        const loaded = loadConversationScenarioById(scenarioId);
+        storage.upsertScenario({
+            id: loaded.definition.id,
+            name: loaded.definition.name,
+            suite: loaded.definition.suite,
+            difficulty: loaded.definition.difficulty,
+            description: loaded.definition.description,
+        }, loaded.definition, loaded.filePath, loaded.fileHash);
+        const agentLabel = label ?? httpConfig.label ?? httpConfig.name;
+        const agentConfig = { provider: "http", url: httpConfig.url, agentName: httpConfig.name };
+        const agentVersion = {
+            id: createAgentVersionId(agentLabel, agentConfig),
+            label: agentLabel,
+            provider: "http",
+            config: agentConfig,
+        };
+        storage.upsertAgentVersion(agentVersion);
+        const bundle = await runConversation({
+            httpConfig,
+            agentVersion,
+            scenario: loaded.definition,
+            scenarioFileHash: loaded.fileHash,
+        });
+        bundle.run.suiteBatchId = suiteBatchId;
+        bundle.agentVersion = agentVersion;
+        storage.saveRun(bundle);
+        printConversationSummary(bundle, httpConfig.url, loaded.definition.steps.length);
+        return bundle;
+    }
+    finally {
+        storage.close();
+    }
+}
+function printSuiteSummary(suite, runs, suiteBatchId) {
+    const passed = runs.filter((bundle) => bundle.run.status === "pass").length;
+    const failed = runs.filter((bundle) => bundle.run.status === "fail").length;
+    const errored = runs.filter((bundle) => bundle.run.status === "error").length;
+    const avgScore = Math.round(runs.reduce((sum, bundle) => sum + bundle.run.score, 0) / runs.length);
+    console.log(`Suite: ${suite}`);
+    console.log(`Passed: ${passed}/${runs.length}`);
+    console.log(`Failed: ${failed}/${runs.length}`);
+    console.log(`Errored: ${errored}/${runs.length}`);
+    console.log(`Average score: ${avgScore}`);
+    console.log(`Suite batch: ${suiteBatchId}`);
+}
+function printConversationSummary(bundle, agentUrl, totalSteps) {
+    const statusLabel = bundle.run.status.toUpperCase();
+    console.log(`run ${bundle.run.scenarioId} — ${statusLabel}`);
+    console.log(`  agent: ${bundle.agentVersion?.label ?? bundle.run.agentVersionId} (${agentUrl})`);
+    console.log(`  turns completed: ${bundle.run.totalSteps}/${totalSteps}`);
+    const stepEvals = bundle.evaluatorResults.filter((r) => r.evaluatorId.startsWith("step_"));
+    const stepIndices = new Set(stepEvals.map((r) => {
+        const match = r.evaluatorId.match(/^step_(\d+)_/);
+        return match ? parseInt(match[1], 10) : -1;
+    }));
+    for (const stepIndex of [...stepIndices].sort((a, b) => a - b)) {
+        const resultsForStep = stepEvals.filter((r) => r.evaluatorId.startsWith(`step_${stepIndex}_`));
+        const allPass = resultsForStep.every((r) => r.status === "pass");
+        const stepStatus = allPass ? "pass" : "FAIL";
+        const details = resultsForStep.map((r) => {
+            if (r.evaluatorType === "response_latency_max") {
+                const latencyMatch = r.message.match(/(\d+)ms/);
+                return latencyMatch ? `latency ${latencyMatch[1]}ms ✓` : r.message;
+            }
+            return `${r.evaluatorType} ${r.status === "pass" ? "✓" : "✗"}`;
+        });
+        console.log(`  step ${stepIndex + 1}: ${stepStatus}${details.length > 0 ? ` (${details.join(", ")})` : ""}`);
+    }
+    if (bundle.run.status !== "pass") {
+        console.log(`  run stopped (${bundle.run.terminationReason})`);
+    }
+    console.log(`  run id: ${bundle.run.id}`);
 }
 async function handleUi() {
     const { startUiServer } = await import("./ui/server.js");
@@ -145,6 +357,9 @@ function printRunSummary(bundle) {
     if (bundle.agentVersion?.command) {
         console.log(`Command: ${bundle.agentVersion.command} ${(bundle.agentVersion.args ?? []).join(" ")}`.trim());
     }
+    for (const line of formatRunIdentityLines(bundle)) {
+        console.log(line);
+    }
     console.log(`Runtime: ${bundle.run.durationMs}ms`);
     if (bundle.run.status !== "pass") {
         console.log(`Reason: ${bundle.run.terminationReason}`);
@@ -152,6 +367,13 @@ function printRunSummary(bundle) {
         if (errorDetail) {
             console.log(`Error: ${errorDetail}`);
         }
+        const failedEvaluators = getFailedEvaluatorSummaries(bundle);
+        if (failedEvaluators.length > 0) {
+            console.log("Failed evaluators:");
+            for (const summary of failedEvaluators) {
+                console.log(`- ${summary}`);
+            }
+        }
     }
 }
 async function handleShow(args) {
@@ -161,69 +383,129 @@ async function handleShow(args) {
     }
     const { Storage } = await import("./storage.js");
     const storage = new Storage();
-    const bundle = storage.getRun(runId);
-    if (!bundle) {
-        throw new Error(`Run '${runId}' not found.`);
-    }
-    console.log(`Run: ${bundle.run.id}`);
-    console.log(`Scenario: ${bundle.run.scenarioId}`);
-    console.log(`Status: ${bundle.run.status.toUpperCase()}`);
-    console.log(`Score: ${bundle.run.score}/100`);
-    if (bundle.agentVersion) {
-        console.log(`Provider: ${bundle.agentVersion.provider ?? "unknown"}`);
-        console.log(`Model: ${bundle.agentVersion.modelId ?? "unknown"}`);
-        if (bundle.agentVersion.command) {
-            console.log(`Command: ${bundle.agentVersion.command} ${(bundle.agentVersion.args ?? []).join(" ")}`.trim());
+    try {
+        const bundle = storage.getRun(runId);
+        if (!bundle) {
+            throw new Error(`Run '${runId}' not found.`);
+        }
+        console.log(`Run: ${bundle.run.id}`);
+        console.log(`Scenario: ${bundle.run.scenarioId}`);
+        console.log(`Status: ${bundle.run.status.toUpperCase()}`);
+        console.log(`Score: ${bundle.run.score}/100`);
+        if (bundle.agentVersion) {
+            console.log(`Provider: ${bundle.agentVersion.provider ?? "unknown"}`);
+            console.log(`Model: ${bundle.agentVersion.modelId ?? "unknown"}`);
+            if (bundle.agentVersion.command) {
+                console.log(`Command: ${bundle.agentVersion.command} ${(bundle.agentVersion.args ?? []).join(" ")}`.trim());
+            }
+        }
+        console.log(`Termination: ${bundle.run.terminationReason}`);
+        const errorDetail = getRunErrorDetail(bundle);
+        if (errorDetail) {
+            console.log(`Error: ${errorDetail}`);
+        }
+        console.log(`Final output: ${bundle.run.finalOutput}`);
+        console.log("Evaluators:");
+        for (const result of bundle.evaluatorResults) {
+            console.log(`- ${result.evaluatorId}: ${result.status.toUpperCase()} - ${result.message}`);
         }
     }
-    console.log(`Termination: ${bundle.run.terminationReason}`);
-    const errorDetail = getRunErrorDetail(bundle);
-    if (errorDetail) {
-        console.log(`Error: ${errorDetail}`);
-    }
-    console.log(`Final output: ${bundle.run.finalOutput}`);
-    console.log("Evaluators:");
-    for (const result of bundle.evaluatorResults) {
-        console.log(`- ${result.evaluatorId}: ${result.status.toUpperCase()} - ${result.message}`);
+    finally {
+        storage.close();
     }
 }
 async function handleCompare(args) {
-    const [baselineRunId, candidateRunId] = args;
-    if (!baselineRunId || !candidateRunId) {
-        throw new Error("Missing baseline or candidate run id.");
-    }
+    const isSuiteCompare = args[0] === "--suite";
     const { Storage } = await import("./storage.js");
     const storage = new Storage();
-    const comparison = storage.compareRuns(baselineRunId, candidateRunId);
-    console.log(`Scenario: ${comparison.baseline.run.scenarioId}`);
-    console.log(`Baseline: ${comparison.baseline.run.id} (${comparison.baseline.run.status.toUpperCase()} ${comparison.baseline.run.score}/100)`);
-    console.log(`Candidate: ${comparison.candidate.run.id} (${comparison.candidate.run.status.toUpperCase()} ${comparison.candidate.run.score}/100)`);
-    console.log("Changes:");
-    if (comparison.notes.length === 0) {
-        console.log("- No material changes.");
-    }
-    else {
-        for (const note of comparison.notes) {
-            console.log(`- ${note}`);
+    try {
+        if (isSuiteCompare) {
+            const baselineBatchId = args[1];
+            const candidateBatchId = args[2];
+            if (!baselineBatchId || !candidateBatchId) {
+                throw new Error("Missing baseline or candidate suite batch id.");
+            }
+            const comparison = storage.compareSuites(baselineBatchId, candidateBatchId);
+            console.log(`Suite: ${comparison.suite}`);
+            console.log(`Baseline batch: ${comparison.baselineBatchId}`);
+            console.log(`Candidate batch: ${comparison.candidateBatchId}`);
+            console.log(`Classification: ${comparison.classification.toUpperCase()}`);
+            console.log(`Pass delta: ${signedMetric(comparison.deltas.pass)}`);
+            console.log(`Fail delta: ${signedMetric(comparison.deltas.fail)}`);
+            console.log(`Error delta: ${signedMetric(comparison.deltas.error)}`);
+            console.log(`Average score delta: ${signedMetric(comparison.deltas.averageScore)}`);
+            console.log(`Average runtime delta: ${signedMetric(comparison.deltas.averageRuntimeMs)}ms`);
+            console.log(`Average steps delta: ${signedMetric(comparison.deltas.averageSteps)}`);
+            if (comparison.notes.length > 0) {
+                console.log("Notes:");
+                for (const note of comparison.notes) {
+                    console.log(`- ${note}`);
+                }
+            }
+            if (comparison.regressions.length > 0) {
+                console.log("Regressions:");
+                for (const regression of comparison.regressions) {
+                    console.log(`- ${regression.scenarioId}: ${regression.comparison.classification}`);
+                }
+            }
+            if (comparison.improvements.length > 0) {
+                console.log("Improvements:");
+                for (const improvement of comparison.improvements) {
+                    console.log(`- ${improvement.scenarioId}: ${improvement.comparison.classification}`);
+                }
+            }
+            if (comparison.missingFromCandidate.length > 0) {
+                console.log(`Missing from candidate: ${comparison.missingFromCandidate.join(", ")}`);
+            }
+            if (comparison.missingFromBaseline.length > 0) {
+                console.log(`Missing from baseline: ${comparison.missingFromBaseline.join(", ")}`);
+            }
+            return;
         }
-    }
-    if (comparison.evaluatorDiffs.length > 0) {
-        console.log("Evaluator diffs:");
-        for (const diff of comparison.evaluatorDiffs) {
-            console.log(`- ${diff.note}`);
+        const [baselineRunId, candidateRunId] = args;
+        if (!baselineRunId || !candidateRunId) {
+            throw new Error("Missing baseline or candidate run id.");
         }
-    }
-    if (comparison.toolDiffs.length > 0) {
-        console.log("Tool diffs:");
-        for (const diff of comparison.toolDiffs) {
-            console.log(`- ${diff.note}`);
+        const comparison = storage.compareRuns(baselineRunId, candidateRunId);
+        console.log(`Scenario: ${comparison.baseline.run.scenarioId}`);
+        console.log(`Baseline: ${comparison.baseline.run.id} (${comparison.baseline.run.status.toUpperCase()} ${comparison.baseline.run.score}/100)`);
+        console.log(`Candidate: ${comparison.candidate.run.id} (${comparison.candidate.run.status.toUpperCase()} ${comparison.candidate.run.score}/100)`);
+        console.log(`Classification: ${comparison.classification.toUpperCase()}`);
+        console.log("Changes:");
+        if (comparison.notes.length === 0) {
+            console.log("- No material changes.");
+        }
+        else {
+            for (const note of comparison.notes) {
+                console.log(`- ${note}`);
+            }
+        }
+        if (comparison.evaluatorDiffs.length > 0) {
+            console.log("Evaluator diffs:");
+            for (const diff of comparison.evaluatorDiffs) {
+                console.log(`- ${diff.note}`);
+            }
         }
+        if (comparison.toolDiffs.length > 0) {
+            console.log("Tool diffs:");
+            for (const diff of comparison.toolDiffs) {
+                console.log(`- ${diff.note}`);
+            }
+        }
+    }
+    finally {
+        storage.close();
     }
 }
+function signedMetric(value) {
+    return value > 0 ? `+${value}` : `${value}`;
+}
 function parseRunArgs(args) {
     const runtimeConfig = { provider: "mock" };
     let scenarioId;
     let suite;
+    let suiteDefinition;
+    let variantSetName;
     for (let index = 0; index < args.length; index += 1) {
         const arg = args[index];
         if (arg === "--suite") {
@@ -231,9 +513,19 @@ function parseRunArgs(args) {
             index += 1;
             continue;
         }
+        if (arg === "--suite-def") {
+            suiteDefinition = args[index + 1];
+            index += 1;
+            continue;
+        }
+        if (arg === "--variant-set") {
+            variantSetName = args[index + 1];
+            index += 1;
+            continue;
+        }
         if (arg === "--provider") {
             const provider = args[index + 1];
-            if (provider !== "mock" && provider !== "openai" && provider !== "external_process") {
+            if (provider !== "mock" && provider !== "openai" && provider !== "external_process" && provider !== "http") {
                 throw new Error(`Unsupported provider '${String(provider)}'.`);
             }
             runtimeConfig.provider = provider;
@@ -261,17 +553,26 @@ function parseRunArgs(args) {
         }
         throw new Error(`Unexpected argument '${arg}'.`);
     }
-    return { scenarioId, suite, runtimeConfig };
+    return { scenarioId, suite, suiteDefinition, variantSetName, runtimeConfig };
 }
 function validateRuntimeConfig(config) {
     if (config.agentName) {
         const registration = getAgentRegistration(config.agentName);
         config.provider = registration.provider;
-        config.model = config.model ?? registration.model;
         config.label = config.label ?? registration.label ?? registration.name;
-        config.command = registration.command;
-        config.args = registration.args;
-        config.envAllowlist = registration.envAllowlist;
+        if (registration.provider !== "http") {
+            config.model = config.model ?? registration.model;
+            config.command = registration.command;
+            config.args = registration.args;
+            config.envAllowlist = registration.envAllowlist;
+        }
+        else {
+            config.url = registration.url;
+            config.request_template = registration.request_template;
+            config.response_field = registration.response_field;
+            config.headers = registration.headers;
+            config.timeout_ms = registration.timeout_ms;
+        }
     }
     if (config.provider === "openai") {
         if (!process.env.OPENAI_API_KEY) {
@@ -288,9 +589,25 @@ function validateRuntimeConfig(config) {
         }
         config.label = config.label ?? config.agentName ?? "external-process-agent";
     }
+    if (config.provider === "http") {
+        if (!config.url) {
+            throw new Error("HTTP agents require a configured url. Use --agent <name> with provider: http in agentlab.config.yaml.");
+        }
+        config.label = config.label ?? config.agentName ?? "http-agent";
+    }
     return config;
 }
-main().catch((error) => {
-    console.error(error instanceof Error ? error.message : String(error));
-    process.exitCode = 1;
-});
+if (isEntrypoint()) {
+    main().catch((error) => {
+        const message = error instanceof Error ? error.message : String(error);
+        console.error(formatCliErrorMessage(message));
+        process.exitCode = 1;
+    });
+}
+function isEntrypoint() {
+    const entry = process.argv[1];
+    if (!entry) {
+        return false;
+    }
+    return import.meta.url === pathToFileURL(entry).href;
+}