agent-regression-lab 0.1.0

package/dist/lib/id.js

@@ -0,0 +1,16 @@
+import { createHash, randomUUID } from "node:crypto";
+export function hashText(text) {
+    return createHash("sha256").update(text).digest("hex");
+}
+export function createRunId() {
+    return `run_${Date.now()}`;
+}
+export function createEventId() {
+    return `evt_${randomUUID()}`;
+}
+export function createToolCallId() {
+    return `tool_${randomUUID()}`;
+}
+export function createAgentVersionId(label, config) {
+    return `agent_${hashText(`${label}:${JSON.stringify(config)}`).slice(0, 12)}`;
+}

package/dist/runOutput.js

@@ -0,0 +1,13 @@
+export function getRunErrorDetail(bundle) {
+    for (const event of [...bundle.traceEvents].reverse()) {
+        if (event.type === "agent_error") {
+            const message = event.payload.message;
+            return typeof message === "string" ? message : undefined;
+        }
+        if (event.type === "tool_call_failed") {
+            const error = event.payload.error;
+            return typeof error === "string" ? error : undefined;
+        }
+    }
+    return undefined;
+}

package/dist/runner.js

@@ -0,0 +1,199 @@
+import { performance } from "node:perf_hooks";
+import { createToolCallId, createRunId } from "./lib/id.js";
+import { evaluateScenario } from "./evaluators.js";
+import { computeScore } from "./scoring.js";
+import { TraceRecorder } from "./trace.js";
+export async function runScenario(deps) {
+    const runId = createRunId();
+    const startedAt = new Date().toISOString();
+    const runStart = performance.now();
+    const trace = new TraceRecorder(runId, deps.scenario.id);
+    const toolCalls = [];
+    const maxSteps = deps.scenario.runtime?.max_steps ?? 8;
+    const timeoutSeconds = deps.scenario.runtime?.timeout_seconds;
+    const deadline = timeoutSeconds ? Date.now() + timeoutSeconds * 1000 : undefined;
+    trace.record("runner", "run_started", {
+        agentVersionId: deps.agentVersion.id,
+        provider: deps.agentVersion.provider ?? "unknown",
+        modelId: deps.agentVersion.modelId ?? "unknown",
+        command: deps.agentVersion.command,
+        args: deps.agentVersion.args,
+        scenarioVersionHash: deps.scenarioFileHash,
+        maxSteps,
+        timeoutSeconds,
+    });
+    const availableTools = deps.toolSpecs.filter((tool) => deps.scenario.tools.allowed.includes(tool.name));
+    const session = await deps.agentAdapter.startRun({
+        instructions: deps.scenario.task.instructions,
+        availableTools,
+        context: deps.scenario.context?.variables ?? {},
+        maxSteps,
+        metadata: {
+            scenarioId: deps.scenario.id,
+            provider: deps.agentVersion.provider,
+            model: deps.agentVersion.modelId,
+        },
+    });
+    let finalOutput = "";
+    let terminationReason = "completed";
+    let status = "pass";
+    let loopCount = 0;
+    let event = {
+        type: "run_started",
+    };
+    while (loopCount < maxSteps) {
+        if (hasTimedOut(deadline)) {
+            status = "error";
+            terminationReason = "timeout_exceeded";
+            trace.record("runner", "timeout_exceeded", { timeoutSeconds });
+            break;
+        }
+        loopCount += 1;
+        trace.record("agent", "agent_turn_started", { loopCount });
+        const turn = await raceWithTimeout(session.next(event), deadline, "Agent turn timed out.");
+        if (turn.type === "error") {
+            status = "error";
+            terminationReason = "agent_error";
+            trace.record("agent", "agent_error", { message: turn.message });
+            break;
+        }
+        if (turn.type === "final") {
+            finalOutput = turn.output;
+            trace.record("agent", "agent_final_output", { output: turn.output, metadata: turn.metadata ?? {} });
+            break;
+        }
+        const toolName = turn.toolName;
+        const toolCallId = createToolCallId();
+        trace.record("agent", "agent_message", { content: String(turn.metadata?.message ?? `Requesting ${toolName}`) });
+        trace.record("agent", "tool_call_requested", { toolCallId, toolName, input: turn.input });
+        if (!deps.scenario.tools.allowed.includes(toolName) || deps.scenario.tools.forbidden?.includes(toolName)) {
+            status = "fail";
+            terminationReason = "forbidden_tool_used";
+            trace.record("runner", "forbidden_tool_attempted", { toolName });
+            break;
+        }
+        const handler = deps.tools[toolName];
+        if (!handler) {
+            status = "error";
+            terminationReason = "tool_error";
+            trace.record("tool", "tool_call_failed", { toolCallId, toolName, error: "Tool handler missing" });
+            break;
+        }
+        const started = performance.now();
+        trace.record("tool", "tool_call_started", { toolCallId, toolName, input: turn.input });
+        try {
+            const result = await raceWithTimeout(handler(turn.input, { scenarioId: deps.scenario.id }), deadline, `Tool '${toolName}' timed out.`);
+            const durationMs = Math.round(performance.now() - started);
+            toolCalls.push({
+                id: toolCallId,
+                stepIndex: trace.getStepCount() + 1,
+                toolName,
+                input: turn.input,
+                output: result,
+                status: "pass",
+                durationMs,
+            });
+            trace.record("tool", "tool_call_completed", { toolCallId, toolName, input: turn.input, output: result, durationMs });
+            event = { type: "tool_result", toolName, result };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            if (deadline && Date.now() >= deadline) {
+                status = "error";
+                terminationReason = "timeout_exceeded";
+                trace.record("runner", "timeout_exceeded", { timeoutSeconds, message });
+            }
+            else {
+                status = "error";
+                terminationReason = "tool_error";
+            }
+            toolCalls.push({
+                id: toolCallId,
+                stepIndex: trace.getStepCount() + 1,
+                toolName,
+                input: turn.input,
+                status: "fail",
+                errorMessage: message,
+            });
+            trace.record("tool", "tool_call_failed", { toolCallId, toolName, error: message });
+            break;
+        }
+    }
+    if (!finalOutput && status !== "error" && loopCount >= maxSteps) {
+        status = "fail";
+        terminationReason = "step_limit_exceeded";
+        trace.record("runner", "step_budget_exceeded", { maxSteps });
+    }
+    const finishedAt = new Date().toISOString();
+    const durationMs = Math.round(performance.now() - runStart);
+    const run = {
+        id: runId,
+        scenarioId: deps.scenario.id,
+        scenarioFileHash: deps.scenarioFileHash,
+        agentVersionId: deps.agentVersion.id,
+        status,
+        terminationReason,
+        finalOutput,
+        totalSteps: trace.getStepCount(),
+        totalToolCalls: toolCalls.length,
+        durationMs,
+        score: 0,
+        startedAt,
+        finishedAt,
+    };
+    let bundle = {
+        run,
+        traceEvents: trace.getEvents(),
+        toolCalls,
+        evaluatorResults: [],
+    };
+    trace.record("evaluator", "evaluation_started", {});
+    const evaluatorResults = evaluateScenario(bundle, deps.scenario.evaluators);
+    for (const result of evaluatorResults) {
+        trace.record("evaluator", "evaluation_result", {
+            evaluatorId: result.evaluatorId,
+            status: result.status,
+            message: result.message,
+        });
+    }
+    trace.record("evaluator", "evaluation_finished", {});
+    const finalScoring = computeScore(evaluatorResults);
+    run.score = finalScoring.score;
+    if (run.status !== "error") {
+        run.status = finalScoring.status;
+        if (run.status === "fail" && terminationReason === "completed") {
+            run.terminationReason = "evaluator_failed";
+        }
+    }
+    trace.record("runner", "run_finished", {
+        status: run.status,
+        terminationReason: run.terminationReason,
+        totalSteps: run.totalSteps,
+        durationMs: run.durationMs,
+    });
+    bundle = {
+        run,
+        traceEvents: trace.getEvents(),
+        toolCalls,
+        evaluatorResults,
+    };
+    return bundle;
+}
+function hasTimedOut(deadline) {
+    return deadline !== undefined && Date.now() >= deadline;
+}
+async function raceWithTimeout(promise, deadline, message) {
+    if (deadline === undefined) {
+        return promise;
+    }
+    const remainingMs = deadline - Date.now();
+    if (remainingMs <= 0) {
+        throw new Error(message);
+    }
+    return await Promise.race([
+        promise,
+        new Promise((_, reject) => {
+            setTimeout(() => reject(new Error(message)), remainingMs);
+        }),
+    ]);
+}

package/dist/scenarios.js

@@ -0,0 +1,155 @@
+import { readFileSync, readdirSync, statSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { join, relative, resolve } from "node:path";
+import { parse } from "yaml";
+import { loadAgentLabConfig } from "./config.js";
+import { getBuiltinToolSpecs } from "./tools.js";
+const SCENARIOS_ROOT = resolve("scenarios");
+export function listScenarioFiles(root = SCENARIOS_ROOT) {
+    if (!safeExists(root)) {
+        return [];
+    }
+    const results = [];
+    function walk(dir) {
+        const entries = readdirSync(dir, { withFileTypes: true });
+        for (const entry of entries) {
+            const fullPath = join(dir, entry.name);
+            if (entry.isDirectory()) {
+                walk(fullPath);
+            }
+            else if (entry.isFile() && (entry.name.endsWith(".yaml") || entry.name.endsWith(".yml"))) {
+                results.push(fullPath);
+            }
+        }
+    }
+    walk(root);
+    return results.sort();
+}
+export function listScenarios() {
+    return listScenarioFiles().map((filePath) => {
+        const { definition } = loadScenarioByPath(filePath, getKnownToolNames());
+        return {
+            id: definition.id,
+            name: definition.name,
+            suite: definition.suite,
+            difficulty: definition.difficulty,
+            description: definition.description,
+        };
+    });
+}
+export function loadScenarioById(scenarioId) {
+    for (const filePath of listScenarioFiles()) {
+        const loaded = loadScenarioByPath(filePath, getKnownToolNames());
+        if (loaded.definition.id === scenarioId) {
+            return loaded;
+        }
+    }
+    throw new Error(`Scenario '${scenarioId}' not found.`);
+}
+export function loadScenariosBySuite(suite) {
+    return listScenarioFiles()
+        .map((filePath) => loadScenarioByPath(filePath, getKnownToolNames()))
+        .filter(({ definition }) => definition.suite === suite);
+}
+export function loadScenarioByPath(filePath, knownToolNames = getKnownToolNames()) {
+    const absolutePath = resolve(filePath);
+    const raw = readFileSync(absolutePath, "utf8");
+    const parsed = parse(raw);
+    validateScenario(parsed, absolutePath, knownToolNames);
+    return {
+        definition: parsed,
+        filePath: relative(process.cwd(), absolutePath),
+        fileHash: createHash("sha256").update(raw).digest("hex"),
+    };
+}
+function validateScenario(value, filePath, knownToolNames) {
+    if (!isObject(value)) {
+        throw new Error(`Scenario file '${filePath}' must contain a YAML object.`);
+    }
+    const requiredStrings = [
+        ["id", value.id],
+        ["name", value.name],
+        ["suite", value.suite],
+    ];
+    for (const [field, candidate] of requiredStrings) {
+        if (typeof candidate !== "string" || candidate.length === 0) {
+            throw new Error(`Scenario file '${filePath}' is missing required string field '${field}'.`);
+        }
+    }
+    if (!isObject(value.task) || typeof value.task.instructions !== "string" || value.task.instructions.length === 0) {
+        throw new Error(`Scenario file '${filePath}' must define task.instructions.`);
+    }
+    if (!isObject(value.tools) || !Array.isArray(value.tools.allowed) || value.tools.allowed.length === 0) {
+        throw new Error(`Scenario file '${filePath}' must define at least one allowed tool.`);
+    }
+    for (const toolName of value.tools.allowed) {
+        if (typeof toolName !== "string") {
+            throw new Error(`Scenario file '${filePath}' contains a non-string tool name in tools.allowed.`);
+        }
+        if (!knownToolNames.has(toolName)) {
+            throw new Error(`Scenario file '${filePath}' references unknown allowed tool '${toolName}'.`);
+        }
+    }
+    if (Array.isArray(value.tools.forbidden)) {
+        for (const toolName of value.tools.forbidden) {
+            if (typeof toolName !== "string") {
+                throw new Error(`Scenario file '${filePath}' contains a non-string tool name in tools.forbidden.`);
+            }
+        }
+    }
+    if (!Array.isArray(value.evaluators) || value.evaluators.length === 0) {
+        throw new Error(`Scenario file '${filePath}' must define at least one evaluator.`);
+    }
+    const evaluatorIds = new Set();
+    for (const evaluator of value.evaluators) {
+        if (!isObject(evaluator) || typeof evaluator.id !== "string" || typeof evaluator.type !== "string") {
+            throw new Error(`Scenario file '${filePath}' has an invalid evaluator entry.`);
+        }
+        if (evaluatorIds.has(evaluator.id)) {
+            throw new Error(`Scenario file '${filePath}' defines duplicate evaluator id '${evaluator.id}'.`);
+        }
+        evaluatorIds.add(evaluator.id);
+    }
+    if (isObject(value.runtime)) {
+        validatePositiveInt(value.runtime.max_steps, "runtime.max_steps", filePath);
+        validatePositiveInt(value.runtime.timeout_seconds, "runtime.timeout_seconds", filePath);
+    }
+    if (isObject(value.context) && Array.isArray(value.context.fixtures)) {
+        for (const fixturePath of value.context.fixtures) {
+            if (typeof fixturePath !== "string") {
+                throw new Error(`Scenario file '${filePath}' contains a non-string fixture path.`);
+            }
+            const resolvedPath = resolve(fixturePath);
+            if (!safeExists(resolvedPath) || !statSync(resolvedPath).isFile()) {
+                throw new Error(`Scenario file '${filePath}' references missing fixture '${fixturePath}'.`);
+            }
+        }
+    }
+}
+function validatePositiveInt(value, field, filePath) {
+    if (value === undefined) {
+        return;
+    }
+    if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
+        throw new Error(`Scenario file '${filePath}' field '${field}' must be a positive integer.`);
+    }
+}
+function isObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function safeExists(path) {
+    try {
+        statSync(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function getKnownToolNames() {
+    const names = new Set(getBuiltinToolSpecs().map((tool) => tool.name));
+    for (const tool of loadAgentLabConfig().tools ?? []) {
+        names.add(tool.name);
+    }
+    return names;
+}

package/dist/scoring.js

@@ -0,0 +1,18 @@
+const PASS_THRESHOLD = 80;
+export function computeScore(results) {
+    const hardGateFailure = results.some((result) => result.mode === "hard_gate" && result.status === "fail");
+    const weighted = results.filter((result) => result.mode === "weighted");
+    let score = 100;
+    if (weighted.length > 0) {
+        const totalWeight = weighted.reduce((sum, result) => sum + (result.weight ?? 0), 0);
+        const earnedWeight = weighted.reduce((sum, result) => {
+            const weight = result.weight ?? 0;
+            return sum + (result.status === "pass" ? weight : 0);
+        }, 0);
+        score = totalWeight === 0 ? 100 : Math.round((earnedWeight / totalWeight) * 100);
+    }
+    if (hardGateFailure) {
+        return { score, status: "fail" };
+    }
+    return { score, status: score >= PASS_THRESHOLD ? "pass" : "fail" };
+}