agent-regression-lab 0.1.1 → 0.3.0

package/dist/lib/id.js

@@ -5,6 +5,9 @@ export function hashText(text) {
 export function createRunId() {
     return `run_${Date.now()}`;
 }
+export function createSuiteBatchId() {
+    return `suite_${Date.now()}_${randomUUID().slice(0, 8)}`;
+}
 export function createEventId() {
     return `evt_${randomUUID()}`;
 }
@@ -14,3 +17,6 @@ export function createToolCallId() {
 export function createAgentVersionId(label, config) {
     return `agent_${hashText(`${label}:${JSON.stringify(config)}`).slice(0, 12)}`;
 }
+export function createConfigHash(input) {
+    return createAgentVersionId("config", input);
+}

package/dist/runOutput.js

@@ -1,5 +1,11 @@
 export function getRunErrorDetail(bundle) {
     for (const event of [...bundle.traceEvents].reverse()) {
+        if (event.type === "conversation_finished") {
+            const errorMessage = event.payload.errorMessage;
+            if (typeof errorMessage === "string") {
+                return errorMessage;
+            }
+        }
         if (event.type === "agent_error") {
             const message = event.payload.message;
             return typeof message === "string" ? message : undefined;
@@ -11,3 +17,43 @@ export function getRunErrorDetail(bundle) {
     }
     return undefined;
 }
+export function formatCliErrorMessage(message) {
+    if (message.includes("database is locked")) {
+        return "SQLite database is locked. Retry the run sequentially or wait for the current run to finish.";
+    }
+    return message;
+}
+export function getFailedEvaluatorSummaries(bundle) {
+    return bundle.evaluatorResults
+        .filter((result) => result.status === "fail")
+        .map((result) => `${result.evaluatorId}: ${result.message}`);
+}
+export function formatRunIdentityLines(bundle) {
+    const lines = [];
+    const run = bundle.run;
+    if (run.variantSetName) {
+        lines.push(`Variant set: ${run.variantSetName}`);
+    }
+    if (run.variantLabel) {
+        lines.push(`Variant: ${run.variantLabel}`);
+    }
+    if (run.promptVersion) {
+        lines.push(`Prompt version: ${run.promptVersion}`);
+    }
+    if (run.modelVersion) {
+        lines.push(`Model version: ${run.modelVersion}`);
+    }
+    if (run.toolSchemaVersion) {
+        lines.push(`Tool schema version: ${run.toolSchemaVersion}`);
+    }
+    if (run.configLabel) {
+        lines.push(`Config label: ${run.configLabel}`);
+    }
+    if (run.runtimeProfileName) {
+        lines.push(`Runtime profile: ${run.runtimeProfileName}`);
+    }
+    if (run.suiteDefinitionName) {
+        lines.push(`Suite definition: ${run.suiteDefinitionName}`);
+    }
+    return lines;
+}

package/dist/runner.js

@@ -1,7 +1,9 @@
 import { performance } from "node:perf_hooks";
+import { getRuntimeProfile } from "./config.js";
 import { createToolCallId, createRunId } from "./lib/id.js";
 import { evaluateScenario } from "./evaluators.js";
 import { computeScore } from "./scoring.js";
+import { applyRuntimeProfileToTools } from "./tools.js";
 import { TraceRecorder } from "./trace.js";
 export async function runScenario(deps) {
     const runId = createRunId();
@@ -9,6 +11,8 @@ export async function runScenario(deps) {
     const runStart = performance.now();
     const trace = new TraceRecorder(runId, deps.scenario.id);
     const toolCalls = [];
+    const runtimeProfile = deps.scenario.runtime_profile ? getRuntimeProfile(deps.scenario.runtime_profile) : undefined;
+    const tools = applyRuntimeProfileToTools(deps.tools, runtimeProfile, trace);
     const maxSteps = deps.scenario.runtime?.max_steps ?? 8;
     const timeoutSeconds = deps.scenario.runtime?.timeout_seconds;
     const deadline = timeoutSeconds ? Date.now() + timeoutSeconds * 1000 : undefined;
@@ -22,6 +26,9 @@ export async function runScenario(deps) {
         maxSteps,
         timeoutSeconds,
     });
+    trace.record("system", "runtime_profile_applied", {
+        name: runtimeProfile?.name ?? null,
+    }, { countStep: false });
     const availableTools = deps.toolSpecs.filter((tool) => deps.scenario.tools.allowed.includes(tool.name));
     const session = await deps.agentAdapter.startRun({
         instructions: deps.scenario.task.instructions,
@@ -72,7 +79,7 @@ export async function runScenario(deps) {
             trace.record("runner", "forbidden_tool_attempted", { toolName });
             break;
         }
-        const handler = deps.tools[toolName];
+        const handler = tools[toolName];
         if (!handler) {
             status = "error";
             terminationReason = "tool_error";
@@ -98,7 +105,8 @@ export async function runScenario(deps) {
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);
-            if (deadline && Date.now() >= deadline) {
+            const isInjectedTimeout = error instanceof Error && error.code === "timeout_exceeded";
+            if (isInjectedTimeout || (deadline && Date.now() >= deadline)) {
                 status = "error";
                 terminationReason = "timeout_exceeded";
                 trace.record("runner", "timeout_exceeded", { timeoutSeconds, message });
@@ -182,18 +190,32 @@ export async function runScenario(deps) {
 function hasTimedOut(deadline) {
     return deadline !== undefined && Date.now() >= deadline;
 }
+function toolRaceTimeoutError(message) {
+    const error = new Error(message);
+    error.code = "timeout_exceeded";
+    return error;
+}
 async function raceWithTimeout(promise, deadline, message) {
     if (deadline === undefined) {
         return promise;
     }
     const remainingMs = deadline - Date.now();
     if (remainingMs <= 0) {
-        throw new Error(message);
+        throw toolRaceTimeoutError(message);
+    }
+    let timeoutHandle;
+    try {
+        return await Promise.race([
+            promise,
+            new Promise((_, reject) => {
+                timeoutHandle = setTimeout(() => reject(toolRaceTimeoutError(message)), remainingMs);
+                timeoutHandle.unref?.();
+            }),
+        ]);
+    }
+    finally {
+        if (timeoutHandle !== undefined) {
+            clearTimeout(timeoutHandle);
+        }
     }
-    return await Promise.race([
-        promise,
-        new Promise((_, reject) => {
-            setTimeout(() => reject(new Error(message)), remainingMs);
-        }),
-    ]);
 }

package/dist/scenarios.js

@@ -2,9 +2,20 @@ import { readFileSync, readdirSync, statSync } from "node:fs";
 import { createHash } from "node:crypto";
 import { join, relative, resolve } from "node:path";
 import { parse } from "yaml";
-import { loadAgentLabConfig } from "./config.js";
+import { getRuntimeProfile, getSuiteDefinition, loadAgentLabConfig } from "./config.js";
 import { getBuiltinToolSpecs } from "./tools.js";
 const SCENARIOS_ROOT = resolve("scenarios");
+const VALID_TASK_EVALUATOR_TYPES = new Set([
+    "exact_final_answer",
+    "final_answer_contains",
+    "forbidden_tool",
+    "tool_call_assertion",
+    "step_count_max",
+    "tool_call_count_max",
+    "tool_repeat_max",
+    "cost_max",
+]);
+const VALID_EVALUATOR_MODES = new Set(["hard_gate", "weighted"]);
 export function listScenarioFiles(root = SCENARIOS_ROOT) {
     if (!safeExists(root)) {
         return [];
@@ -26,19 +37,37 @@ export function listScenarioFiles(root = SCENARIOS_ROOT) {
     return results.sort();
 }
 export function listScenarios() {
-    return listScenarioFiles().map((filePath) => {
-        const { definition } = loadScenarioByPath(filePath, getKnownToolNames());
-        return {
-            id: definition.id,
-            name: definition.name,
-            suite: definition.suite,
-            difficulty: definition.difficulty,
-            description: definition.description,
-        };
+    return listScenarioFiles().flatMap((filePath) => {
+        try {
+            const scenarioType = getScenarioType(filePath);
+            if (scenarioType === "conversation") {
+                const { definition } = loadConversationScenarioByPath(filePath);
+                return [{
+                        id: definition.id,
+                        name: definition.name,
+                        suite: definition.suite,
+                        difficulty: definition.difficulty,
+                        description: definition.description,
+                    }];
+            }
+            const { definition } = loadScenarioByPath(filePath, getKnownToolNames());
+            return [{
+                    id: definition.id,
+                    name: definition.name,
+                    suite: definition.suite,
+                    difficulty: definition.difficulty,
+                    description: definition.description,
+                }];
+        }
+        catch {
+            return [];
+        }
     });
 }
 export function loadScenarioById(scenarioId) {
     for (const filePath of listScenarioFiles()) {
+        if (getScenarioType(filePath) !== "task")
+            continue;
         const loaded = loadScenarioByPath(filePath, getKnownToolNames());
         if (loaded.definition.id === scenarioId) {
             return loaded;
@@ -48,9 +77,24 @@ export function loadScenarioById(scenarioId) {
 }
 export function loadScenariosBySuite(suite) {
     return listScenarioFiles()
+        .filter((filePath) => getScenarioType(filePath) === "task")
         .map((filePath) => loadScenarioByPath(filePath, getKnownToolNames()))
         .filter(({ definition }) => definition.suite === suite);
 }
+export function loadScenariosBySuiteDefinition(name) {
+    const suiteDefinition = getSuiteDefinition(name);
+    const knownToolNames = getKnownToolNames();
+    const scenarioFiles = listScenarioFiles(resolve("scenarios"));
+    const loadedScenarios = scenarioFiles.map((filePath) => loadScenarioRecordByPath(filePath, knownToolNames));
+    const included = loadedScenarios
+        .filter(({ definition }) => matchesSuiteDefinitionInclude(definition, suiteDefinition));
+    const excludedIds = new Set(loadedScenarios
+        .filter(({ definition }) => matchesSuiteDefinitionExclude(definition, suiteDefinition))
+        .map(({ definition }) => definition.id));
+    return included
+        .filter(({ definition }) => !excludedIds.has(definition.id))
+        .sort((left, right) => left.definition.id.localeCompare(right.definition.id));
+}
 export function loadScenarioByPath(filePath, knownToolNames = getKnownToolNames()) {
     const absolutePath = resolve(filePath);
     const raw = readFileSync(absolutePath, "utf8");
@@ -90,7 +134,10 @@ function validateScenario(value, filePath, knownToolNames) {
             throw new Error(`Scenario file '${filePath}' references unknown allowed tool '${toolName}'.`);
         }
     }
-    if (Array.isArray(value.tools.forbidden)) {
+    if (value.tools.forbidden !== undefined) {
+        if (!Array.isArray(value.tools.forbidden)) {
+            throw new Error(`Scenario file '${filePath}' field 'tools.forbidden' must be an array of strings.`);
+        }
         for (const toolName of value.tools.forbidden) {
             if (typeof toolName !== "string") {
                 throw new Error(`Scenario file '${filePath}' contains a non-string tool name in tools.forbidden.`);
@@ -105,6 +152,17 @@ function validateScenario(value, filePath, knownToolNames) {
         if (!isObject(evaluator) || typeof evaluator.id !== "string" || typeof evaluator.type !== "string") {
             throw new Error(`Scenario file '${filePath}' has an invalid evaluator entry.`);
         }
+        if (!VALID_TASK_EVALUATOR_TYPES.has(evaluator.type)) {
+            throw new Error(`Scenario file '${filePath}' evaluator '${evaluator.id}' has invalid type '${evaluator.type}'. ` +
+                `Valid types: ${[...VALID_TASK_EVALUATOR_TYPES].join(", ")}.`);
+        }
+        if (!VALID_EVALUATOR_MODES.has(evaluator.mode)) {
+            throw new Error(`Scenario file '${filePath}' evaluator '${evaluator.id}' has invalid mode '${String(evaluator.mode)}'. ` +
+                `Valid modes: hard_gate, weighted.`);
+        }
+        if (!isObject(evaluator.config)) {
+            throw new Error(`Scenario file '${filePath}' evaluator '${evaluator.id}' must define an object config.`);
+        }
         if (evaluatorIds.has(evaluator.id)) {
             throw new Error(`Scenario file '${filePath}' defines duplicate evaluator id '${evaluator.id}'.`);
         }
@@ -114,6 +172,12 @@ function validateScenario(value, filePath, knownToolNames) {
         validatePositiveInt(value.runtime.max_steps, "runtime.max_steps", filePath);
         validatePositiveInt(value.runtime.timeout_seconds, "runtime.timeout_seconds", filePath);
     }
+    if (value.runtime_profile !== undefined) {
+        if (typeof value.runtime_profile !== "string" || value.runtime_profile.length === 0) {
+            throw new Error(`Scenario file '${filePath}' field 'runtime_profile' must be a non-empty string.`);
+        }
+        getRuntimeProfile(value.runtime_profile);
+    }
     if (isObject(value.context) && Array.isArray(value.context.fixtures)) {
         for (const fixturePath of value.context.fixtures) {
             if (typeof fixturePath !== "string") {
@@ -153,3 +217,139 @@ function getKnownToolNames() {
     }
     return names;
 }
+export function getScenarioType(filePath) {
+    const absolutePath = resolve(filePath);
+    const raw = readFileSync(absolutePath, "utf8");
+    const parsed = parse(raw);
+    if (isObject(parsed) && parsed.type === "conversation") {
+        return "conversation";
+    }
+    return "task";
+}
+export function loadConversationScenarioByPath(filePath) {
+    const absolutePath = resolve(filePath);
+    const raw = readFileSync(absolutePath, "utf8");
+    const parsed = parse(raw);
+    validateConversationScenario(parsed, absolutePath);
+    return {
+        definition: parsed,
+        filePath: relative(process.cwd(), absolutePath),
+        fileHash: createHash("sha256").update(raw).digest("hex"),
+    };
+}
+export function loadConversationScenarioById(scenarioId) {
+    for (const filePath of listScenarioFiles()) {
+        const absolutePath = resolve(filePath);
+        const raw = readFileSync(absolutePath, "utf8");
+        const parsed = parse(raw);
+        if (parsed.type === "conversation" && parsed.id === scenarioId) {
+            return loadConversationScenarioByPath(filePath);
+        }
+    }
+    throw new Error(`Conversation scenario '${scenarioId}' not found.`);
+}
+const VALID_CONVERSATION_EVALUATOR_TYPES = new Set([
+    "response_contains",
+    "response_not_contains",
+    "response_matches_regex",
+    "response_latency_max",
+    "step_count_max",
+    "exact_final_answer",
+    "final_answer_contains",
+]);
+function validateConversationEvaluatorList(evaluators, context, filePath) {
+    if (!Array.isArray(evaluators)) {
+        throw new Error(`Conversation scenario '${filePath}' ${context} evaluators must be an array.`);
+    }
+    for (let i = 0; i < evaluators.length; i += 1) {
+        const ev = evaluators[i];
+        if (!isObject(ev)) {
+            throw new Error(`Conversation scenario '${filePath}' ${context} evaluator ${i} must be an object.`);
+        }
+        if (typeof ev.type !== "string" || !VALID_CONVERSATION_EVALUATOR_TYPES.has(ev.type)) {
+            throw new Error(`Conversation scenario '${filePath}' ${context} evaluator ${i} has invalid type '${String(ev.type)}'. ` +
+                `Valid types: ${[...VALID_CONVERSATION_EVALUATOR_TYPES].join(", ")}.`);
+        }
+        if (ev.mode !== "hard_gate" && ev.mode !== "weighted") {
+            throw new Error(`Conversation scenario '${filePath}' ${context} evaluator ${i} must have mode: hard_gate or weighted.`);
+        }
+        if (ev.type === "response_contains" || ev.type === "response_not_contains") {
+            if (!isObject(ev.config)) {
+                throw new Error(`Conversation scenario '${filePath}' ${context} evaluator ${i} must define an object config.`);
+            }
+            if ("text" in ev.config) {
+                throw new Error(`Conversation scenario '${filePath}' ${context} evaluator ${i} uses stale 'config.text'; use 'config.keywords: string[]'.`);
+            }
+            if (!Array.isArray(ev.config.keywords) || ev.config.keywords.some((kw) => typeof kw !== "string")) {
+                throw new Error(`Conversation scenario '${filePath}' ${context} evaluator ${i} must define config.keywords as a string array.`);
+            }
+        }
+    }
+}
+function validateConversationScenario(value, filePath) {
+    if (!isObject(value)) {
+        throw new Error(`Scenario file '${filePath}' must contain a YAML object.`);
+    }
+    for (const field of ["id", "name", "suite"]) {
+        if (typeof value[field] !== "string" || value[field].length === 0) {
+            throw new Error(`Conversation scenario '${filePath}' is missing required string field '${field}'.`);
+        }
+    }
+    if (value.type !== "conversation") {
+        throw new Error(`Scenario file '${filePath}' does not have type: conversation.`);
+    }
+    if (value.runtime_profile !== undefined) {
+        if (typeof value.runtime_profile !== "string" || value.runtime_profile.length === 0) {
+            throw new Error(`Conversation scenario '${filePath}' field 'runtime_profile' must be a non-empty string.`);
+        }
+        getRuntimeProfile(value.runtime_profile);
+    }
+    if ("tools" in value) {
+        throw new Error(`Conversation scenario '${filePath}' must not define 'tools'. HTTP agents manage their own tools internally.`);
+    }
+    if (!Array.isArray(value.steps) || value.steps.length === 0) {
+        throw new Error(`Conversation scenario '${filePath}' must define at least one step.`);
+    }
+    for (let i = 0; i < value.steps.length; i += 1) {
+        const step = value.steps[i];
+        if (!isObject(step)) {
+            throw new Error(`Conversation scenario '${filePath}' step ${i} must be an object.`);
+        }
+        if (step.role !== "user") {
+            throw new Error(`Conversation scenario '${filePath}' step ${i} must have role: user.`);
+        }
+        if (typeof step.message !== "string" || step.message.length === 0) {
+            throw new Error(`Conversation scenario '${filePath}' step ${i} must have a non-empty message.`);
+        }
+        if (step.evaluators !== undefined) {
+            validateConversationEvaluatorList(step.evaluators, `step ${i}`, filePath);
+        }
+    }
+    if (value.evaluators !== undefined) {
+        validateConversationEvaluatorList(value.evaluators, "end-of-run evaluators", filePath);
+    }
+}
+function loadScenarioRecordByPath(filePath, knownToolNames = getKnownToolNames()) {
+    if (getScenarioType(filePath) === "conversation") {
+        return loadConversationScenarioByPath(filePath);
+    }
+    return loadScenarioByPath(filePath, knownToolNames);
+}
+function matchesSuiteDefinitionInclude(definition, suiteDefinition) {
+    return matchesSuiteDefinitionSelectors(definition, suiteDefinition.include);
+}
+function matchesSuiteDefinitionExclude(definition, suiteDefinition) {
+    return suiteDefinition.exclude !== undefined && matchesSuiteDefinitionSelectors(definition, suiteDefinition.exclude);
+}
+function matchesSuiteDefinitionSelectors(definition, selectors) {
+    if (selectors.scenarios?.includes(definition.id)) {
+        return true;
+    }
+    if (selectors.tags?.some((tag) => definition.tags?.includes(tag) ?? false)) {
+        return true;
+    }
+    if (selectors.suites?.includes(definition.suite)) {
+        return true;
+    }
+    return false;
+}

package/dist/scoring.js

@@ -4,9 +4,9 @@ export function computeScore(results) {
     const weighted = results.filter((result) => result.mode === "weighted");
     let score = 100;
     if (weighted.length > 0) {
-        const totalWeight = weighted.reduce((sum, result) => sum + (result.weight ?? 0), 0);
+        const totalWeight = weighted.reduce((sum, result) => sum + (result.weight ?? 1), 0);
         const earnedWeight = weighted.reduce((sum, result) => {
-            const weight = result.weight ?? 0;
+            const weight = result.weight ?? 1;
             return sum + (result.status === "pass" ? weight : 0);
         }, 0);
         score = totalWeight === 0 ? 100 : Math.round((earnedWeight / totalWeight) * 100);