agent-regression-lab 0.1.1 → 0.3.0

package/dist/agent/mockAdapter.js

@@ -1,8 +1,9 @@
 class MockAgentSession {
     input;
-    state = { step: "start" };
+    state;
     constructor(input) {
         this.input = input;
+        this.state = { domain: detectDomain(input), step: "start" };
     }
     hasTool(toolName) {
         return this.input.availableTools.some((tool) => tool.name === toolName);
@@ -11,9 +12,23 @@ class MockAgentSession {
         if (event.type === "runner_error") {
             return { type: "error", message: event.message };
         }
+        switch (this.state.domain) {
+            case "support":
+                return this.nextSupport(event);
+            case "coding":
+                return this.nextCoding(event);
+            case "research":
+                return this.nextResearch(event);
+            case "ops":
+                return this.nextOps(event);
+            default:
+                return { type: "error", message: "Unsupported mock domain." };
+        }
+    }
+    async nextSupport(event) {
         if (this.state.step === "start") {
             const email = String(this.input.context.customer_email ?? "");
-            this.state = { step: "listed_customer" };
+            this.state.step = "listed_customer";
             return {
                 type: "tool_call",
                 toolName: "crm.search_customer",
@@ -22,36 +37,70 @@ class MockAgentSession {
             };
         }
         if (this.state.step === "listed_customer") {
-            if (event.type !== "tool_result") {
+            if (event.type !== "tool_result" || typeof event.result !== "object" || event.result === null) {
                 return { type: "error", message: "Expected customer lookup result." };
             }
-            const result = event.result;
+            const customerId = String(event.result.id ?? "");
+            if (this.hasTool("accounts.get_profile") && this.hasTool("accounts.update_newsletter")) {
+                this.state.step = "newsletter_lookup";
+                return {
+                    type: "tool_call",
+                    toolName: "accounts.get_profile",
+                    input: { customer_id: customerId },
+                    metadata: { message: "Checking newsletter settings." },
+                };
+            }
+            if (this.hasTool("subscriptions.cancel")) {
+                this.state.step = "cancel_done";
+                return {
+                    type: "tool_call",
+                    toolName: "subscriptions.cancel",
+                    input: { customer_id: customerId },
+                    metadata: { message: "Cancelling active subscription." },
+                };
+            }
             if (this.hasTool("support.find_duplicate_charge")) {
-                this.state = { step: "found_duplicate" };
+                this.state.step = "found_duplicate";
                 return {
                     type: "tool_call",
                     toolName: "support.find_duplicate_charge",
-                    input: { customer_id: String(result.id ?? "") },
+                    input: { customer_id: customerId },
                     metadata: { message: "Looking up the duplicated order directly." },
                 };
             }
-            this.state = { step: "listed_orders" };
+            this.state.step = "listed_orders";
             return {
                 type: "tool_call",
                 toolName: "orders.list",
-                input: { customer_id: String(result.id ?? "") },
+                input: { customer_id: customerId },
                 metadata: { message: "Listing customer orders." },
             };
         }
+        if (this.state.step === "newsletter_lookup") {
+            if (event.type !== "tool_result" || typeof event.result !== "object" || event.result === null) {
+                return { type: "error", message: "Expected account lookup result." };
+            }
+            this.state.step = "done";
+            return {
+                type: "tool_call",
+                toolName: "accounts.update_newsletter",
+                input: {
+                    customer_id: String(this.input.context.customer_id ?? event.result.customer_id ?? ""),
+                    subscribed: false,
+                },
+                metadata: { message: "Turning off newsletter subscription." },
+            };
+        }
         if (this.state.step === "listed_orders") {
             if (event.type !== "tool_result" || !Array.isArray(event.result)) {
                 return { type: "error", message: "Expected order list result." };
             }
-            const duplicate = event.result.find((order) => typeof order === "object" && order !== null && order.id === "ord_1024");
+            const targetOrderId = String(this.input.context.target_order_id ?? "ord_1024");
+            const duplicate = event.result.find((order) => typeof order === "object" && order !== null && order.id === targetOrderId);
             if (!duplicate?.id) {
                 return { type: "error", message: "Could not identify duplicate order." };
             }
-            this.state = { step: "done" };
+            this.state.step = "done";
             return {
                 type: "tool_call",
                 toolName: "orders.refund",
@@ -67,7 +116,7 @@ class MockAgentSession {
             if (!result.order_id) {
                 return { type: "error", message: "Duplicate lookup did not return an order id." };
             }
-            this.state = { step: "done" };
+            this.state.step = "done";
             return {
                 type: "tool_call",
                 toolName: "orders.refund",
@@ -75,9 +124,26 @@ class MockAgentSession {
                 metadata: { message: "Refunding the duplicated charge." },
             };
         }
+        if (this.state.step === "cancel_done") {
+            if (event.type !== "tool_result" || typeof event.result !== "object" || event.result === null) {
+                return { type: "error", message: "Expected cancellation result." };
+            }
+            return {
+                type: "final",
+                output: `Cancelled subscription ${String(event.result.subscription_id ?? "unknown")}.`,
+                metadata: { completed: true },
+            };
+        }
         if (this.state.step === "done") {
             if (event.type !== "tool_result" || typeof event.result !== "object" || event.result === null) {
-                return { type: "error", message: "Expected refund result." };
+                return { type: "error", message: "Expected final support tool result." };
+            }
+            if ("newsletter_subscribed" in event.result) {
+                return {
+                    type: "final",
+                    output: "Disabled the newsletter subscription for the customer.",
+                    metadata: { completed: true },
+                };
             }
             const refund = event.result;
             return {
@@ -86,8 +152,139 @@ class MockAgentSession {
                 metadata: { completed: true },
             };
         }
-        return { type: "error", message: "Unexpected session state." };
+        return { type: "error", message: "Unexpected support session state." };
+    }
+    async nextCoding(event) {
+        const targetPath = String(this.input.context.target_path ?? "");
+        const replacement = String(this.input.context.replacement ?? "");
+        if (this.state.step === "start") {
+            this.state.step = "listed_files";
+            return {
+                type: "tool_call",
+                toolName: "repo.list_files",
+                input: {},
+                metadata: { message: "Listing repository files." },
+            };
+        }
+        if (this.state.step === "listed_files") {
+            this.state.step = "read_file";
+            return {
+                type: "tool_call",
+                toolName: "repo.read_file",
+                input: { path: targetPath },
+                metadata: { message: "Reading target file." },
+            };
+        }
+        if (this.state.step === "read_file") {
+            this.state.step = "patched";
+            return {
+                type: "tool_call",
+                toolName: "repo.apply_patch",
+                input: { path: targetPath, replacement },
+                metadata: { message: "Applying deterministic patch." },
+            };
+        }
+        if (this.state.step === "patched") {
+            return {
+                type: "final",
+                output: `Updated ${targetPath} with replacement '${replacement}'.`,
+                metadata: { completed: true },
+            };
+        }
+        return { type: "error", message: "Unexpected coding session state." };
+    }
+    async nextResearch(event) {
+        const query = String(this.input.context.query ?? "");
+        if (this.state.step === "start") {
+            this.state.step = "searched";
+            return {
+                type: "tool_call",
+                toolName: "docs.search",
+                input: { query },
+                metadata: { message: "Searching documents." },
+            };
+        }
+        if (this.state.step === "searched") {
+            if (event.type !== "tool_result" || !Array.isArray(event.result) || event.result.length === 0) {
+                return { type: "error", message: "Expected document search results." };
+            }
+            const first = event.result[0];
+            this.state.step = "read_doc";
+            return {
+                type: "tool_call",
+                toolName: "docs.read",
+                input: { doc_id: String(first.id ?? "") },
+                metadata: { message: "Reading top matching document." },
+            };
+        }
+        if (this.state.step === "read_doc") {
+            if (event.type !== "tool_result" || typeof event.result !== "object" || event.result === null) {
+                return { type: "error", message: "Expected document read result." };
+            }
+            const doc = event.result;
+            return {
+                type: "final",
+                output: `${String(this.input.context.answer_prefix ?? "Answer")}: ${String(this.input.context.expected_answer ?? "")} (source: ${String(doc.id ?? "")})`,
+                metadata: { completed: true },
+            };
+        }
+        return { type: "error", message: "Unexpected research session state." };
+    }
+    async nextOps(event) {
+        const service = String(this.input.context.service ?? "");
+        if (this.state.step === "start") {
+            this.state.step = "alerts_loaded";
+            return {
+                type: "tool_call",
+                toolName: "alerts.list_active",
+                input: {},
+                metadata: { message: "Loading active alerts." },
+            };
+        }
+        if (this.state.step === "alerts_loaded") {
+            this.state.step = "logs_loaded";
+            return {
+                type: "tool_call",
+                toolName: "logs.query_service",
+                input: { service },
+                metadata: { message: "Querying service logs." },
+            };
+        }
+        if (this.state.step === "logs_loaded") {
+            this.state.step = "status_loaded";
+            return {
+                type: "tool_call",
+                toolName: "status.get_service",
+                input: { service },
+                metadata: { message: "Loading service ownership metadata." },
+            };
+        }
+        if (this.state.step === "status_loaded") {
+            if (event.type !== "tool_result" || typeof event.result !== "object" || event.result === null) {
+                return { type: "error", message: "Expected service status result." };
+            }
+            const owner = String(event.result.owner ?? "");
+            return {
+                type: "final",
+                output: `${String(this.input.context.expected_summary ?? "")} Escalate to ${owner}.`,
+                metadata: { completed: true },
+            };
+        }
+        return { type: "error", message: "Unexpected ops session state." };
+    }
+}
+function detectDomain(input) {
+    const toolNames = new Set(input.availableTools.map((tool) => tool.name));
+    if (toolNames.has("repo.list_files")) {
+        return "coding";
+    }
+    if (toolNames.has("docs.search")) {
+        return "research";
+    }
+    if (toolNames.has("alerts.list_active")) {
+        return "ops";
     }
+    return "support";
 }
 export class MockAgentAdapter {
     async startRun(input) {

package/dist/config.js

@@ -1,12 +1,12 @@
 import { statSync, readFileSync } from "node:fs";
 import { resolve, relative, sep } from "node:path";
 import { parse } from "yaml";
-const CONFIG_PATH = resolve("agentlab.config.yaml");
 export function loadAgentLabConfig() {
-    if (!exists(CONFIG_PATH)) {
+    const configPath = resolve("agentlab.config.yaml");
+    if (!exists(configPath)) {
         return {};
     }
-    const raw = readFileSync(CONFIG_PATH, "utf8");
+    const raw = readFileSync(configPath, "utf8");
     const parsed = parse(raw);
     validateConfig(parsed);
     return parsed;
@@ -41,6 +41,47 @@ function validateConfig(value) {
             names.add(agent.name);
         }
     }
+    const agents = (value.agents ?? []);
+    const agentNames = new Set(agents.map((agent) => agent.name));
+    if (value.variant_sets !== undefined) {
+        if (!Array.isArray(value.variant_sets)) {
+            throw new Error("agentlab.config.yaml field 'variant_sets' must be an array.");
+        }
+        const names = new Set();
+        for (const variantSet of value.variant_sets) {
+            validateVariantSetDefinition(variantSet, agentNames);
+            if (names.has(variantSet.name)) {
+                throw new Error(`agentlab.config.yaml defines duplicate variant set '${variantSet.name}'.`);
+            }
+            names.add(variantSet.name);
+        }
+    }
+    if (value.runtime_profiles !== undefined) {
+        if (!Array.isArray(value.runtime_profiles)) {
+            throw new Error("agentlab.config.yaml field 'runtime_profiles' must be an array.");
+        }
+        const names = new Set();
+        for (const runtimeProfile of value.runtime_profiles) {
+            validateRuntimeProfileDefinition(runtimeProfile);
+            if (names.has(runtimeProfile.name)) {
+                throw new Error(`agentlab.config.yaml defines duplicate runtime profile '${runtimeProfile.name}'.`);
+            }
+            names.add(runtimeProfile.name);
+        }
+    }
+    if (value.suite_definitions !== undefined) {
+        if (!Array.isArray(value.suite_definitions)) {
+            throw new Error("agentlab.config.yaml field 'suite_definitions' must be an array.");
+        }
+        const names = new Set();
+        for (const suiteDefinition of value.suite_definitions) {
+            validateSuiteDefinition(suiteDefinition);
+            if (names.has(suiteDefinition.name)) {
+                throw new Error(`agentlab.config.yaml defines duplicate suite definition '${suiteDefinition.name}'.`);
+            }
+            names.add(suiteDefinition.name);
+        }
+    }
 }
 function validateToolRegistration(value) {
     if (!isObject(value)) {
@@ -77,12 +118,16 @@ function validateAgentRegistration(value) {
     if (typeof value.name !== "string" || value.name.length === 0) {
         throw new Error("Each agent registration must define a non-empty 'name'.");
     }
-    if (value.provider !== "mock" && value.provider !== "openai" && value.provider !== "external_process") {
+    if (value.provider !== "mock" && value.provider !== "openai" && value.provider !== "external_process" && value.provider !== "http") {
         throw new Error(`Agent '${value.name}' uses unsupported provider '${String(value.provider)}'.`);
     }
     if (value.label !== undefined && (typeof value.label !== "string" || value.label.length === 0)) {
         throw new Error(`Agent '${value.name}' must define a non-empty 'label' when provided.`);
     }
+    if (value.provider === "http") {
+        validateHttpAgentConfig(value);
+        return;
+    }
     if (value.provider === "openai" && value.model !== undefined && (typeof value.model !== "string" || value.model.length === 0)) {
         throw new Error(`Agent '${value.name}' must define a non-empty 'model' when provided.`);
     }
@@ -102,6 +147,38 @@ function validateAgentRegistration(value) {
         }
     }
 }
+export function validateHttpAgentConfig(value) {
+    const name = String(value.name ?? "");
+    if (typeof value.url !== "string" || value.url.length === 0) {
+        throw new Error(`Agent '${name}' with provider 'http' must define a non-empty 'url'.`);
+    }
+    if (value.timeout_ms !== undefined && (typeof value.timeout_ms !== "number" || value.timeout_ms <= 0)) {
+        throw new Error(`Agent '${name}' field 'timeout_ms' must be a positive number.`);
+    }
+    if (value.request_template !== undefined) {
+        if (!isObject(value.request_template)) {
+            throw new Error(`Agent '${name}' field 'request_template' must be an object.`);
+        }
+        for (const [k, v] of Object.entries(value.request_template)) {
+            if (typeof v !== "string") {
+                throw new Error(`Agent '${name}' request_template field '${k}' must be a string value.`);
+            }
+        }
+    }
+    if (value.response_field !== undefined && (typeof value.response_field !== "string" || value.response_field.length === 0)) {
+        throw new Error(`Agent '${name}' field 'response_field' must be a non-empty string.`);
+    }
+    if (value.headers !== undefined) {
+        if (!isObject(value.headers)) {
+            throw new Error(`Agent '${name}' field 'headers' must be an object.`);
+        }
+        for (const [k, v] of Object.entries(value.headers)) {
+            if (typeof v !== "string") {
+                throw new Error(`Agent '${name}' headers field '${k}' must be a string value.`);
+            }
+        }
+    }
+}
 export function getAgentRegistration(name) {
     const match = loadAgentLabConfig().agents?.find((agent) => agent.name === name);
     if (!match) {
@@ -109,6 +186,148 @@ export function getAgentRegistration(name) {
     }
     return match;
 }
+export function getVariantSet(name) {
+    const match = loadAgentLabConfig().variant_sets?.find((variantSet) => variantSet.name === name);
+    if (!match) {
+        throw new Error(`agentlab.config.yaml does not define variant set '${name}'.`);
+    }
+    return match;
+}
+export function getRuntimeProfile(name) {
+    const match = loadAgentLabConfig().runtime_profiles?.find((runtimeProfile) => runtimeProfile.name === name);
+    if (!match) {
+        throw new Error(`agentlab.config.yaml does not define runtime profile '${name}'.`);
+    }
+    return match;
+}
+export function getSuiteDefinition(name) {
+    const match = loadAgentLabConfig().suite_definitions?.find((suiteDefinition) => suiteDefinition.name === name);
+    if (!match) {
+        throw new Error(`agentlab.config.yaml does not define suite definition '${name}'.`);
+    }
+    return match;
+}
+function validateVariantSetDefinition(value, agentNames) {
+    if (!isObject(value)) {
+        throw new Error("Each variant set definition in agentlab.config.yaml must be an object.");
+    }
+    if (typeof value.name !== "string" || value.name.length === 0) {
+        throw new Error("Each variant set definition must define a non-empty 'name'.");
+    }
+    if (!Array.isArray(value.variants)) {
+        throw new Error(`Variant set '${value.name}' must define a 'variants' array.`);
+    }
+    const labels = new Set();
+    for (const variant of value.variants) {
+        if (!isObject(variant)) {
+            throw new Error(`Variant set '${value.name}' contains a non-object variant definition.`);
+        }
+        if (typeof variant.agent !== "string" || variant.agent.length === 0) {
+            throw new Error(`Variant set '${value.name}' contains a variant with a non-empty 'agent' required.`);
+        }
+        if (!agentNames.has(variant.agent)) {
+            throw new Error(`Variant set '${value.name}' references unknown agent '${variant.agent}'.`);
+        }
+        if (typeof variant.label !== "string" || variant.label.length === 0) {
+            throw new Error(`Variant set '${value.name}' contains a variant with a non-empty 'label' required.`);
+        }
+        if (labels.has(variant.label)) {
+            throw new Error(`Variant set '${value.name}' defines duplicate variant label '${variant.label}'.`);
+        }
+        labels.add(variant.label);
+    }
+}
+function validateRuntimeProfileDefinition(value) {
+    if (!isObject(value)) {
+        throw new Error("Each runtime profile definition in agentlab.config.yaml must be an object.");
+    }
+    if (typeof value.name !== "string" || value.name.length === 0) {
+        throw new Error("Each runtime profile definition must define a non-empty 'name'.");
+    }
+    if (value.tool_faults !== undefined) {
+        if (!Array.isArray(value.tool_faults)) {
+            throw new Error(`Runtime profile '${value.name}' field 'tool_faults' must be an array.`);
+        }
+        for (const fault of value.tool_faults) {
+            if (!isObject(fault)) {
+                throw new Error(`Runtime profile '${value.name}' contains a non-object tool fault definition.`);
+            }
+            if (typeof fault.tool !== "string" || fault.tool.length === 0) {
+                throw new Error(`Runtime profile '${value.name}' contains a tool fault with a non-empty 'tool' required.`);
+            }
+            if (fault.mode !== "timeout" && fault.mode !== "error" && fault.mode !== "malformed_output" && fault.mode !== "partial_output") {
+                throw new Error(`Runtime profile '${value.name}' uses invalid tool fault mode '${String(fault.mode)}'.`);
+            }
+            if (fault.error_message !== undefined && (typeof fault.error_message !== "string" || fault.error_message.length === 0)) {
+                throw new Error(`Runtime profile '${value.name}' tool fault for '${fault.tool}' field 'error_message' must be a non-empty string.`);
+            }
+            if (fault.timeout_ms !== undefined && (typeof fault.timeout_ms !== "number" || fault.timeout_ms <= 0)) {
+                throw new Error(`Runtime profile '${value.name}' tool fault for '${fault.tool}' field 'timeout_ms' must be a positive number.`);
+            }
+            if (fault.partial_output !== undefined && !isObject(fault.partial_output)) {
+                throw new Error(`Runtime profile '${value.name}' tool fault for '${fault.tool}' field 'partial_output' must be an object.`);
+            }
+        }
+    }
+    if (value.state !== undefined) {
+        if (!isObject(value.state)) {
+            throw new Error(`Runtime profile '${value.name}' field 'state' must be an object.`);
+        }
+        if (value.state.reset !== "per_run" && value.state.reset !== "per_variant_run" && value.state.reset !== "manual") {
+            throw new Error(`Runtime profile '${value.name}' field 'state.reset' must be one of 'per_run', 'per_variant_run', or 'manual'.`);
+        }
+        if (value.state.seeded_messages !== undefined) {
+            if (!Array.isArray(value.state.seeded_messages)) {
+                throw new Error(`Runtime profile '${value.name}' field 'state.seeded_messages' must be an array.`);
+            }
+            for (const message of value.state.seeded_messages) {
+                if (!isObject(message)) {
+                    throw new Error(`Runtime profile '${value.name}' contains a non-object seeded message.`);
+                }
+                if (message.role !== "user" && message.role !== "assistant") {
+                    throw new Error(`Runtime profile '${value.name}' seeded message role must be 'user' or 'assistant'.`);
+                }
+                if (typeof message.message !== "string" || message.message.length === 0) {
+                    throw new Error(`Runtime profile '${value.name}' seeded message must define a non-empty 'message'.`);
+                }
+            }
+        }
+        if (value.state.memory_blob !== undefined && !isObject(value.state.memory_blob)) {
+            throw new Error(`Runtime profile '${value.name}' field 'state.memory_blob' must be an object.`);
+        }
+    }
+}
+function validateSuiteDefinition(value) {
+    if (!isObject(value)) {
+        throw new Error("Each suite definition in agentlab.config.yaml must be an object.");
+    }
+    if (typeof value.name !== "string" || value.name.length === 0) {
+        throw new Error("Each suite definition must define a non-empty 'name'.");
+    }
+    if (!isObject(value.include)) {
+        throw new Error(`Suite definition '${value.name}' must define an object 'include'.`);
+    }
+    validateSuiteSelectorArray(value.include, value.name, "include.scenarios");
+    validateSuiteSelectorArray(value.include, value.name, "include.tags");
+    validateSuiteSelectorArray(value.include, value.name, "include.suites");
+    if (value.exclude !== undefined) {
+        if (!isObject(value.exclude)) {
+            throw new Error(`Suite definition '${value.name}' field 'exclude' must be an object.`);
+        }
+        validateSuiteSelectorArray(value.exclude, value.name, "exclude.scenarios");
+        validateSuiteSelectorArray(value.exclude, value.name, "exclude.tags");
+        validateSuiteSelectorArray(value.exclude, value.name, "exclude.suites");
+    }
+}
+function validateSuiteSelectorArray(value, suiteName, key) {
+    const fieldName = key.split(".")[1];
+    const selector = value[fieldName];
+    if (selector !== undefined) {
+        if (!Array.isArray(selector) || selector.some((item) => typeof item !== "string")) {
+            throw new Error(`Suite definition '${suiteName}' field '${key}' must be an array of strings.`);
+        }
+    }
+}
 function exists(path) {
     try {
         statSync(path);