@agentforge/testing 0.16.21 → 0.16.23

package/dist/index.d.cts

@@ -499,10 +499,25 @@ declare function getToolByName(name: string): Tool<{
     query: string;
 }, string> | undefined;
 
+/**
+ * Agent-like contract used by the test runner.
+ */
+interface AgentTestAgent<TInput = unknown, TState = unknown> {
+    invoke(input: TInput): TState | Promise<TState>;
+}
+/**
+ * Captured runner step. The current runner preserves an empty step list, but
+ * this contract gives future step capture a typed state boundary.
+ */
+interface AgentTestRunnerStep<TState = unknown> {
+    state: TState;
+    messages: BaseMessage[];
+    timestamp: number;
+}
 /**
  * Configuration for agent test runner
  */
-interface AgentTestConfig {
+interface AgentTestConfig<TState = unknown> {
     /**
      * Maximum time to wait for agent response (ms)
      */
@@ -518,16 +533,16 @@ interface AgentTestConfig {
     /**
      * Custom state validator
      */
-    stateValidator?: (state: any) => boolean | Promise<boolean>;
+    stateValidator?: (state: TState | undefined) => boolean | Promise<boolean>;
 }
 /**
  * Result from agent test run
  */
-interface AgentTestResult {
+interface AgentTestResult<TState = unknown, TStep = AgentTestRunnerStep<TState>> {
     /**
      * Final state after execution
      */
-    finalState: any;
+    finalState: TState | undefined;
     /**
      * Messages exchanged
      */
@@ -539,7 +554,7 @@ interface AgentTestResult {
     /**
      * Intermediate steps (if captured)
      */
-    steps?: any[];
+    steps?: TStep[];
     /**
      * Whether the test passed
      */
@@ -567,23 +582,23 @@ interface AgentTestResult {
  * expect(result.messages.length).toBeGreaterThan(1);
  * ```
  */
-declare class AgentTestRunner {
+declare class AgentTestRunner<TInput = unknown, TState = unknown, TStep = AgentTestRunnerStep<TState>> {
     private agent;
     private config;
-    constructor(agent: any, config?: AgentTestConfig);
+    constructor(agent: AgentTestAgent<TInput, TState>, config?: AgentTestConfig<TState>);
     /**
      * Run the agent with given input
      */
-    run(input: any): Promise<AgentTestResult>;
+    run(input: TInput): Promise<AgentTestResult<TState, TStep>>;
     /**
      * Run multiple test cases
      */
-    runMany(inputs: any[]): Promise<AgentTestResult[]>;
+    runMany(inputs: TInput[]): Promise<AgentTestResult<TState, TStep>[]>;
 }
 /**
  * Create an agent test runner
  */
-declare function createAgentTestRunner(agent: any, config?: AgentTestConfig): AgentTestRunner;
+declare function createAgentTestRunner<TInput = unknown, TState = unknown, TStep = AgentTestRunnerStep<TState>>(agent: AgentTestAgent<TInput, TState>, config?: AgentTestConfig<TState>): AgentTestRunner<TInput, TState, TStep>;
 
 /**
  * Configuration for conversation simulator
@@ -635,6 +650,9 @@ interface ConversationResult {
      */
     error?: Error;
 }
+interface ConversationSimulatorInput {
+    messages: BaseMessage[];
+}
 /**
  * Conversation simulator for testing multi-turn interactions
  *
@@ -655,10 +673,10 @@ interface ConversationResult {
  * expect(result.completed).toBe(true);
  * ```
  */
-declare class ConversationSimulator {
+declare class ConversationSimulator<TState = unknown> {
     private agent;
     private config;
-    constructor(agent: any, config?: ConversationSimulatorConfig);
+    constructor(agent: AgentTestAgent<ConversationSimulatorInput, TState>, config?: ConversationSimulatorConfig);
     /**
      * Simulate a conversation with predefined user inputs
      */
@@ -671,7 +689,7 @@ declare class ConversationSimulator {
 /**
  * Create a conversation simulator
  */
-declare function createConversationSimulator(agent: any, config?: ConversationSimulatorConfig): ConversationSimulator;
+declare function createConversationSimulator<TState = unknown>(agent: AgentTestAgent<ConversationSimulatorInput, TState>, config?: ConversationSimulatorConfig): ConversationSimulator<TState>;
 
 type SnapshotObject = Record<string, unknown>;
 interface SnapshotDiff {
@@ -741,4 +759,4 @@ declare function createStateDiff<TState1 = unknown, TState2 = unknown>(state1: T
  */
 declare function assertStateChanged<TStateBefore = unknown, TStateAfter = unknown>(stateBefore: TStateBefore, stateAfter: TStateAfter, expectedChanges: string[], config?: SnapshotConfig): void;
 
-export { type AgentTestConfig, type AgentTestResult, AgentTestRunner, type AssertedMessage, type ConversationResult, ConversationSimulator, type ConversationSimulatorConfig, type MessageSnapshot, MockLLM, type MockLLMConfig, type MockToolConfig, type PlanningStep, type PlanningTestState, ROOT_SNAPSHOT_DIFF_KEY, type ReActTestState, type SnapshotConfig, type SnapshotDiff, type SnapshotObject, StateBuilder, type TestToolCall, type TestToolResult, assertAlternatingMessages, assertCompletesWithin, assertHasKeys, assertInRange, assertIsMessage, assertIterationsWithinLimit, assertLastMessageContains, assertMatchesSnapshot, assertMessageContains, assertMessagesMatchSnapshot, assertNotEmpty, assertStateChanged, assertStateHasFields, assertStateSnapshot, assertThrowsWithMessage, assertToolCalled, calculatorTool, compareStates, complexReasoningConversation, createAgentTestRunner, createCalculatorTool, createConversation, createConversationSimulator, createConversationState, createConversationWithSystem, createDelayedTool, createEchoLLM, createEchoTool, createErrorLLM, createErrorTool, createMessageSnapshot, createMockLLM, createMockTool, createPlanningState, createReActState, createSnapshot, createStateBuilder, createStateDiff, databaseQueryTool, errorHandlingConversation, fileReaderTool, getToolByName, getToolsByCategory, longContextConversation, multiTurnConversation, sampleData, sampleTools, searchTool, simpleGreeting, timeTool, toolUsageConversation, weatherTool };
+export { type AgentTestAgent, type AgentTestConfig, type AgentTestResult, AgentTestRunner, type AgentTestRunnerStep, type AssertedMessage, type ConversationResult, ConversationSimulator, type ConversationSimulatorConfig, type MessageSnapshot, MockLLM, type MockLLMConfig, type MockToolConfig, type PlanningStep, type PlanningTestState, ROOT_SNAPSHOT_DIFF_KEY, type ReActTestState, type SnapshotConfig, type SnapshotDiff, type SnapshotObject, StateBuilder, type TestToolCall, type TestToolResult, assertAlternatingMessages, assertCompletesWithin, assertHasKeys, assertInRange, assertIsMessage, assertIterationsWithinLimit, assertLastMessageContains, assertMatchesSnapshot, assertMessageContains, assertMessagesMatchSnapshot, assertNotEmpty, assertStateChanged, assertStateHasFields, assertStateSnapshot, assertThrowsWithMessage, assertToolCalled, calculatorTool, compareStates, complexReasoningConversation, createAgentTestRunner, createCalculatorTool, createConversation, createConversationSimulator, createConversationState, createConversationWithSystem, createDelayedTool, createEchoLLM, createEchoTool, createErrorLLM, createErrorTool, createMessageSnapshot, createMockLLM, createMockTool, createPlanningState, createReActState, createSnapshot, createStateBuilder, createStateDiff, databaseQueryTool, errorHandlingConversation, fileReaderTool, getToolByName, getToolsByCategory, longContextConversation, multiTurnConversation, sampleData, sampleTools, searchTool, simpleGreeting, timeTool, toolUsageConversation, weatherTool };

package/dist/index.d.ts

@@ -499,10 +499,25 @@ declare function getToolByName(name: string): Tool<{
     query: string;
 }, string> | undefined;
 
+/**
+ * Agent-like contract used by the test runner.
+ */
+interface AgentTestAgent<TInput = unknown, TState = unknown> {
+    invoke(input: TInput): TState | Promise<TState>;
+}
+/**
+ * Captured runner step. The current runner preserves an empty step list, but
+ * this contract gives future step capture a typed state boundary.
+ */
+interface AgentTestRunnerStep<TState = unknown> {
+    state: TState;
+    messages: BaseMessage[];
+    timestamp: number;
+}
 /**
  * Configuration for agent test runner
  */
-interface AgentTestConfig {
+interface AgentTestConfig<TState = unknown> {
     /**
      * Maximum time to wait for agent response (ms)
      */
@@ -518,16 +533,16 @@ interface AgentTestConfig {
     /**
      * Custom state validator
      */
-    stateValidator?: (state: any) => boolean | Promise<boolean>;
+    stateValidator?: (state: TState | undefined) => boolean | Promise<boolean>;
 }
 /**
  * Result from agent test run
  */
-interface AgentTestResult {
+interface AgentTestResult<TState = unknown, TStep = AgentTestRunnerStep<TState>> {
     /**
      * Final state after execution
      */
-    finalState: any;
+    finalState: TState | undefined;
     /**
      * Messages exchanged
      */
@@ -539,7 +554,7 @@ interface AgentTestResult {
     /**
      * Intermediate steps (if captured)
      */
-    steps?: any[];
+    steps?: TStep[];
     /**
      * Whether the test passed
      */
@@ -567,23 +582,23 @@ interface AgentTestResult {
  * expect(result.messages.length).toBeGreaterThan(1);
  * ```
  */
-declare class AgentTestRunner {
+declare class AgentTestRunner<TInput = unknown, TState = unknown, TStep = AgentTestRunnerStep<TState>> {
     private agent;
     private config;
-    constructor(agent: any, config?: AgentTestConfig);
+    constructor(agent: AgentTestAgent<TInput, TState>, config?: AgentTestConfig<TState>);
     /**
      * Run the agent with given input
      */
-    run(input: any): Promise<AgentTestResult>;
+    run(input: TInput): Promise<AgentTestResult<TState, TStep>>;
     /**
      * Run multiple test cases
      */
-    runMany(inputs: any[]): Promise<AgentTestResult[]>;
+    runMany(inputs: TInput[]): Promise<AgentTestResult<TState, TStep>[]>;
 }
 /**
  * Create an agent test runner
  */
-declare function createAgentTestRunner(agent: any, config?: AgentTestConfig): AgentTestRunner;
+declare function createAgentTestRunner<TInput = unknown, TState = unknown, TStep = AgentTestRunnerStep<TState>>(agent: AgentTestAgent<TInput, TState>, config?: AgentTestConfig<TState>): AgentTestRunner<TInput, TState, TStep>;
 
 /**
  * Configuration for conversation simulator
@@ -635,6 +650,9 @@ interface ConversationResult {
      */
     error?: Error;
 }
+interface ConversationSimulatorInput {
+    messages: BaseMessage[];
+}
 /**
  * Conversation simulator for testing multi-turn interactions
  *
@@ -655,10 +673,10 @@ interface ConversationResult {
  * expect(result.completed).toBe(true);
  * ```
  */
-declare class ConversationSimulator {
+declare class ConversationSimulator<TState = unknown> {
     private agent;
     private config;
-    constructor(agent: any, config?: ConversationSimulatorConfig);
+    constructor(agent: AgentTestAgent<ConversationSimulatorInput, TState>, config?: ConversationSimulatorConfig);
     /**
      * Simulate a conversation with predefined user inputs
      */
@@ -671,7 +689,7 @@ declare class ConversationSimulator {
 /**
  * Create a conversation simulator
  */
-declare function createConversationSimulator(agent: any, config?: ConversationSimulatorConfig): ConversationSimulator;
+declare function createConversationSimulator<TState = unknown>(agent: AgentTestAgent<ConversationSimulatorInput, TState>, config?: ConversationSimulatorConfig): ConversationSimulator<TState>;
 
 type SnapshotObject = Record<string, unknown>;
 interface SnapshotDiff {
@@ -741,4 +759,4 @@ declare function createStateDiff<TState1 = unknown, TState2 = unknown>(state1: T
  */
 declare function assertStateChanged<TStateBefore = unknown, TStateAfter = unknown>(stateBefore: TStateBefore, stateAfter: TStateAfter, expectedChanges: string[], config?: SnapshotConfig): void;
 
-export { type AgentTestConfig, type AgentTestResult, AgentTestRunner, type AssertedMessage, type ConversationResult, ConversationSimulator, type ConversationSimulatorConfig, type MessageSnapshot, MockLLM, type MockLLMConfig, type MockToolConfig, type PlanningStep, type PlanningTestState, ROOT_SNAPSHOT_DIFF_KEY, type ReActTestState, type SnapshotConfig, type SnapshotDiff, type SnapshotObject, StateBuilder, type TestToolCall, type TestToolResult, assertAlternatingMessages, assertCompletesWithin, assertHasKeys, assertInRange, assertIsMessage, assertIterationsWithinLimit, assertLastMessageContains, assertMatchesSnapshot, assertMessageContains, assertMessagesMatchSnapshot, assertNotEmpty, assertStateChanged, assertStateHasFields, assertStateSnapshot, assertThrowsWithMessage, assertToolCalled, calculatorTool, compareStates, complexReasoningConversation, createAgentTestRunner, createCalculatorTool, createConversation, createConversationSimulator, createConversationState, createConversationWithSystem, createDelayedTool, createEchoLLM, createEchoTool, createErrorLLM, createErrorTool, createMessageSnapshot, createMockLLM, createMockTool, createPlanningState, createReActState, createSnapshot, createStateBuilder, createStateDiff, databaseQueryTool, errorHandlingConversation, fileReaderTool, getToolByName, getToolsByCategory, longContextConversation, multiTurnConversation, sampleData, sampleTools, searchTool, simpleGreeting, timeTool, toolUsageConversation, weatherTool };
+export { type AgentTestAgent, type AgentTestConfig, type AgentTestResult, AgentTestRunner, type AgentTestRunnerStep, type AssertedMessage, type ConversationResult, ConversationSimulator, type ConversationSimulatorConfig, type MessageSnapshot, MockLLM, type MockLLMConfig, type MockToolConfig, type PlanningStep, type PlanningTestState, ROOT_SNAPSHOT_DIFF_KEY, type ReActTestState, type SnapshotConfig, type SnapshotDiff, type SnapshotObject, StateBuilder, type TestToolCall, type TestToolResult, assertAlternatingMessages, assertCompletesWithin, assertHasKeys, assertInRange, assertIsMessage, assertIterationsWithinLimit, assertLastMessageContains, assertMatchesSnapshot, assertMessageContains, assertMessagesMatchSnapshot, assertNotEmpty, assertStateChanged, assertStateHasFields, assertStateSnapshot, assertThrowsWithMessage, assertToolCalled, calculatorTool, compareStates, complexReasoningConversation, createAgentTestRunner, createCalculatorTool, createConversation, createConversationSimulator, createConversationState, createConversationWithSystem, createDelayedTool, createEchoLLM, createEchoTool, createErrorLLM, createErrorTool, createMessageSnapshot, createMockLLM, createMockTool, createPlanningState, createReActState, createSnapshot, createStateBuilder, createStateDiff, databaseQueryTool, errorHandlingConversation, fileReaderTool, getToolByName, getToolsByCategory, longContextConversation, multiTurnConversation, sampleData, sampleTools, searchTool, simpleGreeting, timeTool, toolUsageConversation, weatherTool };

package/dist/index.js

@@ -12806,14 +12806,14 @@ function withTimeout(fn2, timeout, isHook = false) {
   if (timeout <= 0 || timeout === Number.POSITIVE_INFINITY) {
     return fn2;
   }
-  const { setTimeout: setTimeout2, clearTimeout } = getSafeTimers();
+  const { setTimeout: setTimeout2, clearTimeout: clearTimeout2 } = getSafeTimers();
   return function runWithTimeout(...args) {
     return Promise.race([
       fn2(...args),
       new Promise((resolve4, reject) => {
         var _a;
         const timer = setTimeout2(() => {
-          clearTimeout(timer);
+          clearTimeout2(timer);
           reject(new Error(makeTimeoutMsg(isHook, timeout)));
         }, timeout);
         (_a = timer.unref) == null ? void 0 : _a.call(timer);
@@ -15671,9 +15671,9 @@ function createExpectPoll(expect2) {
           const promise = () => new Promise((resolve4, reject) => {
             let intervalId;
             let lastError;
-            const { setTimeout: setTimeout2, clearTimeout } = getSafeTimers();
+            const { setTimeout: setTimeout2, clearTimeout: clearTimeout2 } = getSafeTimers();
             const timeoutId = setTimeout2(() => {
-              clearTimeout(intervalId);
+              clearTimeout2(intervalId);
               reject(
                 copyStackTrace$1(
                   new Error(`Matcher did not succeed in ${timeout}ms`, {
@@ -15689,8 +15689,8 @@ function createExpectPoll(expect2) {
                 const obj = await fn2();
                 utils_exports.flag(assertion, "object", obj);
                 resolve4(await assertionFunction.call(assertion, ...args));
-                clearTimeout(intervalId);
-                clearTimeout(timeoutId);
+                clearTimeout2(intervalId);
+                clearTimeout2(timeoutId);
               } catch (err) {
                 lastError = err;
                 intervalId = setTimeout2(check, interval);
@@ -17281,7 +17281,7 @@ To automatically clean-up native timers, use \`shouldClearNativeTimers\`.`
           });
         };
       }
-      clock.clearTimeout = function clearTimeout(timerId) {
+      clock.clearTimeout = function clearTimeout2(timerId) {
         return clearTimer(clock, timerId, "Timeout");
       };
       clock.nextTick = function nextTick(func) {
@@ -17924,7 +17924,7 @@ function copyStackTrace(target, source) {
   return target;
 }
 function waitFor(callback, options = {}) {
-  const { setTimeout: setTimeout2, setInterval, clearTimeout, clearInterval } = getSafeTimers();
+  const { setTimeout: setTimeout2, setInterval, clearTimeout: clearTimeout2, clearInterval } = getSafeTimers();
   const { interval = 50, timeout = 1e3 } = typeof options === "number" ? { timeout: options } : options;
   const STACK_TRACE_ERROR = new Error("STACK_TRACE_ERROR");
   return new Promise((resolve4, reject) => {
@@ -17934,7 +17934,7 @@ function waitFor(callback, options = {}) {
     let intervalId;
     const onResolve = (result) => {
       if (timeoutId) {
-        clearTimeout(timeoutId);
+        clearTimeout2(timeoutId);
       }
       if (intervalId) {
         clearInterval(intervalId);
@@ -17992,7 +17992,7 @@ function waitFor(callback, options = {}) {
   });
 }
 function waitUntil(callback, options = {}) {
-  const { setTimeout: setTimeout2, setInterval, clearTimeout, clearInterval } = getSafeTimers();
+  const { setTimeout: setTimeout2, setInterval, clearTimeout: clearTimeout2, clearInterval } = getSafeTimers();
   const { interval = 50, timeout = 1e3 } = typeof options === "number" ? { timeout: options } : options;
   const STACK_TRACE_ERROR = new Error("STACK_TRACE_ERROR");
   return new Promise((resolve4, reject) => {
@@ -18016,7 +18016,7 @@ function waitUntil(callback, options = {}) {
         return;
       }
       if (timeoutId) {
-        clearTimeout(timeoutId);
+        clearTimeout2(timeoutId);
       }
       if (intervalId) {
         clearInterval(intervalId);
@@ -18602,20 +18602,21 @@ var AgentTestRunner = class {
     let messages = [];
     let passed = true;
     let error;
+    let timeoutId;
     try {
-      const timeout = this.config.timeout || 3e4;
+      const timeout = this.config.timeout ?? 3e4;
       const timeoutPromise = new Promise((_, reject) => {
-        setTimeout(() => reject(new Error("Agent test timeout")), timeout);
+        timeoutId = setTimeout(() => reject(new Error("Agent test timeout")), timeout);
       });
       const runPromise = (async () => {
         if (this.config.captureSteps) {
           const result = await this.agent.invoke(input);
           finalState = result;
-          messages = result.messages || [];
+          messages = extractMessages(result);
         } else {
           const result = await this.agent.invoke(input);
           finalState = result;
-          messages = result.messages || [];
+          messages = extractMessages(result);
         }
         if (this.config.validateState && this.config.stateValidator) {
           const isValid = await this.config.stateValidator(finalState);
@@ -18624,7 +18625,13 @@ var AgentTestRunner = class {
           }
         }
       })();
-      await Promise.race([runPromise, timeoutPromise]);
+      try {
+        await Promise.race([runPromise, timeoutPromise]);
+      } finally {
+        if (timeoutId !== void 0) {
+          clearTimeout(timeoutId);
+        }
+      }
     } catch (err) {
       passed = false;
       error = err;
@@ -18649,6 +18656,13 @@ var AgentTestRunner = class {
 function createAgentTestRunner(agent, config2) {
   return new AgentTestRunner(agent, config2);
 }
+function extractMessages(state) {
+  if (typeof state !== "object" || state === null) {
+    return [];
+  }
+  const { messages } = state;
+  return Array.isArray(messages) ? messages : [];
+}
 var ConversationSimulator = class {
   constructor(agent, config2 = {}) {
     this.agent = agent;
@@ -18677,7 +18691,7 @@ var ConversationSimulator = class {
           console.log(`User: ${input}`);
         }
         const result = await this.agent.invoke({ messages });
-        const aiMessage = result.messages[result.messages.length - 1];
+        const aiMessage = extractLatestMessage(result);
         messages.push(aiMessage);
         if (this.config.verbose) {
           console.log(`AI: ${aiMessage.content}`);
@@ -18726,7 +18740,7 @@ var ConversationSimulator = class {
         const userMessage = new HumanMessage(input);
         messages.push(userMessage);
         const result = await this.agent.invoke({ messages });
-        const aiMessage = result.messages[result.messages.length - 1];
+        const aiMessage = extractLatestMessage(result);
         messages.push(aiMessage);
         turns++;
       }
@@ -18749,6 +18763,14 @@ var ConversationSimulator = class {
 function createConversationSimulator(agent, config2) {
   return new ConversationSimulator(agent, config2);
 }
+function extractLatestMessage(state) {
+  const messages = extractMessages(state);
+  const latestMessage = messages[messages.length - 1];
+  if (!latestMessage) {
+    throw new Error("Agent response did not include any messages");
+  }
+  return latestMessage;
+}
 var ROOT_SNAPSHOT_DIFF_KEY = "$root";
 function createSnapshotObject() {
   return /* @__PURE__ */ Object.create(null);