@agentforge/testing 0.16.20 → 0.16.22

package/dist/index.d.cts

@@ -499,10 +499,25 @@ declare function getToolByName(name: string): Tool<{
     query: string;
 }, string> | undefined;
 
+/**
+ * Agent-like contract used by the test runner.
+ */
+interface AgentTestAgent<TInput = unknown, TState = unknown> {
+    invoke(input: TInput): TState | Promise<TState>;
+}
+/**
+ * Captured runner step. The current runner preserves an empty step list, but
+ * this contract gives future step capture a typed state boundary.
+ */
+interface AgentTestRunnerStep<TState = unknown> {
+    state: TState;
+    messages: BaseMessage[];
+    timestamp: number;
+}
 /**
  * Configuration for agent test runner
  */
-interface AgentTestConfig {
+interface AgentTestConfig<TState = unknown> {
     /**
      * Maximum time to wait for agent response (ms)
      */
@@ -518,16 +533,16 @@ interface AgentTestConfig {
     /**
      * Custom state validator
      */
-    stateValidator?: (state: any) => boolean | Promise<boolean>;
+    stateValidator?: (state: TState | undefined) => boolean | Promise<boolean>;
 }
 /**
  * Result from agent test run
  */
-interface AgentTestResult {
+interface AgentTestResult<TState = unknown, TStep = AgentTestRunnerStep<TState>> {
     /**
      * Final state after execution
      */
-    finalState: any;
+    finalState: TState | undefined;
     /**
      * Messages exchanged
      */
@@ -539,7 +554,7 @@ interface AgentTestResult {
     /**
      * Intermediate steps (if captured)
      */
-    steps?: any[];
+    steps?: TStep[];
     /**
      * Whether the test passed
      */
@@ -567,23 +582,23 @@ interface AgentTestResult {
  * expect(result.messages.length).toBeGreaterThan(1);
  * ```
  */
-declare class AgentTestRunner {
+declare class AgentTestRunner<TInput = unknown, TState = unknown, TStep = AgentTestRunnerStep<TState>> {
     private agent;
     private config;
-    constructor(agent: any, config?: AgentTestConfig);
+    constructor(agent: AgentTestAgent<TInput, TState>, config?: AgentTestConfig<TState>);
     /**
      * Run the agent with given input
      */
-    run(input: any): Promise<AgentTestResult>;
+    run(input: TInput): Promise<AgentTestResult<TState, TStep>>;
     /**
      * Run multiple test cases
      */
-    runMany(inputs: any[]): Promise<AgentTestResult[]>;
+    runMany(inputs: TInput[]): Promise<AgentTestResult<TState, TStep>[]>;
 }
 /**
  * Create an agent test runner
  */
-declare function createAgentTestRunner(agent: any, config?: AgentTestConfig): AgentTestRunner;
+declare function createAgentTestRunner<TInput = unknown, TState = unknown, TStep = AgentTestRunnerStep<TState>>(agent: AgentTestAgent<TInput, TState>, config?: AgentTestConfig<TState>): AgentTestRunner<TInput, TState, TStep>;
 
 /**
  * Configuration for conversation simulator
@@ -741,4 +756,4 @@ declare function createStateDiff<TState1 = unknown, TState2 = unknown>(state1: T
  */
 declare function assertStateChanged<TStateBefore = unknown, TStateAfter = unknown>(stateBefore: TStateBefore, stateAfter: TStateAfter, expectedChanges: string[], config?: SnapshotConfig): void;
 
-export { type AgentTestConfig, type AgentTestResult, AgentTestRunner, type AssertedMessage, type ConversationResult, ConversationSimulator, type ConversationSimulatorConfig, type MessageSnapshot, MockLLM, type MockLLMConfig, type MockToolConfig, type PlanningStep, type PlanningTestState, ROOT_SNAPSHOT_DIFF_KEY, type ReActTestState, type SnapshotConfig, type SnapshotDiff, type SnapshotObject, StateBuilder, type TestToolCall, type TestToolResult, assertAlternatingMessages, assertCompletesWithin, assertHasKeys, assertInRange, assertIsMessage, assertIterationsWithinLimit, assertLastMessageContains, assertMatchesSnapshot, assertMessageContains, assertMessagesMatchSnapshot, assertNotEmpty, assertStateChanged, assertStateHasFields, assertStateSnapshot, assertThrowsWithMessage, assertToolCalled, calculatorTool, compareStates, complexReasoningConversation, createAgentTestRunner, createCalculatorTool, createConversation, createConversationSimulator, createConversationState, createConversationWithSystem, createDelayedTool, createEchoLLM, createEchoTool, createErrorLLM, createErrorTool, createMessageSnapshot, createMockLLM, createMockTool, createPlanningState, createReActState, createSnapshot, createStateBuilder, createStateDiff, databaseQueryTool, errorHandlingConversation, fileReaderTool, getToolByName, getToolsByCategory, longContextConversation, multiTurnConversation, sampleData, sampleTools, searchTool, simpleGreeting, timeTool, toolUsageConversation, weatherTool };
+export { type AgentTestAgent, type AgentTestConfig, type AgentTestResult, AgentTestRunner, type AgentTestRunnerStep, type AssertedMessage, type ConversationResult, ConversationSimulator, type ConversationSimulatorConfig, type MessageSnapshot, MockLLM, type MockLLMConfig, type MockToolConfig, type PlanningStep, type PlanningTestState, ROOT_SNAPSHOT_DIFF_KEY, type ReActTestState, type SnapshotConfig, type SnapshotDiff, type SnapshotObject, StateBuilder, type TestToolCall, type TestToolResult, assertAlternatingMessages, assertCompletesWithin, assertHasKeys, assertInRange, assertIsMessage, assertIterationsWithinLimit, assertLastMessageContains, assertMatchesSnapshot, assertMessageContains, assertMessagesMatchSnapshot, assertNotEmpty, assertStateChanged, assertStateHasFields, assertStateSnapshot, assertThrowsWithMessage, assertToolCalled, calculatorTool, compareStates, complexReasoningConversation, createAgentTestRunner, createCalculatorTool, createConversation, createConversationSimulator, createConversationState, createConversationWithSystem, createDelayedTool, createEchoLLM, createEchoTool, createErrorLLM, createErrorTool, createMessageSnapshot, createMockLLM, createMockTool, createPlanningState, createReActState, createSnapshot, createStateBuilder, createStateDiff, databaseQueryTool, errorHandlingConversation, fileReaderTool, getToolByName, getToolsByCategory, longContextConversation, multiTurnConversation, sampleData, sampleTools, searchTool, simpleGreeting, timeTool, toolUsageConversation, weatherTool };

package/dist/index.d.ts

@@ -499,10 +499,25 @@ declare function getToolByName(name: string): Tool<{
     query: string;
 }, string> | undefined;
 
+/**
+ * Agent-like contract used by the test runner.
+ */
+interface AgentTestAgent<TInput = unknown, TState = unknown> {
+    invoke(input: TInput): TState | Promise<TState>;
+}
+/**
+ * Captured runner step. The current runner preserves an empty step list, but
+ * this contract gives future step capture a typed state boundary.
+ */
+interface AgentTestRunnerStep<TState = unknown> {
+    state: TState;
+    messages: BaseMessage[];
+    timestamp: number;
+}
 /**
  * Configuration for agent test runner
  */
-interface AgentTestConfig {
+interface AgentTestConfig<TState = unknown> {
     /**
      * Maximum time to wait for agent response (ms)
      */
@@ -518,16 +533,16 @@ interface AgentTestConfig {
     /**
      * Custom state validator
      */
-    stateValidator?: (state: any) => boolean | Promise<boolean>;
+    stateValidator?: (state: TState | undefined) => boolean | Promise<boolean>;
 }
 /**
  * Result from agent test run
  */
-interface AgentTestResult {
+interface AgentTestResult<TState = unknown, TStep = AgentTestRunnerStep<TState>> {
     /**
      * Final state after execution
      */
-    finalState: any;
+    finalState: TState | undefined;
     /**
      * Messages exchanged
      */
@@ -539,7 +554,7 @@ interface AgentTestResult {
     /**
      * Intermediate steps (if captured)
      */
-    steps?: any[];
+    steps?: TStep[];
     /**
      * Whether the test passed
      */
@@ -567,23 +582,23 @@ interface AgentTestResult {
  * expect(result.messages.length).toBeGreaterThan(1);
  * ```
  */
-declare class AgentTestRunner {
+declare class AgentTestRunner<TInput = unknown, TState = unknown, TStep = AgentTestRunnerStep<TState>> {
     private agent;
     private config;
-    constructor(agent: any, config?: AgentTestConfig);
+    constructor(agent: AgentTestAgent<TInput, TState>, config?: AgentTestConfig<TState>);
     /**
      * Run the agent with given input
      */
-    run(input: any): Promise<AgentTestResult>;
+    run(input: TInput): Promise<AgentTestResult<TState, TStep>>;
     /**
      * Run multiple test cases
      */
-    runMany(inputs: any[]): Promise<AgentTestResult[]>;
+    runMany(inputs: TInput[]): Promise<AgentTestResult<TState, TStep>[]>;
 }
 /**
  * Create an agent test runner
  */
-declare function createAgentTestRunner(agent: any, config?: AgentTestConfig): AgentTestRunner;
+declare function createAgentTestRunner<TInput = unknown, TState = unknown, TStep = AgentTestRunnerStep<TState>>(agent: AgentTestAgent<TInput, TState>, config?: AgentTestConfig<TState>): AgentTestRunner<TInput, TState, TStep>;
 
 /**
  * Configuration for conversation simulator
@@ -741,4 +756,4 @@ declare function createStateDiff<TState1 = unknown, TState2 = unknown>(state1: T
  */
 declare function assertStateChanged<TStateBefore = unknown, TStateAfter = unknown>(stateBefore: TStateBefore, stateAfter: TStateAfter, expectedChanges: string[], config?: SnapshotConfig): void;
 
-export { type AgentTestConfig, type AgentTestResult, AgentTestRunner, type AssertedMessage, type ConversationResult, ConversationSimulator, type ConversationSimulatorConfig, type MessageSnapshot, MockLLM, type MockLLMConfig, type MockToolConfig, type PlanningStep, type PlanningTestState, ROOT_SNAPSHOT_DIFF_KEY, type ReActTestState, type SnapshotConfig, type SnapshotDiff, type SnapshotObject, StateBuilder, type TestToolCall, type TestToolResult, assertAlternatingMessages, assertCompletesWithin, assertHasKeys, assertInRange, assertIsMessage, assertIterationsWithinLimit, assertLastMessageContains, assertMatchesSnapshot, assertMessageContains, assertMessagesMatchSnapshot, assertNotEmpty, assertStateChanged, assertStateHasFields, assertStateSnapshot, assertThrowsWithMessage, assertToolCalled, calculatorTool, compareStates, complexReasoningConversation, createAgentTestRunner, createCalculatorTool, createConversation, createConversationSimulator, createConversationState, createConversationWithSystem, createDelayedTool, createEchoLLM, createEchoTool, createErrorLLM, createErrorTool, createMessageSnapshot, createMockLLM, createMockTool, createPlanningState, createReActState, createSnapshot, createStateBuilder, createStateDiff, databaseQueryTool, errorHandlingConversation, fileReaderTool, getToolByName, getToolsByCategory, longContextConversation, multiTurnConversation, sampleData, sampleTools, searchTool, simpleGreeting, timeTool, toolUsageConversation, weatherTool };
+export { type AgentTestAgent, type AgentTestConfig, type AgentTestResult, AgentTestRunner, type AgentTestRunnerStep, type AssertedMessage, type ConversationResult, ConversationSimulator, type ConversationSimulatorConfig, type MessageSnapshot, MockLLM, type MockLLMConfig, type MockToolConfig, type PlanningStep, type PlanningTestState, ROOT_SNAPSHOT_DIFF_KEY, type ReActTestState, type SnapshotConfig, type SnapshotDiff, type SnapshotObject, StateBuilder, type TestToolCall, type TestToolResult, assertAlternatingMessages, assertCompletesWithin, assertHasKeys, assertInRange, assertIsMessage, assertIterationsWithinLimit, assertLastMessageContains, assertMatchesSnapshot, assertMessageContains, assertMessagesMatchSnapshot, assertNotEmpty, assertStateChanged, assertStateHasFields, assertStateSnapshot, assertThrowsWithMessage, assertToolCalled, calculatorTool, compareStates, complexReasoningConversation, createAgentTestRunner, createCalculatorTool, createConversation, createConversationSimulator, createConversationState, createConversationWithSystem, createDelayedTool, createEchoLLM, createEchoTool, createErrorLLM, createErrorTool, createMessageSnapshot, createMockLLM, createMockTool, createPlanningState, createReActState, createSnapshot, createStateBuilder, createStateDiff, databaseQueryTool, errorHandlingConversation, fileReaderTool, getToolByName, getToolsByCategory, longContextConversation, multiTurnConversation, sampleData, sampleTools, searchTool, simpleGreeting, timeTool, toolUsageConversation, weatherTool };

package/dist/index.js

@@ -12806,14 +12806,14 @@ function withTimeout(fn2, timeout, isHook = false) {
   if (timeout <= 0 || timeout === Number.POSITIVE_INFINITY) {
     return fn2;
   }
-  const { setTimeout: setTimeout2, clearTimeout } = getSafeTimers();
+  const { setTimeout: setTimeout2, clearTimeout: clearTimeout2 } = getSafeTimers();
   return function runWithTimeout(...args) {
     return Promise.race([
       fn2(...args),
       new Promise((resolve4, reject) => {
         var _a;
         const timer = setTimeout2(() => {
-          clearTimeout(timer);
+          clearTimeout2(timer);
           reject(new Error(makeTimeoutMsg(isHook, timeout)));
         }, timeout);
         (_a = timer.unref) == null ? void 0 : _a.call(timer);
@@ -15671,9 +15671,9 @@ function createExpectPoll(expect2) {
           const promise = () => new Promise((resolve4, reject) => {
             let intervalId;
             let lastError;
-            const { setTimeout: setTimeout2, clearTimeout } = getSafeTimers();
+            const { setTimeout: setTimeout2, clearTimeout: clearTimeout2 } = getSafeTimers();
             const timeoutId = setTimeout2(() => {
-              clearTimeout(intervalId);
+              clearTimeout2(intervalId);
               reject(
                 copyStackTrace$1(
                   new Error(`Matcher did not succeed in ${timeout}ms`, {
@@ -15689,8 +15689,8 @@ function createExpectPoll(expect2) {
                 const obj = await fn2();
                 utils_exports.flag(assertion, "object", obj);
                 resolve4(await assertionFunction.call(assertion, ...args));
-                clearTimeout(intervalId);
-                clearTimeout(timeoutId);
+                clearTimeout2(intervalId);
+                clearTimeout2(timeoutId);
               } catch (err) {
                 lastError = err;
                 intervalId = setTimeout2(check, interval);
@@ -17281,7 +17281,7 @@ To automatically clean-up native timers, use \`shouldClearNativeTimers\`.`
           });
         };
       }
-      clock.clearTimeout = function clearTimeout(timerId) {
+      clock.clearTimeout = function clearTimeout2(timerId) {
         return clearTimer(clock, timerId, "Timeout");
       };
       clock.nextTick = function nextTick(func) {
@@ -17924,7 +17924,7 @@ function copyStackTrace(target, source) {
   return target;
 }
 function waitFor(callback, options = {}) {
-  const { setTimeout: setTimeout2, setInterval, clearTimeout, clearInterval } = getSafeTimers();
+  const { setTimeout: setTimeout2, setInterval, clearTimeout: clearTimeout2, clearInterval } = getSafeTimers();
   const { interval = 50, timeout = 1e3 } = typeof options === "number" ? { timeout: options } : options;
   const STACK_TRACE_ERROR = new Error("STACK_TRACE_ERROR");
   return new Promise((resolve4, reject) => {
@@ -17934,7 +17934,7 @@ function waitFor(callback, options = {}) {
     let intervalId;
     const onResolve = (result) => {
       if (timeoutId) {
-        clearTimeout(timeoutId);
+        clearTimeout2(timeoutId);
       }
       if (intervalId) {
         clearInterval(intervalId);
@@ -17992,7 +17992,7 @@ function waitFor(callback, options = {}) {
   });
 }
 function waitUntil(callback, options = {}) {
-  const { setTimeout: setTimeout2, setInterval, clearTimeout, clearInterval } = getSafeTimers();
+  const { setTimeout: setTimeout2, setInterval, clearTimeout: clearTimeout2, clearInterval } = getSafeTimers();
   const { interval = 50, timeout = 1e3 } = typeof options === "number" ? { timeout: options } : options;
   const STACK_TRACE_ERROR = new Error("STACK_TRACE_ERROR");
   return new Promise((resolve4, reject) => {
@@ -18016,7 +18016,7 @@ function waitUntil(callback, options = {}) {
         return;
       }
       if (timeoutId) {
-        clearTimeout(timeoutId);
+        clearTimeout2(timeoutId);
       }
       if (intervalId) {
         clearInterval(intervalId);
@@ -18602,20 +18602,21 @@ var AgentTestRunner = class {
     let messages = [];
     let passed = true;
     let error;
+    let timeoutId;
     try {
-      const timeout = this.config.timeout || 3e4;
+      const timeout = this.config.timeout ?? 3e4;
       const timeoutPromise = new Promise((_, reject) => {
-        setTimeout(() => reject(new Error("Agent test timeout")), timeout);
+        timeoutId = setTimeout(() => reject(new Error("Agent test timeout")), timeout);
       });
       const runPromise = (async () => {
         if (this.config.captureSteps) {
           const result = await this.agent.invoke(input);
           finalState = result;
-          messages = result.messages || [];
+          messages = extractMessages(result);
         } else {
           const result = await this.agent.invoke(input);
           finalState = result;
-          messages = result.messages || [];
+          messages = extractMessages(result);
         }
         if (this.config.validateState && this.config.stateValidator) {
           const isValid = await this.config.stateValidator(finalState);
@@ -18624,7 +18625,13 @@ var AgentTestRunner = class {
           }
         }
       })();
-      await Promise.race([runPromise, timeoutPromise]);
+      try {
+        await Promise.race([runPromise, timeoutPromise]);
+      } finally {
+        if (timeoutId !== void 0) {
+          clearTimeout(timeoutId);
+        }
+      }
     } catch (err) {
       passed = false;
       error = err;
@@ -18649,6 +18656,13 @@ var AgentTestRunner = class {
 function createAgentTestRunner(agent, config2) {
   return new AgentTestRunner(agent, config2);
 }
+function extractMessages(state) {
+  if (typeof state !== "object" || state === null) {
+    return [];
+  }
+  const { messages } = state;
+  return Array.isArray(messages) ? messages : [];
+}
 var ConversationSimulator = class {
   constructor(agent, config2 = {}) {
     this.agent = agent;