@od-oneapp/ai-platform 0.1.0

package/src/internal/testing/index.ts

@@ -0,0 +1,960 @@
+/**
+ * Testing & Evaluation Module
+ *
+ * AI testing and evaluation patterns:
+ * - Mock providers and tools
+ * - LLM-as-judge evaluation
+ * - Automated test harness
+ * - Regression testing
+ *
+ * @module testing
+ */
+
+import { type z } from 'zod/v4';
+
+// -----------------------------------------------------------------------------
+// Types
+// -----------------------------------------------------------------------------
+
+/** Mock response configuration */
+export interface MockResponse {
+  readonly text?: string;
+  readonly object?: unknown;
+  readonly toolCalls?: MockToolCall[];
+  readonly error?: Error;
+  readonly delay?: number;
+  readonly usage?: { promptTokens: number; completionTokens: number };
+}
+
+/** Mock tool call */
+export interface MockToolCall {
+  readonly toolName: string;
+  readonly args: Record<string, unknown>;
+  readonly result?: unknown;
+}
+
+/** Evaluation result */
+export interface EvalResult {
+  readonly id: string;
+  readonly input: string;
+  readonly output: string;
+  readonly expected?: string;
+  readonly scores: Record<string, number>;
+  readonly passed: boolean;
+  readonly feedback?: string;
+  readonly metadata?: Record<string, unknown>;
+}
+
+/** Evaluation criteria */
+export interface EvalCriteria {
+  readonly name: string;
+  readonly description: string;
+  readonly weight?: number;
+  evaluate: (input: string, output: string, expected?: string) => Promise<EvalScore>;
+}
+
+/** Evaluation score */
+export interface EvalScore {
+  readonly score: number; // 0-1
+  readonly reason?: string;
+  readonly details?: Record<string, unknown>;
+}
+
+/** Test case */
+export interface TestCase {
+  readonly id: string;
+  readonly name: string;
+  readonly input: string;
+  readonly expected?: string;
+  readonly metadata?: Record<string, unknown>;
+  validate?: (output: string) => boolean;
+}
+
+/** Test suite */
+export interface TestSuite {
+  readonly name: string;
+  readonly description?: string;
+  readonly cases: TestCase[];
+  readonly criteria: EvalCriteria[];
+}
+
+/** Test run result */
+export interface TestRunResult {
+  readonly suiteId: string;
+  readonly timestamp: number;
+  readonly duration: number;
+  readonly results: EvalResult[];
+  readonly summary: TestSummary;
+}
+
+/** Test summary */
+export interface TestSummary {
+  readonly total: number;
+  readonly passed: number;
+  readonly failed: number;
+  readonly avgScores: Record<string, number>;
+  readonly passRate: number;
+}
+
+// -----------------------------------------------------------------------------
+// Mock Provider
+// -----------------------------------------------------------------------------
+
+/**
+ * Creates a mock AI provider for testing.
+ *
+ * @example
+ * ```typescript
+ * const mock = createMockProvider({
+ *   responses: [
+ *     { text: 'Hello, how can I help?' },
+ *     { text: 'The answer is 42.' }
+ *   ]
+ * });
+ *
+ * // Use in tests
+ * const result = await generateText({ model: mock, prompt: 'Hi' });
+ * expect(result.text).toBe('Hello, how can I help?');
+ * ```
+ */
+export function createMockProvider(options?: {
+  responses?: MockResponse[];
+  defaultResponse?: MockResponse;
+  onGenerate?: (prompt: string) => MockResponse | Promise<MockResponse>;
+}): MockProvider {
+  const { responses = [], defaultResponse, onGenerate } = options ?? {};
+  let callIndex = 0;
+  const callHistory: { prompt: string; response: MockResponse }[] = [];
+
+  async function generate(prompt: string): Promise<MockResponse> {
+    let response: MockResponse;
+
+    if (onGenerate) {
+      response = await onGenerate(prompt);
+    } else if (callIndex < responses.length) {
+      const queued = responses[callIndex];
+      if (!queued) {
+        throw new Error(`Mock response missing at index ${callIndex}`);
+      }
+      response = queued;
+    } else if (defaultResponse) {
+      response = defaultResponse;
+    } else {
+      response = { text: `Mock response ${callIndex + 1}` };
+    }
+
+    callIndex++;
+    callHistory.push({ prompt, response });
+
+    if (response.delay) {
+      await new Promise(resolve => setTimeout(resolve, response.delay));
+    }
+
+    if (response.error) {
+      throw response.error;
+    }
+
+    return response;
+  }
+
+  function reset(): void {
+    callIndex = 0;
+    callHistory.length = 0;
+  }
+
+  function getCalls(): { prompt: string; response: MockResponse }[] {
+    return [...callHistory];
+  }
+
+  function getCallCount(): number {
+    return callHistory.length;
+  }
+
+  return {
+    generate,
+    reset,
+    getCalls,
+    getCallCount,
+    modelId: 'mock-provider',
+  };
+}
+
+export interface MockProvider {
+  generate: (prompt: string) => Promise<MockResponse>;
+  reset: () => void;
+  getCalls: () => { prompt: string; response: MockResponse }[];
+  getCallCount: () => number;
+  readonly modelId: string;
+}
+
+// -----------------------------------------------------------------------------
+// Mock Tools
+// -----------------------------------------------------------------------------
+
+/**
+ * Creates a mock tool for testing.
+ *
+ * @example
+ * ```typescript
+ * const mockSearch = createMockTool({
+ *   name: 'search',
+ *   results: [
+ *     { query: 'weather', result: { temp: 72, condition: 'sunny' } }
+ *   ]
+ * });
+ * ```
+ */
+export function createMockTool<TInput extends z.ZodType, TOutput>(options: {
+  name: string;
+  description?: string;
+  inputSchema: TInput;
+  results?: { args: Partial<z.infer<TInput>>; result: TOutput }[];
+  defaultResult?: TOutput;
+  onExecute?: (args: z.infer<TInput>) => TOutput | Promise<TOutput>;
+}): MockTool<TInput, TOutput> {
+  const { name, description, inputSchema, results = [], defaultResult, onExecute } = options;
+  const callHistory: { args: z.infer<TInput>; result: TOutput }[] = [];
+
+  async function execute(args: z.infer<TInput>): Promise<TOutput> {
+    // Validate input
+    const parsed = inputSchema.safeParse(args);
+    if (!parsed.success) {
+      throw new Error(`Invalid tool input: ${JSON.stringify(parsed.error)}`);
+    }
+
+    let result: TOutput;
+
+    if (onExecute) {
+      result = await onExecute(args);
+    } else {
+      // Find matching result
+      const match = results.find(r =>
+        Object.entries(r.args).every(
+          ([key, value]) => (args as Record<string, unknown>)[key] === value,
+        ),
+      );
+
+      if (match) {
+        const { result: matchedResult } = match;
+        result = matchedResult;
+      } else if (defaultResult !== undefined) {
+        result = defaultResult;
+      } else {
+        throw new Error(`No mock result found for args: ${JSON.stringify(args)}`);
+      }
+    }
+
+    callHistory.push({ args, result });
+    return result;
+  }
+
+  function reset(): void {
+    callHistory.length = 0;
+  }
+
+  function getCalls(): { args: z.infer<TInput>; result: TOutput }[] {
+    return [...callHistory];
+  }
+
+  return {
+    name,
+    description: description ?? `Mock tool: ${name}`,
+    inputSchema,
+    execute,
+    reset,
+    getCalls,
+  };
+}
+
+export interface MockTool<TInput extends z.ZodType, TOutput> {
+  readonly name: string;
+  readonly description: string;
+  readonly inputSchema: TInput;
+  execute: (args: z.infer<TInput>) => Promise<TOutput>;
+  reset: () => void;
+  getCalls: () => { args: z.infer<TInput>; result: TOutput }[];
+}
+
+// -----------------------------------------------------------------------------
+// Built-in Evaluation Criteria
+// -----------------------------------------------------------------------------
+
+/**
+ * Creates exact match evaluation criteria.
+ */
+export function exactMatch(): EvalCriteria {
+  return {
+    name: 'exact_match',
+    description: 'Checks if output exactly matches expected',
+    evaluate: async (_, output, expected) => {
+      if (!expected) {
+        return { score: 1, reason: 'No expected value provided' };
+      }
+      const matches = output.trim() === expected.trim();
+      return {
+        score: matches ? 1 : 0,
+        reason: matches ? 'Exact match' : 'Does not match expected output',
+      };
+    },
+  };
+}
+
+/**
+ * Creates contains evaluation criteria.
+ */
+export function contains(substring: string | string[]): EvalCriteria {
+  const substrings = Array.isArray(substring) ? substring : [substring];
+
+  return {
+    name: 'contains',
+    description: `Checks if output contains: ${substrings.join(', ')}`,
+    evaluate: async (_, output) => {
+      const found = substrings.filter(s => output.includes(s));
+      const score = found.length / substrings.length;
+      return {
+        score,
+        reason:
+          score === 1
+            ? 'All substrings found'
+            : `Found ${found.length}/${substrings.length} substrings`,
+        details: { found, missing: substrings.filter(s => !output.includes(s)) },
+      };
+    },
+  };
+}
+
+/**
+ * Creates regex match evaluation criteria.
+ */
+export function regexMatch(pattern: RegExp): EvalCriteria {
+  return {
+    name: 'regex_match',
+    description: `Matches pattern: ${pattern.toString()}`,
+    evaluate: async (_, output) => {
+      const matches = pattern.test(output);
+      return {
+        score: matches ? 1 : 0,
+        reason: matches ? 'Pattern matched' : 'Pattern did not match',
+      };
+    },
+  };
+}
+
+/**
+ * Creates length constraint evaluation criteria.
+ */
+export function lengthConstraint(options: {
+  min?: number;
+  max?: number;
+  unit?: 'chars' | 'words' | 'sentences';
+}): EvalCriteria {
+  const { min = 0, max = Infinity, unit = 'chars' } = options;
+
+  return {
+    name: 'length_constraint',
+    description: `Length between ${min}-${max} ${unit}`,
+    evaluate: async (_, output) => {
+      let length: number;
+
+      switch (unit) {
+        case 'words': {
+          const { length: wordCount } = output.split(/\s+/).filter(Boolean);
+          length = wordCount;
+          break;
+        }
+        case 'sentences': {
+          const { length: sentenceCount } = output.split(/[.!?]+/).filter(Boolean);
+          length = sentenceCount;
+          break;
+        }
+        case 'chars': {
+          const { length: charCount } = output;
+          length = charCount;
+          break;
+        }
+        default: {
+          const { length: charCount } = output;
+          length = charCount;
+        }
+      }
+
+      const inRange = length >= min && length <= max;
+      return {
+        score: inRange ? 1 : 0,
+        reason: inRange
+          ? `Length ${length} is within range`
+          : `Length ${length} is outside range [${min}, ${max}]`,
+        details: { length, min, max, unit },
+      };
+    },
+  };
+}
+
+/**
+ * Creates JSON validity evaluation criteria.
+ */
+export function validJson<T extends z.ZodType>(schema?: T): EvalCriteria {
+  return {
+    name: 'valid_json',
+    description: schema ? 'Validates JSON against schema' : 'Checks if output is valid JSON',
+    evaluate: async (_, output) => {
+      try {
+        const parsed = JSON.parse(output);
+
+        if (schema) {
+          const result = schema.safeParse(parsed);
+          return {
+            score: result.success ? 1 : 0,
+            reason: result.success
+              ? 'Valid JSON matching schema'
+              : `Schema validation failed: ${JSON.stringify(result.error)}`,
+          };
+        }
+
+        return { score: 1, reason: 'Valid JSON' };
+      } catch (e) {
+        return {
+          score: 0,
+          reason: `Invalid JSON: ${e instanceof Error ? e.message : String(e)}`,
+        };
+      }
+    },
+  };
+}
+
+// -----------------------------------------------------------------------------
+// LLM-as-Judge
+// -----------------------------------------------------------------------------
+
+/**
+ * Creates an LLM-as-judge evaluation criteria.
+ *
+ * @example
+ * ```typescript
+ * const relevance = llmJudge({
+ *   name: 'relevance',
+ *   prompt: 'Rate how relevant the response is to the question (0-10)',
+ *   generate: (prompt) => generateText({ model, prompt }),
+ *   parseScore: (response) => parseInt(response) / 10
+ * });
+ * ```
+ */
+export function llmJudge(options: {
+  name: string;
+  description?: string;
+  prompt: string | ((input: string, output: string, expected?: string) => string);
+  generate: (prompt: string) => Promise<string>;
+  parseScore: (response: string) => number;
+}): EvalCriteria {
+  const { name, description, prompt: promptTemplate, generate, parseScore } = options;
+
+  return {
+    name,
+    description: description ?? `LLM judge: ${name}`,
+    evaluate: async (input, output, expected) => {
+      const prompt =
+        typeof promptTemplate === 'function'
+          ? promptTemplate(input, output, expected)
+          : `${promptTemplate}
+
+Input: ${input}
+Output: ${output}
+${expected ? `Expected: ${expected}` : ''}
+
+Evaluation:`;
+
+      try {
+        const response = await generate(prompt);
+        const score = Math.max(0, Math.min(1, parseScore(response)));
+
+        return {
+          score,
+          reason: response.slice(0, 200), // First 200 chars as reason
+          details: { fullResponse: response },
+        };
+      } catch (e) {
+        return {
+          score: 0,
+          reason: `Judge error: ${e instanceof Error ? e.message : String(e)}`,
+        };
+      }
+    },
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Test Runner
+// -----------------------------------------------------------------------------
+
+/**
+ * Creates a test runner for evaluating AI outputs.
+ *
+ * @example
+ * ```typescript
+ * const runner = createTestRunner({
+ *   generate: (input) => generateText({ model, prompt: input }),
+ *   parallel: 5
+ * });
+ *
+ * const suite: TestSuite = {
+ *   name: 'QA Tests',
+ *   cases: [
+ *     { id: '1', name: 'Capital', input: 'What is the capital of France?', expected: 'Paris' }
+ *   ],
+ *   criteria: [exactMatch(), contains('Paris')]
+ * };
+ *
+ * const results = await runner.run(suite);
+ * console.log(`Pass rate: ${results.summary.passRate}%`);
+ * ```
+ */
+export function createTestRunner(options: {
+  generate: (input: string) => Promise<string>;
+  parallel?: number;
+  passThreshold?: number;
+  onProgress?: (completed: number, total: number) => void;
+}): TestRunner {
+  const { generate, parallel = 1, passThreshold = 0.7, onProgress } = options;
+
+  async function run(suite: TestSuite): Promise<TestRunResult> {
+    const startTime = Date.now();
+    const results: EvalResult[] = [];
+
+    // Run tests in parallel batches
+    for (let i = 0; i < suite.cases.length; i += parallel) {
+      const batch = suite.cases.slice(i, i + parallel);
+
+      const batchResults = await Promise.all(
+        batch.map(async testCase => {
+          try {
+            const output = await generate(testCase.input);
+            return evaluateCase(testCase, output, suite.criteria);
+          } catch (e) {
+            return createErrorResult(testCase, e);
+          }
+        }),
+      );
+
+      results.push(...batchResults);
+      onProgress?.(results.length, suite.cases.length);
+    }
+
+    const summary = computeSummary(results, suite.criteria);
+
+    return {
+      suiteId: suite.name,
+      timestamp: startTime,
+      duration: Date.now() - startTime,
+      results,
+      summary,
+    };
+  }
+
+  async function evaluateCase(
+    testCase: TestCase,
+    output: string,
+    criteria: EvalCriteria[],
+  ): Promise<EvalResult> {
+    const scores: Record<string, number> = {};
+    const feedbackParts: string[] = [];
+
+    let totalWeight = 0;
+    let weightedSum = 0;
+
+    for (const criterion of criteria) {
+      const score = await criterion.evaluate(testCase.input, output, testCase.expected);
+      scores[criterion.name] = score.score;
+
+      const weight = criterion.weight ?? 1;
+      totalWeight += weight;
+      weightedSum += score.score * weight;
+
+      if (score.reason) {
+        feedbackParts.push(`${criterion.name}: ${score.reason}`);
+      }
+    }
+
+    const avgScore = totalWeight > 0 ? weightedSum / totalWeight : 0;
+    const customValid = testCase.validate ? testCase.validate(output) : true;
+
+    return {
+      id: testCase.id,
+      input: testCase.input,
+      output,
+      expected: testCase.expected,
+      scores,
+      passed: avgScore >= passThreshold && customValid,
+      feedback: feedbackParts.join('\n'),
+      metadata: testCase.metadata,
+    };
+  }
+
+  function createErrorResult(testCase: TestCase, error: unknown): EvalResult {
+    return {
+      id: testCase.id,
+      input: testCase.input,
+      output: '',
+      expected: testCase.expected,
+      scores: {},
+      passed: false,
+      feedback: `Error: ${error instanceof Error ? error.message : String(error)}`,
+      metadata: testCase.metadata,
+    };
+  }
+
+  function computeSummary(results: EvalResult[], criteria: EvalCriteria[]): TestSummary {
+    const passed = results.filter(r => r.passed).length;
+    const avgScores: Record<string, number> = {};
+
+    for (const criterion of criteria) {
+      const scores = results
+        .map(r => r.scores[criterion.name])
+        .filter((s): s is number => s !== undefined);
+
+      avgScores[criterion.name] =
+        scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0;
+    }
+
+    return {
+      total: results.length,
+      passed,
+      failed: results.length - passed,
+      avgScores,
+      passRate: results.length > 0 ? (passed / results.length) * 100 : 0,
+    };
+  }
+
+  async function runSingle(
+    input: string,
+    criteria: EvalCriteria[],
+    expected?: string,
+  ): Promise<EvalResult> {
+    const output = await generate(input);
+    return evaluateCase({ id: 'single', name: 'Single test', input, expected }, output, criteria);
+  }
+
+  return {
+    run,
+    runSingle,
+  };
+}
+
+export interface TestRunner {
+  run: (suite: TestSuite) => Promise<TestRunResult>;
+  runSingle: (input: string, criteria: EvalCriteria[], expected?: string) => Promise<EvalResult>;
+}
+
+// -----------------------------------------------------------------------------
+// Assertion Helpers
+// -----------------------------------------------------------------------------
+
+/**
+ * Assertion helpers for test frameworks.
+ */
+export const assertions = {
+  /**
+   * Asserts that the eval result passed.
+   */
+  assertPassed(result: EvalResult, message?: string): void {
+    if (!result.passed) {
+      throw new Error(
+        message ??
+          `Evaluation failed. Feedback: ${result.feedback}\nScores: ${JSON.stringify(result.scores)}`,
+      );
+    }
+  },
+
+  /**
+   * Asserts that a specific criterion score is above threshold.
+   */
+  assertScore(result: EvalResult, criterion: string, minScore: number, message?: string): void {
+    const score = result.scores[criterion];
+    if (score === undefined) {
+      throw new Error(message ?? `Criterion "${criterion}" not found in scores`);
+    }
+    if (score < minScore) {
+      throw new Error(
+        message ?? `Score for "${criterion}" (${score}) is below threshold (${minScore})`,
+      );
+    }
+  },
+
+  /**
+   * Asserts that pass rate is above threshold.
+   */
+  assertPassRate(result: TestRunResult, minRate: number, message?: string): void {
+    if (result.summary.passRate < minRate) {
+      throw new Error(
+        message ??
+          `Pass rate ${result.summary.passRate.toFixed(1)}% is below threshold ${minRate}%`,
+      );
+    }
+  },
+};
+
+// -----------------------------------------------------------------------------
+// Tool Sequence Assertions
+// -----------------------------------------------------------------------------
+
+/** Tool call record for sequence assertions */
+export interface ToolSequenceRecord {
+  toolName: string;
+  toolCallId?: string;
+  args?: Record<string, unknown>;
+  result?: unknown;
+  timestamp?: number;
+}
+
+/**
+ * Tool sequence assertion helpers for agent testing.
+ *
+ * @example
+ * ```typescript
+ * const calls = agent.getToolCalls();
+ *
+ * // Assert exact sequence
+ * toolSequence.assertSequence(calls, ['search', 'browse', 'summarize']);
+ *
+ * // Assert tool was called
+ * toolSequence.assertCalled(calls, 'search');
+ *
+ * // Assert tool was called with specific args
+ * toolSequence.assertCalledWith(calls, 'search', { query: 'AI news' });
+ *
+ * // Assert call count
+ * toolSequence.assertCallCount(calls, 'search', 2);
+ * ```
+ */
+export const toolSequence = {
+  /**
+   * Assert that tools were called in a specific order.
+   */
+  assertSequence(
+    calls: ToolSequenceRecord[],
+    expected: string[],
+    options?: { strict?: boolean },
+  ): void {
+    const { strict = false } = options ?? {};
+    const actualNames = calls.map(c => c.toolName);
+
+    if (strict) {
+      // Exact sequence match
+      if (actualNames.length !== expected.length) {
+        throw new Error(
+          `Tool sequence length mismatch. Expected ${expected.length} calls, got ${actualNames.length}.\n` +
+            `Expected: ${expected.join(' -> ')}\n` +
+            `Actual: ${actualNames.join(' -> ')}`,
+        );
+      }
+
+      for (let i = 0; i < expected.length; i++) {
+        if (actualNames[i] !== expected[i]) {
+          throw new Error(
+            `Tool sequence mismatch at position ${i}.\n` +
+              `Expected: ${expected[i]}\n` +
+              `Actual: ${actualNames[i]}\n` +
+              `Full sequence: ${actualNames.join(' -> ')}`,
+          );
+        }
+      }
+    } else {
+      // Subsequence match (tools appear in order, but other tools may be in between)
+      let expectedIndex = 0;
+      for (const actualName of actualNames) {
+        if (actualName === expected[expectedIndex]) {
+          expectedIndex++;
+          if (expectedIndex === expected.length) break;
+        }
+      }
+
+      if (expectedIndex !== expected.length) {
+        throw new Error(
+          `Tool sequence not found.\n` +
+            `Expected subsequence: ${expected.join(' -> ')}\n` +
+            `Actual sequence: ${actualNames.join(' -> ')}\n` +
+            `Matched up to: ${expected.slice(0, expectedIndex).join(' -> ')}`,
+        );
+      }
+    }
+  },
+
+  /**
+   * Assert that a tool was called at least once.
+   */
+  assertCalled(calls: ToolSequenceRecord[], toolName: string): void {
+    const found = calls.some(c => c.toolName === toolName);
+    if (!found) {
+      const availableTools = [...new Set(calls.map(c => c.toolName))].join(', ');
+      throw new Error(
+        `Tool "${toolName}" was not called.\n` + `Tools called: ${availableTools || 'none'}`,
+      );
+    }
+  },
+
+  /**
+   * Assert that a tool was NOT called.
+   */
+  assertNotCalled(calls: ToolSequenceRecord[], toolName: string): void {
+    const found = calls.some(c => c.toolName === toolName);
+    if (found) {
+      throw new Error(`Tool "${toolName}" should not have been called but was.`);
+    }
+  },
+
+  /**
+   * Assert that a tool was called with specific arguments.
+   */
+  assertCalledWith(
+    calls: ToolSequenceRecord[],
+    toolName: string,
+    expectedArgs: Record<string, unknown>,
+  ): void {
+    const matchingCalls = calls.filter(c => c.toolName === toolName);
+
+    if (matchingCalls.length === 0) {
+      throw new Error(`Tool "${toolName}" was not called.`);
+    }
+
+    const found = matchingCalls.some(call => {
+      if (!call.args) return false;
+      return Object.entries(expectedArgs).every(
+        ([key, value]) => JSON.stringify(call.args?.[key]) === JSON.stringify(value),
+      );
+    });
+
+    if (!found) {
+      const actualArgs = matchingCalls.map(c => JSON.stringify(c.args)).join('\n  ');
+      throw new Error(
+        `Tool "${toolName}" was not called with expected args.\n` +
+          `Expected: ${JSON.stringify(expectedArgs)}\n` +
+          `Actual calls:\n  ${actualArgs}`,
+      );
+    }
+  },
+
+  /**
+   * Assert tool was called a specific number of times.
+   */
+  assertCallCount(calls: ToolSequenceRecord[], toolName: string, expected: number): void {
+    const count = calls.filter(c => c.toolName === toolName).length;
+    if (count !== expected) {
+      throw new Error(`Tool "${toolName}" was called ${count} times, expected ${expected}.`);
+    }
+  },
+
+  /**
+   * Assert tool was called at least N times.
+   */
+  assertMinCalls(calls: ToolSequenceRecord[], toolName: string, min: number): void {
+    const count = calls.filter(c => c.toolName === toolName).length;
+    if (count < min) {
+      throw new Error(`Tool "${toolName}" was called ${count} times, expected at least ${min}.`);
+    }
+  },
+
+  /**
+   * Assert tool was called at most N times.
+   */
+  assertMaxCalls(calls: ToolSequenceRecord[], toolName: string, max: number): void {
+    const count = calls.filter(c => c.toolName === toolName).length;
+    if (count > max) {
+      throw new Error(`Tool "${toolName}" was called ${count} times, expected at most ${max}.`);
+    }
+  },
+
+  /**
+   * Assert a tool was called before another tool.
+   */
+  assertCalledBefore(calls: ToolSequenceRecord[], firstTool: string, secondTool: string): void {
+    const firstIndex = calls.findIndex(c => c.toolName === firstTool);
+    const secondIndex = calls.findIndex(c => c.toolName === secondTool);
+
+    if (firstIndex === -1) {
+      throw new Error(`Tool "${firstTool}" was not called.`);
+    }
+    if (secondIndex === -1) {
+      throw new Error(`Tool "${secondTool}" was not called.`);
+    }
+    if (firstIndex >= secondIndex) {
+      throw new Error(
+        `Tool "${firstTool}" was not called before "${secondTool}".\n` +
+          `"${firstTool}" at index ${firstIndex}, "${secondTool}" at index ${secondIndex}.`,
+      );
+    }
+  },
+
+  /**
+   * Assert a tool returned a specific result.
+   */
+  assertResult(
+    calls: ToolSequenceRecord[],
+    toolName: string,
+    expectedResult: unknown,
+    options?: { callIndex?: number },
+  ): void {
+    const { callIndex } = options ?? {};
+    const matchingCalls = calls.filter(c => c.toolName === toolName);
+
+    if (matchingCalls.length === 0) {
+      throw new Error(`Tool "${toolName}" was not called.`);
+    }
+
+    const call = callIndex !== undefined ? matchingCalls[callIndex] : matchingCalls[0];
+
+    if (!call) {
+      throw new Error(`Tool "${toolName}" call at index ${callIndex} not found.`);
+    }
+
+    const actualJson = JSON.stringify(call.result);
+    const expectedJson = JSON.stringify(expectedResult);
+
+    if (actualJson !== expectedJson) {
+      throw new Error(
+        `Tool "${toolName}" result mismatch.\n` +
+          `Expected: ${expectedJson}\n` +
+          `Actual: ${actualJson}`,
+      );
+    }
+  },
+
+  /**
+   * Get all calls for a specific tool.
+   */
+  getCallsFor(calls: ToolSequenceRecord[], toolName: string): ToolSequenceRecord[] {
+    return calls.filter(c => c.toolName === toolName);
+  },
+
+  /**
+   * Get the last call for a specific tool.
+   */
+  getLastCall(calls: ToolSequenceRecord[], toolName: string): ToolSequenceRecord | undefined {
+    const matching = calls.filter(c => c.toolName === toolName);
+    return matching[matching.length - 1];
+  },
+
+  /**
+   * Create a summary of tool calls.
+   */
+  summarize(calls: ToolSequenceRecord[]): {
+    totalCalls: number;
+    uniqueTools: string[];
+    callCounts: Record<string, number>;
+    sequence: string[];
+  } {
+    const callCounts: Record<string, number> = {};
+    for (const call of calls) {
+      callCounts[call.toolName] = (callCounts[call.toolName] ?? 0) + 1;
+    }
+
+    return {
+      totalCalls: calls.length,
+      uniqueTools: [...new Set(calls.map(c => c.toolName))],
+      callCounts,
+      sequence: calls.map(c => c.toolName),
+    };
+  },
+};
+
+// Evaluators available via direct import
+// import { createLLMJudge, EvaluationCriteria } from '@od-oneapp/ai-platform/testing/evaluators';
+// Note: Requires SDK v6 compatibility fixes