@livekit/agents 1.0.36-dev.0 → 1.0.37

package/dist/voice/testing/run_result.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/voice/testing/run_result.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { z } from 'zod';\nimport type { AgentHandoffItem, ChatItem, ChatRole } from '../../llm/chat_context.js';\nimport { ChatContext } from '../../llm/chat_context.js';\nimport type { LLM } from '../../llm/llm.js';\nimport { tool } from '../../llm/tool_context.js';\nimport type { Task } from '../../utils.js';\nimport { Future } from '../../utils.js';\nimport type { Agent } from '../agent.js';\nimport { type SpeechHandle, isSpeechHandle } from '../speech_handle.js';\nimport {\n  type AgentHandoffAssertOptions,\n  type AgentHandoffEvent,\n  type ChatMessageEvent,\n  type EventType,\n  type FunctionCallAssertOptions,\n  type FunctionCallEvent,\n  type FunctionCallOutputAssertOptions,\n  type FunctionCallOutputEvent,\n  type MessageAssertOptions,\n  type RunEvent,\n  isAgentHandoffEvent,\n  isChatMessageEvent,\n  isFunctionCallEvent,\n  isFunctionCallOutputEvent,\n} from './types.js';\n\n// Type for agent constructor (used in assertions)\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AgentConstructor = new (...args: any[]) => Agent;\n\n// Environment variable for verbose output\nconst evalsVerbose = parseInt(process.env.LIVEKIT_EVALS_VERBOSE || '0', 10);\n\n/**\n * Result of a test run containing recorded events and assertion utilities.\n *\n * @example\n * ```typescript\n * const result = await session.run({ userInput: 'Hello' });\n * result.expect.nextEvent().isMessage({ role: 'assistant' });\n * result.expect.noMoreEvents();\n * ```\n */\nexport class RunResult<T = unknown> {\n  private _events: RunEvent[] = [];\n  private doneFut = new Future<void>();\n  private userInput?: string;\n\n  private handles: Set<SpeechHandle | Task<void>> = new Set();\n  private lastSpeechHandle?: SpeechHandle;\n  private runAssert?: RunAssert;\n\n  // TODO(brian): Add typed output support for parity with Python\n  // - Add outputType?: new (...args: unknown[]) => T\n  // - Add finalOutput?: T\n  // - Implement markDone() to extract final_output from SpeechHandle.maybeRunFinalOutput\n  // - See Python: run_result.py lines 182-201\n\n  constructor(options?: { userInput?: string }) {\n    this.userInput = options?.userInput;\n  }\n\n  /**\n   * List of all recorded events generated during the run.\n   */\n  get events(): RunEvent[] {\n    return this._events;\n  }\n\n  /**\n   * Provides an assertion helper for verifying the run events.\n   */\n  get expect(): RunAssert {\n    if (evalsVerbose) {\n      const eventsStr = formatEvents(this._events)\n        .map((line) => `      ${line}`)\n        .join('\\n');\n      console.log(\n        `\\n+ RunResult {\\n    userInput: \"${this.userInput}\"\\n    events: [\\n${eventsStr}\\n    ]\\n  }`,\n      );\n    }\n\n    // Cache the RunAssert so cursor position persists across multiple .expect accesses\n    if (!this.runAssert) {\n      this.runAssert = new RunAssert(this);\n    }\n    return this.runAssert;\n  }\n\n  /**\n   * Returns the final output of the run after completion.\n   *\n   * @throws Error - Not implemented yet.\n   */\n  get finalOutput(): T {\n    // TODO(brian): Implement typed output support after AgentTask is implemented.\n    throw new Error('finalOutput is not yet implemented in JS.');\n  }\n\n  /**\n   * Indicates whether the run has finished processing all events.\n   */\n  done(): boolean {\n    return this.doneFut.done;\n  }\n\n  /**\n   * Wait for the RunResult to complete. Returns `this` for method chaining.\n   *\n   * @example\n   * ```ts\n   * const result = session.run({ userInput: 'Hi!' });\n   * await result.wait();  // waits for completion\n   * result.expect.nextEvent().isMessage({ role: 'assistant' });\n   * ```\n   */\n  async wait(): Promise<this> {\n    await this.doneFut.await;\n    return this;\n  }\n\n  /**\n   * @internal\n   * Records an agent handoff event.\n   */\n  _agentHandoff(params: { item: AgentHandoffItem; oldAgent?: Agent; newAgent: Agent }): void {\n    const event: AgentHandoffEvent = {\n      type: 'agent_handoff',\n      item: params.item,\n      oldAgent: params.oldAgent,\n      newAgent: params.newAgent,\n    };\n    const index = this._findInsertionIndex(event.item.createdAt);\n    this._events.splice(index, 0, event);\n  }\n\n  /**\n   * @internal\n   * Called when a chat item is added during the run.\n   */\n  _itemAdded(item: ChatItem): void {\n    if (this.doneFut.done) {\n      return;\n    }\n\n    let event: RunEvent | undefined;\n\n    if (item.type === 'message') {\n      event = { type: 'message', item };\n    } else if (item.type === 'function_call') {\n      event = { type: 'function_call', item };\n    } else if (item.type === 'function_call_output') {\n      event = { type: 'function_call_output', item };\n    }\n\n    if (event) {\n      const index = this._findInsertionIndex(item.createdAt);\n      this._events.splice(index, 0, event);\n    }\n  }\n\n  /**\n   * @internal\n   * Watch a speech handle or task for completion.\n   */\n  _watchHandle(handle: SpeechHandle | Task<void>): void {\n    this.handles.add(handle);\n\n    if (isSpeechHandle(handle)) {\n      handle._addItemAddedCallback(this._itemAdded.bind(this));\n    }\n\n    handle.addDoneCallback(() => {\n      this._markDoneIfNeeded(handle);\n    });\n  }\n\n  /**\n   * @internal\n   * Unwatch a handle.\n   */\n  _unwatchHandle(handle: SpeechHandle | Task<void>): void {\n    this.handles.delete(handle);\n\n    if (isSpeechHandle(handle)) {\n      handle._removeItemAddedCallback(this._itemAdded.bind(this));\n    }\n  }\n\n  private _markDoneIfNeeded(handle: SpeechHandle | Task<void>): void {\n    if (isSpeechHandle(handle)) {\n      this.lastSpeechHandle = handle;\n    }\n\n    if ([...this.handles].every((h) => (isSpeechHandle(h) ? h.done() : h.done))) {\n      this._markDone();\n    }\n  }\n\n  private _markDone(): void {\n    // TODO(brian): Implement final output support after AgentTask is implemented.\n    // See Python run_result.py _mark_done() for reference:\n    // - Check lastSpeechHandle._maybeRunFinalOutput\n    // - Validate output type matches expected type\n    // - Set exception or resolve based on output\n    if (!this.doneFut.done) {\n      this.doneFut.resolve();\n    }\n  }\n\n  /**\n   * Find the correct insertion index to maintain chronological order.\n   */\n  private _findInsertionIndex(createdAt: number): number {\n    for (let i = this._events.length - 1; i >= 0; i--) {\n      if (this._events[i]!.item.createdAt <= createdAt) {\n        return i + 1;\n      }\n    }\n    return 0;\n  }\n}\n\n/**\n * Assertion helper for verifying run events in sequence.\n */\nexport class RunAssert {\n  private _events: RunEvent[];\n  private _currentIndex = 0;\n\n  constructor(runResult: RunResult) {\n    this._events = runResult.events;\n  }\n\n  /**\n   * Access a specific event by index for assertions.\n   * Supports negative indices (e.g., -1 for last event).\n   *\n   * @example\n   * ```typescript\n   * result.expect.at(0).isMessage({ role: 'user' });\n   * result.expect.at(-1).isMessage({ role: 'assistant' });\n   * ```\n   */\n  at(index: number): EventAssert {\n    let normalizedIndex = index;\n    if (index < 0) {\n      normalizedIndex = this._events.length + index;\n    }\n\n    if (normalizedIndex < 0 || normalizedIndex >= this._events.length) {\n      this._raiseWithDebugInfo(\n        `at(${index}) out of range (total events: ${this._events.length})`,\n        normalizedIndex,\n      );\n    }\n\n    return new EventAssert(this._events[normalizedIndex]!, this, normalizedIndex);\n  }\n\n  /**\n   * Advance to the next event, optionally filtering by type.\n   *\n   * @example\n   * ```typescript\n   * result.expect.nextEvent().isMessage({ role: 'assistant' });\n   * result.expect.nextEvent({ type: 'function_call' }).isFunctionCall({ name: 'foo' });\n   * ```\n   */\n  nextEvent(options?: { type?: EventType }): EventAssert {\n    while (true) {\n      const evAssert = this._currentEvent();\n      this._currentIndex++;\n\n      if (!options?.type || evAssert.event().type === options.type) {\n        return evAssert;\n      }\n    }\n  }\n\n  /**\n   * Skip a specified number of upcoming events without assertions.\n   *\n   * @example\n   * ```typescript\n   * result.expect.skipNext(2);\n   * ```\n   */\n  skipNext(count: number = 1): this {\n    for (let i = 0; i < count; i++) {\n      if (this._currentIndex >= this._events.length) {\n        this._raiseWithDebugInfo(`Tried to skip ${count} event(s), but only ${i} were available.`);\n      }\n      this._currentIndex++;\n    }\n    return this;\n  }\n\n  /**\n   * Conditionally skip the next event if it matches the specified criteria.\n   * Returns the event assertion if matched and skipped, or undefined if not matched.\n   *\n   * @example\n   * ```typescript\n   * // Skip optional assistant message before function call\n   * result.expect.skipNextEventIf({ type: 'message', role: 'assistant' });\n   * result.expect.nextEvent().isFunctionCall({ name: 'foo' });\n   * ```\n   */\n  skipNextEventIf(\n    options:\n      | { type: 'message'; role?: ChatRole }\n      | { type: 'function_call'; name?: string; args?: Record<string, unknown> }\n      | { type: 'function_call_output'; output?: string; isError?: boolean }\n      | { type: 'agent_handoff'; newAgentType?: AgentConstructor },\n  ):\n    | MessageAssert\n    | FunctionCallAssert\n    | FunctionCallOutputAssert\n    | AgentHandoffAssert\n    | undefined {\n    if (this._currentIndex >= this._events.length) {\n      return undefined;\n    }\n\n    try {\n      const evAssert = this._currentEvent();\n\n      if (options.type === 'message') {\n        const { role } = options;\n        const result = evAssert.isMessage({ role });\n        this._currentIndex++;\n        return result;\n      } else if (options.type === 'function_call') {\n        const { name, args } = options;\n        const result = evAssert.isFunctionCall({\n          name,\n          args,\n        });\n        this._currentIndex++;\n        return result;\n      } else if (options.type === 'function_call_output') {\n        const { output, isError } = options;\n        const result = evAssert.isFunctionCallOutput({\n          output,\n          isError,\n        });\n        this._currentIndex++;\n        return result;\n      } else if (options.type === 'agent_handoff') {\n        const { newAgentType } = options;\n        const result = evAssert.isAgentHandoff({ newAgentType });\n        this._currentIndex++;\n        return result;\n      }\n    } catch {\n      // Assertion failed, event doesn't match criteria\n      return undefined;\n    }\n\n    return undefined;\n  }\n\n  /**\n   * Get an EventRangeAssert for a range of events.\n   * Similar to Python's slice access: expect[0:3] or expect[:]\n   *\n   * @param start - Start index (inclusive), defaults to 0\n   * @param end - End index (exclusive), defaults to events.length\n   *\n   * @example\n   * ```typescript\n   * // Search all events\n   * result.expect.range().containsFunctionCall({ name: 'foo' });\n   * // Search first 3 events\n   * result.expect.range(0, 3).containsMessage({ role: 'assistant' });\n   * ```\n   */\n  range(start?: number, end?: number): EventRangeAssert {\n    const startIdx = start ?? 0;\n    const endIdx = end ?? this._events.length;\n    const events = this._events.slice(startIdx, endIdx);\n    return new EventRangeAssert(events, this, { start: startIdx, end: endIdx });\n  }\n\n  /**\n   * Assert that a function call matching criteria exists anywhere in the events.\n   *\n   * @example\n   * ```typescript\n   * result.expect.containsFunctionCall({ name: 'order_item' });\n   * ```\n   */\n  containsFunctionCall(options?: FunctionCallAssertOptions): FunctionCallAssert {\n    return this.range().containsFunctionCall(options);\n  }\n\n  /**\n   * Assert that a message matching criteria exists anywhere in the events.\n   *\n   * @example\n   * ```typescript\n   * result.expect.containsMessage({ role: 'assistant' });\n   * ```\n   */\n  containsMessage(options?: MessageAssertOptions): MessageAssert {\n    return this.range().containsMessage(options);\n  }\n\n  /**\n   * Assert that a function call output matching criteria exists anywhere in the events.\n   *\n   * @example\n   * ```typescript\n   * result.expect.containsFunctionCallOutput({ isError: false });\n   * ```\n   */\n  containsFunctionCallOutput(options?: FunctionCallOutputAssertOptions): FunctionCallOutputAssert {\n    return this.range().containsFunctionCallOutput(options);\n  }\n\n  /**\n   * Assert that an agent handoff matching criteria exists anywhere in the events.\n   *\n   * @example\n   * ```typescript\n   * result.expect.containsAgentHandoff({ newAgentType: MyAgent });\n   * ```\n   */\n  containsAgentHandoff(options?: AgentHandoffAssertOptions): AgentHandoffAssert {\n    return this.range().containsAgentHandoff(options);\n  }\n\n  /**\n   * Assert that there are no further events.\n   *\n   * @example\n   * ```typescript\n   * result.expect.noMoreEvents();\n   * ```\n   */\n  noMoreEvents(): void {\n    if (this._currentIndex < this._events.length) {\n      const event = this._events[this._currentIndex]!;\n      this._raiseWithDebugInfo(`Expected no more events, but found: ${event.type}`);\n    }\n  }\n\n  private _currentEvent(): EventAssert {\n    if (this._currentIndex >= this._events.length) {\n      this._raiseWithDebugInfo('Expected another event, but none left.');\n    }\n    return this.at(this._currentIndex);\n  }\n\n  /** @internal */\n  _raiseWithDebugInfo(message: string, index?: number): never {\n    const markerIndex = index ?? this._currentIndex;\n    const eventsStr = formatEvents(this._events, markerIndex).join('\\n');\n    throw new AssertionError(`${message}\\nContext around failure:\\n${eventsStr}`);\n  }\n}\n\n/**\n * Assertion wrapper for a single event.\n */\nexport class EventAssert {\n  protected _event: RunEvent;\n  protected _parent: RunAssert;\n  protected _index: number;\n\n  constructor(event: RunEvent, parent: RunAssert, index: number) {\n    this._event = event;\n    this._parent = parent;\n    this._index = index;\n  }\n\n  /**\n   * Get the underlying event.\n   */\n  event(): RunEvent {\n    return this._event;\n  }\n\n  protected _raise(message: string): never {\n    this._parent._raiseWithDebugInfo(message, this._index);\n  }\n\n  /**\n   * Verify this event is a message with optional role matching.\n   *\n   * @example\n   * ```typescript\n   * result.expect.nextEvent().isMessage({ role: 'assistant' });\n   * ```\n   */\n  isMessage(options?: MessageAssertOptions): MessageAssert {\n    if (!isChatMessageEvent(this._event)) {\n      this._raise(`Expected ChatMessageEvent, got ${this._event.type}`);\n    }\n\n    if (options?.role && this._event.item.role !== options.role) {\n      this._raise(`Expected role '${options.role}', got '${this._event.item.role}'`);\n    }\n\n    return new MessageAssert(this._event, this._parent, this._index);\n  }\n\n  /**\n   * Verify this event is a function call with optional name/args matching.\n   *\n   * @example\n   * ```typescript\n   * result.expect.nextEvent().isFunctionCall({ name: 'order_item', args: { id: 'big_mac' } });\n   * ```\n   */\n  isFunctionCall(options?: FunctionCallAssertOptions): FunctionCallAssert {\n    if (!isFunctionCallEvent(this._event)) {\n      this._raise(`Expected FunctionCallEvent, got ${this._event.type}`);\n    }\n\n    if (options?.name && this._event.item.name !== options.name) {\n      this._raise(`Expected call name '${options.name}', got '${this._event.item.name}'`);\n    }\n\n    if (options?.args) {\n      let actual: Record<string, unknown>;\n      try {\n        actual = JSON.parse(this._event.item.args);\n      } catch {\n        this._raise(`Failed to parse function call arguments: ${this._event.item.args}`);\n      }\n\n      for (const [key, value] of Object.entries(options.args)) {\n        if (!(key in actual) || actual[key] !== value) {\n          this._raise(\n            `For key '${key}', expected ${JSON.stringify(value)}, got ${JSON.stringify(actual[key])}`,\n          );\n        }\n      }\n    }\n\n    return new FunctionCallAssert(this._event, this._parent, this._index);\n  }\n\n  /**\n   * Verify this event is a function call output with optional matching.\n   *\n   * @example\n   * ```typescript\n   * result.expect.nextEvent().isFunctionCallOutput({ isError: false });\n   * ```\n   */\n  isFunctionCallOutput(options?: FunctionCallOutputAssertOptions): FunctionCallOutputAssert {\n    if (!isFunctionCallOutputEvent(this._event)) {\n      this._raise(`Expected FunctionCallOutputEvent, got ${this._event.type}`);\n    }\n\n    if (options?.output !== undefined && this._event.item.output !== options.output) {\n      this._raise(`Expected output '${options.output}', got '${this._event.item.output}'`);\n    }\n\n    if (options?.isError !== undefined && this._event.item.isError !== options.isError) {\n      this._raise(`Expected isError=${options.isError}, got ${this._event.item.isError}`);\n    }\n\n    return new FunctionCallOutputAssert(this._event, this._parent, this._index);\n  }\n\n  /**\n   * Verify this event is an agent handoff with optional type matching.\n   *\n   * @example\n   * ```typescript\n   * result.expect.nextEvent().isAgentHandoff({ newAgentType: MyAgent });\n   * ```\n   */\n  isAgentHandoff(options?: AgentHandoffAssertOptions): AgentHandoffAssert {\n    if (!isAgentHandoffEvent(this._event)) {\n      this._raise(`Expected AgentHandoffEvent, got ${this._event.type}`);\n    }\n\n    const event = this._event;\n\n    if (options?.newAgentType) {\n      const actualType = event.newAgent.constructor.name;\n      if (!(event.newAgent instanceof options.newAgentType)) {\n        this._raise(`Expected new_agent '${options.newAgentType.name}', got '${actualType}'`);\n      }\n    }\n\n    return new AgentHandoffAssert(event, this._parent, this._index);\n  }\n}\n\n/**\n * Assertion wrapper for a range of events.\n * Provides contains*() methods to search within the range.\n */\nexport class EventRangeAssert {\n  private _events: RunEvent[];\n  private _parent: RunAssert;\n  private _range: { start: number; end: number };\n\n  constructor(events: RunEvent[], parent: RunAssert, range: { start: number; end: number }) {\n    this._events = events;\n    this._parent = parent;\n    this._range = range;\n  }\n\n  /**\n   * Assert that a function call matching criteria exists in this event range.\n   *\n   * @example\n   * ```typescript\n   * result.expect.range(0, 3).containsFunctionCall({ name: 'foo' });\n   * ```\n   */\n  containsFunctionCall(options?: FunctionCallAssertOptions): FunctionCallAssert {\n    for (let idx = 0; idx < this._events.length; idx++) {\n      const ev = this._events[idx]!;\n      const candidate = new EventAssert(ev, this._parent, this._range.start + idx);\n      try {\n        return candidate.isFunctionCall(options);\n      } catch {\n        // Continue searching\n      }\n    }\n\n    this._parent._raiseWithDebugInfo(\n      `No FunctionCallEvent satisfying criteria found in range [${this._range.start}:${this._range.end}]`,\n    );\n  }\n\n  /**\n   * Assert that a message matching criteria exists in this event range.\n   *\n   * @example\n   * ```typescript\n   * result.expect.range(0, 2).containsMessage({ role: 'assistant' });\n   * ```\n   */\n  containsMessage(options?: MessageAssertOptions): MessageAssert {\n    for (let idx = 0; idx < this._events.length; idx++) {\n      const ev = this._events[idx]!;\n      const candidate = new EventAssert(ev, this._parent, this._range.start + idx);\n      try {\n        return candidate.isMessage(options);\n      } catch {\n        // Continue searching\n      }\n    }\n\n    this._parent._raiseWithDebugInfo(\n      `No ChatMessageEvent matching criteria found in range [${this._range.start}:${this._range.end}]`,\n    );\n  }\n\n  /**\n   * Assert that a function call output matching criteria exists in this event range.\n   *\n   * @example\n   * ```typescript\n   * result.expect.range(1, 4).containsFunctionCallOutput({ isError: true });\n   * ```\n   */\n  containsFunctionCallOutput(options?: FunctionCallOutputAssertOptions): FunctionCallOutputAssert {\n    for (let idx = 0; idx < this._events.length; idx++) {\n      const ev = this._events[idx]!;\n      const candidate = new EventAssert(ev, this._parent, this._range.start + idx);\n      try {\n        return candidate.isFunctionCallOutput(options);\n      } catch {\n        // Continue searching\n      }\n    }\n\n    this._parent._raiseWithDebugInfo(\n      `No FunctionCallOutputEvent matching criteria found in range [${this._range.start}:${this._range.end}]`,\n    );\n  }\n\n  /**\n   * Assert that an agent handoff matching criteria exists in this event range.\n   *\n   * @example\n   * ```typescript\n   * result.expect.range(0, 3).containsAgentHandoff({ newAgentType: MyAgent });\n   * ```\n   */\n  containsAgentHandoff(options?: AgentHandoffAssertOptions): AgentHandoffAssert {\n    for (let idx = 0; idx < this._events.length; idx++) {\n      const ev = this._events[idx]!;\n      const candidate = new EventAssert(ev, this._parent, this._range.start + idx);\n      try {\n        return candidate.isAgentHandoff(options);\n      } catch {\n        // Continue searching\n      }\n    }\n\n    this._parent._raiseWithDebugInfo(\n      `No AgentHandoffEvent matching criteria found in range [${this._range.start}:${this._range.end}]`,\n    );\n  }\n}\n\n/**\n * Assertion wrapper for message events.\n */\nexport class MessageAssert extends EventAssert {\n  protected declare _event: ChatMessageEvent;\n\n  constructor(event: ChatMessageEvent, parent: RunAssert, index: number) {\n    super(event, parent, index);\n  }\n\n  override event(): ChatMessageEvent {\n    return this._event;\n  }\n\n  /**\n   * Evaluate whether the message fulfills the given intent using an LLM.\n   *\n   * @param llm - LLM instance for judgment\n   * @param options - Options containing the intent description\n   * @returns Self for chaining further assertions\n   *\n   * @example\n   * ```typescript\n   * await result.expect\n   *   .nextEvent()\n   *   .isMessage({ role: 'assistant' })\n   *   .judge(llm, { intent: 'should ask for the drink size' });\n   * ```\n   */\n  async judge(llm: LLM, options: { intent: string }): Promise<MessageAssert> {\n    const { intent } = options;\n\n    // Extract text content from message\n    const content = this._event.item.content;\n    const msgContent =\n      typeof content === 'string'\n        ? content\n        : Array.isArray(content)\n          ? content.filter((c): c is string => typeof c === 'string').join(' ')\n          : '';\n\n    if (!msgContent) {\n      this._raise('The chat message is empty.');\n    }\n\n    if (!intent) {\n      this._raise('Intent is required to judge the message.');\n    }\n\n    // Create the check_intent tool\n    const checkIntentTool = tool({\n      description:\n        'Determines whether the message correctly fulfills the given intent. ' +\n        'Returns success=true if the message satisfies the intent, false otherwise. ' +\n        'Provide a concise reason justifying the result.',\n      parameters: z.object({\n        success: z.boolean().describe('Whether the message satisfies the intent'),\n        reason: z.string().describe('A concise explanation justifying the result'),\n      }),\n      execute: async ({ success, reason }: { success: boolean; reason: string }) => {\n        return { success, reason };\n      },\n    });\n\n    // Create chat context for the judge\n    const chatCtx = ChatContext.empty();\n    chatCtx.addMessage({\n      role: 'system',\n      content:\n        'You are a test evaluator for conversational agents.\\n' +\n        'You will be shown a message and a target intent. Determine whether the message accomplishes the intent.\\n' +\n        'Only respond by calling the `check_intent(success: bool, reason: str)` function with your final judgment.\\n' +\n        'Be strict: if the message does not clearly fulfill the intent, return `success = false` and explain why.',\n    });\n    chatCtx.addMessage({\n      role: 'user',\n      content:\n        'Check if the following message fulfills the given intent.\\n\\n' +\n        `Intent:\\n${intent}\\n\\n` +\n        `Message:\\n${msgContent}`,\n    });\n\n    // Call the LLM with the check_intent tool\n    let toolArgs: { success: boolean; reason: string } | undefined;\n\n    const stream = llm.chat({\n      chatCtx,\n      toolCtx: { check_intent: checkIntentTool },\n      toolChoice: { type: 'function', function: { name: 'check_intent' } },\n      extraKwargs: { temperature: 0 },\n    });\n\n    for await (const chunk of stream) {\n      if (!chunk.delta) continue;\n\n      if (chunk.delta.toolCalls && chunk.delta.toolCalls.length > 0) {\n        const toolCall = chunk.delta.toolCalls[0]!;\n        if (toolCall.args) {\n          try {\n            toolArgs = JSON.parse(toolCall.args);\n          } catch {\n            // Args might be streamed incrementally, keep the last valid parse\n          }\n        }\n      }\n    }\n\n    if (!toolArgs) {\n      this._raise('LLM did not return any arguments for evaluation.');\n    }\n\n    const { success, reason } = toolArgs;\n\n    if (!success) {\n      this._raise(`Judgment failed: ${reason}`);\n    } else if (evalsVerbose) {\n      const printMsg =\n        msgContent.length > 30 ? msgContent.slice(0, 30).replace(/\\n/g, '\\\\n') + '...' : msgContent;\n      console.log(`- Judgment succeeded for \\`${printMsg}\\`: \\`${reason}\\``);\n    }\n\n    return this;\n  }\n}\n\n/**\n * Assertion wrapper for function call events.\n */\nexport class FunctionCallAssert extends EventAssert {\n  protected declare _event: FunctionCallEvent;\n\n  constructor(event: FunctionCallEvent, parent: RunAssert, index: number) {\n    super(event, parent, index);\n  }\n\n  override event(): FunctionCallEvent {\n    return this._event;\n  }\n}\n\n/**\n * Assertion wrapper for function call output events.\n */\nexport class FunctionCallOutputAssert extends EventAssert {\n  protected declare _event: FunctionCallOutputEvent;\n\n  constructor(event: FunctionCallOutputEvent, parent: RunAssert, index: number) {\n    super(event, parent, index);\n  }\n\n  override event(): FunctionCallOutputEvent {\n    return this._event;\n  }\n}\n\n/**\n * Assertion wrapper for agent handoff events.\n */\nexport class AgentHandoffAssert extends EventAssert {\n  protected declare _event: AgentHandoffEvent;\n\n  constructor(event: AgentHandoffEvent, parent: RunAssert, index: number) {\n    super(event, parent, index);\n  }\n\n  override event(): AgentHandoffEvent {\n    return this._event;\n  }\n}\n\n/**\n * Custom assertion error for test failures.\n */\nexport class AssertionError extends Error {\n  constructor(message: string) {\n    super(message);\n    this.name = 'AssertionError';\n    Error.captureStackTrace?.(this, AssertionError);\n  }\n}\n\n// TODO: mockTools() utility for mocking tool implementations in tests\n// Will be implemented for test suites.\n// See Python run_result.py lines 1010-1031 for reference.\n\n/**\n * Format events for debug output, optionally marking a selected index.\n */\nfunction formatEvents(events: RunEvent[], selectedIndex?: number): string[] {\n  const lines: string[] = [];\n\n  for (let i = 0; i < events.length; i++) {\n    const event = events[i]!;\n    let prefix = '';\n    if (selectedIndex !== undefined) {\n      prefix = i === selectedIndex ? '>>>' : '   ';\n    }\n\n    let line: string;\n    if (isChatMessageEvent(event)) {\n      const { role, content, interrupted } = event.item;\n      const textContent =\n        typeof content === 'string'\n          ? content\n          : Array.isArray(content)\n            ? content.filter((c): c is string => typeof c === 'string').join(' ')\n            : '';\n      const truncated = textContent.length > 50 ? textContent.slice(0, 50) + '...' : textContent;\n      line = `${prefix}[${i}] { type: \"message\", role: \"${role}\", content: \"${truncated}\", interrupted: ${interrupted} }`;\n    } else if (isFunctionCallEvent(event)) {\n      const { name, args } = event.item;\n      line = `${prefix}[${i}] { type: \"function_call\", name: \"${name}\", args: ${args} }`;\n    } else if (isFunctionCallOutputEvent(event)) {\n      const { output, isError } = event.item;\n      const truncated = output.length > 50 ? output.slice(0, 50) + '...' : output;\n      line = `${prefix}[${i}] { type: \"function_call_output\", output: \"${truncated}\", isError: ${isError} }`;\n    } else if (isAgentHandoffEvent(event)) {\n      line = `${prefix}[${i}] { type: \"agent_handoff\", oldAgent: \"${event.oldAgent?.constructor.name}\", newAgent: \"${event.newAgent.constructor.name}\" }`;\n    } else {\n      line = `${prefix}[${i}] ${event}`;\n    }\n\n    lines.push(line);\n  }\n\n  return lines;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,iBAAkB;AAElB,0BAA4B;AAE5B,0BAAqB;AAErB,mBAAuB;AAEvB,2BAAkD;AAClD,mBAeO;AAOP,MAAM,eAAe,SAAS,QAAQ,IAAI,yBAAyB,KAAK,EAAE;AAYnE,MAAM,UAAuB;AAAA,EAC1B,UAAsB,CAAC;AAAA,EACvB,UAAU,IAAI,oBAAa;AAAA,EAC3B;AAAA,EAEA,UAA0C,oBAAI,IAAI;AAAA,EAClD;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQR,YAAY,SAAkC;AAC5C,SAAK,YAAY,mCAAS;AAAA,EAC5B;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,SAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKA,IAAI,SAAoB;AACtB,QAAI,cAAc;AAChB,YAAM,YAAY,aAAa,KAAK,OAAO,EACxC,IAAI,CAAC,SAAS,SAAS,IAAI,EAAE,EAC7B,KAAK,IAAI;AACZ,cAAQ;AAAA,QACN;AAAA;AAAA,kBAAoC,KAAK,SAAS;AAAA;AAAA,EAAqB,SAAS;AAAA;AAAA;AAAA,MAClF;AAAA,IACF;AAGA,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY,IAAI,UAAU,IAAI;AAAA,IACrC;AACA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,IAAI,cAAiB;AAEnB,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AAAA;AAAA;AAAA;AAAA,EAKA,OAAgB;AACd,WAAO,KAAK,QAAQ;AAAA,EACtB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,OAAsB;AAC1B,UAAM,KAAK,QAAQ;AACnB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,cAAc,QAA6E;AACzF,UAAM,QAA2B;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,OAAO;AAAA,MACb,UAAU,OAAO;AAAA,MACjB,UAAU,OAAO;AAAA,IACnB;AACA,UAAM,QAAQ,KAAK,oBAAoB,MAAM,KAAK,SAAS;AAC3D,SAAK,QAAQ,OAAO,OAAO,GAAG,KAAK;AAAA,EACrC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,WAAW,MAAsB;AAC/B,QAAI,KAAK,QAAQ,MAAM;AACrB;AAAA,IACF;AAEA,QAAI;AAEJ,QAAI,KAAK,SAAS,WAAW;AAC3B,cAAQ,EAAE,MAAM,WAAW,KAAK;AAAA,IAClC,WAAW,KAAK,SAAS,iBAAiB;AACxC,cAAQ,EAAE,MAAM,iBAAiB,KAAK;AAAA,IACxC,WAAW,KAAK,SAAS,wBAAwB;AAC/C,cAAQ,EAAE,MAAM,wBAAwB,KAAK;AAAA,IAC/C;AAEA,QAAI,OAAO;AACT,YAAM,QAAQ,KAAK,oBAAoB,KAAK,SAAS;AACrD,WAAK,QAAQ,OAAO,OAAO,GAAG,KAAK;AAAA,IACrC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,aAAa,QAAyC;AACpD,SAAK,QAAQ,IAAI,MAAM;AAEvB,YAAI,qCAAe,MAAM,GAAG;AAC1B,aAAO,sBAAsB,KAAK,WAAW,KAAK,IAAI,CAAC;AAAA,IACzD;AAEA,WAAO,gBAAgB,MAAM;AAC3B,WAAK,kBAAkB,MAAM;AAAA,IAC/B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,eAAe,QAAyC;AACtD,SAAK,QAAQ,OAAO,MAAM;AAE1B,YAAI,qCAAe,MAAM,GAAG;AAC1B,aAAO,yBAAyB,KAAK,WAAW,KAAK,IAAI,CAAC;AAAA,IAC5D;AAAA,EACF;AAAA,EAEQ,kBAAkB,QAAyC;AACjE,YAAI,qCAAe,MAAM,GAAG;AAC1B,WAAK,mBAAmB;AAAA,IAC1B;AAEA,QAAI,CAAC,GAAG,KAAK,OAAO,EAAE,MAAM,CAAC,UAAO,qCAAe,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE,IAAK,GAAG;AAC3E,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AAAA,EAEQ,YAAkB;AAMxB,QAAI,CAAC,KAAK,QAAQ,MAAM;AACtB,WAAK,QAAQ,QAAQ;AAAA,IACvB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,oBAAoB,WAA2B;AACrD,aAAS,IAAI,KAAK,QAAQ,SAAS,GAAG,KAAK,GAAG,KAAK;AACjD,UAAI,KAAK,QAAQ,CAAC,EAAG,KAAK,aAAa,WAAW;AAChD,eAAO,IAAI;AAAA,MACb;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;AAKO,MAAM,UAAU;AAAA,EACb;AAAA,EACA,gBAAgB;AAAA,EAExB,YAAY,WAAsB;AAChC,SAAK,UAAU,UAAU;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,GAAG,OAA4B;AAC7B,QAAI,kBAAkB;AACtB,QAAI,QAAQ,GAAG;AACb,wBAAkB,KAAK,QAAQ,SAAS;AAAA,IAC1C;AAEA,QAAI,kBAAkB,KAAK,mBAAmB,KAAK,QAAQ,QAAQ;AACjE,WAAK;AAAA,QACH,MAAM,KAAK,iCAAiC,KAAK,QAAQ,MAAM;AAAA,QAC/D;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,YAAY,KAAK,QAAQ,eAAe,GAAI,MAAM,eAAe;AAAA,EAC9E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,UAAU,SAA6C;AACrD,WAAO,MAAM;AACX,YAAM,WAAW,KAAK,cAAc;AACpC,WAAK;AAEL,UAAI,EAAC,mCAAS,SAAQ,SAAS,MAAM,EAAE,SAAS,QAAQ,MAAM;AAC5D,eAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,SAAS,QAAgB,GAAS;AAChC,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,UAAI,KAAK,iBAAiB,KAAK,QAAQ,QAAQ;AAC7C,aAAK,oBAAoB,iBAAiB,KAAK,uBAAuB,CAAC,kBAAkB;AAAA,MAC3F;AACA,WAAK;AAAA,IACP;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,gBACE,SAUY;AACZ,QAAI,KAAK,iBAAiB,KAAK,QAAQ,QAAQ;AAC7C,aAAO;AAAA,IACT;AAEA,QAAI;AACF,YAAM,WAAW,KAAK,cAAc;AAEpC,UAAI,QAAQ,SAAS,WAAW;AAC9B,cAAM,EAAE,KAAK,IAAI;AACjB,cAAM,SAAS,SAAS,UAAU,EAAE,KAAK,CAAC;AAC1C,aAAK;AACL,eAAO;AAAA,MACT,WAAW,QAAQ,SAAS,iBAAiB;AAC3C,cAAM,EAAE,MAAM,KAAK,IAAI;AACvB,cAAM,SAAS,SAAS,eAAe;AAAA,UACrC;AAAA,UACA;AAAA,QACF,CAAC;AACD,aAAK;AACL,eAAO;AAAA,MACT,WAAW,QAAQ,SAAS,wBAAwB;AAClD,cAAM,EAAE,QAAQ,QAAQ,IAAI;AAC5B,cAAM,SAAS,SAAS,qBAAqB;AAAA,UAC3C;AAAA,UACA;AAAA,QACF,CAAC;AACD,aAAK;AACL,eAAO;AAAA,MACT,WAAW,QAAQ,SAAS,iBAAiB;AAC3C,cAAM,EAAE,aAAa,IAAI;AACzB,cAAM,SAAS,SAAS,eAAe,EAAE,aAAa,CAAC;AACvD,aAAK;AACL,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAEN,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAiBA,MAAM,OAAgB,KAAgC;AACpD,UAAM,WAAW,SAAS;AAC1B,UAAM,SAAS,OAAO,KAAK,QAAQ;AACnC,UAAM,SAAS,KAAK,QAAQ,MAAM,UAAU,MAAM;AAClD,WAAO,IAAI,iBAAiB,QAAQ,MAAM,EAAE,OAAO,UAAU,KAAK,OAAO,CAAC;AAAA,EAC5E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,qBAAqB,SAAyD;AAC5E,WAAO,KAAK,MAAM,EAAE,qBAAqB,OAAO;AAAA,EAClD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,gBAAgB,SAA+C;AAC7D,WAAO,KAAK,MAAM,EAAE,gBAAgB,OAAO;AAAA,EAC7C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,2BAA2B,SAAqE;AAC9F,WAAO,KAAK,MAAM,EAAE,2BAA2B,OAAO;AAAA,EACxD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,qBAAqB,SAAyD;AAC5E,WAAO,KAAK,MAAM,EAAE,qBAAqB,OAAO;AAAA,EAClD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,eAAqB;AACnB,QAAI,KAAK,gBAAgB,KAAK,QAAQ,QAAQ;AAC5C,YAAM,QAAQ,KAAK,QAAQ,KAAK,aAAa;AAC7C,WAAK,oBAAoB,uCAAuC,MAAM,IAAI,EAAE;AAAA,IAC9E;AAAA,EACF;AAAA,EAEQ,gBAA6B;AACnC,QAAI,KAAK,iBAAiB,KAAK,QAAQ,QAAQ;AAC7C,WAAK,oBAAoB,wCAAwC;AAAA,IACnE;AACA,WAAO,KAAK,GAAG,KAAK,aAAa;AAAA,EACnC;AAAA;AAAA,EAGA,oBAAoB,SAAiB,OAAuB;AAC1D,UAAM,cAAc,SAAS,KAAK;AAClC,UAAM,YAAY,aAAa,KAAK,SAAS,WAAW,EAAE,KAAK,IAAI;AACnE,UAAM,IAAI,eAAe,GAAG,OAAO;AAAA;AAAA,EAA8B,SAAS,EAAE;AAAA,EAC9E;AACF;AAKO,MAAM,YAAY;AAAA,EACb;AAAA,EACA;AAAA,EACA;AAAA,EAEV,YAAY,OAAiB,QAAmB,OAAe;AAC7D,SAAK,SAAS;AACd,SAAK,UAAU;AACf,SAAK,SAAS;AAAA,EAChB;AAAA;AAAA;AAAA;AAAA,EAKA,QAAkB;AAChB,WAAO,KAAK;AAAA,EACd;AAAA,EAEU,OAAO,SAAwB;AACvC,SAAK,QAAQ,oBAAoB,SAAS,KAAK,MAAM;AAAA,EACvD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,UAAU,SAA+C;AACvD,QAAI,KAAC,iCAAmB,KAAK,MAAM,GAAG;AACpC,WAAK,OAAO,kCAAkC,KAAK,OAAO,IAAI,EAAE;AAAA,IAClE;AAEA,SAAI,mCAAS,SAAQ,KAAK,OAAO,KAAK,SAAS,QAAQ,MAAM;AAC3D,WAAK,OAAO,kBAAkB,QAAQ,IAAI,WAAW,KAAK,OAAO,KAAK,IAAI,GAAG;AAAA,IAC/E;AAEA,WAAO,IAAI,cAAc,KAAK,QAAQ,KAAK,SAAS,KAAK,MAAM;AAAA,EACjE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,eAAe,SAAyD;AACtE,QAAI,KAAC,kCAAoB,KAAK,MAAM,GAAG;AACrC,WAAK,OAAO,mCAAmC,KAAK,OAAO,IAAI,EAAE;AAAA,IACnE;AAEA,SAAI,mCAAS,SAAQ,KAAK,OAAO,KAAK,SAAS,QAAQ,MAAM;AAC3D,WAAK,OAAO,uBAAuB,QAAQ,IAAI,WAAW,KAAK,OAAO,KAAK,IAAI,GAAG;AAAA,IACpF;AAEA,QAAI,mCAAS,MAAM;AACjB,UAAI;AACJ,UAAI;AACF,iBAAS,KAAK,MAAM,KAAK,OAAO,KAAK,IAAI;AAAA,MAC3C,QAAQ;AACN,aAAK,OAAO,4CAA4C,KAAK,OAAO,KAAK,IAAI,EAAE;AAAA,MACjF;AAEA,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,QAAQ,IAAI,GAAG;AACvD,YAAI,EAAE,OAAO,WAAW,OAAO,GAAG,MAAM,OAAO;AAC7C,eAAK;AAAA,YACH,YAAY,GAAG,eAAe,KAAK,UAAU,KAAK,CAAC,SAAS,KAAK,UAAU,OAAO,GAAG,CAAC,CAAC;AAAA,UACzF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,mBAAmB,KAAK,QAAQ,KAAK,SAAS,KAAK,MAAM;AAAA,EACtE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,qBAAqB,SAAqE;AACxF,QAAI,KAAC,wCAA0B,KAAK,MAAM,GAAG;AAC3C,WAAK,OAAO,yCAAyC,KAAK,OAAO,IAAI,EAAE;AAAA,IACzE;AAEA,SAAI,mCAAS,YAAW,UAAa,KAAK,OAAO,KAAK,WAAW,QAAQ,QAAQ;AAC/E,WAAK,OAAO,oBAAoB,QAAQ,MAAM,WAAW,KAAK,OAAO,KAAK,MAAM,GAAG;AAAA,IACrF;AAEA,SAAI,mCAAS,aAAY,UAAa,KAAK,OAAO,KAAK,YAAY,QAAQ,SAAS;AAClF,WAAK,OAAO,oBAAoB,QAAQ,OAAO,SAAS,KAAK,OAAO,KAAK,OAAO,EAAE;AAAA,IACpF;AAEA,WAAO,IAAI,yBAAyB,KAAK,QAAQ,KAAK,SAAS,KAAK,MAAM;AAAA,EAC5E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,eAAe,SAAyD;AACtE,QAAI,KAAC,kCAAoB,KAAK,MAAM,GAAG;AACrC,WAAK,OAAO,mCAAmC,KAAK,OAAO,IAAI,EAAE;AAAA,IACnE;AAEA,UAAM,QAAQ,KAAK;AAEnB,QAAI,mCAAS,cAAc;AACzB,YAAM,aAAa,MAAM,SAAS,YAAY;AAC9C,UAAI,EAAE,MAAM,oBAAoB,QAAQ,eAAe;AACrD,aAAK,OAAO,uBAAuB,QAAQ,aAAa,IAAI,WAAW,UAAU,GAAG;AAAA,MACtF;AAAA,IACF;AAEA,WAAO,IAAI,mBAAmB,OAAO,KAAK,SAAS,KAAK,MAAM;AAAA,EAChE;AACF;AAMO,MAAM,iBAAiB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,QAAoB,QAAmB,OAAuC;AACxF,SAAK,UAAU;AACf,SAAK,UAAU;AACf,SAAK,SAAS;AAAA,EAChB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,qBAAqB,SAAyD;AAC5E,aAAS,MAAM,GAAG,MAAM,KAAK,QAAQ,QAAQ,OAAO;AAClD,YAAM,KAAK,KAAK,QAAQ,GAAG;AAC3B,YAAM,YAAY,IAAI,YAAY,IAAI,KAAK,SAAS,KAAK,OAAO,QAAQ,GAAG;AAC3E,UAAI;AACF,eAAO,UAAU,eAAe,OAAO;AAAA,MACzC,QAAQ;AAAA,MAER;AAAA,IACF;AAEA,SAAK,QAAQ;AAAA,MACX,4DAA4D,KAAK,OAAO,KAAK,IAAI,KAAK,OAAO,GAAG;AAAA,IAClG;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,gBAAgB,SAA+C;AAC7D,aAAS,MAAM,GAAG,MAAM,KAAK,QAAQ,QAAQ,OAAO;AAClD,YAAM,KAAK,KAAK,QAAQ,GAAG;AAC3B,YAAM,YAAY,IAAI,YAAY,IAAI,KAAK,SAAS,KAAK,OAAO,QAAQ,GAAG;AAC3E,UAAI;AACF,eAAO,UAAU,UAAU,OAAO;AAAA,MACpC,QAAQ;AAAA,MAER;AAAA,IACF;AAEA,SAAK,QAAQ;AAAA,MACX,yDAAyD,KAAK,OAAO,KAAK,IAAI,KAAK,OAAO,GAAG;AAAA,IAC/F;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,2BAA2B,SAAqE;AAC9F,aAAS,MAAM,GAAG,MAAM,KAAK,QAAQ,QAAQ,OAAO;AAClD,YAAM,KAAK,KAAK,QAAQ,GAAG;AAC3B,YAAM,YAAY,IAAI,YAAY,IAAI,KAAK,SAAS,KAAK,OAAO,QAAQ,GAAG;AAC3E,UAAI;AACF,eAAO,UAAU,qBAAqB,OAAO;AAAA,MAC/C,QAAQ;AAAA,MAER;AAAA,IACF;AAEA,SAAK,QAAQ;AAAA,MACX,gEAAgE,KAAK,OAAO,KAAK,IAAI,KAAK,OAAO,GAAG;AAAA,IACtG;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,qBAAqB,SAAyD;AAC5E,aAAS,MAAM,GAAG,MAAM,KAAK,QAAQ,QAAQ,OAAO;AAClD,YAAM,KAAK,KAAK,QAAQ,GAAG;AAC3B,YAAM,YAAY,IAAI,YAAY,IAAI,KAAK,SAAS,KAAK,OAAO,QAAQ,GAAG;AAC3E,UAAI;AACF,eAAO,UAAU,eAAe,OAAO;AAAA,MACzC,QAAQ;AAAA,MAER;AAAA,IACF;AAEA,SAAK,QAAQ;AAAA,MACX,0DAA0D,KAAK,OAAO,KAAK,IAAI,KAAK,OAAO,GAAG;AAAA,IAChG;AAAA,EACF;AACF;AAKO,MAAM,sBAAsB,YAAY;AAAA,EAG7C,YAAY,OAAyB,QAAmB,OAAe;AACrE,UAAM,OAAO,QAAQ,KAAK;AAAA,EAC5B;AAAA,EAES,QAA0B;AACjC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAiBA,MAAM,MAAM,KAAU,SAAqD;AACzE,UAAM,EAAE,OAAO,IAAI;AAGnB,UAAM,UAAU,KAAK,OAAO,KAAK;AACjC,UAAM,aACJ,OAAO,YAAY,WACf,UACA,MAAM,QAAQ,OAAO,IACnB,QAAQ,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,EAAE,KAAK,GAAG,IAClE;AAER,QAAI,CAAC,YAAY;AACf,WAAK,OAAO,4BAA4B;AAAA,IAC1C;AAEA,QAAI,CAAC,QAAQ;AACX,WAAK,OAAO,0CAA0C;AAAA,IACxD;AAGA,UAAM,sBAAkB,0BAAK;AAAA,MAC3B,aACE;AAAA,MAGF,YAAY,aAAE,OAAO;AAAA,QACnB,SAAS,aAAE,QAAQ,EAAE,SAAS,0CAA0C;AAAA,QACxE,QAAQ,aAAE,OAAO,EAAE,SAAS,6CAA6C;AAAA,MAC3E,CAAC;AAAA,MACD,SAAS,OAAO,EAAE,SAAAA,UAAS,QAAAC,QAAO,MAA4C;AAC5E,eAAO,EAAE,SAAAD,UAAS,QAAAC,QAAO;AAAA,MAC3B;AAAA,IACF,CAAC;AAGD,UAAM,UAAU,gCAAY,MAAM;AAClC,YAAQ,WAAW;AAAA,MACjB,MAAM;AAAA,MACN,SACE;AAAA,IAIJ,CAAC;AACD,YAAQ,WAAW;AAAA,MACjB,MAAM;AAAA,MACN,SACE;AAAA;AAAA;AAAA,EACY,MAAM;AAAA;AAAA;AAAA,EACL,UAAU;AAAA,IAC3B,CAAC;AAGD,QAAI;AAEJ,UAAM,SAAS,IAAI,KAAK;AAAA,MACtB;AAAA,MACA,SAAS,EAAE,cAAc,gBAAgB;AAAA,MACzC,YAAY,EAAE,MAAM,YAAY,UAAU,EAAE,MAAM,eAAe,EAAE;AAAA,MACnE,aAAa,EAAE,aAAa,EAAE;AAAA,IAChC,CAAC;AAED,qBAAiB,SAAS,QAAQ;AAChC,UAAI,CAAC,MAAM,MAAO;AAElB,UAAI,MAAM,MAAM,aAAa,MAAM,MAAM,UAAU,SAAS,GAAG;AAC7D,cAAM,WAAW,MAAM,MAAM,UAAU,CAAC;AACxC,YAAI,SAAS,MAAM;AACjB,cAAI;AACF,uBAAW,KAAK,MAAM,SAAS,IAAI;AAAA,UACrC,QAAQ;AAAA,UAER;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,UAAU;AACb,WAAK,OAAO,kDAAkD;AAAA,IAChE;AAEA,UAAM,EAAE,SAAS,OAAO,IAAI;AAE5B,QAAI,CAAC,SAAS;AACZ,WAAK,OAAO,oBAAoB,MAAM,EAAE;AAAA,IAC1C,WAAW,cAAc;AACvB,YAAM,WACJ,WAAW,SAAS,KAAK,WAAW,MAAM,GAAG,EAAE,EAAE,QAAQ,OAAO,KAAK,IAAI,QAAQ;AACnF,cAAQ,IAAI,8BAA8B,QAAQ,SAAS,MAAM,IAAI;AAAA,IACvE;AAEA,WAAO;AAAA,EACT;AACF;AAKO,MAAM,2BAA2B,YAAY;AAAA,EAGlD,YAAY,OAA0B,QAAmB,OAAe;AACtE,UAAM,OAAO,QAAQ,KAAK;AAAA,EAC5B;AAAA,EAES,QAA2B;AAClC,WAAO,KAAK;AAAA,EACd;AACF;AAKO,MAAM,iCAAiC,YAAY;AAAA,EAGxD,YAAY,OAAgC,QAAmB,OAAe;AAC5E,UAAM,OAAO,QAAQ,KAAK;AAAA,EAC5B;AAAA,EAES,QAAiC;AACxC,WAAO,KAAK;AAAA,EACd;AACF;AAKO,MAAM,2BAA2B,YAAY;AAAA,EAGlD,YAAY,OAA0B,QAAmB,OAAe;AACtE,UAAM,OAAO,QAAQ,KAAK;AAAA,EAC5B;AAAA,EAES,QAA2B;AAClC,WAAO,KAAK;AAAA,EACd;AACF;AAKO,MAAM,uBAAuB,MAAM;AAAA,EACxC,YAAY,SAAiB;AAp3B/B;AAq3BI,UAAM,OAAO;AACb,SAAK,OAAO;AACZ,gBAAM,sBAAN,+BAA0B,MAAM;AAAA,EAClC;AACF;AASA,SAAS,aAAa,QAAoB,eAAkC;AAl4B5E;AAm4BE,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,UAAM,QAAQ,OAAO,CAAC;AACtB,QAAI,SAAS;AACb,QAAI,kBAAkB,QAAW;AAC/B,eAAS,MAAM,gBAAgB,QAAQ;AAAA,IACzC;AAEA,QAAI;AACJ,YAAI,iCAAmB,KAAK,GAAG;AAC7B,YAAM,EAAE,MAAM,SAAS,YAAY,IAAI,MAAM;AAC7C,YAAM,cACJ,OAAO,YAAY,WACf,UACA,MAAM,QAAQ,OAAO,IACnB,QAAQ,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,EAAE,KAAK,GAAG,IAClE;AACR,YAAM,YAAY,YAAY,SAAS,KAAK,YAAY,MAAM,GAAG,EAAE,IAAI,QAAQ;AAC/E,aAAO,GAAG,MAAM,IAAI,CAAC,+BAA+B,IAAI,gBAAgB,SAAS,mBAAmB,WAAW;AAAA,IACjH,eAAW,kCAAoB,KAAK,GAAG;AACrC,YAAM,EAAE,MAAM,KAAK,IAAI,MAAM;AAC7B,aAAO,GAAG,MAAM,IAAI,CAAC,qCAAqC,IAAI,YAAY,IAAI;AAAA,IAChF,eAAW,wCAA0B,KAAK,GAAG;AAC3C,YAAM,EAAE,QAAQ,QAAQ,IAAI,MAAM;AAClC,YAAM,YAAY,OAAO,SAAS,KAAK,OAAO,MAAM,GAAG,EAAE,IAAI,QAAQ;AACrE,aAAO,GAAG,MAAM,IAAI,CAAC,8CAA8C,SAAS,eAAe,OAAO;AAAA,IACpG,eAAW,kCAAoB,KAAK,GAAG;AACrC,aAAO,GAAG,MAAM,IAAI,CAAC,0CAAyC,WAAM,aAAN,mBAAgB,YAAY,IAAI,iBAAiB,MAAM,SAAS,YAAY,IAAI;AAAA,IAChJ,OAAO;AACL,aAAO,GAAG,MAAM,IAAI,CAAC,KAAK,KAAK;AAAA,IACjC;AAEA,UAAM,KAAK,IAAI;AAAA,EACjB;AAEA,SAAO;AACT;","names":["success","reason"]}

package/dist/voice/testing/run_result.d.cts

@@ -0,0 +1,374 @@
+import type { AgentHandoffItem, ChatItem, ChatRole } from '../../llm/chat_context.js';
+import type { LLM } from '../../llm/llm.js';
+import type { Task } from '../../utils.js';
+import type { Agent } from '../agent.js';
+import { type SpeechHandle } from '../speech_handle.js';
+import { type AgentHandoffAssertOptions, type AgentHandoffEvent, type ChatMessageEvent, type EventType, type FunctionCallAssertOptions, type FunctionCallEvent, type FunctionCallOutputAssertOptions, type FunctionCallOutputEvent, type MessageAssertOptions, type RunEvent } from './types.js';
+type AgentConstructor = new (...args: any[]) => Agent;
+/**
+ * Result of a test run containing recorded events and assertion utilities.
+ *
+ * @example
+ * ```typescript
+ * const result = await session.run({ userInput: 'Hello' });
+ * result.expect.nextEvent().isMessage({ role: 'assistant' });
+ * result.expect.noMoreEvents();
+ * ```
+ */
+export declare class RunResult<T = unknown> {
+    private _events;
+    private doneFut;
+    private userInput?;
+    private handles;
+    private lastSpeechHandle?;
+    private runAssert?;
+    constructor(options?: {
+        userInput?: string;
+    });
+    /**
+     * List of all recorded events generated during the run.
+     */
+    get events(): RunEvent[];
+    /**
+     * Provides an assertion helper for verifying the run events.
+     */
+    get expect(): RunAssert;
+    /**
+     * Returns the final output of the run after completion.
+     *
+     * @throws Error - Not implemented yet.
+     */
+    get finalOutput(): T;
+    /**
+     * Indicates whether the run has finished processing all events.
+     */
+    done(): boolean;
+    /**
+     * Wait for the RunResult to complete. Returns `this` for method chaining.
+     *
+     * @example
+     * ```ts
+     * const result = session.run({ userInput: 'Hi!' });
+     * await result.wait();  // waits for completion
+     * result.expect.nextEvent().isMessage({ role: 'assistant' });
+     * ```
+     */
+    wait(): Promise<this>;
+    /**
+     * @internal
+     * Records an agent handoff event.
+     */
+    _agentHandoff(params: {
+        item: AgentHandoffItem;
+        oldAgent?: Agent;
+        newAgent: Agent;
+    }): void;
+    /**
+     * @internal
+     * Called when a chat item is added during the run.
+     */
+    _itemAdded(item: ChatItem): void;
+    /**
+     * @internal
+     * Watch a speech handle or task for completion.
+     */
+    _watchHandle(handle: SpeechHandle | Task<void>): void;
+    /**
+     * @internal
+     * Unwatch a handle.
+     */
+    _unwatchHandle(handle: SpeechHandle | Task<void>): void;
+    private _markDoneIfNeeded;
+    private _markDone;
+    /**
+     * Find the correct insertion index to maintain chronological order.
+     */
+    private _findInsertionIndex;
+}
+/**
+ * Assertion helper for verifying run events in sequence.
+ */
+export declare class RunAssert {
+    private _events;
+    private _currentIndex;
+    constructor(runResult: RunResult);
+    /**
+     * Access a specific event by index for assertions.
+     * Supports negative indices (e.g., -1 for last event).
+     *
+     * @example
+     * ```typescript
+     * result.expect.at(0).isMessage({ role: 'user' });
+     * result.expect.at(-1).isMessage({ role: 'assistant' });
+     * ```
+     */
+    at(index: number): EventAssert;
+    /**
+     * Advance to the next event, optionally filtering by type.
+     *
+     * @example
+     * ```typescript
+     * result.expect.nextEvent().isMessage({ role: 'assistant' });
+     * result.expect.nextEvent({ type: 'function_call' }).isFunctionCall({ name: 'foo' });
+     * ```
+     */
+    nextEvent(options?: {
+        type?: EventType;
+    }): EventAssert;
+    /**
+     * Skip a specified number of upcoming events without assertions.
+     *
+     * @example
+     * ```typescript
+     * result.expect.skipNext(2);
+     * ```
+     */
+    skipNext(count?: number): this;
+    /**
+     * Conditionally skip the next event if it matches the specified criteria.
+     * Returns the event assertion if matched and skipped, or undefined if not matched.
+     *
+     * @example
+     * ```typescript
+     * // Skip optional assistant message before function call
+     * result.expect.skipNextEventIf({ type: 'message', role: 'assistant' });
+     * result.expect.nextEvent().isFunctionCall({ name: 'foo' });
+     * ```
+     */
+    skipNextEventIf(options: {
+        type: 'message';
+        role?: ChatRole;
+    } | {
+        type: 'function_call';
+        name?: string;
+        args?: Record<string, unknown>;
+    } | {
+        type: 'function_call_output';
+        output?: string;
+        isError?: boolean;
+    } | {
+        type: 'agent_handoff';
+        newAgentType?: AgentConstructor;
+    }): MessageAssert | FunctionCallAssert | FunctionCallOutputAssert | AgentHandoffAssert | undefined;
+    /**
+     * Get an EventRangeAssert for a range of events.
+     * Similar to Python's slice access: expect[0:3] or expect[:]
+     *
+     * @param start - Start index (inclusive), defaults to 0
+     * @param end - End index (exclusive), defaults to events.length
+     *
+     * @example
+     * ```typescript
+     * // Search all events
+     * result.expect.range().containsFunctionCall({ name: 'foo' });
+     * // Search first 3 events
+     * result.expect.range(0, 3).containsMessage({ role: 'assistant' });
+     * ```
+     */
+    range(start?: number, end?: number): EventRangeAssert;
+    /**
+     * Assert that a function call matching criteria exists anywhere in the events.
+     *
+     * @example
+     * ```typescript
+     * result.expect.containsFunctionCall({ name: 'order_item' });
+     * ```
+     */
+    containsFunctionCall(options?: FunctionCallAssertOptions): FunctionCallAssert;
+    /**
+     * Assert that a message matching criteria exists anywhere in the events.
+     *
+     * @example
+     * ```typescript
+     * result.expect.containsMessage({ role: 'assistant' });
+     * ```
+     */
+    containsMessage(options?: MessageAssertOptions): MessageAssert;
+    /**
+     * Assert that a function call output matching criteria exists anywhere in the events.
+     *
+     * @example
+     * ```typescript
+     * result.expect.containsFunctionCallOutput({ isError: false });
+     * ```
+     */
+    containsFunctionCallOutput(options?: FunctionCallOutputAssertOptions): FunctionCallOutputAssert;
+    /**
+     * Assert that an agent handoff matching criteria exists anywhere in the events.
+     *
+     * @example
+     * ```typescript
+     * result.expect.containsAgentHandoff({ newAgentType: MyAgent });
+     * ```
+     */
+    containsAgentHandoff(options?: AgentHandoffAssertOptions): AgentHandoffAssert;
+    /**
+     * Assert that there are no further events.
+     *
+     * @example
+     * ```typescript
+     * result.expect.noMoreEvents();
+     * ```
+     */
+    noMoreEvents(): void;
+    private _currentEvent;
+    /** @internal */
+    _raiseWithDebugInfo(message: string, index?: number): never;
+}
+/**
+ * Assertion wrapper for a single event.
+ */
+export declare class EventAssert {
+    protected _event: RunEvent;
+    protected _parent: RunAssert;
+    protected _index: number;
+    constructor(event: RunEvent, parent: RunAssert, index: number);
+    /**
+     * Get the underlying event.
+     */
+    event(): RunEvent;
+    protected _raise(message: string): never;
+    /**
+     * Verify this event is a message with optional role matching.
+     *
+     * @example
+     * ```typescript
+     * result.expect.nextEvent().isMessage({ role: 'assistant' });
+     * ```
+     */
+    isMessage(options?: MessageAssertOptions): MessageAssert;
+    /**
+     * Verify this event is a function call with optional name/args matching.
+     *
+     * @example
+     * ```typescript
+     * result.expect.nextEvent().isFunctionCall({ name: 'order_item', args: { id: 'big_mac' } });
+     * ```
+     */
+    isFunctionCall(options?: FunctionCallAssertOptions): FunctionCallAssert;
+    /**
+     * Verify this event is a function call output with optional matching.
+     *
+     * @example
+     * ```typescript
+     * result.expect.nextEvent().isFunctionCallOutput({ isError: false });
+     * ```
+     */
+    isFunctionCallOutput(options?: FunctionCallOutputAssertOptions): FunctionCallOutputAssert;
+    /**
+     * Verify this event is an agent handoff with optional type matching.
+     *
+     * @example
+     * ```typescript
+     * result.expect.nextEvent().isAgentHandoff({ newAgentType: MyAgent });
+     * ```
+     */
+    isAgentHandoff(options?: AgentHandoffAssertOptions): AgentHandoffAssert;
+}
+/**
+ * Assertion wrapper for a range of events.
+ * Provides contains*() methods to search within the range.
+ */
+export declare class EventRangeAssert {
+    private _events;
+    private _parent;
+    private _range;
+    constructor(events: RunEvent[], parent: RunAssert, range: {
+        start: number;
+        end: number;
+    });
+    /**
+     * Assert that a function call matching criteria exists in this event range.
+     *
+     * @example
+     * ```typescript
+     * result.expect.range(0, 3).containsFunctionCall({ name: 'foo' });
+     * ```
+     */
+    containsFunctionCall(options?: FunctionCallAssertOptions): FunctionCallAssert;
+    /**
+     * Assert that a message matching criteria exists in this event range.
+     *
+     * @example
+     * ```typescript
+     * result.expect.range(0, 2).containsMessage({ role: 'assistant' });
+     * ```
+     */
+    containsMessage(options?: MessageAssertOptions): MessageAssert;
+    /**
+     * Assert that a function call output matching criteria exists in this event range.
+     *
+     * @example
+     * ```typescript
+     * result.expect.range(1, 4).containsFunctionCallOutput({ isError: true });
+     * ```
+     */
+    containsFunctionCallOutput(options?: FunctionCallOutputAssertOptions): FunctionCallOutputAssert;
+    /**
+     * Assert that an agent handoff matching criteria exists in this event range.
+     *
+     * @example
+     * ```typescript
+     * result.expect.range(0, 3).containsAgentHandoff({ newAgentType: MyAgent });
+     * ```
+     */
+    containsAgentHandoff(options?: AgentHandoffAssertOptions): AgentHandoffAssert;
+}
+/**
+ * Assertion wrapper for message events.
+ */
+export declare class MessageAssert extends EventAssert {
+    protected _event: ChatMessageEvent;
+    constructor(event: ChatMessageEvent, parent: RunAssert, index: number);
+    event(): ChatMessageEvent;
+    /**
+     * Evaluate whether the message fulfills the given intent using an LLM.
+     *
+     * @param llm - LLM instance for judgment
+     * @param options - Options containing the intent description
+     * @returns Self for chaining further assertions
+     *
+     * @example
+     * ```typescript
+     * await result.expect
+     *   .nextEvent()
+     *   .isMessage({ role: 'assistant' })
+     *   .judge(llm, { intent: 'should ask for the drink size' });
+     * ```
+     */
+    judge(llm: LLM, options: {
+        intent: string;
+    }): Promise<MessageAssert>;
+}
+/**
+ * Assertion wrapper for function call events.
+ */
+export declare class FunctionCallAssert extends EventAssert {
+    protected _event: FunctionCallEvent;
+    constructor(event: FunctionCallEvent, parent: RunAssert, index: number);
+    event(): FunctionCallEvent;
+}
+/**
+ * Assertion wrapper for function call output events.
+ */
+export declare class FunctionCallOutputAssert extends EventAssert {
+    protected _event: FunctionCallOutputEvent;
+    constructor(event: FunctionCallOutputEvent, parent: RunAssert, index: number);
+    event(): FunctionCallOutputEvent;
+}
+/**
+ * Assertion wrapper for agent handoff events.
+ */
+export declare class AgentHandoffAssert extends EventAssert {
+    protected _event: AgentHandoffEvent;
+    constructor(event: AgentHandoffEvent, parent: RunAssert, index: number);
+    event(): AgentHandoffEvent;
+}
+/**
+ * Custom assertion error for test failures.
+ */
+export declare class AssertionError extends Error {
+    constructor(message: string);
+}
+export {};
+//# sourceMappingURL=run_result.d.ts.map