@livekit/agents 1.0.36-dev.0 → 1.0.37

package/dist/voice/testing/run_result.js

@@ -0,0 +1,739 @@
+import { z } from "zod";
+import { ChatContext } from "../../llm/chat_context.js";
+import { tool } from "../../llm/tool_context.js";
+import { Future } from "../../utils.js";
+import { isSpeechHandle } from "../speech_handle.js";
+import {
+  isAgentHandoffEvent,
+  isChatMessageEvent,
+  isFunctionCallEvent,
+  isFunctionCallOutputEvent
+} from "./types.js";
+const evalsVerbose = parseInt(process.env.LIVEKIT_EVALS_VERBOSE || "0", 10);
+class RunResult {
+  _events = [];
+  doneFut = new Future();
+  userInput;
+  handles = /* @__PURE__ */ new Set();
+  lastSpeechHandle;
+  runAssert;
+  // TODO(brian): Add typed output support for parity with Python
+  // - Add outputType?: new (...args: unknown[]) => T
+  // - Add finalOutput?: T
+  // - Implement markDone() to extract final_output from SpeechHandle.maybeRunFinalOutput
+  // - See Python: run_result.py lines 182-201
+  constructor(options) {
+    this.userInput = options == null ? void 0 : options.userInput;
+  }
+  /**
+   * List of all recorded events generated during the run.
+   */
+  get events() {
+    return this._events;
+  }
+  /**
+   * Provides an assertion helper for verifying the run events.
+   */
+  get expect() {
+    if (evalsVerbose) {
+      const eventsStr = formatEvents(this._events).map((line) => `      ${line}`).join("\n");
+      console.log(
+        `
++ RunResult {
+    userInput: "${this.userInput}"
+    events: [
+${eventsStr}
+    ]
+  }`
+      );
+    }
+    if (!this.runAssert) {
+      this.runAssert = new RunAssert(this);
+    }
+    return this.runAssert;
+  }
+  /**
+   * Returns the final output of the run after completion.
+   *
+   * @throws Error - Not implemented yet.
+   */
+  get finalOutput() {
+    throw new Error("finalOutput is not yet implemented in JS.");
+  }
+  /**
+   * Indicates whether the run has finished processing all events.
+   */
+  done() {
+    return this.doneFut.done;
+  }
+  /**
+   * Wait for the RunResult to complete. Returns `this` for method chaining.
+   *
+   * @example
+   * ```ts
+   * const result = session.run({ userInput: 'Hi!' });
+   * await result.wait();  // waits for completion
+   * result.expect.nextEvent().isMessage({ role: 'assistant' });
+   * ```
+   */
+  async wait() {
+    await this.doneFut.await;
+    return this;
+  }
+  /**
+   * @internal
+   * Records an agent handoff event.
+   */
+  _agentHandoff(params) {
+    const event = {
+      type: "agent_handoff",
+      item: params.item,
+      oldAgent: params.oldAgent,
+      newAgent: params.newAgent
+    };
+    const index = this._findInsertionIndex(event.item.createdAt);
+    this._events.splice(index, 0, event);
+  }
+  /**
+   * @internal
+   * Called when a chat item is added during the run.
+   */
+  _itemAdded(item) {
+    if (this.doneFut.done) {
+      return;
+    }
+    let event;
+    if (item.type === "message") {
+      event = { type: "message", item };
+    } else if (item.type === "function_call") {
+      event = { type: "function_call", item };
+    } else if (item.type === "function_call_output") {
+      event = { type: "function_call_output", item };
+    }
+    if (event) {
+      const index = this._findInsertionIndex(item.createdAt);
+      this._events.splice(index, 0, event);
+    }
+  }
+  /**
+   * @internal
+   * Watch a speech handle or task for completion.
+   */
+  _watchHandle(handle) {
+    this.handles.add(handle);
+    if (isSpeechHandle(handle)) {
+      handle._addItemAddedCallback(this._itemAdded.bind(this));
+    }
+    handle.addDoneCallback(() => {
+      this._markDoneIfNeeded(handle);
+    });
+  }
+  /**
+   * @internal
+   * Unwatch a handle.
+   */
+  _unwatchHandle(handle) {
+    this.handles.delete(handle);
+    if (isSpeechHandle(handle)) {
+      handle._removeItemAddedCallback(this._itemAdded.bind(this));
+    }
+  }
+  _markDoneIfNeeded(handle) {
+    if (isSpeechHandle(handle)) {
+      this.lastSpeechHandle = handle;
+    }
+    if ([...this.handles].every((h) => isSpeechHandle(h) ? h.done() : h.done)) {
+      this._markDone();
+    }
+  }
+  _markDone() {
+    if (!this.doneFut.done) {
+      this.doneFut.resolve();
+    }
+  }
+  /**
+   * Find the correct insertion index to maintain chronological order.
+   */
+  _findInsertionIndex(createdAt) {
+    for (let i = this._events.length - 1; i >= 0; i--) {
+      if (this._events[i].item.createdAt <= createdAt) {
+        return i + 1;
+      }
+    }
+    return 0;
+  }
+}
+class RunAssert {
+  _events;
+  _currentIndex = 0;
+  constructor(runResult) {
+    this._events = runResult.events;
+  }
+  /**
+   * Access a specific event by index for assertions.
+   * Supports negative indices (e.g., -1 for last event).
+   *
+   * @example
+   * ```typescript
+   * result.expect.at(0).isMessage({ role: 'user' });
+   * result.expect.at(-1).isMessage({ role: 'assistant' });
+   * ```
+   */
+  at(index) {
+    let normalizedIndex = index;
+    if (index < 0) {
+      normalizedIndex = this._events.length + index;
+    }
+    if (normalizedIndex < 0 || normalizedIndex >= this._events.length) {
+      this._raiseWithDebugInfo(
+        `at(${index}) out of range (total events: ${this._events.length})`,
+        normalizedIndex
+      );
+    }
+    return new EventAssert(this._events[normalizedIndex], this, normalizedIndex);
+  }
+  /**
+   * Advance to the next event, optionally filtering by type.
+   *
+   * @example
+   * ```typescript
+   * result.expect.nextEvent().isMessage({ role: 'assistant' });
+   * result.expect.nextEvent({ type: 'function_call' }).isFunctionCall({ name: 'foo' });
+   * ```
+   */
+  nextEvent(options) {
+    while (true) {
+      const evAssert = this._currentEvent();
+      this._currentIndex++;
+      if (!(options == null ? void 0 : options.type) || evAssert.event().type === options.type) {
+        return evAssert;
+      }
+    }
+  }
+  /**
+   * Skip a specified number of upcoming events without assertions.
+   *
+   * @example
+   * ```typescript
+   * result.expect.skipNext(2);
+   * ```
+   */
+  skipNext(count = 1) {
+    for (let i = 0; i < count; i++) {
+      if (this._currentIndex >= this._events.length) {
+        this._raiseWithDebugInfo(`Tried to skip ${count} event(s), but only ${i} were available.`);
+      }
+      this._currentIndex++;
+    }
+    return this;
+  }
+  /**
+   * Conditionally skip the next event if it matches the specified criteria.
+   * Returns the event assertion if matched and skipped, or undefined if not matched.
+   *
+   * @example
+   * ```typescript
+   * // Skip optional assistant message before function call
+   * result.expect.skipNextEventIf({ type: 'message', role: 'assistant' });
+   * result.expect.nextEvent().isFunctionCall({ name: 'foo' });
+   * ```
+   */
+  skipNextEventIf(options) {
+    if (this._currentIndex >= this._events.length) {
+      return void 0;
+    }
+    try {
+      const evAssert = this._currentEvent();
+      if (options.type === "message") {
+        const { role } = options;
+        const result = evAssert.isMessage({ role });
+        this._currentIndex++;
+        return result;
+      } else if (options.type === "function_call") {
+        const { name, args } = options;
+        const result = evAssert.isFunctionCall({
+          name,
+          args
+        });
+        this._currentIndex++;
+        return result;
+      } else if (options.type === "function_call_output") {
+        const { output, isError } = options;
+        const result = evAssert.isFunctionCallOutput({
+          output,
+          isError
+        });
+        this._currentIndex++;
+        return result;
+      } else if (options.type === "agent_handoff") {
+        const { newAgentType } = options;
+        const result = evAssert.isAgentHandoff({ newAgentType });
+        this._currentIndex++;
+        return result;
+      }
+    } catch {
+      return void 0;
+    }
+    return void 0;
+  }
+  /**
+   * Get an EventRangeAssert for a range of events.
+   * Similar to Python's slice access: expect[0:3] or expect[:]
+   *
+   * @param start - Start index (inclusive), defaults to 0
+   * @param end - End index (exclusive), defaults to events.length
+   *
+   * @example
+   * ```typescript
+   * // Search all events
+   * result.expect.range().containsFunctionCall({ name: 'foo' });
+   * // Search first 3 events
+   * result.expect.range(0, 3).containsMessage({ role: 'assistant' });
+   * ```
+   */
+  range(start, end) {
+    const startIdx = start ?? 0;
+    const endIdx = end ?? this._events.length;
+    const events = this._events.slice(startIdx, endIdx);
+    return new EventRangeAssert(events, this, { start: startIdx, end: endIdx });
+  }
+  /**
+   * Assert that a function call matching criteria exists anywhere in the events.
+   *
+   * @example
+   * ```typescript
+   * result.expect.containsFunctionCall({ name: 'order_item' });
+   * ```
+   */
+  containsFunctionCall(options) {
+    return this.range().containsFunctionCall(options);
+  }
+  /**
+   * Assert that a message matching criteria exists anywhere in the events.
+   *
+   * @example
+   * ```typescript
+   * result.expect.containsMessage({ role: 'assistant' });
+   * ```
+   */
+  containsMessage(options) {
+    return this.range().containsMessage(options);
+  }
+  /**
+   * Assert that a function call output matching criteria exists anywhere in the events.
+   *
+   * @example
+   * ```typescript
+   * result.expect.containsFunctionCallOutput({ isError: false });
+   * ```
+   */
+  containsFunctionCallOutput(options) {
+    return this.range().containsFunctionCallOutput(options);
+  }
+  /**
+   * Assert that an agent handoff matching criteria exists anywhere in the events.
+   *
+   * @example
+   * ```typescript
+   * result.expect.containsAgentHandoff({ newAgentType: MyAgent });
+   * ```
+   */
+  containsAgentHandoff(options) {
+    return this.range().containsAgentHandoff(options);
+  }
+  /**
+   * Assert that there are no further events.
+   *
+   * @example
+   * ```typescript
+   * result.expect.noMoreEvents();
+   * ```
+   */
+  noMoreEvents() {
+    if (this._currentIndex < this._events.length) {
+      const event = this._events[this._currentIndex];
+      this._raiseWithDebugInfo(`Expected no more events, but found: ${event.type}`);
+    }
+  }
+  _currentEvent() {
+    if (this._currentIndex >= this._events.length) {
+      this._raiseWithDebugInfo("Expected another event, but none left.");
+    }
+    return this.at(this._currentIndex);
+  }
+  /** @internal */
+  _raiseWithDebugInfo(message, index) {
+    const markerIndex = index ?? this._currentIndex;
+    const eventsStr = formatEvents(this._events, markerIndex).join("\n");
+    throw new AssertionError(`${message}
+Context around failure:
+${eventsStr}`);
+  }
+}
+class EventAssert {
+  _event;
+  _parent;
+  _index;
+  constructor(event, parent, index) {
+    this._event = event;
+    this._parent = parent;
+    this._index = index;
+  }
+  /**
+   * Get the underlying event.
+   */
+  event() {
+    return this._event;
+  }
+  _raise(message) {
+    this._parent._raiseWithDebugInfo(message, this._index);
+  }
+  /**
+   * Verify this event is a message with optional role matching.
+   *
+   * @example
+   * ```typescript
+   * result.expect.nextEvent().isMessage({ role: 'assistant' });
+   * ```
+   */
+  isMessage(options) {
+    if (!isChatMessageEvent(this._event)) {
+      this._raise(`Expected ChatMessageEvent, got ${this._event.type}`);
+    }
+    if ((options == null ? void 0 : options.role) && this._event.item.role !== options.role) {
+      this._raise(`Expected role '${options.role}', got '${this._event.item.role}'`);
+    }
+    return new MessageAssert(this._event, this._parent, this._index);
+  }
+  /**
+   * Verify this event is a function call with optional name/args matching.
+   *
+   * @example
+   * ```typescript
+   * result.expect.nextEvent().isFunctionCall({ name: 'order_item', args: { id: 'big_mac' } });
+   * ```
+   */
+  isFunctionCall(options) {
+    if (!isFunctionCallEvent(this._event)) {
+      this._raise(`Expected FunctionCallEvent, got ${this._event.type}`);
+    }
+    if ((options == null ? void 0 : options.name) && this._event.item.name !== options.name) {
+      this._raise(`Expected call name '${options.name}', got '${this._event.item.name}'`);
+    }
+    if (options == null ? void 0 : options.args) {
+      let actual;
+      try {
+        actual = JSON.parse(this._event.item.args);
+      } catch {
+        this._raise(`Failed to parse function call arguments: ${this._event.item.args}`);
+      }
+      for (const [key, value] of Object.entries(options.args)) {
+        if (!(key in actual) || actual[key] !== value) {
+          this._raise(
+            `For key '${key}', expected ${JSON.stringify(value)}, got ${JSON.stringify(actual[key])}`
+          );
+        }
+      }
+    }
+    return new FunctionCallAssert(this._event, this._parent, this._index);
+  }
+  /**
+   * Verify this event is a function call output with optional matching.
+   *
+   * @example
+   * ```typescript
+   * result.expect.nextEvent().isFunctionCallOutput({ isError: false });
+   * ```
+   */
+  isFunctionCallOutput(options) {
+    if (!isFunctionCallOutputEvent(this._event)) {
+      this._raise(`Expected FunctionCallOutputEvent, got ${this._event.type}`);
+    }
+    if ((options == null ? void 0 : options.output) !== void 0 && this._event.item.output !== options.output) {
+      this._raise(`Expected output '${options.output}', got '${this._event.item.output}'`);
+    }
+    if ((options == null ? void 0 : options.isError) !== void 0 && this._event.item.isError !== options.isError) {
+      this._raise(`Expected isError=${options.isError}, got ${this._event.item.isError}`);
+    }
+    return new FunctionCallOutputAssert(this._event, this._parent, this._index);
+  }
+  /**
+   * Verify this event is an agent handoff with optional type matching.
+   *
+   * @example
+   * ```typescript
+   * result.expect.nextEvent().isAgentHandoff({ newAgentType: MyAgent });
+   * ```
+   */
+  isAgentHandoff(options) {
+    if (!isAgentHandoffEvent(this._event)) {
+      this._raise(`Expected AgentHandoffEvent, got ${this._event.type}`);
+    }
+    const event = this._event;
+    if (options == null ? void 0 : options.newAgentType) {
+      const actualType = event.newAgent.constructor.name;
+      if (!(event.newAgent instanceof options.newAgentType)) {
+        this._raise(`Expected new_agent '${options.newAgentType.name}', got '${actualType}'`);
+      }
+    }
+    return new AgentHandoffAssert(event, this._parent, this._index);
+  }
+}
+class EventRangeAssert {
+  _events;
+  _parent;
+  _range;
+  constructor(events, parent, range) {
+    this._events = events;
+    this._parent = parent;
+    this._range = range;
+  }
+  /**
+   * Assert that a function call matching criteria exists in this event range.
+   *
+   * @example
+   * ```typescript
+   * result.expect.range(0, 3).containsFunctionCall({ name: 'foo' });
+   * ```
+   */
+  containsFunctionCall(options) {
+    for (let idx = 0; idx < this._events.length; idx++) {
+      const ev = this._events[idx];
+      const candidate = new EventAssert(ev, this._parent, this._range.start + idx);
+      try {
+        return candidate.isFunctionCall(options);
+      } catch {
+      }
+    }
+    this._parent._raiseWithDebugInfo(
+      `No FunctionCallEvent satisfying criteria found in range [${this._range.start}:${this._range.end}]`
+    );
+  }
+  /**
+   * Assert that a message matching criteria exists in this event range.
+   *
+   * @example
+   * ```typescript
+   * result.expect.range(0, 2).containsMessage({ role: 'assistant' });
+   * ```
+   */
+  containsMessage(options) {
+    for (let idx = 0; idx < this._events.length; idx++) {
+      const ev = this._events[idx];
+      const candidate = new EventAssert(ev, this._parent, this._range.start + idx);
+      try {
+        return candidate.isMessage(options);
+      } catch {
+      }
+    }
+    this._parent._raiseWithDebugInfo(
+      `No ChatMessageEvent matching criteria found in range [${this._range.start}:${this._range.end}]`
+    );
+  }
+  /**
+   * Assert that a function call output matching criteria exists in this event range.
+   *
+   * @example
+   * ```typescript
+   * result.expect.range(1, 4).containsFunctionCallOutput({ isError: true });
+   * ```
+   */
+  containsFunctionCallOutput(options) {
+    for (let idx = 0; idx < this._events.length; idx++) {
+      const ev = this._events[idx];
+      const candidate = new EventAssert(ev, this._parent, this._range.start + idx);
+      try {
+        return candidate.isFunctionCallOutput(options);
+      } catch {
+      }
+    }
+    this._parent._raiseWithDebugInfo(
+      `No FunctionCallOutputEvent matching criteria found in range [${this._range.start}:${this._range.end}]`
+    );
+  }
+  /**
+   * Assert that an agent handoff matching criteria exists in this event range.
+   *
+   * @example
+   * ```typescript
+   * result.expect.range(0, 3).containsAgentHandoff({ newAgentType: MyAgent });
+   * ```
+   */
+  containsAgentHandoff(options) {
+    for (let idx = 0; idx < this._events.length; idx++) {
+      const ev = this._events[idx];
+      const candidate = new EventAssert(ev, this._parent, this._range.start + idx);
+      try {
+        return candidate.isAgentHandoff(options);
+      } catch {
+      }
+    }
+    this._parent._raiseWithDebugInfo(
+      `No AgentHandoffEvent matching criteria found in range [${this._range.start}:${this._range.end}]`
+    );
+  }
+}
+class MessageAssert extends EventAssert {
+  constructor(event, parent, index) {
+    super(event, parent, index);
+  }
+  event() {
+    return this._event;
+  }
+  /**
+   * Evaluate whether the message fulfills the given intent using an LLM.
+   *
+   * @param llm - LLM instance for judgment
+   * @param options - Options containing the intent description
+   * @returns Self for chaining further assertions
+   *
+   * @example
+   * ```typescript
+   * await result.expect
+   *   .nextEvent()
+   *   .isMessage({ role: 'assistant' })
+   *   .judge(llm, { intent: 'should ask for the drink size' });
+   * ```
+   */
+  async judge(llm, options) {
+    const { intent } = options;
+    const content = this._event.item.content;
+    const msgContent = typeof content === "string" ? content : Array.isArray(content) ? content.filter((c) => typeof c === "string").join(" ") : "";
+    if (!msgContent) {
+      this._raise("The chat message is empty.");
+    }
+    if (!intent) {
+      this._raise("Intent is required to judge the message.");
+    }
+    const checkIntentTool = tool({
+      description: "Determines whether the message correctly fulfills the given intent. Returns success=true if the message satisfies the intent, false otherwise. Provide a concise reason justifying the result.",
+      parameters: z.object({
+        success: z.boolean().describe("Whether the message satisfies the intent"),
+        reason: z.string().describe("A concise explanation justifying the result")
+      }),
+      execute: async ({ success: success2, reason: reason2 }) => {
+        return { success: success2, reason: reason2 };
+      }
+    });
+    const chatCtx = ChatContext.empty();
+    chatCtx.addMessage({
+      role: "system",
+      content: "You are a test evaluator for conversational agents.\nYou will be shown a message and a target intent. Determine whether the message accomplishes the intent.\nOnly respond by calling the `check_intent(success: bool, reason: str)` function with your final judgment.\nBe strict: if the message does not clearly fulfill the intent, return `success = false` and explain why."
+    });
+    chatCtx.addMessage({
+      role: "user",
+      content: `Check if the following message fulfills the given intent.
+
+Intent:
+${intent}
+
+Message:
+${msgContent}`
+    });
+    let toolArgs;
+    const stream = llm.chat({
+      chatCtx,
+      toolCtx: { check_intent: checkIntentTool },
+      toolChoice: { type: "function", function: { name: "check_intent" } },
+      extraKwargs: { temperature: 0 }
+    });
+    for await (const chunk of stream) {
+      if (!chunk.delta) continue;
+      if (chunk.delta.toolCalls && chunk.delta.toolCalls.length > 0) {
+        const toolCall = chunk.delta.toolCalls[0];
+        if (toolCall.args) {
+          try {
+            toolArgs = JSON.parse(toolCall.args);
+          } catch {
+          }
+        }
+      }
+    }
+    if (!toolArgs) {
+      this._raise("LLM did not return any arguments for evaluation.");
+    }
+    const { success, reason } = toolArgs;
+    if (!success) {
+      this._raise(`Judgment failed: ${reason}`);
+    } else if (evalsVerbose) {
+      const printMsg = msgContent.length > 30 ? msgContent.slice(0, 30).replace(/\n/g, "\\n") + "..." : msgContent;
+      console.log(`- Judgment succeeded for \`${printMsg}\`: \`${reason}\``);
+    }
+    return this;
+  }
+}
+class FunctionCallAssert extends EventAssert {
+  constructor(event, parent, index) {
+    super(event, parent, index);
+  }
+  event() {
+    return this._event;
+  }
+}
+class FunctionCallOutputAssert extends EventAssert {
+  constructor(event, parent, index) {
+    super(event, parent, index);
+  }
+  event() {
+    return this._event;
+  }
+}
+class AgentHandoffAssert extends EventAssert {
+  constructor(event, parent, index) {
+    super(event, parent, index);
+  }
+  event() {
+    return this._event;
+  }
+}
+class AssertionError extends Error {
+  constructor(message) {
+    var _a;
+    super(message);
+    this.name = "AssertionError";
+    (_a = Error.captureStackTrace) == null ? void 0 : _a.call(Error, this, AssertionError);
+  }
+}
+function formatEvents(events, selectedIndex) {
+  var _a;
+  const lines = [];
+  for (let i = 0; i < events.length; i++) {
+    const event = events[i];
+    let prefix = "";
+    if (selectedIndex !== void 0) {
+      prefix = i === selectedIndex ? ">>>" : "   ";
+    }
+    let line;
+    if (isChatMessageEvent(event)) {
+      const { role, content, interrupted } = event.item;
+      const textContent = typeof content === "string" ? content : Array.isArray(content) ? content.filter((c) => typeof c === "string").join(" ") : "";
+      const truncated = textContent.length > 50 ? textContent.slice(0, 50) + "..." : textContent;
+      line = `${prefix}[${i}] { type: "message", role: "${role}", content: "${truncated}", interrupted: ${interrupted} }`;
+    } else if (isFunctionCallEvent(event)) {
+      const { name, args } = event.item;
+      line = `${prefix}[${i}] { type: "function_call", name: "${name}", args: ${args} }`;
+    } else if (isFunctionCallOutputEvent(event)) {
+      const { output, isError } = event.item;
+      const truncated = output.length > 50 ? output.slice(0, 50) + "..." : output;
+      line = `${prefix}[${i}] { type: "function_call_output", output: "${truncated}", isError: ${isError} }`;
+    } else if (isAgentHandoffEvent(event)) {
+      line = `${prefix}[${i}] { type: "agent_handoff", oldAgent: "${(_a = event.oldAgent) == null ? void 0 : _a.constructor.name}", newAgent: "${event.newAgent.constructor.name}" }`;
+    } else {
+      line = `${prefix}[${i}] ${event}`;
+    }
+    lines.push(line);
+  }
+  return lines;
+}
+export {
+  AgentHandoffAssert,
+  AssertionError,
+  EventAssert,
+  EventRangeAssert,
+  FunctionCallAssert,
+  FunctionCallOutputAssert,
+  MessageAssert,
+  RunAssert,
+  RunResult
+};
+//# sourceMappingURL=run_result.js.map