@livekit/agents 1.0.33 → 1.0.35

package/src/inference/llm.ts

@@ -27,7 +27,14 @@ export type OpenAIModels =
   | 'openai/gpt-4o-mini'
   | 'openai/gpt-oss-120b';
 
-export type GoogleModels = 'google/gemini-2.0-flash-lite';
+export type GoogleModels =
+  | 'google/gemini-3-pro-preview'
+  | 'google/gemini-3-flash-preview'
+  | 'google/gemini-2.5-pro'
+  | 'google/gemini-2.5-flash'
+  | 'google/gemini-2.5-flash-lite'
+  | 'google/gemini-2.0-flash'
+  | 'google/gemini-2.0-flash-lite';
 
 export type QwenModels = 'qwen/qwen3-235b-a22b-instruct';
 
@@ -235,6 +242,7 @@ export class LLMStream extends llm.LLMStream {
   private toolIndex?: number;
   private fncName?: string;
   private fncRawArguments?: string;
+  private toolExtra?: Record<string, unknown>;
 
   constructor(
     llm: LLM,
@@ -277,6 +285,7 @@ export class LLMStream extends llm.LLMStream {
     // (defined inside the run method to make sure the state is reset for each run/attempt)
     let retryable = true;
     this.toolCallId = this.fncName = this.fncRawArguments = this.toolIndex = undefined;
+    this.toolExtra = undefined;
 
     try {
       const messages = (await this.chatCtx.toProviderFormat(
@@ -386,8 +395,6 @@ export class LLMStream extends llm.LLMStream {
           options: { retryable },
         });
       }
-    } finally {
-      this.queue.close();
     }
   }
 
@@ -430,6 +437,7 @@ export class LLMStream extends llm.LLMStream {
         if (this.toolCallId && tool.id && tool.index !== this.toolIndex) {
           callChunk = this.createRunningToolCallChunk(id, delta);
           this.toolCallId = this.fncName = this.fncRawArguments = undefined;
+          this.toolExtra = undefined;
         }
 
         // Start or continue building the current tool call
@@ -438,6 +446,10 @@ export class LLMStream extends llm.LLMStream {
           this.toolCallId = tool.id;
           this.fncName = tool.function.name;
           this.fncRawArguments = tool.function.arguments || '';
+          // Extract extra from tool call (e.g., Google thought signatures)
+          this.toolExtra =
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ((tool as any).extra_content as Record<string, unknown> | undefined) ?? undefined;
         } else if (tool.function.arguments) {
           this.fncRawArguments = (this.fncRawArguments || '') + tool.function.arguments;
         }
@@ -456,11 +468,17 @@ export class LLMStream extends llm.LLMStream {
     ) {
       const callChunk = this.createRunningToolCallChunk(id, delta);
       this.toolCallId = this.fncName = this.fncRawArguments = undefined;
+      this.toolExtra = undefined;
       return callChunk;
     }
 
+    // Extract extra from delta (e.g., Google thought signatures on text parts)
+    const deltaExtra =
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      ((delta as any).extra_content as Record<string, unknown> | undefined) ?? undefined;
+
     // Regular content message
-    if (!delta.content) {
+    if (!delta.content && !deltaExtra) {
       return undefined;
     }
 
@@ -468,7 +486,8 @@ export class LLMStream extends llm.LLMStream {
       id,
       delta: {
         role: 'assistant',
-        content: delta.content,
+        content: delta.content || undefined,
+        extra: deltaExtra,
       },
     };
   }
@@ -477,19 +496,37 @@ export class LLMStream extends llm.LLMStream {
     id: string,
     delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta,
   ): llm.ChatChunk {
+    const toolExtra = this.toolExtra ? { ...this.toolExtra } : {};
+    const thoughtSignature = this.extractThoughtSignature(toolExtra);
+    const deltaExtra =
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      ((delta as any).extra_content as Record<string, unknown> | undefined) ?? undefined;
+
     return {
       id,
       delta: {
         role: 'assistant',
         content: delta.content || undefined,
+        extra: deltaExtra,
         toolCalls: [
           llm.FunctionCall.create({
             callId: this.toolCallId || '',
             name: this.fncName || '',
             args: this.fncRawArguments || '',
+            extra: toolExtra,
+            thoughtSignature,
           }),
         ],
       },
     };
   }
+
+  private extractThoughtSignature(extra?: Record<string, unknown>): string | undefined {
+    const googleExtra = extra?.google;
+    if (googleExtra && typeof googleExtra === 'object') {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      return (googleExtra as any).thoughtSignature || (googleExtra as any).thought_signature;
+    }
+    return undefined;
+  }
 }

package/src/ipc/job_proc_lazy_main.ts

@@ -136,7 +136,7 @@ const startJob = (
       shutdownTasks.push(callback());
     }
     await Promise.all(shutdownTasks).catch((error) =>
-      logger.error('error while shutting down the job', error),
+      logger.error({ error }, 'error while shutting down the job'),
     );
 
     process.send!({ case: 'done' });

package/src/llm/chat_context.ts

@@ -189,6 +189,12 @@ export class FunctionCall {
 
   createdAt: number;
 
+  extra: Record<string, unknown>;
+  /**
+   * Optional grouping identifier for parallel tool calls.
+   */
+  groupId?: string;
+
   /**
    * Opaque signature for Gemini thinking mode.
    * When using Gemini 3+ models with thinking enabled, this signature must be
@@ -202,6 +208,8 @@ export class FunctionCall {
     args: string;
     id?: string;
     createdAt?: number;
+    extra?: Record<string, unknown>;
+    groupId?: string;
     thoughtSignature?: string;
   }) {
     const {
@@ -210,6 +218,8 @@ export class FunctionCall {
       args,
       id = shortuuid('item_'),
       createdAt = Date.now(),
+      extra = {},
+      groupId,
       thoughtSignature,
     } = params;
     this.id = id;
@@ -217,7 +227,15 @@ export class FunctionCall {
     this.args = args;
     this.name = name;
     this.createdAt = createdAt;
-    this.thoughtSignature = thoughtSignature;
+    this.extra = { ...extra };
+    this.groupId = groupId;
+    this.thoughtSignature =
+      thoughtSignature ??
+      (typeof this.extra.google === 'object' && this.extra.google !== null
+        ? // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          (this.extra.google as any).thoughtSignature ||
+          (this.extra.google as any).thought_signature
+        : undefined);
   }
 
   static create(params: {
@@ -226,6 +244,8 @@ export class FunctionCall {
     args: string;
     id?: string;
     createdAt?: number;
+    extra?: Record<string, unknown>;
+    groupId?: string;
     thoughtSignature?: string;
   }) {
     return new FunctionCall(params);
@@ -241,6 +261,14 @@ export class FunctionCall {
       args: this.args,
     };
 
+    if (Object.keys(this.extra).length > 0) {
+      result.extra = this.extra as JSONValue;
+    }
+
+    if (this.groupId) {
+      result.groupId = this.groupId;
+    }
+
     if (this.thoughtSignature) {
       result.thoughtSignature = this.thoughtSignature;
     }
@@ -627,7 +655,9 @@ export class ChatContext {
           a.name !== b.name ||
           a.callId !== b.callId ||
           a.args !== b.args ||
-          a.thoughtSignature !== b.thoughtSignature
+          a.thoughtSignature !== b.thoughtSignature ||
+          a.groupId !== b.groupId ||
+          JSON.stringify(a.extra) !== JSON.stringify(b.extra)
         ) {
           return false;
         }

package/src/llm/fallback_adapter.test.ts

@@ -0,0 +1,238 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { beforeAll, describe, expect, it, vi } from 'vitest';
+import { APIConnectionError, APIError } from '../_exceptions.js';
+import { initializeLogger } from '../log.js';
+import type { APIConnectOptions } from '../types.js';
+import { delay } from '../utils.js';
+import type { ChatContext } from './chat_context.js';
+import { FallbackAdapter } from './fallback_adapter.js';
+import { type ChatChunk, LLM, LLMStream } from './llm.js';
+import type { ToolChoice, ToolContext } from './tool_context.js';
+
+class MockLLMStream extends LLMStream {
+  public myLLM: LLM;
+
+  constructor(
+    llm: LLM,
+    opts: {
+      chatCtx: ChatContext;
+      toolCtx?: ToolContext;
+      connOptions: APIConnectOptions;
+    },
+    private shouldFail: boolean = false,
+    private failAfterChunks: number = 0,
+  ) {
+    super(llm, opts);
+    this.myLLM = llm;
+  }
+
+  protected async run(): Promise<void> {
+    if (this.shouldFail && this.failAfterChunks === 0) {
+      throw new APIError('Mock LLM failed immediately');
+    }
+
+    const chunk: ChatChunk = {
+      id: 'test-id',
+      delta: { role: 'assistant', content: 'chunk' },
+    };
+
+    for (let i = 0; i < 3; i++) {
+      if (this.shouldFail && i === this.failAfterChunks) {
+        throw new APIError('Mock LLM failed after chunks');
+      }
+      this.queue.put(chunk);
+      await delay(10);
+    }
+  }
+}
+
+class MockLLM extends LLM {
+  shouldFail: boolean = false;
+  failAfterChunks: number = 0;
+  private _label: string;
+
+  constructor(label: string) {
+    super();
+    this._label = label;
+  }
+
+  label(): string {
+    return this._label;
+  }
+
+  chat(opts: {
+    chatCtx: ChatContext;
+    toolCtx?: ToolContext;
+    connOptions?: APIConnectOptions;
+    parallelToolCalls?: boolean;
+    toolChoice?: ToolChoice;
+    extraKwargs?: Record<string, unknown>;
+  }): LLMStream {
+    return new MockLLMStream(
+      this,
+      {
+        chatCtx: opts.chatCtx,
+        toolCtx: opts.toolCtx,
+        connOptions: opts.connOptions!,
+      },
+      this.shouldFail,
+      this.failAfterChunks,
+    );
+  }
+}
+
+describe('FallbackAdapter', () => {
+  beforeAll(() => {
+    initializeLogger({ pretty: false });
+    // Suppress unhandled rejections from LLMStream background tasks
+    process.on('unhandledRejection', () => {});
+  });
+
+  it('should initialize correctly', () => {
+    const llm1 = new MockLLM('llm1');
+    const adapter = new FallbackAdapter({ llms: [llm1] });
+    expect(adapter.llms).toHaveLength(1);
+    expect(adapter.llms[0]).toBe(llm1);
+  });
+
+  it('should throw if no LLMs provided', () => {
+    expect(() => new FallbackAdapter({ llms: [] })).toThrow();
+  });
+
+  it('should use primary LLM if successful', async () => {
+    const llm1 = new MockLLM('llm1');
+    const llm2 = new MockLLM('llm2');
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+
+    const stream = adapter.chat({
+      chatCtx: {} as ChatContext,
+    });
+
+    const chunks: ChatChunk[] = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+
+    expect(chunks).toHaveLength(3);
+    // Should verify it used llm1 (we can check logs or spy, but simple success is good first step)
+  });
+
+  it('should fallback to second LLM if first fails immediately', async () => {
+    const llm1 = new MockLLM('llm1');
+    llm1.shouldFail = true;
+    const llm2 = new MockLLM('llm2');
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+
+    const stream = adapter.chat({
+      chatCtx: {} as ChatContext,
+    });
+
+    const chunks: ChatChunk[] = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+
+    expect(chunks).toHaveLength(3);
+    expect(adapter._status[0]!.available).toBe(false);
+    expect(adapter._status[1]!.available).toBe(true);
+  });
+
+  it('should fail if all LLMs fail', async () => {
+    const llm1 = new MockLLM('llm1');
+    llm1.shouldFail = true;
+    const llm2 = new MockLLM('llm2');
+    llm2.shouldFail = true;
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+
+    const stream = adapter.chat({
+      chatCtx: {} as ChatContext,
+    });
+
+    const errorPromise = new Promise<Error>((resolve) => {
+      adapter.on('error', (e) => resolve(e.error));
+    });
+
+    for await (const _ of stream) {
+      // consume
+    }
+
+    const error = await errorPromise;
+    expect(error).toBeInstanceOf(APIConnectionError);
+  });
+
+  it('should fail if chunks sent and retryOnChunkSent is false', async () => {
+    const llm1 = new MockLLM('llm1');
+    llm1.shouldFail = true;
+    llm1.failAfterChunks = 1; // Fail after 1 chunk
+    const llm2 = new MockLLM('llm2');
+    const adapter = new FallbackAdapter({
+      llms: [llm1, llm2],
+      retryOnChunkSent: false,
+    });
+
+    const stream = adapter.chat({
+      chatCtx: {} as ChatContext,
+    });
+
+    const errorPromise = new Promise<Error>((resolve) => {
+      adapter.on('error', (e) => resolve(e.error));
+    });
+
+    for await (const _ of stream) {
+      // consume
+    }
+
+    const error = await errorPromise;
+    expect(error).toBeInstanceOf(APIError);
+  });
+
+  it('should fallback if chunks sent and retryOnChunkSent is true', async () => {
+    const llm1 = new MockLLM('llm1');
+    llm1.shouldFail = true;
+    llm1.failAfterChunks = 1;
+    const llm2 = new MockLLM('llm2');
+    const adapter = new FallbackAdapter({
+      llms: [llm1, llm2],
+      retryOnChunkSent: true,
+    });
+
+    const stream = adapter.chat({
+      chatCtx: {} as ChatContext,
+    });
+
+    const chunks: ChatChunk[] = [];
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+
+    // 1 chunk from failed llm1 + 3 chunks from llm2
+    expect(chunks).toHaveLength(4);
+  });
+
+  it('should emit availability changed events', async () => {
+    const llm1 = new MockLLM('llm1');
+    llm1.shouldFail = true;
+    const llm2 = new MockLLM('llm2');
+    const adapter = new FallbackAdapter({ llms: [llm1, llm2] });
+
+    const eventSpy = vi.fn();
+    (adapter as any).on('llm_availability_changed', eventSpy);
+
+    const stream = adapter.chat({
+      chatCtx: {} as ChatContext,
+    });
+
+    for await (const _ of stream) {
+      // consume
+    }
+
+    expect(eventSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        llm: llm1,
+        available: false,
+      }),
+    );
+  });
+});