@livekit/agents 1.0.34 → 1.0.36-dev.0

package/src/inference/interruption/interruption.ts

@@ -0,0 +1,88 @@
+import { slidingWindowMinMax } from '../utils.js';
+import { MIN_INTERRUPTION_DURATION } from './defaults.js';
+
+export enum InterruptionEventType {
+  INTERRUPTION = 'interruption',
+  OVERLAP_SPEECH_ENDED = 'overlap_speech_ended',
+}
+export interface InterruptionEvent {
+  type: InterruptionEventType;
+  timestamp: number;
+  isInterruption: boolean;
+  totalDuration: number;
+  predictionDuration: number;
+  detectionDelay: number;
+  overlapSpeechStartedAt?: number;
+  speechInput?: Int16Array;
+  probabilities?: Float32Array;
+  probability: number;
+}
+
+export class InterruptionDetectionError extends Error {
+  readonly type = 'InterruptionDetectionError';
+
+  readonly timestamp: number;
+  readonly label: string;
+  readonly recoverable: boolean;
+
+  constructor(message: string, timestamp: number, label: string, recoverable: boolean) {
+    super(message);
+    this.name = 'InterruptionDetectionError';
+    this.timestamp = timestamp;
+    this.label = label;
+    this.recoverable = recoverable;
+  }
+
+  toString(): string {
+    return `${this.name}: ${this.message} (label=${this.label}, timestamp=${this.timestamp}, recoverable=${this.recoverable})`;
+  }
+}
+
+function estimateProbability(
+  probabilities: Float32Array,
+  windowSize: number = MIN_INTERRUPTION_DURATION,
+): number {
+  const minWindow = Math.ceil(windowSize / 0.025); // 25ms per frame
+  if (probabilities.length < minWindow) {
+    return 0;
+  }
+
+  return slidingWindowMinMax(probabilities, windowSize);
+}
+
+/**
+ * Typed cache entry for interruption inference results.
+ */
+export class InterruptionCacheEntry {
+  readonly createdAt: number;
+  readonly totalDuration: number;
+  readonly predictionDuration: number;
+  readonly detectionDelay: number;
+  readonly speechInput?: Int16Array;
+  readonly probabilities?: Float32Array;
+  readonly isInterruption?: boolean;
+  readonly probability: number;
+
+  constructor(params: {
+    createdAt: number;
+    speechInput?: Int16Array;
+    totalDuration?: number;
+    predictionDuration?: number;
+    detectionDelay?: number;
+    probabilities?: Float32Array;
+    isInterruption?: boolean;
+  }) {
+    this.createdAt = params.createdAt;
+    this.totalDuration = params.totalDuration ?? 0;
+    this.predictionDuration = params.predictionDuration ?? 0;
+    this.detectionDelay = params.detectionDelay ?? 0;
+    this.speechInput = params.speechInput;
+    this.probabilities = params.probabilities;
+    this.isInterruption = params.isInterruption;
+    this.probability = this.probabilities ? estimateProbability(this.probabilities) : 0;
+  }
+
+  static default(): InterruptionCacheEntry {
+    return new InterruptionCacheEntry({ createdAt: 0 });
+  }
+}

package/src/inference/llm.ts

@@ -27,7 +27,14 @@ export type OpenAIModels =
   | 'openai/gpt-4o-mini'
   | 'openai/gpt-oss-120b';
 
-export type GoogleModels = 'google/gemini-2.0-flash-lite';
+export type GoogleModels =
+  | 'google/gemini-3-pro-preview'
+  | 'google/gemini-3-flash-preview'
+  | 'google/gemini-2.5-pro'
+  | 'google/gemini-2.5-flash'
+  | 'google/gemini-2.5-flash-lite'
+  | 'google/gemini-2.0-flash'
+  | 'google/gemini-2.0-flash-lite';
 
 export type QwenModels = 'qwen/qwen3-235b-a22b-instruct';
 
@@ -235,6 +242,7 @@ export class LLMStream extends llm.LLMStream {
   private toolIndex?: number;
   private fncName?: string;
   private fncRawArguments?: string;
+  private toolExtra?: Record<string, unknown>;
 
   constructor(
     llm: LLM,
@@ -277,6 +285,7 @@ export class LLMStream extends llm.LLMStream {
     // (defined inside the run method to make sure the state is reset for each run/attempt)
     let retryable = true;
     this.toolCallId = this.fncName = this.fncRawArguments = this.toolIndex = undefined;
+    this.toolExtra = undefined;
 
     try {
       const messages = (await this.chatCtx.toProviderFormat(
@@ -428,6 +437,7 @@ export class LLMStream extends llm.LLMStream {
         if (this.toolCallId && tool.id && tool.index !== this.toolIndex) {
           callChunk = this.createRunningToolCallChunk(id, delta);
           this.toolCallId = this.fncName = this.fncRawArguments = undefined;
+          this.toolExtra = undefined;
         }
 
         // Start or continue building the current tool call
@@ -436,6 +446,10 @@ export class LLMStream extends llm.LLMStream {
           this.toolCallId = tool.id;
           this.fncName = tool.function.name;
           this.fncRawArguments = tool.function.arguments || '';
+          // Extract extra from tool call (e.g., Google thought signatures)
+          this.toolExtra =
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ((tool as any).extra_content as Record<string, unknown> | undefined) ?? undefined;
         } else if (tool.function.arguments) {
           this.fncRawArguments = (this.fncRawArguments || '') + tool.function.arguments;
         }
@@ -454,11 +468,17 @@ export class LLMStream extends llm.LLMStream {
     ) {
       const callChunk = this.createRunningToolCallChunk(id, delta);
       this.toolCallId = this.fncName = this.fncRawArguments = undefined;
+      this.toolExtra = undefined;
       return callChunk;
     }
 
+    // Extract extra from delta (e.g., Google thought signatures on text parts)
+    const deltaExtra =
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      ((delta as any).extra_content as Record<string, unknown> | undefined) ?? undefined;
+
     // Regular content message
-    if (!delta.content) {
+    if (!delta.content && !deltaExtra) {
       return undefined;
     }
 
@@ -466,7 +486,8 @@ export class LLMStream extends llm.LLMStream {
       id,
       delta: {
         role: 'assistant',
-        content: delta.content,
+        content: delta.content || undefined,
+        extra: deltaExtra,
       },
     };
   }
@@ -475,19 +496,37 @@ export class LLMStream extends llm.LLMStream {
     id: string,
     delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta,
   ): llm.ChatChunk {
+    const toolExtra = this.toolExtra ? { ...this.toolExtra } : {};
+    const thoughtSignature = this.extractThoughtSignature(toolExtra);
+    const deltaExtra =
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      ((delta as any).extra_content as Record<string, unknown> | undefined) ?? undefined;
+
     return {
       id,
       delta: {
         role: 'assistant',
         content: delta.content || undefined,
+        extra: deltaExtra,
         toolCalls: [
           llm.FunctionCall.create({
             callId: this.toolCallId || '',
             name: this.fncName || '',
             args: this.fncRawArguments || '',
+            extra: toolExtra,
+            thoughtSignature,
           }),
         ],
       },
     };
   }
+
+  private extractThoughtSignature(extra?: Record<string, unknown>): string | undefined {
+    const googleExtra = extra?.google;
+    if (googleExtra && typeof googleExtra === 'object') {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      return (googleExtra as any).thoughtSignature || (googleExtra as any).thought_signature;
+    }
+    return undefined;
+  }
 }

package/src/inference/utils.test.ts

@@ -0,0 +1,31 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { slidingWindowMinMax } from './utils.js';
+
+describe('slidingWindowMinMax', () => {
+  it('returns -Infinity when array is shorter than window size', () => {
+    expect(slidingWindowMinMax([0.5, 0.6], 3)).toBe(-Infinity);
+    expect(slidingWindowMinMax([], 1)).toBe(-Infinity);
+  });
+
+  it('returns the max value when window size is 1', () => {
+    // With window size 1, min of each window is the element itself,
+    // so max of mins is just the max of the array
+    expect(slidingWindowMinMax([0.1, 0.5, 0.3, 0.8, 0.2], 1)).toBe(0.8);
+  });
+
+  it('finds the best sustained probability across windows', () => {
+    // Windows of size 3: [0.2, 0.8, 0.7], [0.8, 0.7, 0.3], [0.7, 0.3, 0.9]
+    // Mins:              0.2,             0.3,             0.3
+    // Max of mins: 0.3
+    expect(slidingWindowMinMax([0.2, 0.8, 0.7, 0.3, 0.9], 3)).toBe(0.3);
+  });
+
+  it('returns the single element when array length equals window size', () => {
+    // Only one window covering the entire array, return min of that window
+    expect(slidingWindowMinMax([0.5, 0.9, 0.7], 3)).toBe(0.5);
+    expect(slidingWindowMinMax([0.8], 1)).toBe(0.8);
+  });
+});

package/src/inference/utils.ts

@@ -64,3 +64,18 @@ export async function connectWs(
     socket.once('close', onClose);
   });
 }
+
+export function slidingWindowMinMax(probabilities: Float32Array, minWindow: number): number {
+  if (probabilities.length < minWindow) {
+    return -Infinity;
+  }
+
+  let maxOfMins = -Infinity;
+
+  for (let i = 0; i <= probabilities.length - minWindow; i++) {
+    const windowMin = Math.min(...probabilities.slice(i, i + minWindow));
+    maxOfMins = Math.max(maxOfMins, windowMin);
+  }
+
+  return maxOfMins;
+}

package/src/ipc/job_proc_lazy_main.ts

@@ -136,7 +136,7 @@ const startJob = (
       shutdownTasks.push(callback());
     }
     await Promise.all(shutdownTasks).catch((error) =>
-      logger.error('error while shutting down the job', error),
+      logger.error({ error }, 'error while shutting down the job'),
     );
 
     process.send!({ case: 'done' });

package/src/llm/chat_context.ts

@@ -189,6 +189,12 @@ export class FunctionCall {
 
   createdAt: number;
 
+  extra: Record<string, unknown>;
+  /**
+   * Optional grouping identifier for parallel tool calls.
+   */
+  groupId?: string;
+
   /**
    * Opaque signature for Gemini thinking mode.
    * When using Gemini 3+ models with thinking enabled, this signature must be
@@ -202,6 +208,8 @@ export class FunctionCall {
     args: string;
     id?: string;
     createdAt?: number;
+    extra?: Record<string, unknown>;
+    groupId?: string;
     thoughtSignature?: string;
   }) {
     const {
@@ -210,6 +218,8 @@ export class FunctionCall {
       args,
       id = shortuuid('item_'),
       createdAt = Date.now(),
+      extra = {},
+      groupId,
       thoughtSignature,
     } = params;
     this.id = id;
@@ -217,7 +227,15 @@ export class FunctionCall {
     this.args = args;
     this.name = name;
     this.createdAt = createdAt;
-    this.thoughtSignature = thoughtSignature;
+    this.extra = { ...extra };
+    this.groupId = groupId;
+    this.thoughtSignature =
+      thoughtSignature ??
+      (typeof this.extra.google === 'object' && this.extra.google !== null
+        ? // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          (this.extra.google as any).thoughtSignature ||
+          (this.extra.google as any).thought_signature
+        : undefined);
   }
 
   static create(params: {
@@ -226,6 +244,8 @@ export class FunctionCall {
     args: string;
     id?: string;
     createdAt?: number;
+    extra?: Record<string, unknown>;
+    groupId?: string;
     thoughtSignature?: string;
   }) {
     return new FunctionCall(params);
@@ -241,6 +261,14 @@ export class FunctionCall {
       args: this.args,
     };
 
+    if (Object.keys(this.extra).length > 0) {
+      result.extra = this.extra as JSONValue;
+    }
+
+    if (this.groupId) {
+      result.groupId = this.groupId;
+    }
+
     if (this.thoughtSignature) {
       result.thoughtSignature = this.thoughtSignature;
     }
@@ -627,7 +655,9 @@ export class ChatContext {
           a.name !== b.name ||
           a.callId !== b.callId ||
           a.args !== b.args ||
-          a.thoughtSignature !== b.thoughtSignature
+          a.thoughtSignature !== b.thoughtSignature ||
+          a.groupId !== b.groupId ||
+          JSON.stringify(a.extra) !== JSON.stringify(b.extra)
         ) {
           return false;
         }

package/src/llm/llm.ts

@@ -17,6 +17,7 @@ export interface ChoiceDelta {
   role: ChatRole;
   content?: string;
   toolCalls?: FunctionCall[];
+  extra?: Record<string, unknown>;
 }
 
 export interface CompletionUsage {

package/src/llm/provider_format/openai.test.ts

@@ -258,6 +258,46 @@ describe('toChatCtx', () => {
     ]);
   });
 
+  it('should include provider-specific extra content on tool calls', async () => {
+    const ctx = ChatContext.empty();
+    const msg = ctx.addMessage({ role: 'assistant', content: 'Running tool' });
+
+    const toolCall = FunctionCall.create({
+      id: `${msg.id}/tool_1`,
+      callId: 'call_789',
+      name: 'google_call',
+      args: '{}',
+      extra: { google: { thoughtSignature: 'sig-123' } },
+    });
+    const toolOutput = FunctionCallOutput.create({
+      callId: 'call_789',
+      output: '{"result": "ok"}',
+      isError: false,
+    });
+
+    ctx.insert([toolCall, toolOutput]);
+
+    const result = await toChatCtx(ctx);
+
+    expect(result[0]).toEqual({
+      role: 'assistant',
+      content: 'Running tool',
+      tool_calls: [
+        {
+          type: 'function',
+          id: 'call_789',
+          function: { name: 'google_call', arguments: '{}' },
+          extra_content: { google: { thoughtSignature: 'sig-123' } },
+        },
+      ],
+    });
+    expect(result[1]).toEqual({
+      role: 'tool',
+      tool_call_id: 'call_789',
+      content: '{"result": "ok"}',
+    });
+  });
+
   it('should handle multiple tool calls in one message', async () => {
     const ctx = ChatContext.empty();
 

package/src/llm/provider_format/openai.ts

@@ -17,11 +17,20 @@ export async function toChatCtx(chatCtx: ChatContext, injectDummyUserMessage: bo
       ? await toChatItem(group.message)
       : { role: 'assistant' };
 
-    const toolCalls = group.toolCalls.map((toolCall) => ({
-      type: 'function',
-      id: toolCall.callId,
-      function: { name: toolCall.name, arguments: toolCall.args },
-    }));
+    const toolCalls = group.toolCalls.map((toolCall) => {
+      const tc: Record<string, any> = {
+        type: 'function',
+        id: toolCall.callId,
+        function: { name: toolCall.name, arguments: toolCall.args },
+      };
+
+      // Include provider-specific extra content (e.g., Google thought signatures)
+      const googleExtra = getGoogleExtra(toolCall);
+      if (googleExtra) {
+        tc.extra_content = { google: googleExtra };
+      }
+      return tc;
+    });
 
     if (toolCalls.length > 0) {
       message['tool_calls'] = toolCalls;
@@ -53,24 +62,33 @@ async function toChatItem(item: ChatItem) {
       }
     }
 
-    const content =
-      listContent.length == 0
-        ? textContent
-        : textContent.length == 0
-          ? listContent
-          : [...listContent, { type: 'text', text: textContent }];
+    const result: Record<string, any> = { role: item.role };
+    if (listContent.length === 0) {
+      result.content = textContent;
+    } else {
+      if (textContent.length > 0) {
+        listContent.push({ type: 'text', text: textContent });
+      }
+      result.content = listContent;
+    }
 
-    return { role: item.role, content };
+    return result;
   } else if (item.type === 'function_call') {
+    const tc: Record<string, any> = {
+      id: item.callId,
+      type: 'function',
+      function: { name: item.name, arguments: item.args },
+    };
+
+    // Include provider-specific extra content (e.g., Google thought signatures)
+    const googleExtra = getGoogleExtra(item);
+    if (googleExtra) {
+      tc.extra_content = { google: googleExtra };
+    }
+
     return {
       role: 'assistant',
-      tool_calls: [
-        {
-          id: item.callId,
-          type: 'function',
-          function: { name: item.name, arguments: item.args },
-        },
-      ],
+      tool_calls: [tc],
     };
   } else if (item.type === 'function_call_output') {
     return {
@@ -84,6 +102,15 @@ async function toChatItem(item: ChatItem) {
   throw new Error(`Unsupported item type: ${item['type']}`);
 }
 
+function getGoogleExtra(
+  item: Partial<{ extra?: Record<string, unknown>; thoughtSignature?: string }>,
+): Record<string, unknown> | undefined {
+  const googleExtra =
+    (item.extra?.google as Record<string, unknown> | undefined) ||
+    (item.thoughtSignature ? { thoughtSignature: item.thoughtSignature } : undefined);
+  return googleExtra;
+}
+
 async function toImageContent(content: ImageContent) {
   const cacheKey = 'serialized_image'; // TODO: use hash of encoding options if available
   let serialized: SerializedImage;

package/src/llm/provider_format/utils.ts

@@ -133,7 +133,11 @@ export function groupToolCalls(chatCtx: ChatContext) {
 
     if (isAssistantMessage || isFunctionCall) {
       // only assistant messages and function calls can be grouped
-      const groupId = item.id.split('/')[0]!;
+      // For function calls, use group_id if available (for parallel function calls),
+      // otherwise fall back to id-based grouping for backwards compatibility
+      const groupId =
+        item.type === 'function_call' && item.groupId ? item.groupId : item.id.split('/')[0]!;
+
       if (itemGroups[groupId] === undefined) {
         itemGroups[groupId] = ChatItemGroup.create();
 

package/src/stream/stream_channel.ts

@@ -4,14 +4,15 @@
 import type { ReadableStream } from 'node:stream/web';
 import { IdentityTransform } from './identity_transform.js';
 
-export interface StreamChannel<T> {
+export interface StreamChannel<T, E extends Error = Error> {
   write(chunk: T): Promise<void>;
   close(): Promise<void>;
   stream(): ReadableStream<T>;
+  abort(error: E): Promise<void>;
   readonly closed: boolean;
 }
 
-export function createStreamChannel<T>(): StreamChannel<T> {
+export function createStreamChannel<T, E extends Error = Error>(): StreamChannel<T, E> {
   const transform = new IdentityTransform<T>();
   const writer = transform.writable.getWriter();
   let isClosed = false;
@@ -19,6 +20,9 @@ export function createStreamChannel<T>(): StreamChannel<T> {
   return {
     write: (chunk: T) => writer.write(chunk),
     stream: () => transform.readable,
+    abort: (error: E) => {
+      return writer.abort(error);
+    },
     close: async () => {
       try {
         const result = await writer.close();

package/src/telemetry/trace_types.ts

@@ -51,6 +51,13 @@ export const ATTR_TRANSCRIPT_CONFIDENCE = 'lk.transcript_confidence';
 export const ATTR_TRANSCRIPTION_DELAY = 'lk.transcription_delay';
 export const ATTR_END_OF_TURN_DELAY = 'lk.end_of_turn_delay';
 
+// Adaptive Interruption attributes
+export const ATTR_IS_INTERRUPTION = 'lk.is_interruption';
+export const ATTR_INTERRUPTION_PROBABILITY = 'lk.interruption.probability';
+export const ATTR_INTERRUPTION_TOTAL_DURATION = 'lk.interruption.total_duration';
+export const ATTR_INTERRUPTION_PREDICTION_DURATION = 'lk.interruption.prediction_duration';
+export const ATTR_INTERRUPTION_DETECTION_DELAY = 'lk.interruption.detection_delay';
+
 // metrics
 export const ATTR_LLM_METRICS = 'lk.llm_metrics';
 export const ATTR_TTS_METRICS = 'lk.tts_metrics';