@livekit/agents 1.1.0 → 1.2.0

package/dist/inference/interruption/ws_transport.js

@@ -0,0 +1,308 @@
+import { TransformStream } from "stream/web";
+import WebSocket from "ws";
+import { z } from "zod";
+import { APIConnectionError, APIStatusError, APITimeoutError } from "../../_exceptions.js";
+import { log } from "../../log.js";
+import { createAccessToken } from "../utils.js";
+import { InterruptionCacheEntry } from "./interruption_cache_entry.js";
+const MSG_SESSION_CREATE = "session.create";
+const MSG_SESSION_CLOSE = "session.close";
+const MSG_SESSION_CREATED = "session.created";
+const MSG_SESSION_CLOSED = "session.closed";
+const MSG_INTERRUPTION_DETECTED = "bargein_detected";
+const MSG_INFERENCE_DONE = "inference_done";
+const MSG_ERROR = "error";
+const wsMessageSchema = z.discriminatedUnion("type", [
+  z.object({
+    type: z.literal(MSG_SESSION_CREATED)
+  }),
+  z.object({
+    type: z.literal(MSG_SESSION_CLOSED)
+  }),
+  z.object({
+    type: z.literal(MSG_INTERRUPTION_DETECTED),
+    created_at: z.number(),
+    probabilities: z.array(z.number()).default([]),
+    prediction_duration: z.number().default(0)
+  }),
+  z.object({
+    type: z.literal(MSG_INFERENCE_DONE),
+    created_at: z.number(),
+    probabilities: z.array(z.number()).default([]),
+    prediction_duration: z.number().default(0),
+    is_bargein: z.boolean().optional()
+  }),
+  z.object({
+    type: z.literal(MSG_ERROR),
+    message: z.string(),
+    code: z.number().optional(),
+    session_id: z.string().optional()
+  })
+]);
+async function connectWebSocket(options) {
+  const baseUrl = options.baseUrl.replace(/^http/, "ws");
+  const token = await createAccessToken(options.apiKey, options.apiSecret);
+  const url = `${baseUrl}/bargein`;
+  const ws = new WebSocket(url, {
+    headers: { Authorization: `Bearer ${token}` }
+  });
+  await new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      ws.terminate();
+      reject(
+        new APITimeoutError({
+          message: "WebSocket connection timeout",
+          options: { retryable: false }
+        })
+      );
+    }, options.timeout);
+    ws.once("open", () => {
+      clearTimeout(timeout);
+      resolve();
+    });
+    ws.once("unexpected-response", (_req, res) => {
+      clearTimeout(timeout);
+      ws.terminate();
+      const statusCode = res.statusCode ?? -1;
+      reject(
+        new APIStatusError({
+          message: `WebSocket connection rejected with status ${statusCode}`,
+          options: { statusCode, retryable: false }
+        })
+      );
+    });
+    ws.once("error", (err) => {
+      clearTimeout(timeout);
+      ws.terminate();
+      reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));
+    });
+  });
+  return ws;
+}
+function createWsTransport(options, getState, setState, updateUserSpeakingSpan, onRequestSent, getAndResetNumRequests) {
+  const logger = log();
+  let ws = null;
+  let outputController = null;
+  function setupMessageHandler(socket) {
+    socket.on("message", (data) => {
+      try {
+        const message = wsMessageSchema.parse(JSON.parse(data.toString()));
+        handleMessage(message);
+      } catch {
+        logger.warn({ data: data.toString() }, "Failed to parse WebSocket message");
+      }
+    });
+    socket.on("error", (err) => {
+      outputController == null ? void 0 : outputController.error(
+        new APIConnectionError({ message: `WebSocket error: ${err.message}` })
+      );
+    });
+    socket.on("close", (code, reason) => {
+      logger.debug({ code, reason: reason.toString() }, "WebSocket closed");
+    });
+  }
+  async function ensureConnection() {
+    if (ws && ws.readyState === WebSocket.OPEN) return;
+    ws = await connectWebSocket(options);
+    setupMessageHandler(ws);
+    const sessionCreateMsg = JSON.stringify({
+      type: MSG_SESSION_CREATE,
+      settings: {
+        sample_rate: options.sampleRate,
+        num_channels: 1,
+        threshold: options.threshold,
+        min_frames: options.minFrames,
+        encoding: "s16le"
+      }
+    });
+    ws.send(sessionCreateMsg);
+  }
+  function handleMessage(message) {
+    const state = getState();
+    switch (message.type) {
+      case MSG_SESSION_CREATED:
+        logger.debug("WebSocket session created");
+        break;
+      case MSG_INTERRUPTION_DETECTED: {
+        const createdAt = message.created_at;
+        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;
+        if (state.overlapSpeechStarted && overlapSpeechStartedAt !== void 0) {
+          const existing = state.cache.get(createdAt);
+          const totalDurationInS = (existing == null ? void 0 : existing.requestStartedAt) !== void 0 ? (performance.now() - existing.requestStartedAt) / 1e3 : (performance.now() - createdAt) / 1e3;
+          const entry = state.cache.setOrUpdate(
+            createdAt,
+            () => new InterruptionCacheEntry({ createdAt }),
+            {
+              speechInput: existing == null ? void 0 : existing.speechInput,
+              requestStartedAt: existing == null ? void 0 : existing.requestStartedAt,
+              totalDurationInS,
+              probabilities: message.probabilities,
+              isInterruption: true,
+              predictionDurationInS: message.prediction_duration,
+              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1e3
+            }
+          );
+          if (updateUserSpeakingSpan) {
+            updateUserSpeakingSpan(entry);
+          }
+          logger.debug(
+            {
+              totalDuration: entry.totalDurationInS,
+              predictionDuration: entry.predictionDurationInS,
+              detectionDelay: entry.detectionDelayInS,
+              probability: entry.probability
+            },
+            "interruption detected"
+          );
+          const event = {
+            type: "overlapping_speech",
+            detectedAt: Date.now(),
+            isInterruption: true,
+            totalDurationInS: entry.totalDurationInS,
+            predictionDurationInS: entry.predictionDurationInS,
+            overlapStartedAt: overlapSpeechStartedAt,
+            speechInput: entry.speechInput,
+            probabilities: entry.probabilities,
+            detectionDelayInS: entry.detectionDelayInS,
+            probability: entry.probability,
+            numRequests: (getAndResetNumRequests == null ? void 0 : getAndResetNumRequests()) ?? 0
+          };
+          outputController == null ? void 0 : outputController.enqueue(event);
+          setState({ overlapSpeechStarted: false });
+        }
+        break;
+      }
+      case MSG_INFERENCE_DONE: {
+        const createdAt = message.created_at;
+        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;
+        if (state.overlapSpeechStarted && overlapSpeechStartedAt !== void 0) {
+          const existing = state.cache.get(createdAt);
+          const totalDurationInS = (existing == null ? void 0 : existing.requestStartedAt) !== void 0 ? (performance.now() - existing.requestStartedAt) / 1e3 : (performance.now() - createdAt) / 1e3;
+          const entry = state.cache.setOrUpdate(
+            createdAt,
+            () => new InterruptionCacheEntry({ createdAt }),
+            {
+              speechInput: existing == null ? void 0 : existing.speechInput,
+              requestStartedAt: existing == null ? void 0 : existing.requestStartedAt,
+              totalDurationInS,
+              predictionDurationInS: message.prediction_duration,
+              probabilities: message.probabilities,
+              isInterruption: message.is_bargein ?? false,
+              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1e3
+            }
+          );
+          logger.debug(
+            {
+              totalDurationInS: entry.totalDurationInS,
+              predictionDurationInS: entry.predictionDurationInS
+            },
+            "interruption inference done"
+          );
+        }
+        break;
+      }
+      case MSG_SESSION_CLOSED:
+        logger.debug("WebSocket session closed");
+        break;
+      case MSG_ERROR:
+        outputController == null ? void 0 : outputController.error(
+          new APIStatusError({
+            message: `LiveKit Adaptive Interruption error: ${message.message}`,
+            options: { statusCode: message.code ?? -1 }
+          })
+        );
+        break;
+    }
+  }
+  function sendAudioData(audioSlice) {
+    if (!ws || ws.readyState !== WebSocket.OPEN) {
+      throw new APIConnectionError({ message: "WebSocket not connected" });
+    }
+    const state = getState();
+    const createdAt = Math.floor(performance.now());
+    state.cache.set(
+      createdAt,
+      new InterruptionCacheEntry({
+        createdAt,
+        requestStartedAt: performance.now(),
+        speechInput: audioSlice
+      })
+    );
+    const header = new ArrayBuffer(8);
+    const view = new DataView(header);
+    view.setUint32(0, createdAt >>> 0, true);
+    view.setUint32(4, Math.floor(createdAt / 4294967296) >>> 0, true);
+    const audioBytes = new Uint8Array(
+      audioSlice.buffer,
+      audioSlice.byteOffset,
+      audioSlice.byteLength
+    );
+    const combined = new Uint8Array(8 + audioBytes.length);
+    combined.set(new Uint8Array(header), 0);
+    combined.set(audioBytes, 8);
+    ws.send(combined);
+    onRequestSent == null ? void 0 : onRequestSent();
+  }
+  function close() {
+    if ((ws == null ? void 0 : ws.readyState) === WebSocket.OPEN) {
+      const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });
+      try {
+        ws.send(closeMsg);
+      } catch (e) {
+        logger.error(e, "failed to send close message");
+      }
+    }
+    ws == null ? void 0 : ws.close(1e3);
+    ws = null;
+  }
+  async function reconnect() {
+    close();
+  }
+  const transport = new TransformStream(
+    {
+      async start(controller) {
+        outputController = controller;
+        await ensureConnection();
+      },
+      transform(chunk, controller) {
+        if (!(chunk instanceof Int16Array)) {
+          controller.enqueue(chunk);
+          return;
+        }
+        const state = getState();
+        if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;
+        if (options.timeout > 0) {
+          const now = performance.now();
+          for (const [, entry] of state.cache.entries()) {
+            if (entry.totalDurationInS !== 0) continue;
+            if (now - entry.createdAt > options.timeout) {
+              controller.error(
+                new APIStatusError({
+                  message: `interruption inference timed out after ${((now - entry.createdAt) / 1e3).toFixed(1)}s (ws)`,
+                  options: { statusCode: 408, retryable: false }
+                })
+              );
+              return;
+            }
+            break;
+          }
+        }
+        try {
+          sendAudioData(chunk);
+        } catch (err) {
+          controller.error(err);
+        }
+      },
+      flush() {
+        close();
+      }
+    },
+    { highWaterMark: 2 },
+    { highWaterMark: 2 }
+  );
+  return { transport, reconnect };
+}
+export {
+  createWsTransport
+};
+//# sourceMappingURL=ws_transport.js.map

package/dist/inference/interruption/ws_transport.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/inference/interruption/ws_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TransformStream } from 'stream/web';\nimport WebSocket from 'ws';\nimport { z } from 'zod';\nimport { APIConnectionError, APIStatusError, APITimeoutError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\n// WebSocket message types\nconst MSG_SESSION_CREATE = 'session.create';\nconst MSG_SESSION_CLOSE = 'session.close';\nconst MSG_SESSION_CREATED = 'session.created';\nconst MSG_SESSION_CLOSED = 'session.closed';\nconst MSG_INTERRUPTION_DETECTED = 'bargein_detected';\nconst MSG_INFERENCE_DONE = 'inference_done';\nconst MSG_ERROR = 'error';\n\nexport interface WsTransportOptions {\n  baseUrl: string;\n  apiKey: string;\n  apiSecret: string;\n  sampleRate: number;\n  threshold: number;\n  minFrames: number;\n  timeout: number;\n  maxRetries?: number;\n}\n\nexport interface WsTransportState {\n  overlapSpeechStarted: boolean;\n  overlapSpeechStartedAt: number | undefined;\n  cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\nconst wsMessageSchema = z.discriminatedUnion('type', [\n  z.object({\n    type: z.literal(MSG_SESSION_CREATED),\n  }),\n  z.object({\n    type: z.literal(MSG_SESSION_CLOSED),\n  }),\n  z.object({\n    type: z.literal(MSG_INTERRUPTION_DETECTED),\n    created_at: z.number(),\n    probabilities: z.array(z.number()).default([]),\n    prediction_duration: z.number().default(0),\n  }),\n  z.object({\n    type: z.literal(MSG_INFERENCE_DONE),\n    created_at: z.number(),\n    probabilities: z.array(z.number()).default([]),\n    prediction_duration: z.number().default(0),\n    is_bargein: z.boolean().optional(),\n  }),\n  z.object({\n    type: z.literal(MSG_ERROR),\n    message: z.string(),\n    code: z.number().optional(),\n    session_id: z.string().optional(),\n  }),\n]);\n\ntype WsMessage = z.infer<typeof wsMessageSchema>;\n\n/**\n * Creates a WebSocket connection and waits for it to open.\n */\nasync function connectWebSocket(options: WsTransportOptions): Promise<WebSocket> {\n  const baseUrl = options.baseUrl.replace(/^http/, 'ws');\n  const token = await createAccessToken(options.apiKey, options.apiSecret);\n  const url = `${baseUrl}/bargein`;\n\n  const ws = new WebSocket(url, {\n    headers: { Authorization: `Bearer ${token}` },\n  });\n\n  await new Promise<void>((resolve, reject) => {\n    const timeout = setTimeout(() => {\n      ws.terminate();\n      reject(\n        new APITimeoutError({\n          message: 'WebSocket connection timeout',\n          options: { retryable: false },\n        }),\n      );\n    }, options.timeout);\n    ws.once('open', () => {\n      clearTimeout(timeout);\n      resolve();\n    });\n    ws.once('unexpected-response', (_req, res) => {\n      clearTimeout(timeout);\n      ws.terminate();\n      const statusCode = res.statusCode ?? -1;\n      reject(\n        new APIStatusError({\n          message: `WebSocket connection rejected with status ${statusCode}`,\n          options: { statusCode, retryable: false },\n        }),\n      );\n    });\n    ws.once('error', (err: Error) => {\n      clearTimeout(timeout);\n      ws.terminate();\n      reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` }));\n    });\n  });\n\n  return ws;\n}\n\nexport interface WsTransportResult {\n  transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;\n  reconnect: () => Promise<void>;\n}\n\n/**\n * Creates a WebSocket transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * It maintains a persistent WebSocket connection with automatic retry on failure.\n * Returns both the transport and a reconnect function for option updates.\n */\nexport function createWsTransport(\n  options: WsTransportOptions,\n  getState: () => WsTransportState,\n  setState: (partial: Partial<WsTransportState>) => void,\n  updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n  onRequestSent?: () => void,\n  getAndResetNumRequests?: () => number,\n): WsTransportResult {\n  const logger = log();\n  let ws: WebSocket | null = null;\n  let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;\n\n  function setupMessageHandler(socket: WebSocket): void {\n    socket.on('message', (data: WebSocket.Data) => {\n      try {\n        const message = wsMessageSchema.parse(JSON.parse(data.toString()));\n        handleMessage(message);\n      } catch {\n        logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');\n      }\n    });\n\n    socket.on('error', (err: Error) => {\n      outputController?.error(\n        new APIConnectionError({ message: `WebSocket error: ${err.message}` }),\n      );\n    });\n\n    socket.on('close', (code: number, reason: Buffer) => {\n      logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');\n    });\n  }\n\n  async function ensureConnection(): Promise<void> {\n    if (ws && ws.readyState === WebSocket.OPEN) return;\n\n    ws = await connectWebSocket(options);\n    setupMessageHandler(ws);\n\n    const sessionCreateMsg = JSON.stringify({\n      type: MSG_SESSION_CREATE,\n      settings: {\n        sample_rate: options.sampleRate,\n        num_channels: 1,\n        threshold: options.threshold,\n        min_frames: options.minFrames,\n        encoding: 's16le',\n      },\n    });\n    ws.send(sessionCreateMsg);\n  }\n\n  function handleMessage(message: WsMessage): void {\n    const state = getState();\n\n    switch (message.type) {\n      case MSG_SESSION_CREATED:\n        logger.debug('WebSocket session created');\n        break;\n\n      case MSG_INTERRUPTION_DETECTED: {\n        const createdAt = message.created_at;\n        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n        if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n          const existing = state.cache.get(createdAt);\n\n          const totalDurationInS =\n            existing?.requestStartedAt !== undefined\n              ? (performance.now() - existing.requestStartedAt) / 1000\n              : (performance.now() - createdAt) / 1000;\n\n          const entry = state.cache.setOrUpdate(\n            createdAt,\n            () => new InterruptionCacheEntry({ createdAt }),\n            {\n              speechInput: existing?.speechInput,\n              requestStartedAt: existing?.requestStartedAt,\n              totalDurationInS,\n              probabilities: message.probabilities,\n              isInterruption: true,\n              predictionDurationInS: message.prediction_duration,\n              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n            },\n          );\n\n          if (updateUserSpeakingSpan) {\n            updateUserSpeakingSpan(entry);\n          }\n\n          logger.debug(\n            {\n              totalDuration: entry.totalDurationInS,\n              predictionDuration: entry.predictionDurationInS,\n              detectionDelay: entry.detectionDelayInS,\n              probability: entry.probability,\n            },\n            'interruption detected',\n          );\n\n          const event: OverlappingSpeechEvent = {\n            type: 'overlapping_speech',\n            detectedAt: Date.now(),\n            isInterruption: true,\n            totalDurationInS: entry.totalDurationInS,\n            predictionDurationInS: entry.predictionDurationInS,\n            overlapStartedAt: overlapSpeechStartedAt,\n            speechInput: entry.speechInput,\n            probabilities: entry.probabilities,\n            detectionDelayInS: entry.detectionDelayInS,\n            probability: entry.probability,\n            numRequests: getAndResetNumRequests?.() ?? 0,\n          };\n\n          outputController?.enqueue(event);\n          setState({ overlapSpeechStarted: false });\n        }\n        break;\n      }\n\n      case MSG_INFERENCE_DONE: {\n        const createdAt = message.created_at;\n        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n        if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n          const existing = state.cache.get(createdAt);\n          const totalDurationInS =\n            existing?.requestStartedAt !== undefined\n              ? (performance.now() - existing.requestStartedAt) / 1000\n              : (performance.now() - createdAt) / 1000;\n          const entry = state.cache.setOrUpdate(\n            createdAt,\n            () => new InterruptionCacheEntry({ createdAt }),\n            {\n              speechInput: existing?.speechInput,\n              requestStartedAt: existing?.requestStartedAt,\n              totalDurationInS,\n              predictionDurationInS: message.prediction_duration,\n              probabilities: message.probabilities,\n              isInterruption: message.is_bargein ?? false,\n              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n            },\n          );\n\n          logger.debug(\n            {\n              totalDurationInS: entry.totalDurationInS,\n              predictionDurationInS: entry.predictionDurationInS,\n            },\n            'interruption inference done',\n          );\n        }\n        break;\n      }\n\n      case MSG_SESSION_CLOSED:\n        logger.debug('WebSocket session closed');\n        break;\n\n      case MSG_ERROR:\n        outputController?.error(\n          new APIStatusError({\n            message: `LiveKit Adaptive Interruption error: ${message.message}`,\n            options: { statusCode: message.code ?? -1 },\n          }),\n        );\n        break;\n    }\n  }\n\n  function sendAudioData(audioSlice: Int16Array): void {\n    if (!ws || ws.readyState !== WebSocket.OPEN) {\n      throw new APIConnectionError({ message: 'WebSocket not connected' });\n    }\n\n    const state = getState();\n    const createdAt = Math.floor(performance.now());\n\n    state.cache.set(\n      createdAt,\n      new InterruptionCacheEntry({\n        createdAt,\n        requestStartedAt: performance.now(),\n        speechInput: audioSlice,\n      }),\n    );\n\n    const header = new ArrayBuffer(8);\n    const view = new DataView(header);\n    view.setUint32(0, createdAt >>> 0, true);\n    view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);\n\n    const audioBytes = new Uint8Array(\n      audioSlice.buffer,\n      audioSlice.byteOffset,\n      audioSlice.byteLength,\n    );\n    const combined = new Uint8Array(8 + audioBytes.length);\n    combined.set(new Uint8Array(header), 0);\n    combined.set(audioBytes, 8);\n\n    ws.send(combined);\n    onRequestSent?.();\n  }\n\n  function close(): void {\n    if (ws?.readyState === WebSocket.OPEN) {\n      const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });\n      try {\n        ws.send(closeMsg);\n      } catch (e: unknown) {\n        logger.error(e, 'failed to send close message');\n      }\n    }\n    ws?.close(1000); // signal normal websocket closure\n    ws = null;\n  }\n\n  /**\n   * Reconnect the WebSocket with updated options.\n   * This is called when options are updated via updateOptions().\n   */\n  async function reconnect(): Promise<void> {\n    close();\n  }\n\n  const transport = new TransformStream<\n    Int16Array | OverlappingSpeechEvent,\n    OverlappingSpeechEvent\n  >(\n    {\n      async start(controller) {\n        outputController = controller;\n        await ensureConnection();\n      },\n\n      transform(chunk, controller) {\n        if (!(chunk instanceof Int16Array)) {\n          controller.enqueue(chunk);\n          return;\n        }\n\n        // Only forwards buffered audio while overlap speech is actively on.\n        const state = getState();\n        if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;\n\n        if (options.timeout > 0) {\n          const now = performance.now();\n          for (const [, entry] of state.cache.entries()) {\n            if (entry.totalDurationInS !== 0) continue;\n            if (now - entry.createdAt > options.timeout) {\n              controller.error(\n                new APIStatusError({\n                  message: `interruption inference timed out after ${((now - entry.createdAt) / 1000).toFixed(1)}s (ws)`,\n                  options: { statusCode: 408, retryable: false },\n                }),\n              );\n              return;\n            }\n            break;\n          }\n        }\n\n        try {\n          sendAudioData(chunk);\n        } catch (err) {\n          controller.error(err);\n        }\n      },\n\n      flush() {\n        close();\n      },\n    },\n    { highWaterMark: 2 },\n    { highWaterMark: 2 },\n  );\n\n  return { transport, reconnect };\n}\n"],"mappings":"AAGA,SAAS,uBAAuB;AAChC,OAAO,eAAe;AACtB,SAAS,SAAS;AAClB,SAAS,oBAAoB,gBAAgB,uBAAuB;AACpE,SAAS,WAAW;AACpB,SAAS,yBAAyB;AAClC,SAAS,8BAA8B;AAKvC,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,4BAA4B;AAClC,MAAM,qBAAqB;AAC3B,MAAM,YAAY;AAmBlB,MAAM,kBAAkB,EAAE,mBAAmB,QAAQ;AAAA,EACnD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,mBAAmB;AAAA,EACrC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,EACpC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,yBAAyB;AAAA,IACzC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAC3C,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,IAClC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,IACzC,YAAY,EAAE,QAAQ,EAAE,SAAS;AAAA,EACnC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,SAAS;AAAA,IACzB,SAAS,EAAE,OAAO;AAAA,IAClB,MAAM,EAAE,OAAO,EAAE,SAAS;AAAA,IAC1B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,CAAC;AACH,CAAC;AAOD,eAAe,iBAAiB,SAAiD;AAC/E,QAAM,UAAU,QAAQ,QAAQ,QAAQ,SAAS,IAAI;AACrD,QAAM,QAAQ,MAAM,kBAAkB,QAAQ,QAAQ,QAAQ,SAAS;AACvE,QAAM,MAAM,GAAG,OAAO;AAEtB,QAAM,KAAK,IAAI,UAAU,KAAK;AAAA,IAC5B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AAED,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,UAAM,UAAU,WAAW,MAAM;AAC/B,SAAG,UAAU;AACb;AAAA,QACE,IAAI,gBAAgB;AAAA,UAClB,SAAS;AAAA,UACT,SAAS,EAAE,WAAW,MAAM;AAAA,QAC9B,CAAC;AAAA,MACH;AAAA,IACF,GAAG,QAAQ,OAAO;AAClB,OAAG,KAAK,QAAQ,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ;AAAA,IACV,CAAC;AACD,OAAG,KAAK,uBAAuB,CAAC,MAAM,QAAQ;AAC5C,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,YAAM,aAAa,IAAI,cAAc;AACrC;AAAA,QACE,IAAI,eAAe;AAAA,UACjB,SAAS,6CAA6C,UAAU;AAAA,UAChE,SAAS,EAAE,YAAY,WAAW,MAAM;AAAA,QAC1C,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AACD,OAAG,KAAK,SAAS,CAAC,QAAe;AAC/B,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,aAAO,IAAI,mBAAmB,EAAE,SAAS,+BAA+B,IAAI,OAAO,GAAG,CAAC,CAAC;AAAA,IAC1F,CAAC;AAAA,EACH,CAAC;AAED,SAAO;AACT;AAcO,SAAS,kBACd,SACA,UACA,UACA,wBACA,eACA,wBACmB;AACnB,QAAM,SAAS,IAAI;AACnB,MAAI,KAAuB;AAC3B,MAAI,mBAAoF;AAExF,WAAS,oBAAoB,QAAyB;AACpD,WAAO,GAAG,WAAW,CAAC,SAAyB;AAC7C,UAAI;AACF,cAAM,UAAU,gBAAgB,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,sBAAc,OAAO;AAAA,MACvB,QAAQ;AACN,eAAO,KAAK,EAAE,MAAM,KAAK,SAAS,EAAE,GAAG,mCAAmC;AAAA,MAC5E;AAAA,IACF,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,QAAe;AACjC,2DAAkB;AAAA,QAChB,IAAI,mBAAmB,EAAE,SAAS,oBAAoB,IAAI,OAAO,GAAG,CAAC;AAAA;AAAA,IAEzE,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,MAAc,WAAmB;AACnD,aAAO,MAAM,EAAE,MAAM,QAAQ,OAAO,SAAS,EAAE,GAAG,kBAAkB;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,iBAAe,mBAAkC;AAC/C,QAAI,MAAM,GAAG,eAAe,UAAU,KAAM;AAE5C,SAAK,MAAM,iBAAiB,OAAO;AACnC,wBAAoB,EAAE;AAEtB,UAAM,mBAAmB,KAAK,UAAU;AAAA,MACtC,MAAM;AAAA,MACN,UAAU;AAAA,QACR,aAAa,QAAQ;AAAA,QACrB,cAAc;AAAA,QACd,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,QACpB,UAAU;AAAA,MACZ;AAAA,IACF,CAAC;AACD,OAAG,KAAK,gBAAgB;AAAA,EAC1B;AAEA,WAAS,cAAc,SAA0B;AAC/C,UAAM,QAAQ,SAAS;AAEvB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,eAAO,MAAM,2BAA2B;AACxC;AAAA,MAEF,KAAK,2BAA2B;AAC9B,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAE1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AAExC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,eAAe,QAAQ;AAAA,cACvB,gBAAgB;AAAA,cAChB,uBAAuB,QAAQ;AAAA,cAC/B,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,cAAI,wBAAwB;AAC1B,mCAAuB,KAAK;AAAA,UAC9B;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,eAAe,MAAM;AAAA,cACrB,oBAAoB,MAAM;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,YACrB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,QAAgC;AAAA,YACpC,MAAM;AAAA,YACN,YAAY,KAAK,IAAI;AAAA,YACrB,gBAAgB;AAAA,YAChB,kBAAkB,MAAM;AAAA,YACxB,uBAAuB,MAAM;AAAA,YAC7B,kBAAkB;AAAA,YAClB,aAAa,MAAM;AAAA,YACnB,eAAe,MAAM;AAAA,YACrB,mBAAmB,MAAM;AAAA,YACzB,aAAa,MAAM;AAAA,YACnB,cAAa,uEAA8B;AAAA,UAC7C;AAEA,+DAAkB,QAAQ;AAC1B,mBAAS,EAAE,sBAAsB,MAAM,CAAC;AAAA,QAC1C;AACA;AAAA,MACF;AAAA,MAEA,KAAK,oBAAoB;AACvB,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAC1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AACxC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,uBAAuB,QAAQ;AAAA,cAC/B,eAAe,QAAQ;AAAA,cACvB,gBAAgB,QAAQ,cAAc;AAAA,cACtC,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,eAAO,MAAM,0BAA0B;AACvC;AAAA,MAEF,KAAK;AACH,6DAAkB;AAAA,UAChB,IAAI,eAAe;AAAA,YACjB,SAAS,wCAAwC,QAAQ,OAAO;AAAA,YAChE,SAAS,EAAE,YAAY,QAAQ,QAAQ,GAAG;AAAA,UAC5C,CAAC;AAAA;AAEH;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,cAAc,YAA8B;AACnD,QAAI,CAAC,MAAM,GAAG,eAAe,UAAU,MAAM;AAC3C,YAAM,IAAI,mBAAmB,EAAE,SAAS,0BAA0B,CAAC;AAAA,IACrE;AAEA,UAAM,QAAQ,SAAS;AACvB,UAAM,YAAY,KAAK,MAAM,YAAY,IAAI,CAAC;AAE9C,UAAM,MAAM;AAAA,MACV;AAAA,MACA,IAAI,uBAAuB;AAAA,QACzB;AAAA,QACA,kBAAkB,YAAY,IAAI;AAAA,QAClC,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,IAAI,YAAY,CAAC;AAChC,UAAM,OAAO,IAAI,SAAS,MAAM;AAChC,SAAK,UAAU,GAAG,cAAc,GAAG,IAAI;AACvC,SAAK,UAAU,GAAG,KAAK,MAAM,YAAY,UAAW,MAAM,GAAG,IAAI;AAEjE,UAAM,aAAa,IAAI;AAAA,MACrB,WAAW;AAAA,MACX,WAAW;AAAA,MACX,WAAW;AAAA,IACb;AACA,UAAM,WAAW,IAAI,WAAW,IAAI,WAAW,MAAM;AACrD,aAAS,IAAI,IAAI,WAAW,MAAM,GAAG,CAAC;AACtC,aAAS,IAAI,YAAY,CAAC;AAE1B,OAAG,KAAK,QAAQ;AAChB;AAAA,EACF;AAEA,WAAS,QAAc;AACrB,SAAI,yBAAI,gBAAe,UAAU,MAAM;AACrC,YAAM,WAAW,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC3D,UAAI;AACF,WAAG,KAAK,QAAQ;AAAA,MAClB,SAAS,GAAY;AACnB,eAAO,MAAM,GAAG,8BAA8B;AAAA,MAChD;AAAA,IACF;AACA,6BAAI,MAAM;AACV,SAAK;AAAA,EACP;AAMA,iBAAe,YAA2B;AACxC,UAAM;AAAA,EACR;AAEA,QAAM,YAAY,IAAI;AAAA,IAIpB;AAAA,MACE,MAAM,MAAM,YAAY;AACtB,2BAAmB;AACnB,cAAM,iBAAiB;AAAA,MACzB;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAGA,cAAM,QAAQ,SAAS;AACvB,YAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,qBAAsB;AAElE,YAAI,QAAQ,UAAU,GAAG;AACvB,gBAAM,MAAM,YAAY,IAAI;AAC5B,qBAAW,CAAC,EAAE,KAAK,KAAK,MAAM,MAAM,QAAQ,GAAG;AAC7C,gBAAI,MAAM,qBAAqB,EAAG;AAClC,gBAAI,MAAM,MAAM,YAAY,QAAQ,SAAS;AAC3C,yBAAW;AAAA,gBACT,IAAI,eAAe;AAAA,kBACjB,SAAS,4CAA4C,MAAM,MAAM,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,kBAC9F,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,gBAC/C,CAAC;AAAA,cACH;AACA;AAAA,YACF;AACA;AAAA,UACF;AAAA,QACF;AAEA,YAAI;AACF,wBAAc,KAAK;AAAA,QACrB,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,MAEA,QAAQ;AACN,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AAEA,SAAO,EAAE,WAAW,UAAU;AAChC;","names":[]}

package/dist/inference/llm.cjs

@@ -33,10 +33,11 @@ __export(llm_exports, {
 });
 module.exports = __toCommonJS(llm_exports);
 var import_openai = __toESM(require("openai"), 1);
-var import__ = require("../index.cjs");
+var import_exceptions = require("../_exceptions.cjs");
 var llm = __toESM(require("../llm/index.cjs"), 1);
-var import_utils = require("./utils.cjs");
-const DEFAULT_BASE_URL = "https://agent-gateway.livekit.cloud/v1";
+var import_types = require("../types.cjs");
+var import_utils = require("../utils.cjs");
+var import_utils2 = require("./utils.cjs");
 class LLM extends llm.LLM {
   client;
   opts;
@@ -44,19 +45,14 @@ class LLM extends llm.LLM {
     super();
     const {
       model,
-      temperature,
-      parallelToolCalls,
-      toolChoice,
-      maxCompletionTokens,
+      provider,
       baseURL,
       apiKey,
       apiSecret,
-      maxRetries,
-      timeout,
-      verbosity,
-      extraKwargs
+      modelOptions,
+      strictToolSchema = false
     } = opts;
-    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkBaseURL = baseURL || (0, import_utils2.getDefaultInferenceUrl)();
     const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
     if (!lkApiKey) {
       throw new Error("apiKey is required: pass apiKey or set LIVEKIT_API_KEY");
@@ -67,20 +63,17 @@ class LLM extends llm.LLM {
     }
     this.opts = {
       model,
-      temperature,
-      parallelToolCalls,
-      toolChoice,
-      verbosity,
-      maxCompletionTokens,
+      provider,
       baseURL: lkBaseURL,
       apiKey: lkApiKey,
       apiSecret: lkApiSecret,
-      extraKwargs: extraKwargs || {}
+      modelOptions: modelOptions || {},
+      strictToolSchema
     };
     this.client = new import_openai.default({
       baseURL: this.opts.baseURL,
-      maxRetries: maxRetries || 0,
-      timeout: timeout || 15e3
+      apiKey: ""
+      // leave a temporary empty string to avoid OpenAI complain about missing key
     });
   }
   label() {
@@ -89,42 +82,41 @@ class LLM extends llm.LLM {
   get model() {
     return this.opts.model;
   }
+  get provider() {
+    return "livekit";
+  }
+  static fromModelString(modelString) {
+    return new LLM({ model: modelString });
+  }
   chat({
     chatCtx,
     toolCtx,
-    connOptions = import__.DEFAULT_API_CONNECT_OPTIONS,
+    connOptions = import_types.DEFAULT_API_CONNECT_OPTIONS,
     parallelToolCalls,
     toolChoice,
     // TODO(AJS-270): Add response_format parameter support
     extraKwargs
   }) {
-    let extras = { ...extraKwargs || {} };
-    if (this.opts.maxCompletionTokens !== void 0) {
-      extras.max_completion_tokens = this.opts.maxCompletionTokens;
-    }
-    if (this.opts.temperature !== void 0) {
-      extras.temperature = this.opts.temperature;
-    }
-    if (this.opts.verbosity !== void 0) {
-      extras.verbosity = this.opts.verbosity;
-    }
-    parallelToolCalls = parallelToolCalls !== void 0 ? parallelToolCalls : this.opts.parallelToolCalls;
+    let modelOptions = { ...extraKwargs || {} };
+    parallelToolCalls = parallelToolCalls !== void 0 ? parallelToolCalls : this.opts.modelOptions.parallel_tool_calls;
     if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== void 0) {
-      extras.parallel_tool_calls = parallelToolCalls;
+      modelOptions.parallel_tool_calls = parallelToolCalls;
     }
-    toolChoice = toolChoice !== void 0 ? toolChoice : this.opts.toolChoice;
+    toolChoice = toolChoice !== void 0 ? toolChoice : this.opts.modelOptions.tool_choice;
     if (toolChoice) {
-      extras.tool_choice = toolChoice;
+      modelOptions.tool_choice = toolChoice;
     }
-    extras = { ...extras, ...this.opts.extraKwargs };
+    modelOptions = { ...modelOptions, ...this.opts.modelOptions };
     return new LLMStream(this, {
       model: this.opts.model,
-      providerFmt: "openai",
+      provider: this.opts.provider,
       client: this.client,
       chatCtx,
       toolCtx,
       connOptions,
-      extraKwargs: extras,
+      modelOptions,
+      strictToolSchema: this.opts.strictToolSchema ?? false,
+      // default to false if not set
       gatewayOptions: {
         apiKey: this.opts.apiKey,
         apiSecret: this.opts.apiSecret
@@ -134,59 +126,80 @@ class LLM extends llm.LLM {
 }
 class LLMStream extends llm.LLMStream {
   model;
+  provider;
   providerFmt;
   client;
-  extraKwargs;
+  modelOptions;
+  strictToolSchema;
   gatewayOptions;
   toolCallId;
   toolIndex;
   fncName;
   fncRawArguments;
+  toolExtra;
   constructor(llm2, {
     model,
-    providerFmt,
+    provider,
     client,
     chatCtx,
     toolCtx,
     gatewayOptions,
     connOptions,
-    extraKwargs
+    modelOptions,
+    providerFmt,
+    strictToolSchema
   }) {
     super(llm2, { chatCtx, toolCtx, connOptions });
     this.client = client;
     this.gatewayOptions = gatewayOptions;
-    this.providerFmt = providerFmt;
-    this.extraKwargs = extraKwargs;
+    this.provider = provider;
+    this.providerFmt = providerFmt || "openai";
+    this.modelOptions = modelOptions;
     this.model = model;
+    this.strictToolSchema = strictToolSchema;
   }
   async run() {
     var _a;
     let retryable = true;
     this.toolCallId = this.fncName = this.fncRawArguments = this.toolIndex = void 0;
+    this.toolExtra = void 0;
     try {
       const messages = await this.chatCtx.toProviderFormat(
         this.providerFmt
       );
-      const tools = this.toolCtx ? Object.entries(this.toolCtx).map(([name, func]) => ({
-        type: "function",
-        function: {
-          name,
-          description: func.description,
-          parameters: llm.toJsonSchema(
-            func.parameters
-          )
+      const tools = this.toolCtx ? Object.entries(this.toolCtx).map(([name, func]) => {
+        const oaiParams = {
+          type: "function",
+          function: {
+            name,
+            description: func.description,
+            parameters: llm.toJsonSchema(
+              func.parameters,
+              true,
+              this.strictToolSchema
+            )
+          }
+        };
+        if (this.strictToolSchema) {
+          oaiParams.function.strict = true;
         }
-      })) : void 0;
-      const requestExtras = { ...this.extraKwargs };
+        return oaiParams;
+      }) : void 0;
+      const requestOptions = { ...this.modelOptions };
       if (!tools) {
-        delete requestExtras.tool_choice;
+        delete requestOptions.tool_choice;
       }
       if (this.gatewayOptions) {
-        this.client.apiKey = await (0, import_utils.createAccessToken)(
+        this.client.apiKey = await (0, import_utils2.createAccessToken)(
           this.gatewayOptions.apiKey,
           this.gatewayOptions.apiSecret
         );
       }
+      if (this.provider) {
+        const extraHeaders = requestOptions.extra_headers ? requestOptions.extra_headers : {};
+        extraHeaders["X-LiveKit-Inference-Provider"] = this.provider;
+        requestOptions.extra_headers = extraHeaders;
+      }
       const stream = await this.client.chat.completions.create(
         {
           model: this.model,
@@ -194,7 +207,7 @@ class LLMStream extends llm.LLMStream {
           tools,
           stream: true,
           stream_options: { include_usage: true },
-          ...requestExtras
+          ...requestOptions
         },
         {
           timeout: this.connOptions.timeoutMs
@@ -227,25 +240,23 @@ class LLMStream extends llm.LLMStream {
       }
     } catch (error) {
       if (error instanceof import_openai.default.APIConnectionTimeoutError) {
-        throw new import__.APITimeoutError({ options: { retryable } });
+        throw new import_exceptions.APITimeoutError({ options: { retryable } });
       } else if (error instanceof import_openai.default.APIError) {
-        throw new import__.APIStatusError({
+        throw new import_exceptions.APIStatusError({
           message: error.message,
           options: {
             statusCode: error.status,
             body: error.error,
-            requestId: error.request_id,
+            requestId: error.requestID,
             retryable
           }
         });
       } else {
-        throw new import__.APIConnectionError({
-          message: (0, import__.toError)(error).message,
+        throw new import_exceptions.APIConnectionError({
+          message: (0, import_utils.toError)(error).message,
           options: { retryable }
         });
       }
-    } finally {
-      this.queue.close();
     }
   }
   parseChoice(id, choice) {
@@ -266,6 +277,13 @@ class LLMStream extends llm.LLMStream {
           this.toolCallId = tool.id;
           this.fncName = tool.function.name;
           this.fncRawArguments = tool.function.arguments || "";
+          const newToolExtra = (
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            tool.extra_content ?? void 0
+          );
+          if (newToolExtra) {
+            this.toolExtra = newToolExtra;
+          }
         } else if (tool.function.arguments) {
           this.fncRawArguments = (this.fncRawArguments || "") + tool.function.arguments;
         }
@@ -277,35 +295,57 @@ class LLMStream extends llm.LLMStream {
     if (choice.finish_reason && ["tool_calls", "stop"].includes(choice.finish_reason) && this.toolCallId !== void 0) {
       const callChunk = this.createRunningToolCallChunk(id, delta);
       this.toolCallId = this.fncName = this.fncRawArguments = void 0;
+      this.toolExtra = void 0;
       return callChunk;
     }
-    if (!delta.content) {
+    const deltaExtra = (
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      delta.extra_content ?? void 0
+    );
+    if (!delta.content && !deltaExtra) {
       return void 0;
     }
     return {
       id,
       delta: {
         role: "assistant",
-        content: delta.content
+        content: delta.content || void 0,
+        extra: deltaExtra
       }
     };
   }
   createRunningToolCallChunk(id, delta) {
+    const toolExtra = this.toolExtra ? { ...this.toolExtra } : {};
+    const thoughtSignature = this.extractThoughtSignature(toolExtra);
+    const deltaExtra = (
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      delta.extra_content ?? void 0
+    );
     return {
       id,
       delta: {
         role: "assistant",
         content: delta.content || void 0,
+        extra: deltaExtra,
         toolCalls: [
           llm.FunctionCall.create({
             callId: this.toolCallId || "",
             name: this.fncName || "",
-            args: this.fncRawArguments || ""
+            args: this.fncRawArguments || "",
+            extra: toolExtra,
+            thoughtSignature
           })
         ]
       }
     };
   }
+  extractThoughtSignature(extra) {
+    const googleExtra = extra == null ? void 0 : extra.google;
+    if (googleExtra && typeof googleExtra === "object") {
+      return googleExtra.thoughtSignature || googleExtra.thought_signature;
+    }
+    return void 0;
+  }
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {