@livekit/agents 1.0.47 → 1.1.0-dev.0

package/dist/inference/interruption/ws_transport.js

@@ -0,0 +1,295 @@
+import { TransformStream } from "stream/web";
+import WebSocket from "ws";
+import { z } from "zod";
+import { log } from "../../log.js";
+import { createAccessToken } from "../utils.js";
+import { intervalForRetry } from "./defaults.js";
+import { InterruptionCacheEntry } from "./interruption_cache_entry.js";
+const MSG_SESSION_CREATE = "session.create";
+const MSG_SESSION_CLOSE = "session.close";
+const MSG_SESSION_CREATED = "session.created";
+const MSG_SESSION_CLOSED = "session.closed";
+const MSG_INTERRUPTION_DETECTED = "bargein_detected";
+const MSG_INFERENCE_DONE = "inference_done";
+const MSG_ERROR = "error";
+const wsMessageSchema = z.discriminatedUnion("type", [
+  z.object({
+    type: z.literal(MSG_SESSION_CREATED)
+  }),
+  z.object({
+    type: z.literal(MSG_SESSION_CLOSED)
+  }),
+  z.object({
+    type: z.literal(MSG_INTERRUPTION_DETECTED),
+    created_at: z.number(),
+    probabilities: z.array(z.number()).default([]),
+    prediction_duration: z.number().default(0)
+  }),
+  z.object({
+    type: z.literal(MSG_INFERENCE_DONE),
+    created_at: z.number(),
+    probabilities: z.array(z.number()).default([]),
+    prediction_duration: z.number().default(0),
+    is_bargein: z.boolean().optional()
+  }),
+  z.object({
+    type: z.literal(MSG_ERROR),
+    message: z.string(),
+    code: z.number().optional(),
+    session_id: z.string().optional()
+  })
+]);
+async function connectWebSocket(options) {
+  const baseUrl = options.baseUrl.replace(/^http/, "ws");
+  const token = await createAccessToken(options.apiKey, options.apiSecret);
+  const url = `${baseUrl}/bargein`;
+  const ws = new WebSocket(url, {
+    headers: { Authorization: `Bearer ${token}` }
+  });
+  await new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      ws.terminate();
+      reject(new Error("WebSocket connection timeout"));
+    }, options.timeout);
+    ws.once("open", () => {
+      clearTimeout(timeout);
+      resolve();
+    });
+    ws.once("error", (err) => {
+      clearTimeout(timeout);
+      ws.terminate();
+      reject(err);
+    });
+  });
+  return ws;
+}
+function createWsTransport(options, getState, setState, updateUserSpeakingSpan, onRequestSent, getAndResetNumRequests) {
+  const logger = log();
+  let ws = null;
+  let outputController = null;
+  function setupMessageHandler(socket) {
+    socket.on("message", (data) => {
+      try {
+        const message = wsMessageSchema.parse(JSON.parse(data.toString()));
+        handleMessage(message);
+      } catch {
+        logger.warn({ data: data.toString() }, "Failed to parse WebSocket message");
+      }
+    });
+    socket.on("error", (err) => {
+      logger.error({ err }, "WebSocket error");
+    });
+    socket.on("close", (code, reason) => {
+      logger.debug({ code, reason: reason.toString() }, "WebSocket closed");
+    });
+  }
+  async function ensureConnection() {
+    if (ws && ws.readyState === WebSocket.OPEN) return;
+    const maxRetries = options.maxRetries ?? 3;
+    let lastError = null;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        ws = await connectWebSocket(options);
+        setupMessageHandler(ws);
+        const sessionCreateMsg = JSON.stringify({
+          type: MSG_SESSION_CREATE,
+          settings: {
+            sample_rate: options.sampleRate,
+            num_channels: 1,
+            threshold: options.threshold,
+            min_frames: options.minFrames,
+            encoding: "s16le"
+          }
+        });
+        ws.send(sessionCreateMsg);
+        return;
+      } catch (err) {
+        lastError = err instanceof Error ? err : new Error(String(err));
+        if (attempt < maxRetries) {
+          const delay = intervalForRetry(attempt);
+          logger.debug(
+            { attempt, delay, err: lastError.message },
+            "WebSocket connection failed, retrying"
+          );
+          await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+      }
+    }
+    throw lastError ?? new Error("Failed to connect to WebSocket after retries");
+  }
+  function handleMessage(message) {
+    const state = getState();
+    switch (message.type) {
+      case MSG_SESSION_CREATED:
+        logger.debug("WebSocket session created");
+        break;
+      case MSG_INTERRUPTION_DETECTED: {
+        const createdAt = message.created_at;
+        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;
+        if (state.overlapSpeechStarted && overlapSpeechStartedAt !== void 0) {
+          const existing = state.cache.get(createdAt);
+          const totalDurationInS = (existing == null ? void 0 : existing.requestStartedAt) !== void 0 ? (performance.now() - existing.requestStartedAt) / 1e3 : (performance.now() - createdAt) / 1e3;
+          const entry = state.cache.setOrUpdate(
+            createdAt,
+            () => new InterruptionCacheEntry({ createdAt }),
+            {
+              speechInput: existing == null ? void 0 : existing.speechInput,
+              requestStartedAt: existing == null ? void 0 : existing.requestStartedAt,
+              totalDurationInS,
+              probabilities: message.probabilities,
+              isInterruption: true,
+              predictionDurationInS: message.prediction_duration,
+              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1e3
+            }
+          );
+          if (updateUserSpeakingSpan) {
+            updateUserSpeakingSpan(entry);
+          }
+          logger.debug(
+            {
+              totalDuration: entry.totalDurationInS,
+              predictionDuration: entry.predictionDurationInS,
+              detectionDelay: entry.detectionDelayInS,
+              probability: entry.probability
+            },
+            "interruption detected"
+          );
+          const event = {
+            type: "user_overlapping_speech",
+            timestamp: Date.now(),
+            isInterruption: true,
+            totalDurationInS: entry.totalDurationInS,
+            predictionDurationInS: entry.predictionDurationInS,
+            overlapStartedAt: overlapSpeechStartedAt,
+            speechInput: entry.speechInput,
+            probabilities: entry.probabilities,
+            detectionDelayInS: entry.detectionDelayInS,
+            probability: entry.probability,
+            numRequests: (getAndResetNumRequests == null ? void 0 : getAndResetNumRequests()) ?? 0
+          };
+          outputController == null ? void 0 : outputController.enqueue(event);
+          setState({ overlapSpeechStarted: false });
+        }
+        break;
+      }
+      case MSG_INFERENCE_DONE: {
+        const createdAt = message.created_at;
+        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;
+        if (state.overlapSpeechStarted && overlapSpeechStartedAt !== void 0) {
+          const existing = state.cache.get(createdAt);
+          const totalDurationInS = (existing == null ? void 0 : existing.requestStartedAt) !== void 0 ? (performance.now() - existing.requestStartedAt) / 1e3 : (performance.now() - createdAt) / 1e3;
+          const entry = state.cache.setOrUpdate(
+            createdAt,
+            () => new InterruptionCacheEntry({ createdAt }),
+            {
+              speechInput: existing == null ? void 0 : existing.speechInput,
+              requestStartedAt: existing == null ? void 0 : existing.requestStartedAt,
+              totalDurationInS,
+              predictionDurationInS: message.prediction_duration,
+              probabilities: message.probabilities,
+              isInterruption: message.is_bargein ?? false,
+              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1e3
+            }
+          );
+          logger.debug(
+            {
+              totalDurationInS: entry.totalDurationInS,
+              predictionDurationInS: entry.predictionDurationInS
+            },
+            "interruption inference done"
+          );
+        }
+        break;
+      }
+      case MSG_SESSION_CLOSED:
+        logger.debug("WebSocket session closed");
+        break;
+      case MSG_ERROR:
+        outputController == null ? void 0 : outputController.error(
+          new Error(
+            `LiveKit Adaptive Interruption error${message.code !== void 0 ? ` (${message.code})` : ""}: ${message.message}`
+          )
+        );
+        break;
+    }
+  }
+  function sendAudioData(audioSlice) {
+    if (!ws || ws.readyState !== WebSocket.OPEN) {
+      throw new Error("WebSocket not connected");
+    }
+    const state = getState();
+    const createdAt = Math.floor(performance.now());
+    state.cache.set(
+      createdAt,
+      new InterruptionCacheEntry({
+        createdAt,
+        requestStartedAt: performance.now(),
+        speechInput: audioSlice
+      })
+    );
+    const header = new ArrayBuffer(8);
+    const view = new DataView(header);
+    view.setUint32(0, createdAt >>> 0, true);
+    view.setUint32(4, Math.floor(createdAt / 4294967296) >>> 0, true);
+    const audioBytes = new Uint8Array(
+      audioSlice.buffer,
+      audioSlice.byteOffset,
+      audioSlice.byteLength
+    );
+    const combined = new Uint8Array(8 + audioBytes.length);
+    combined.set(new Uint8Array(header), 0);
+    combined.set(audioBytes, 8);
+    try {
+      ws.send(combined);
+      onRequestSent == null ? void 0 : onRequestSent();
+    } catch (e) {
+      logger.error(e, `failed to send audio via websocket`);
+    }
+  }
+  function close() {
+    if ((ws == null ? void 0 : ws.readyState) === WebSocket.OPEN) {
+      const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });
+      try {
+        ws.send(closeMsg);
+      } catch (e) {
+        logger.error(e, "failed to send close message");
+      }
+    }
+    ws == null ? void 0 : ws.close(1e3);
+    ws = null;
+  }
+  async function reconnect() {
+    close();
+  }
+  const transport = new TransformStream(
+    {
+      async start(controller) {
+        outputController = controller;
+        await ensureConnection();
+      },
+      transform(chunk, controller) {
+        if (!(chunk instanceof Int16Array)) {
+          controller.enqueue(chunk);
+          return;
+        }
+        const state = getState();
+        if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;
+        try {
+          sendAudioData(chunk);
+        } catch (err) {
+          logger.error({ err }, "Failed to send audio data over WebSocket");
+        }
+      },
+      flush() {
+        close();
+      }
+    },
+    { highWaterMark: 2 },
+    { highWaterMark: 2 }
+  );
+  return { transport, reconnect };
+}
+export {
+  createWsTransport
+};
+//# sourceMappingURL=ws_transport.js.map

package/dist/inference/interruption/ws_transport.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/inference/interruption/ws_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TransformStream } from 'stream/web';\nimport WebSocket from 'ws';\nimport { z } from 'zod';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { intervalForRetry } from './defaults.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\n// WebSocket message types\nconst MSG_SESSION_CREATE = 'session.create';\nconst MSG_SESSION_CLOSE = 'session.close';\nconst MSG_SESSION_CREATED = 'session.created';\nconst MSG_SESSION_CLOSED = 'session.closed';\nconst MSG_INTERRUPTION_DETECTED = 'bargein_detected';\nconst MSG_INFERENCE_DONE = 'inference_done';\nconst MSG_ERROR = 'error';\n\nexport interface WsTransportOptions {\n  baseUrl: string;\n  apiKey: string;\n  apiSecret: string;\n  sampleRate: number;\n  threshold: number;\n  minFrames: number;\n  timeout: number;\n  maxRetries?: number;\n}\n\nexport interface WsTransportState {\n  overlapSpeechStarted: boolean;\n  overlapSpeechStartedAt: number | undefined;\n  cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\nconst wsMessageSchema = z.discriminatedUnion('type', [\n  z.object({\n    type: z.literal(MSG_SESSION_CREATED),\n  }),\n  z.object({\n    type: z.literal(MSG_SESSION_CLOSED),\n  }),\n  z.object({\n    type: z.literal(MSG_INTERRUPTION_DETECTED),\n    created_at: z.number(),\n    probabilities: z.array(z.number()).default([]),\n    prediction_duration: z.number().default(0),\n  }),\n  z.object({\n    type: z.literal(MSG_INFERENCE_DONE),\n    created_at: z.number(),\n    probabilities: z.array(z.number()).default([]),\n    prediction_duration: z.number().default(0),\n    is_bargein: z.boolean().optional(),\n  }),\n  z.object({\n    type: z.literal(MSG_ERROR),\n    message: z.string(),\n    code: z.number().optional(),\n    session_id: z.string().optional(),\n  }),\n]);\n\ntype WsMessage = z.infer<typeof wsMessageSchema>;\n\n/**\n * Creates a WebSocket connection and waits for it to open.\n */\nasync function connectWebSocket(options: WsTransportOptions): Promise<WebSocket> {\n  const baseUrl = options.baseUrl.replace(/^http/, 'ws');\n  const token = await createAccessToken(options.apiKey, options.apiSecret);\n  const url = `${baseUrl}/bargein`;\n\n  const ws = new WebSocket(url, {\n    headers: { Authorization: `Bearer ${token}` },\n  });\n\n  await new Promise<void>((resolve, reject) => {\n    const timeout = setTimeout(() => {\n      ws.terminate();\n      reject(new Error('WebSocket connection timeout'));\n    }, options.timeout);\n    ws.once('open', () => {\n      clearTimeout(timeout);\n      resolve();\n    });\n    ws.once('error', (err: Error) => {\n      clearTimeout(timeout);\n      ws.terminate();\n      reject(err);\n    });\n  });\n\n  return ws;\n}\n\nexport interface WsTransportResult {\n  transport: TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>;\n  reconnect: () => Promise<void>;\n}\n\n/**\n * Creates a WebSocket transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * It maintains a persistent WebSocket connection with automatic retry on failure.\n * Returns both the transport and a reconnect function for option updates.\n */\nexport function createWsTransport(\n  options: WsTransportOptions,\n  getState: () => WsTransportState,\n  setState: (partial: Partial<WsTransportState>) => void,\n  updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n  onRequestSent?: () => void,\n  getAndResetNumRequests?: () => number,\n): WsTransportResult {\n  const logger = log();\n  let ws: WebSocket | null = null;\n  let outputController: TransformStreamDefaultController<OverlappingSpeechEvent> | null = null;\n\n  function setupMessageHandler(socket: WebSocket): void {\n    socket.on('message', (data: WebSocket.Data) => {\n      try {\n        const message = wsMessageSchema.parse(JSON.parse(data.toString()));\n        handleMessage(message);\n      } catch {\n        logger.warn({ data: data.toString() }, 'Failed to parse WebSocket message');\n      }\n    });\n\n    socket.on('error', (err: Error) => {\n      logger.error({ err }, 'WebSocket error');\n    });\n\n    socket.on('close', (code: number, reason: Buffer) => {\n      logger.debug({ code, reason: reason.toString() }, 'WebSocket closed');\n    });\n  }\n\n  async function ensureConnection(): Promise<void> {\n    if (ws && ws.readyState === WebSocket.OPEN) return;\n\n    const maxRetries = options.maxRetries ?? 3;\n    let lastError: Error | null = null;\n\n    for (let attempt = 0; attempt <= maxRetries; attempt++) {\n      try {\n        ws = await connectWebSocket(options);\n        setupMessageHandler(ws);\n\n        // Send session.create message\n        const sessionCreateMsg = JSON.stringify({\n          type: MSG_SESSION_CREATE,\n          settings: {\n            sample_rate: options.sampleRate,\n            num_channels: 1,\n            threshold: options.threshold,\n            min_frames: options.minFrames,\n            encoding: 's16le',\n          },\n        });\n        ws.send(sessionCreateMsg);\n        return;\n      } catch (err) {\n        lastError = err instanceof Error ? err : new Error(String(err));\n        if (attempt < maxRetries) {\n          const delay = intervalForRetry(attempt);\n          logger.debug(\n            { attempt, delay, err: lastError.message },\n            'WebSocket connection failed, retrying',\n          );\n          await new Promise((resolve) => setTimeout(resolve, delay));\n        }\n      }\n    }\n\n    throw lastError ?? new Error('Failed to connect to WebSocket after retries');\n  }\n\n  function handleMessage(message: WsMessage): void {\n    const state = getState();\n\n    switch (message.type) {\n      case MSG_SESSION_CREATED:\n        logger.debug('WebSocket session created');\n        break;\n\n      case MSG_INTERRUPTION_DETECTED: {\n        const createdAt = message.created_at;\n        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n        if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n          const existing = state.cache.get(createdAt);\n\n          const totalDurationInS =\n            existing?.requestStartedAt !== undefined\n              ? (performance.now() - existing.requestStartedAt) / 1000\n              : (performance.now() - createdAt) / 1000;\n\n          const entry = state.cache.setOrUpdate(\n            createdAt,\n            () => new InterruptionCacheEntry({ createdAt }),\n            {\n              speechInput: existing?.speechInput,\n              requestStartedAt: existing?.requestStartedAt,\n              totalDurationInS,\n              probabilities: message.probabilities,\n              isInterruption: true,\n              predictionDurationInS: message.prediction_duration,\n              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n            },\n          );\n\n          if (updateUserSpeakingSpan) {\n            updateUserSpeakingSpan(entry);\n          }\n\n          logger.debug(\n            {\n              totalDuration: entry.totalDurationInS,\n              predictionDuration: entry.predictionDurationInS,\n              detectionDelay: entry.detectionDelayInS,\n              probability: entry.probability,\n            },\n            'interruption detected',\n          );\n\n          const event: OverlappingSpeechEvent = {\n            type: 'user_overlapping_speech',\n            timestamp: Date.now(),\n            isInterruption: true,\n            totalDurationInS: entry.totalDurationInS,\n            predictionDurationInS: entry.predictionDurationInS,\n            overlapStartedAt: overlapSpeechStartedAt,\n            speechInput: entry.speechInput,\n            probabilities: entry.probabilities,\n            detectionDelayInS: entry.detectionDelayInS,\n            probability: entry.probability,\n            numRequests: getAndResetNumRequests?.() ?? 0,\n          };\n\n          outputController?.enqueue(event);\n          setState({ overlapSpeechStarted: false });\n        }\n        break;\n      }\n\n      case MSG_INFERENCE_DONE: {\n        const createdAt = message.created_at;\n        const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n        if (state.overlapSpeechStarted && overlapSpeechStartedAt !== undefined) {\n          const existing = state.cache.get(createdAt);\n          const totalDurationInS =\n            existing?.requestStartedAt !== undefined\n              ? (performance.now() - existing.requestStartedAt) / 1000\n              : (performance.now() - createdAt) / 1000;\n          const entry = state.cache.setOrUpdate(\n            createdAt,\n            () => new InterruptionCacheEntry({ createdAt }),\n            {\n              speechInput: existing?.speechInput,\n              requestStartedAt: existing?.requestStartedAt,\n              totalDurationInS,\n              predictionDurationInS: message.prediction_duration,\n              probabilities: message.probabilities,\n              isInterruption: message.is_bargein ?? false,\n              detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n            },\n          );\n\n          logger.debug(\n            {\n              totalDurationInS: entry.totalDurationInS,\n              predictionDurationInS: entry.predictionDurationInS,\n            },\n            'interruption inference done',\n          );\n        }\n        break;\n      }\n\n      case MSG_SESSION_CLOSED:\n        logger.debug('WebSocket session closed');\n        break;\n\n      case MSG_ERROR:\n        outputController?.error(\n          new Error(\n            `LiveKit Adaptive Interruption error${\n              message.code !== undefined ? ` (${message.code})` : ''\n            }: ${message.message}`,\n          ),\n        );\n        break;\n    }\n  }\n\n  function sendAudioData(audioSlice: Int16Array): void {\n    if (!ws || ws.readyState !== WebSocket.OPEN) {\n      throw new Error('WebSocket not connected');\n    }\n\n    const state = getState();\n    // Use truncated timestamp consistently for both cache key and header\n    // This ensures the server's response created_at matches our cache key\n    const createdAt = Math.floor(performance.now());\n\n    // Store the audio data in cache with truncated timestamp\n    state.cache.set(\n      createdAt,\n      new InterruptionCacheEntry({\n        createdAt,\n        requestStartedAt: performance.now(),\n        speechInput: audioSlice,\n      }),\n    );\n\n    // Create header: 8-byte little-endian uint64 timestamp (milliseconds as integer)\n    const header = new ArrayBuffer(8);\n    const view = new DataView(header);\n    view.setUint32(0, createdAt >>> 0, true);\n    view.setUint32(4, Math.floor(createdAt / 0x100000000) >>> 0, true);\n\n    // Combine header and audio data\n    const audioBytes = new Uint8Array(\n      audioSlice.buffer,\n      audioSlice.byteOffset,\n      audioSlice.byteLength,\n    );\n    const combined = new Uint8Array(8 + audioBytes.length);\n    combined.set(new Uint8Array(header), 0);\n    combined.set(audioBytes, 8);\n\n    try {\n      ws.send(combined);\n      onRequestSent?.();\n    } catch (e: unknown) {\n      logger.error(e, `failed to send audio via websocket`);\n    }\n  }\n\n  function close(): void {\n    if (ws?.readyState === WebSocket.OPEN) {\n      const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE });\n      try {\n        ws.send(closeMsg);\n      } catch (e: unknown) {\n        logger.error(e, 'failed to send close message');\n      }\n    }\n    ws?.close(1000); // signal normal websocket closure\n    ws = null;\n  }\n\n  /**\n   * Reconnect the WebSocket with updated options.\n   * This is called when options are updated via updateOptions().\n   */\n  async function reconnect(): Promise<void> {\n    close();\n  }\n\n  const transport = new TransformStream<\n    Int16Array | OverlappingSpeechEvent,\n    OverlappingSpeechEvent\n  >(\n    {\n      async start(controller) {\n        outputController = controller;\n        await ensureConnection();\n      },\n\n      transform(chunk, controller) {\n        if (!(chunk instanceof Int16Array)) {\n          controller.enqueue(chunk);\n          return;\n        }\n\n        // Only forwards buffered audio while overlap speech is actively on.\n        const state = getState();\n        if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return;\n\n        try {\n          sendAudioData(chunk);\n        } catch (err) {\n          logger.error({ err }, 'Failed to send audio data over WebSocket');\n        }\n      },\n\n      flush() {\n        close();\n      },\n    },\n    { highWaterMark: 2 },\n    { highWaterMark: 2 },\n  );\n\n  return { transport, reconnect };\n}\n"],"mappings":"AAGA,SAAS,uBAAuB;AAChC,OAAO,eAAe;AACtB,SAAS,SAAS;AAClB,SAAS,WAAW;AACpB,SAAS,yBAAyB;AAClC,SAAS,wBAAwB;AACjC,SAAS,8BAA8B;AAKvC,MAAM,qBAAqB;AAC3B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,qBAAqB;AAC3B,MAAM,4BAA4B;AAClC,MAAM,qBAAqB;AAC3B,MAAM,YAAY;AAmBlB,MAAM,kBAAkB,EAAE,mBAAmB,QAAQ;AAAA,EACnD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,mBAAmB;AAAA,EACrC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,EACpC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,yBAAyB;AAAA,IACzC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,EAC3C,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,kBAAkB;AAAA,IAClC,YAAY,EAAE,OAAO;AAAA,IACrB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAC7C,qBAAqB,EAAE,OAAO,EAAE,QAAQ,CAAC;AAAA,IACzC,YAAY,EAAE,QAAQ,EAAE,SAAS;AAAA,EACnC,CAAC;AAAA,EACD,EAAE,OAAO;AAAA,IACP,MAAM,EAAE,QAAQ,SAAS;AAAA,IACzB,SAAS,EAAE,OAAO;AAAA,IAClB,MAAM,EAAE,OAAO,EAAE,SAAS;AAAA,IAC1B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,CAAC;AACH,CAAC;AAOD,eAAe,iBAAiB,SAAiD;AAC/E,QAAM,UAAU,QAAQ,QAAQ,QAAQ,SAAS,IAAI;AACrD,QAAM,QAAQ,MAAM,kBAAkB,QAAQ,QAAQ,QAAQ,SAAS;AACvE,QAAM,MAAM,GAAG,OAAO;AAEtB,QAAM,KAAK,IAAI,UAAU,KAAK;AAAA,IAC5B,SAAS,EAAE,eAAe,UAAU,KAAK,GAAG;AAAA,EAC9C,CAAC;AAED,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,UAAM,UAAU,WAAW,MAAM;AAC/B,SAAG,UAAU;AACb,aAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,IAClD,GAAG,QAAQ,OAAO;AAClB,OAAG,KAAK,QAAQ,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ;AAAA,IACV,CAAC;AACD,OAAG,KAAK,SAAS,CAAC,QAAe;AAC/B,mBAAa,OAAO;AACpB,SAAG,UAAU;AACb,aAAO,GAAG;AAAA,IACZ,CAAC;AAAA,EACH,CAAC;AAED,SAAO;AACT;AAcO,SAAS,kBACd,SACA,UACA,UACA,wBACA,eACA,wBACmB;AACnB,QAAM,SAAS,IAAI;AACnB,MAAI,KAAuB;AAC3B,MAAI,mBAAoF;AAExF,WAAS,oBAAoB,QAAyB;AACpD,WAAO,GAAG,WAAW,CAAC,SAAyB;AAC7C,UAAI;AACF,cAAM,UAAU,gBAAgB,MAAM,KAAK,MAAM,KAAK,SAAS,CAAC,CAAC;AACjE,sBAAc,OAAO;AAAA,MACvB,QAAQ;AACN,eAAO,KAAK,EAAE,MAAM,KAAK,SAAS,EAAE,GAAG,mCAAmC;AAAA,MAC5E;AAAA,IACF,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,QAAe;AACjC,aAAO,MAAM,EAAE,IAAI,GAAG,iBAAiB;AAAA,IACzC,CAAC;AAED,WAAO,GAAG,SAAS,CAAC,MAAc,WAAmB;AACnD,aAAO,MAAM,EAAE,MAAM,QAAQ,OAAO,SAAS,EAAE,GAAG,kBAAkB;AAAA,IACtE,CAAC;AAAA,EACH;AAEA,iBAAe,mBAAkC;AAC/C,QAAI,MAAM,GAAG,eAAe,UAAU,KAAM;AAE5C,UAAM,aAAa,QAAQ,cAAc;AACzC,QAAI,YAA0B;AAE9B,aAAS,UAAU,GAAG,WAAW,YAAY,WAAW;AACtD,UAAI;AACF,aAAK,MAAM,iBAAiB,OAAO;AACnC,4BAAoB,EAAE;AAGtB,cAAM,mBAAmB,KAAK,UAAU;AAAA,UACtC,MAAM;AAAA,UACN,UAAU;AAAA,YACR,aAAa,QAAQ;AAAA,YACrB,cAAc;AAAA,YACd,WAAW,QAAQ;AAAA,YACnB,YAAY,QAAQ;AAAA,YACpB,UAAU;AAAA,UACZ;AAAA,QACF,CAAC;AACD,WAAG,KAAK,gBAAgB;AACxB;AAAA,MACF,SAAS,KAAK;AACZ,oBAAY,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC9D,YAAI,UAAU,YAAY;AACxB,gBAAM,QAAQ,iBAAiB,OAAO;AACtC,iBAAO;AAAA,YACL,EAAE,SAAS,OAAO,KAAK,UAAU,QAAQ;AAAA,YACzC;AAAA,UACF;AACA,gBAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,CAAC;AAAA,QAC3D;AAAA,MACF;AAAA,IACF;AAEA,UAAM,aAAa,IAAI,MAAM,8CAA8C;AAAA,EAC7E;AAEA,WAAS,cAAc,SAA0B;AAC/C,UAAM,QAAQ,SAAS;AAEvB,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,eAAO,MAAM,2BAA2B;AACxC;AAAA,MAEF,KAAK,2BAA2B;AAC9B,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAE1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AAExC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,eAAe,QAAQ;AAAA,cACvB,gBAAgB;AAAA,cAChB,uBAAuB,QAAQ;AAAA,cAC/B,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,cAAI,wBAAwB;AAC1B,mCAAuB,KAAK;AAAA,UAC9B;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,eAAe,MAAM;AAAA,cACrB,oBAAoB,MAAM;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,YACrB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,QAAgC;AAAA,YACpC,MAAM;AAAA,YACN,WAAW,KAAK,IAAI;AAAA,YACpB,gBAAgB;AAAA,YAChB,kBAAkB,MAAM;AAAA,YACxB,uBAAuB,MAAM;AAAA,YAC7B,kBAAkB;AAAA,YAClB,aAAa,MAAM;AAAA,YACnB,eAAe,MAAM;AAAA,YACrB,mBAAmB,MAAM;AAAA,YACzB,aAAa,MAAM;AAAA,YACnB,cAAa,uEAA8B;AAAA,UAC7C;AAEA,+DAAkB,QAAQ;AAC1B,mBAAS,EAAE,sBAAsB,MAAM,CAAC;AAAA,QAC1C;AACA;AAAA,MACF;AAAA,MAEA,KAAK,oBAAoB;AACvB,cAAM,YAAY,QAAQ;AAC1B,cAAM,yBAAyB,MAAM;AACrC,YAAI,MAAM,wBAAwB,2BAA2B,QAAW;AACtE,gBAAM,WAAW,MAAM,MAAM,IAAI,SAAS;AAC1C,gBAAM,oBACJ,qCAAU,sBAAqB,UAC1B,YAAY,IAAI,IAAI,SAAS,oBAAoB,OACjD,YAAY,IAAI,IAAI,aAAa;AACxC,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE,aAAa,qCAAU;AAAA,cACvB,kBAAkB,qCAAU;AAAA,cAC5B;AAAA,cACA,uBAAuB,QAAQ;AAAA,cAC/B,eAAe,QAAQ;AAAA,cACvB,gBAAgB,QAAQ,cAAc;AAAA,cACtC,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,YAC7D;AAAA,UACF;AAEA,iBAAO;AAAA,YACL;AAAA,cACE,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,eAAO,MAAM,0BAA0B;AACvC;AAAA,MAEF,KAAK;AACH,6DAAkB;AAAA,UAChB,IAAI;AAAA,YACF,sCACE,QAAQ,SAAS,SAAY,KAAK,QAAQ,IAAI,MAAM,EACtD,KAAK,QAAQ,OAAO;AAAA,UACtB;AAAA;AAEF;AAAA,IACJ;AAAA,EACF;AAEA,WAAS,cAAc,YAA8B;AACnD,QAAI,CAAC,MAAM,GAAG,eAAe,UAAU,MAAM;AAC3C,YAAM,IAAI,MAAM,yBAAyB;AAAA,IAC3C;AAEA,UAAM,QAAQ,SAAS;AAGvB,UAAM,YAAY,KAAK,MAAM,YAAY,IAAI,CAAC;AAG9C,UAAM,MAAM;AAAA,MACV;AAAA,MACA,IAAI,uBAAuB;AAAA,QACzB;AAAA,QACA,kBAAkB,YAAY,IAAI;AAAA,QAClC,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAGA,UAAM,SAAS,IAAI,YAAY,CAAC;AAChC,UAAM,OAAO,IAAI,SAAS,MAAM;AAChC,SAAK,UAAU,GAAG,cAAc,GAAG,IAAI;AACvC,SAAK,UAAU,GAAG,KAAK,MAAM,YAAY,UAAW,MAAM,GAAG,IAAI;AAGjE,UAAM,aAAa,IAAI;AAAA,MACrB,WAAW;AAAA,MACX,WAAW;AAAA,MACX,WAAW;AAAA,IACb;AACA,UAAM,WAAW,IAAI,WAAW,IAAI,WAAW,MAAM;AACrD,aAAS,IAAI,IAAI,WAAW,MAAM,GAAG,CAAC;AACtC,aAAS,IAAI,YAAY,CAAC;AAE1B,QAAI;AACF,SAAG,KAAK,QAAQ;AAChB;AAAA,IACF,SAAS,GAAY;AACnB,aAAO,MAAM,GAAG,oCAAoC;AAAA,IACtD;AAAA,EACF;AAEA,WAAS,QAAc;AACrB,SAAI,yBAAI,gBAAe,UAAU,MAAM;AACrC,YAAM,WAAW,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC3D,UAAI;AACF,WAAG,KAAK,QAAQ;AAAA,MAClB,SAAS,GAAY;AACnB,eAAO,MAAM,GAAG,8BAA8B;AAAA,MAChD;AAAA,IACF;AACA,6BAAI,MAAM;AACV,SAAK;AAAA,EACP;AAMA,iBAAe,YAA2B;AACxC,UAAM;AAAA,EACR;AAEA,QAAM,YAAY,IAAI;AAAA,IAIpB;AAAA,MACE,MAAM,MAAM,YAAY;AACtB,2BAAmB;AACnB,cAAM,iBAAiB;AAAA,MACzB;AAAA,MAEA,UAAU,OAAO,YAAY;AAC3B,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAGA,cAAM,QAAQ,SAAS;AACvB,YAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,qBAAsB;AAElE,YAAI;AACF,wBAAc,KAAK;AAAA,QACrB,SAAS,KAAK;AACZ,iBAAO,MAAM,EAAE,IAAI,GAAG,0CAA0C;AAAA,QAClE;AAAA,MACF;AAAA,MAEA,QAAQ;AACN,cAAM;AAAA,MACR;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AAEA,SAAO,EAAE,WAAW,UAAU;AAChC;","names":[]}

package/dist/inference/llm.cjs

@@ -33,10 +33,11 @@ __export(llm_exports, {
 });
 module.exports = __toCommonJS(llm_exports);
 var import_openai = __toESM(require("openai"), 1);
-var import__ = require("../index.cjs");
+var import_exceptions = require("../_exceptions.cjs");
 var llm = __toESM(require("../llm/index.cjs"), 1);
-var import_utils = require("./utils.cjs");
-const DEFAULT_BASE_URL = "https://agent-gateway.livekit.cloud/v1";
+var import_types = require("../types.cjs");
+var import_utils = require("../utils.cjs");
+var import_utils2 = require("./utils.cjs");
 class LLM extends llm.LLM {
   client;
   opts;
@@ -51,7 +52,7 @@ class LLM extends llm.LLM {
       modelOptions,
       strictToolSchema = false
     } = opts;
-    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkBaseURL = baseURL || (0, import_utils2.getDefaultInferenceUrl)();
     const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
     if (!lkApiKey) {
       throw new Error("apiKey is required: pass apiKey or set LIVEKIT_API_KEY");
@@ -81,13 +82,16 @@ class LLM extends llm.LLM {
   get model() {
     return this.opts.model;
   }
+  get provider() {
+    return "livekit";
+  }
   static fromModelString(modelString) {
     return new LLM({ model: modelString });
   }
   chat({
     chatCtx,
     toolCtx,
-    connOptions = import__.DEFAULT_API_CONNECT_OPTIONS,
+    connOptions = import_types.DEFAULT_API_CONNECT_OPTIONS,
     parallelToolCalls,
     toolChoice,
     // TODO(AJS-270): Add response_format parameter support
@@ -186,7 +190,7 @@ class LLMStream extends llm.LLMStream {
         delete requestOptions.tool_choice;
       }
       if (this.gatewayOptions) {
-        this.client.apiKey = await (0, import_utils.createAccessToken)(
+        this.client.apiKey = await (0, import_utils2.createAccessToken)(
           this.gatewayOptions.apiKey,
           this.gatewayOptions.apiSecret
         );
@@ -236,9 +240,9 @@ class LLMStream extends llm.LLMStream {
       }
     } catch (error) {
       if (error instanceof import_openai.default.APIConnectionTimeoutError) {
-        throw new import__.APITimeoutError({ options: { retryable } });
+        throw new import_exceptions.APITimeoutError({ options: { retryable } });
       } else if (error instanceof import_openai.default.APIError) {
-        throw new import__.APIStatusError({
+        throw new import_exceptions.APIStatusError({
           message: error.message,
           options: {
             statusCode: error.status,
@@ -248,8 +252,8 @@ class LLMStream extends llm.LLMStream {
           }
         });
       } else {
-        throw new import__.APIConnectionError({
-          message: (0, import__.toError)(error).message,
+        throw new import_exceptions.APIConnectionError({
+          message: (0, import_utils.toError)(error).message,
           options: { retryable }
         });
       }

package/dist/inference/llm.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/inference/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport OpenAI from 'openai';\nimport {\n  APIConnectionError,\n  APIStatusError,\n  APITimeoutError,\n  DEFAULT_API_CONNECT_OPTIONS,\n  type Expand,\n  toError,\n} from '../index.js';\nimport * as llm from '../llm/index.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { type AnyString, createAccessToken } from './utils.js';\n\nconst DEFAULT_BASE_URL = 'https://agent-gateway.livekit.cloud/v1';\n\nexport type OpenAIModels =\n  | 'openai/gpt-5.2'\n  | 'openai/gpt-5.2-chat-latest'\n  | 'openai/gpt-5.1'\n  | 'openai/gpt-5.1-chat-latest'\n  | 'openai/gpt-5'\n  | 'openai/gpt-5-mini'\n  | 'openai/gpt-5-nano'\n  | 'openai/gpt-4.1'\n  | 'openai/gpt-4.1-mini'\n  | 'openai/gpt-4.1-nano'\n  | 'openai/gpt-4o'\n  | 'openai/gpt-4o-mini'\n  | 'openai/gpt-oss-120b';\n\nexport type GoogleModels =\n  | 'google/gemini-3-pro'\n  | 'google/gemini-3-flash'\n  | 'google/gemini-2.5-pro'\n  | 'google/gemini-2.5-flash'\n  | 'google/gemini-2.5-flash-lite'\n  | 'google/gemini-2.0-flash'\n  | 'google/gemini-2.0-flash-lite';\n\nexport type MoonshotModels = 'moonshotai/kimi-k2-instruct';\n\nexport type DeepSeekModels = 'deepseek-ai/deepseek-v3' | 'deepseek-ai/deepseek-v3.2';\n\ntype ChatCompletionPredictionContentParam =\n  Expand<OpenAI.Chat.Completions.ChatCompletionPredictionContent>;\ntype WebSearchOptions = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams.WebSearchOptions>;\ntype ToolChoice = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams['tool_choice']>;\ntype Verbosity = 'low' | 'medium' | 'high';\n\nexport interface ChatCompletionOptions extends Record<string, unknown> {\n  frequency_penalty?: number;\n  logit_bias?: Record<string, number>;\n  logprobs?: boolean;\n  max_completion_tokens?: number;\n  max_tokens?: number;\n  metadata?: Record<string, string>;\n  modalities?: Array<'text' | 'audio'>;\n  n?: number;\n  parallel_tool_calls?: boolean;\n  prediction?: ChatCompletionPredictionContentParam | null;\n  presence_penalty?: number;\n  prompt_cache_key?: string;\n  reasoning_effort?: 'minimal' | 'low' | 'medium' | 'high';\n  safety_identifier?: string;\n  seed?: number;\n  service_tier?: 'auto' | 'default' | 'flex' | 'scale' | 'priority';\n  stop?: string | string[];\n  store?: boolean;\n  temperature?: number;\n  top_logprobs?: number;\n  top_p?: number;\n  user?: string;\n  verbosity?: Verbosity;\n  web_search_options?: WebSearchOptions;\n\n  // livekit-typed arguments\n  tool_choice?: ToolChoice;\n  // TODO(brian): support response format\n  // response_format?: OpenAI.Chat.Completions.ChatCompletionCreateParams['response_format']\n}\n\nexport type LLMModels = OpenAIModels | GoogleModels | MoonshotModels | DeepSeekModels | AnyString;\n\nexport interface InferenceLLMOptions {\n  model: LLMModels;\n  provider?: string;\n  baseURL: string;\n  apiKey: string;\n  apiSecret: string;\n  modelOptions: ChatCompletionOptions;\n  strictToolSchema?: boolean;\n}\n\nexport interface GatewayOptions {\n  apiKey: string;\n  apiSecret: string;\n}\n\n/**\n * Livekit Cloud Inference LLM\n */\nexport class LLM extends llm.LLM {\n  private client: OpenAI;\n  private opts: InferenceLLMOptions;\n\n  constructor(opts: {\n    model: LLMModels;\n    provider?: string;\n    baseURL?: string;\n    apiKey?: string;\n    apiSecret?: string;\n    modelOptions?: InferenceLLMOptions['modelOptions'];\n    strictToolSchema?: boolean;\n  }) {\n    super();\n\n    const {\n      model,\n      provider,\n      baseURL,\n      apiKey,\n      apiSecret,\n      modelOptions,\n      strictToolSchema = false,\n    } = opts;\n\n    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;\n    const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;\n    if (!lkApiKey) {\n      throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');\n    }\n\n    const lkApiSecret =\n      apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;\n    if (!lkApiSecret) {\n      throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET');\n    }\n\n    this.opts = {\n      model,\n      provider,\n      baseURL: lkBaseURL,\n      apiKey: lkApiKey,\n      apiSecret: lkApiSecret,\n      modelOptions: modelOptions || {},\n      strictToolSchema,\n    };\n\n    this.client = new OpenAI({\n      baseURL: this.opts.baseURL,\n      apiKey: '', // leave a temporary empty string to avoid OpenAI complain about missing key\n    });\n  }\n\n  label(): string {\n    return 'inference.LLM';\n  }\n\n  get model(): string {\n    return this.opts.model;\n  }\n\n  static fromModelString(modelString: string): LLM {\n    return new LLM({ model: modelString });\n  }\n\n  chat({\n    chatCtx,\n    toolCtx,\n    connOptions = DEFAULT_API_CONNECT_OPTIONS,\n    parallelToolCalls,\n    toolChoice,\n    // TODO(AJS-270): Add response_format parameter support\n    extraKwargs,\n  }: {\n    chatCtx: llm.ChatContext;\n    toolCtx?: llm.ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: llm.ToolChoice;\n    // TODO(AJS-270): Add responseFormat parameter\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream {\n    let modelOptions: Record<string, unknown> = { ...(extraKwargs || {}) };\n\n    parallelToolCalls =\n      parallelToolCalls !== undefined\n        ? parallelToolCalls\n        : this.opts.modelOptions.parallel_tool_calls;\n\n    if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) {\n      modelOptions.parallel_tool_calls = parallelToolCalls;\n    }\n\n    toolChoice =\n      toolChoice !== undefined\n        ? toolChoice\n        : (this.opts.modelOptions.tool_choice as llm.ToolChoice | undefined);\n\n    if (toolChoice) {\n      modelOptions.tool_choice = toolChoice as ToolChoice;\n    }\n\n    // TODO(AJS-270): Add response_format support here\n\n    modelOptions = { ...modelOptions, ...this.opts.modelOptions };\n\n    return new LLMStream(this, {\n      model: this.opts.model,\n      provider: this.opts.provider,\n      client: this.client,\n      chatCtx,\n      toolCtx,\n      connOptions,\n      modelOptions,\n      strictToolSchema: this.opts.strictToolSchema ?? false, // default to false if not set\n      gatewayOptions: {\n        apiKey: this.opts.apiKey,\n        apiSecret: this.opts.apiSecret,\n      },\n    });\n  }\n}\n\nexport class LLMStream extends llm.LLMStream {\n  private model: LLMModels;\n  private provider?: string;\n  private providerFmt: llm.ProviderFormat;\n  private client: OpenAI;\n  private modelOptions: Record<string, unknown>;\n  private strictToolSchema: boolean;\n\n  private gatewayOptions?: GatewayOptions;\n  private toolCallId?: string;\n  private toolIndex?: number;\n  private fncName?: string;\n  private fncRawArguments?: string;\n  private toolExtra?: Record<string, unknown>;\n\n  constructor(\n    llm: LLM,\n    {\n      model,\n      provider,\n      client,\n      chatCtx,\n      toolCtx,\n      gatewayOptions,\n      connOptions,\n      modelOptions,\n      providerFmt,\n      strictToolSchema,\n    }: {\n      model: LLMModels;\n      provider?: string;\n      client: OpenAI;\n      chatCtx: llm.ChatContext;\n      toolCtx?: llm.ToolContext;\n      gatewayOptions?: GatewayOptions;\n      connOptions: APIConnectOptions;\n      modelOptions: Record<string, unknown>;\n      providerFmt?: llm.ProviderFormat;\n      strictToolSchema: boolean;\n    },\n  ) {\n    super(llm, { chatCtx, toolCtx, connOptions });\n    this.client = client;\n    this.gatewayOptions = gatewayOptions;\n    this.provider = provider;\n    this.providerFmt = providerFmt || 'openai';\n    this.modelOptions = modelOptions;\n    this.model = model;\n    this.strictToolSchema = strictToolSchema;\n  }\n\n  protected async run(): Promise<void> {\n    // current function call that we're waiting for full completion (args are streamed)\n    // (defined inside the run method to make sure the state is reset for each run/attempt)\n    let retryable = true;\n    this.toolCallId = this.fncName = this.fncRawArguments = this.toolIndex = undefined;\n    this.toolExtra = undefined;\n\n    try {\n      const messages = (await this.chatCtx.toProviderFormat(\n        this.providerFmt,\n      )) as OpenAI.ChatCompletionMessageParam[];\n\n      const tools = this.toolCtx\n        ? Object.entries(this.toolCtx).map(([name, func]) => {\n            const oaiParams = {\n              type: 'function' as const,\n              function: {\n                name,\n                description: func.description,\n                parameters: llm.toJsonSchema(\n                  func.parameters,\n                  true,\n                  this.strictToolSchema,\n                ) as unknown as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function']['parameters'],\n              } as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function'],\n            };\n\n            if (this.strictToolSchema) {\n              oaiParams.function.strict = true;\n            }\n\n            return oaiParams;\n          })\n        : undefined;\n\n      const requestOptions: Record<string, unknown> = { ...this.modelOptions };\n      if (!tools) {\n        delete requestOptions.tool_choice;\n      }\n\n      // Dynamically set the access token for the LiveKit Agent Gateway API\n      if (this.gatewayOptions) {\n        this.client.apiKey = await createAccessToken(\n          this.gatewayOptions.apiKey,\n          this.gatewayOptions.apiSecret,\n        );\n      }\n\n      if (this.provider) {\n        const extraHeaders = requestOptions.extra_headers\n          ? (requestOptions.extra_headers as Record<string, string>)\n          : {};\n        extraHeaders['X-LiveKit-Inference-Provider'] = this.provider;\n        requestOptions.extra_headers = extraHeaders;\n      }\n\n      const stream = await this.client.chat.completions.create(\n        {\n          model: this.model,\n          messages,\n          tools,\n          stream: true,\n          stream_options: { include_usage: true },\n          ...requestOptions,\n        },\n        {\n          timeout: this.connOptions.timeoutMs,\n        },\n      );\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          if (this.abortController.signal.aborted) {\n            break;\n          }\n          const chatChunk = this.parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            retryable = false;\n            this.queue.put(chatChunk);\n          }\n        }\n\n        if (chunk.usage) {\n          const usage = chunk.usage;\n          retryable = false;\n          this.queue.put({\n            id: chunk.id,\n            usage: {\n              completionTokens: usage.completion_tokens,\n              promptTokens: usage.prompt_tokens,\n              promptCachedTokens: usage.prompt_tokens_details?.cached_tokens || 0,\n              totalTokens: usage.total_tokens,\n            },\n          });\n        }\n      }\n    } catch (error) {\n      if (error instanceof OpenAI.APIConnectionTimeoutError) {\n        throw new APITimeoutError({ options: { retryable } });\n      } else if (error instanceof OpenAI.APIError) {\n        throw new APIStatusError({\n          message: error.message,\n          options: {\n            statusCode: error.status,\n            body: error.error,\n            requestId: error.requestID,\n            retryable,\n          },\n        });\n      } else {\n        throw new APIConnectionError({\n          message: toError(error).message,\n          options: { retryable },\n        });\n      }\n    }\n  }\n\n  private parseChoice(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    // https://github.com/livekit/agents/issues/688\n    // the delta can be None when using Azure OpenAI (content filtering)\n    if (delta === undefined) return undefined;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        /**\n         * The way OpenAI streams tool calls is a bit tricky.\n         *\n         * For any new tool call, it first emits a delta tool call with id, and function name,\n         * the rest of the delta chunks will only stream the remaining arguments string,\n         * until a new tool call is started or the tool call is finished.\n         * See below for an example.\n         *\n         * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role='assistant', tool_calls=None), finish_reason=None, index=0, logprobs=None)\n         * [ChoiceDeltaToolCall(index=0, id='call_LaVeHWUHpef9K1sd5UO8TtLg', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]\n         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='\\{\"location\": \"P', name=None), type=None)]\n         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='aris\\}', name=None), type=None)]\n         * [ChoiceDeltaToolCall(index=1, id='call_ThU4OmMdQXnnVmpXGOCknXIB', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]\n         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='\\{\"location\": \"T', name=None), type=None)]\n         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='okyo', name=None), type=None)]\n         * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role=None, tool_calls=None), finish_reason='tool_calls', index=0, logprobs=None)\n         */\n        let callChunk: llm.ChatChunk | undefined;\n        // If we have a previous tool call and this is a new one, emit the previous\n        if (this.toolCallId && tool.id && tool.index !== this.toolIndex) {\n          callChunk = this.createRunningToolCallChunk(id, delta);\n          this.toolCallId = this.fncName = this.fncRawArguments = undefined;\n          // Note: We intentionally do NOT reset toolExtra here.\n          // For Gemini 3+, the thought_signature is only provided on the first tool call\n          // in a parallel batch, but must be applied to ALL tool calls in the batch.\n          // We preserve toolExtra so subsequent tool calls inherit the thought_signature.\n        }\n\n        // Start or continue building the current tool call\n        if (tool.function.name) {\n          this.toolIndex = tool.index;\n          this.toolCallId = tool.id;\n          this.fncName = tool.function.name;\n          this.fncRawArguments = tool.function.arguments || '';\n          // Extract extra from tool call (e.g., Google thought signatures)\n          // Only update toolExtra if this tool call has extra_content.\n          // Otherwise, inherit from previous tool call (for parallel Gemini tool calls).\n          const newToolExtra =\n            // eslint-disable-next-line @typescript-eslint/no-explicit-any\n            ((tool as any).extra_content as Record<string, unknown> | undefined) ?? undefined;\n          if (newToolExtra) {\n            this.toolExtra = newToolExtra;\n          }\n        } else if (tool.function.arguments) {\n          this.fncRawArguments = (this.fncRawArguments || '') + tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    // If we're done with tool calls, emit the final one\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.toolCallId !== undefined\n    ) {\n      const callChunk = this.createRunningToolCallChunk(id, delta);\n      this.toolCallId = this.fncName = this.fncRawArguments = undefined;\n      // Reset toolExtra at the end of the response (not between parallel tool calls)\n      this.toolExtra = undefined;\n      return callChunk;\n    }\n\n    // Extract extra from delta (e.g., Google thought signatures on text parts)\n    const deltaExtra =\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      ((delta as any).extra_content as Record<string, unknown> | undefined) ?? undefined;\n\n    // Regular content message\n    if (!delta.content && !deltaExtra) {\n      return undefined;\n    }\n\n    return {\n      id,\n      delta: {\n        role: 'assistant',\n        content: delta.content || undefined,\n        extra: deltaExtra,\n      },\n    };\n  }\n\n  private createRunningToolCallChunk(\n    id: string,\n    delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta,\n  ): llm.ChatChunk {\n    const toolExtra = this.toolExtra ? { ...this.toolExtra } : {};\n    const thoughtSignature = this.extractThoughtSignature(toolExtra);\n    const deltaExtra =\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      ((delta as any).extra_content as Record<string, unknown> | undefined) ?? undefined;\n\n    return {\n      id,\n      delta: {\n        role: 'assistant',\n        content: delta.content || undefined,\n        extra: deltaExtra,\n        toolCalls: [\n          llm.FunctionCall.create({\n            callId: this.toolCallId || '',\n            name: this.fncName || '',\n            args: this.fncRawArguments || '',\n            extra: toolExtra,\n            thoughtSignature,\n          }),\n        ],\n      },\n    };\n  }\n\n  private extractThoughtSignature(extra?: Record<string, unknown>): string | undefined {\n    const googleExtra = extra?.google;\n    if (googleExtra && typeof googleExtra === 'object') {\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      return (googleExtra as any).thoughtSignature || (googleExtra as any).thought_signature;\n    }\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAmB;AACnB,eAOO;AACP,UAAqB;AAErB,mBAAkD;AAElD,MAAM,mBAAmB;AAwFlB,MAAM,YAAY,IAAI,IAAI;AAAA,EACvB;AAAA,EACA;AAAA,EAER,YAAY,MAQT;AACD,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,mBAAmB;AAAA,IACrB,IAAI;AAEJ,UAAM,YAAY,WAAW,QAAQ,IAAI,yBAAyB;AAClE,UAAM,WAAW,UAAU,QAAQ,IAAI,6BAA6B,QAAQ,IAAI;AAChF,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,MAAM,wDAAwD;AAAA,IAC1E;AAEA,UAAM,cACJ,aAAa,QAAQ,IAAI,gCAAgC,QAAQ,IAAI;AACvE,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI,MAAM,iEAAiE;AAAA,IACnF;AAEA,SAAK,OAAO;AAAA,MACV;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,cAAc,gBAAgB,CAAC;AAAA,MAC/B;AAAA,IACF;AAEA,SAAK,SAAS,IAAI,cAAAA,QAAO;AAAA,MACvB,SAAS,KAAK,KAAK;AAAA,MACnB,QAAQ;AAAA;AAAA,IACV,CAAC;AAAA,EACH;AAAA,EAEA,QAAgB;AACd,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA,EAEA,OAAO,gBAAgB,aAA0B;AAC/C,WAAO,IAAI,IAAI,EAAE,OAAO,YAAY,CAAC;AAAA,EACvC;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA,cAAc;AAAA,IACd;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,EACF,GAQc;AACZ,QAAI,eAAwC,EAAE,GAAI,eAAe,CAAC,EAAG;AAErE,wBACE,sBAAsB,SAClB,oBACA,KAAK,KAAK,aAAa;AAE7B,QAAI,WAAW,OAAO,KAAK,OAAO,EAAE,SAAS,KAAK,sBAAsB,QAAW;AACjF,mBAAa,sBAAsB;AAAA,IACrC;AAEA,iBACE,eAAe,SACX,aACC,KAAK,KAAK,aAAa;AAE9B,QAAI,YAAY;AACd,mBAAa,cAAc;AAAA,IAC7B;AAIA,mBAAe,EAAE,GAAG,cAAc,GAAG,KAAK,KAAK,aAAa;AAE5D,WAAO,IAAI,UAAU,MAAM;AAAA,MACzB,OAAO,KAAK,KAAK;AAAA,MACjB,UAAU,KAAK,KAAK;AAAA,MACpB,QAAQ,KAAK;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,kBAAkB,KAAK,KAAK,oBAAoB;AAAA;AAAA,MAChD,gBAAgB;AAAA,QACd,QAAQ,KAAK,KAAK;AAAA,QAClB,WAAW,KAAK,KAAK;AAAA,MACvB;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEO,MAAM,kBAAkB,IAAI,UAAU;AAAA,EACnC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YACEC,MACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAYA;AACA,UAAMA,MAAK,EAAE,SAAS,SAAS,YAAY,CAAC;AAC5C,SAAK,SAAS;AACd,SAAK,iBAAiB;AACtB,SAAK,WAAW;AAChB,SAAK,cAAc,eAAe;AAClC,SAAK,eAAe;AACpB,SAAK,QAAQ;AACb,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,MAAgB,MAAqB;AAtRvC;AAyRI,QAAI,YAAY;AAChB,SAAK,aAAa,KAAK,UAAU,KAAK,kBAAkB,KAAK,YAAY;AACzE,SAAK,YAAY;AAEjB,QAAI;AACF,YAAM,WAAY,MAAM,KAAK,QAAQ;AAAA,QACnC,KAAK;AAAA,MACP;AAEA,YAAM,QAAQ,KAAK,UACf,OAAO,QAAQ,KAAK,OAAO,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,MAAM;AACjD,cAAM,YAAY;AAAA,UAChB,MAAM;AAAA,UACN,UAAU;AAAA,YACR;AAAA,YACA,aAAa,KAAK;AAAA,YAClB,YAAY,IAAI;AAAA,cACd,KAAK;AAAA,cACL;AAAA,cACA,KAAK;AAAA,YACP;AAAA,UACF;AAAA,QACF;AAEA,YAAI,KAAK,kBAAkB;AACzB,oBAAU,SAAS,SAAS;AAAA,QAC9B;AAEA,eAAO;AAAA,MACT,CAAC,IACD;AAEJ,YAAM,iBAA0C,EAAE,GAAG,KAAK,aAAa;AACvE,UAAI,CAAC,OAAO;AACV,eAAO,eAAe;AAAA,MACxB;AAGA,UAAI,KAAK,gBAAgB;AACvB,aAAK,OAAO,SAAS,UAAM;AAAA,UACzB,KAAK,eAAe;AAAA,UACpB,KAAK,eAAe;AAAA,QACtB;AAAA,MACF;AAEA,UAAI,KAAK,UAAU;AACjB,cAAM,eAAe,eAAe,gBAC/B,eAAe,gBAChB,CAAC;AACL,qBAAa,8BAA8B,IAAI,KAAK;AACpD,uBAAe,gBAAgB;AAAA,MACjC;AAEA,YAAM,SAAS,MAAM,KAAK,OAAO,KAAK,YAAY;AAAA,QAChD;AAAA,UACE,OAAO,KAAK;AAAA,UACZ;AAAA,UACA;AAAA,UACA,QAAQ;AAAA,UACR,gBAAgB,EAAE,eAAe,KAAK;AAAA,UACtC,GAAG;AAAA,QACL;AAAA,QACA;AAAA,UACE,SAAS,KAAK,YAAY;AAAA,QAC5B;AAAA,MACF;AAEA,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,cAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,UACF;AACA,gBAAM,YAAY,KAAK,YAAY,MAAM,IAAI,MAAM;AACnD,cAAI,WAAW;AACb,wBAAY;AACZ,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAAA,QACF;AAEA,YAAI,MAAM,OAAO;AACf,gBAAM,QAAQ,MAAM;AACpB,sBAAY;AACZ,eAAK,MAAM,IAAI;AAAA,YACb,IAAI,MAAM;AAAA,YACV,OAAO;AAAA,cACL,kBAAkB,MAAM;AAAA,cACxB,cAAc,MAAM;AAAA,cACpB,sBAAoB,WAAM,0BAAN,mBAA6B,kBAAiB;AAAA,cAClE,aAAa,MAAM;AAAA,YACrB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,cAAAD,QAAO,2BAA2B;AACrD,cAAM,IAAI,yBAAgB,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;AAAA,MACtD,WAAW,iBAAiB,cAAAA,QAAO,UAAU;AAC3C,cAAM,IAAI,wBAAe;AAAA,UACvB,SAAS,MAAM;AAAA,UACf,SAAS;AAAA,YACP,YAAY,MAAM;AAAA,YAClB,MAAM,MAAM;AAAA,YACZ,WAAW,MAAM;AAAA,YACjB;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH,OAAO;AACL,cAAM,IAAI,4BAAmB;AAAA,UAC3B,aAAS,kBAAQ,KAAK,EAAE;AAAA,UACxB,SAAS,EAAE,UAAU;AAAA,QACvB,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,YACN,IACA,QAC2B;AAC3B,UAAM,QAAQ,OAAO;AAIrB,QAAI,UAAU,OAAW,QAAO;AAEhC,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAmBA,YAAI;AAEJ,YAAI,KAAK,cAAc,KAAK,MAAM,KAAK,UAAU,KAAK,WAAW;AAC/D,sBAAY,KAAK,2BAA2B,IAAI,KAAK;AACrD,eAAK,aAAa,KAAK,UAAU,KAAK,kBAAkB;AAAA,QAK1D;AAGA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,YAAY,KAAK;AACtB,eAAK,aAAa,KAAK;AACvB,eAAK,UAAU,KAAK,SAAS;AAC7B,eAAK,kBAAkB,KAAK,SAAS,aAAa;AAIlD,gBAAM;AAAA;AAAA,YAEF,KAAa,iBAAyD;AAAA;AAC1E,cAAI,cAAc;AAChB,iBAAK,YAAY;AAAA,UACnB;AAAA,QACF,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,mBAAmB,KAAK,mBAAmB,MAAM,KAAK,SAAS;AAAA,QACtE;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAGA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,eAAe,QACpB;AACA,YAAM,YAAY,KAAK,2BAA2B,IAAI,KAAK;AAC3D,WAAK,aAAa,KAAK,UAAU,KAAK,kBAAkB;AAExD,WAAK,YAAY;AACjB,aAAO;AAAA,IACT;AAGA,UAAM;AAAA;AAAA,MAEF,MAAc,iBAAyD;AAAA;AAG3E,QAAI,CAAC,MAAM,WAAW,CAAC,YAAY;AACjC,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,MACL;AAAA,MACA,OAAO;AAAA,QACL,MAAM;AAAA,QACN,SAAS,MAAM,WAAW;AAAA,QAC1B,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,2BACN,IACA,OACe;AACf,UAAM,YAAY,KAAK,YAAY,EAAE,GAAG,KAAK,UAAU,IAAI,CAAC;AAC5D,UAAM,mBAAmB,KAAK,wBAAwB,SAAS;AAC/D,UAAM;AAAA;AAAA,MAEF,MAAc,iBAAyD;AAAA;AAE3E,WAAO;AAAA,MACL;AAAA,MACA,OAAO;AAAA,QACL,MAAM;AAAA,QACN,SAAS,MAAM,WAAW;AAAA,QAC1B,OAAO;AAAA,QACP,WAAW;AAAA,UACT,IAAI,aAAa,OAAO;AAAA,YACtB,QAAQ,KAAK,cAAc;AAAA,YAC3B,MAAM,KAAK,WAAW;AAAA,YACtB,MAAM,KAAK,mBAAmB;AAAA,YAC9B,OAAO;AAAA,YACP;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,wBAAwB,OAAqD;AACnF,UAAM,cAAc,+BAAO;AAC3B,QAAI,eAAe,OAAO,gBAAgB,UAAU;AAElD,aAAQ,YAAoB,oBAAqB,YAAoB;AAAA,IACvE;AACA,WAAO;AAAA,EACT;AACF;","names":["OpenAI","llm"]}
+{"version":3,"sources":["../../src/inference/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport OpenAI from 'openai';\nimport { APIConnectionError, APIStatusError, APITimeoutError } from '../_exceptions.js';\nimport * as llm from '../llm/index.js';\nimport { DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { type Expand, toError } from '../utils.js';\nimport { type AnyString, createAccessToken, getDefaultInferenceUrl } from './utils.js';\n\nexport type OpenAIModels =\n  | 'openai/gpt-5.2'\n  | 'openai/gpt-5.2-chat-latest'\n  | 'openai/gpt-5.1'\n  | 'openai/gpt-5.1-chat-latest'\n  | 'openai/gpt-5'\n  | 'openai/gpt-5-mini'\n  | 'openai/gpt-5-nano'\n  | 'openai/gpt-4.1'\n  | 'openai/gpt-4.1-mini'\n  | 'openai/gpt-4.1-nano'\n  | 'openai/gpt-4o'\n  | 'openai/gpt-4o-mini'\n  | 'openai/gpt-oss-120b';\n\nexport type GoogleModels =\n  | 'google/gemini-3-pro'\n  | 'google/gemini-3-flash'\n  | 'google/gemini-2.5-pro'\n  | 'google/gemini-2.5-flash'\n  | 'google/gemini-2.5-flash-lite'\n  | 'google/gemini-2.0-flash'\n  | 'google/gemini-2.0-flash-lite';\n\nexport type MoonshotModels = 'moonshotai/kimi-k2-instruct';\n\nexport type DeepSeekModels = 'deepseek-ai/deepseek-v3' | 'deepseek-ai/deepseek-v3.2';\n\ntype ChatCompletionPredictionContentParam =\n  Expand<OpenAI.Chat.Completions.ChatCompletionPredictionContent>;\ntype WebSearchOptions = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams.WebSearchOptions>;\ntype ToolChoice = Expand<OpenAI.Chat.Completions.ChatCompletionCreateParams['tool_choice']>;\ntype Verbosity = 'low' | 'medium' | 'high';\n\nexport interface ChatCompletionOptions extends Record<string, unknown> {\n  frequency_penalty?: number;\n  logit_bias?: Record<string, number>;\n  logprobs?: boolean;\n  max_completion_tokens?: number;\n  max_tokens?: number;\n  metadata?: Record<string, string>;\n  modalities?: Array<'text' | 'audio'>;\n  n?: number;\n  parallel_tool_calls?: boolean;\n  prediction?: ChatCompletionPredictionContentParam | null;\n  presence_penalty?: number;\n  prompt_cache_key?: string;\n  reasoning_effort?: 'minimal' | 'low' | 'medium' | 'high';\n  safety_identifier?: string;\n  seed?: number;\n  service_tier?: 'auto' | 'default' | 'flex' | 'scale' | 'priority';\n  stop?: string | string[];\n  store?: boolean;\n  temperature?: number;\n  top_logprobs?: number;\n  top_p?: number;\n  user?: string;\n  verbosity?: Verbosity;\n  web_search_options?: WebSearchOptions;\n\n  // livekit-typed arguments\n  tool_choice?: ToolChoice;\n  // TODO(brian): support response format\n  // response_format?: OpenAI.Chat.Completions.ChatCompletionCreateParams['response_format']\n}\n\nexport type LLMModels = OpenAIModels | GoogleModels | MoonshotModels | DeepSeekModels | AnyString;\n\nexport interface InferenceLLMOptions {\n  model: LLMModels;\n  provider?: string;\n  baseURL: string;\n  apiKey: string;\n  apiSecret: string;\n  modelOptions: ChatCompletionOptions;\n  strictToolSchema?: boolean;\n}\n\nexport interface GatewayOptions {\n  apiKey: string;\n  apiSecret: string;\n}\n\n/**\n * Livekit Cloud Inference LLM\n */\nexport class LLM extends llm.LLM {\n  private client: OpenAI;\n  private opts: InferenceLLMOptions;\n\n  constructor(opts: {\n    model: LLMModels;\n    provider?: string;\n    baseURL?: string;\n    apiKey?: string;\n    apiSecret?: string;\n    modelOptions?: InferenceLLMOptions['modelOptions'];\n    strictToolSchema?: boolean;\n  }) {\n    super();\n\n    const {\n      model,\n      provider,\n      baseURL,\n      apiKey,\n      apiSecret,\n      modelOptions,\n      strictToolSchema = false,\n    } = opts;\n\n    const lkBaseURL = baseURL || getDefaultInferenceUrl();\n    const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;\n    if (!lkApiKey) {\n      throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');\n    }\n\n    const lkApiSecret =\n      apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;\n    if (!lkApiSecret) {\n      throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET');\n    }\n\n    this.opts = {\n      model,\n      provider,\n      baseURL: lkBaseURL,\n      apiKey: lkApiKey,\n      apiSecret: lkApiSecret,\n      modelOptions: modelOptions || {},\n      strictToolSchema,\n    };\n\n    this.client = new OpenAI({\n      baseURL: this.opts.baseURL,\n      apiKey: '', // leave a temporary empty string to avoid OpenAI complain about missing key\n    });\n  }\n\n  label(): string {\n    return 'inference.LLM';\n  }\n\n  get model(): string {\n    return this.opts.model;\n  }\n\n  get provider(): string {\n    return 'livekit';\n  }\n\n  static fromModelString(modelString: string): LLM {\n    return new LLM({ model: modelString });\n  }\n\n  chat({\n    chatCtx,\n    toolCtx,\n    connOptions = DEFAULT_API_CONNECT_OPTIONS,\n    parallelToolCalls,\n    toolChoice,\n    // TODO(AJS-270): Add response_format parameter support\n    extraKwargs,\n  }: {\n    chatCtx: llm.ChatContext;\n    toolCtx?: llm.ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: llm.ToolChoice;\n    // TODO(AJS-270): Add responseFormat parameter\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream {\n    let modelOptions: Record<string, unknown> = { ...(extraKwargs || {}) };\n\n    parallelToolCalls =\n      parallelToolCalls !== undefined\n        ? parallelToolCalls\n        : this.opts.modelOptions.parallel_tool_calls;\n\n    if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) {\n      modelOptions.parallel_tool_calls = parallelToolCalls;\n    }\n\n    toolChoice =\n      toolChoice !== undefined\n        ? toolChoice\n        : (this.opts.modelOptions.tool_choice as llm.ToolChoice | undefined);\n\n    if (toolChoice) {\n      modelOptions.tool_choice = toolChoice as ToolChoice;\n    }\n\n    // TODO(AJS-270): Add response_format support here\n\n    modelOptions = { ...modelOptions, ...this.opts.modelOptions };\n\n    return new LLMStream(this, {\n      model: this.opts.model,\n      provider: this.opts.provider,\n      client: this.client,\n      chatCtx,\n      toolCtx,\n      connOptions,\n      modelOptions,\n      strictToolSchema: this.opts.strictToolSchema ?? false, // default to false if not set\n      gatewayOptions: {\n        apiKey: this.opts.apiKey,\n        apiSecret: this.opts.apiSecret,\n      },\n    });\n  }\n}\n\nexport class LLMStream extends llm.LLMStream {\n  private model: LLMModels;\n  private provider?: string;\n  private providerFmt: llm.ProviderFormat;\n  private client: OpenAI;\n  private modelOptions: Record<string, unknown>;\n  private strictToolSchema: boolean;\n\n  private gatewayOptions?: GatewayOptions;\n  private toolCallId?: string;\n  private toolIndex?: number;\n  private fncName?: string;\n  private fncRawArguments?: string;\n  private toolExtra?: Record<string, unknown>;\n\n  constructor(\n    llm: LLM,\n    {\n      model,\n      provider,\n      client,\n      chatCtx,\n      toolCtx,\n      gatewayOptions,\n      connOptions,\n      modelOptions,\n      providerFmt,\n      strictToolSchema,\n    }: {\n      model: LLMModels;\n      provider?: string;\n      client: OpenAI;\n      chatCtx: llm.ChatContext;\n      toolCtx?: llm.ToolContext;\n      gatewayOptions?: GatewayOptions;\n      connOptions: APIConnectOptions;\n      modelOptions: Record<string, unknown>;\n      providerFmt?: llm.ProviderFormat;\n      strictToolSchema: boolean;\n    },\n  ) {\n    super(llm, { chatCtx, toolCtx, connOptions });\n    this.client = client;\n    this.gatewayOptions = gatewayOptions;\n    this.provider = provider;\n    this.providerFmt = providerFmt || 'openai';\n    this.modelOptions = modelOptions;\n    this.model = model;\n    this.strictToolSchema = strictToolSchema;\n  }\n\n  protected async run(): Promise<void> {\n    // current function call that we're waiting for full completion (args are streamed)\n    // (defined inside the run method to make sure the state is reset for each run/attempt)\n    let retryable = true;\n    this.toolCallId = this.fncName = this.fncRawArguments = this.toolIndex = undefined;\n    this.toolExtra = undefined;\n\n    try {\n      const messages = (await this.chatCtx.toProviderFormat(\n        this.providerFmt,\n      )) as OpenAI.ChatCompletionMessageParam[];\n\n      const tools = this.toolCtx\n        ? Object.entries(this.toolCtx).map(([name, func]) => {\n            const oaiParams = {\n              type: 'function' as const,\n              function: {\n                name,\n                description: func.description,\n                parameters: llm.toJsonSchema(\n                  func.parameters,\n                  true,\n                  this.strictToolSchema,\n                ) as unknown as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function']['parameters'],\n              } as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function'],\n            };\n\n            if (this.strictToolSchema) {\n              oaiParams.function.strict = true;\n            }\n\n            return oaiParams;\n          })\n        : undefined;\n\n      const requestOptions: Record<string, unknown> = { ...this.modelOptions };\n      if (!tools) {\n        delete requestOptions.tool_choice;\n      }\n\n      // Dynamically set the access token for the LiveKit Agent Gateway API\n      if (this.gatewayOptions) {\n        this.client.apiKey = await createAccessToken(\n          this.gatewayOptions.apiKey,\n          this.gatewayOptions.apiSecret,\n        );\n      }\n\n      if (this.provider) {\n        const extraHeaders = requestOptions.extra_headers\n          ? (requestOptions.extra_headers as Record<string, string>)\n          : {};\n        extraHeaders['X-LiveKit-Inference-Provider'] = this.provider;\n        requestOptions.extra_headers = extraHeaders;\n      }\n\n      const stream = await this.client.chat.completions.create(\n        {\n          model: this.model,\n          messages,\n          tools,\n          stream: true,\n          stream_options: { include_usage: true },\n          ...requestOptions,\n        },\n        {\n          timeout: this.connOptions.timeoutMs,\n        },\n      );\n\n      for await (const chunk of stream) {\n        for (const choice of chunk.choices) {\n          if (this.abortController.signal.aborted) {\n            break;\n          }\n          const chatChunk = this.parseChoice(chunk.id, choice);\n          if (chatChunk) {\n            retryable = false;\n            this.queue.put(chatChunk);\n          }\n        }\n\n        if (chunk.usage) {\n          const usage = chunk.usage;\n          retryable = false;\n          this.queue.put({\n            id: chunk.id,\n            usage: {\n              completionTokens: usage.completion_tokens,\n              promptTokens: usage.prompt_tokens,\n              promptCachedTokens: usage.prompt_tokens_details?.cached_tokens || 0,\n              totalTokens: usage.total_tokens,\n            },\n          });\n        }\n      }\n    } catch (error) {\n      if (error instanceof OpenAI.APIConnectionTimeoutError) {\n        throw new APITimeoutError({ options: { retryable } });\n      } else if (error instanceof OpenAI.APIError) {\n        throw new APIStatusError({\n          message: error.message,\n          options: {\n            statusCode: error.status,\n            body: error.error,\n            requestId: error.requestID,\n            retryable,\n          },\n        });\n      } else {\n        throw new APIConnectionError({\n          message: toError(error).message,\n          options: { retryable },\n        });\n      }\n    }\n  }\n\n  private parseChoice(\n    id: string,\n    choice: OpenAI.ChatCompletionChunk.Choice,\n  ): llm.ChatChunk | undefined {\n    const delta = choice.delta;\n\n    // https://github.com/livekit/agents/issues/688\n    // the delta can be None when using Azure OpenAI (content filtering)\n    if (delta === undefined) return undefined;\n\n    if (delta.tool_calls) {\n      // check if we have functions to calls\n      for (const tool of delta.tool_calls) {\n        if (!tool.function) {\n          continue; // oai may add other tools in the future\n        }\n\n        /**\n         * The way OpenAI streams tool calls is a bit tricky.\n         *\n         * For any new tool call, it first emits a delta tool call with id, and function name,\n         * the rest of the delta chunks will only stream the remaining arguments string,\n         * until a new tool call is started or the tool call is finished.\n         * See below for an example.\n         *\n         * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role='assistant', tool_calls=None), finish_reason=None, index=0, logprobs=None)\n         * [ChoiceDeltaToolCall(index=0, id='call_LaVeHWUHpef9K1sd5UO8TtLg', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]\n         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='\\{\"location\": \"P', name=None), type=None)]\n         * [ChoiceDeltaToolCall(index=0, id=None, function=ChoiceDeltaToolCallFunction(arguments='aris\\}', name=None), type=None)]\n         * [ChoiceDeltaToolCall(index=1, id='call_ThU4OmMdQXnnVmpXGOCknXIB', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]\n         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='\\{\"location\": \"T', name=None), type=None)]\n         * [ChoiceDeltaToolCall(index=1, id=None, function=ChoiceDeltaToolCallFunction(arguments='okyo', name=None), type=None)]\n         * Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role=None, tool_calls=None), finish_reason='tool_calls', index=0, logprobs=None)\n         */\n        let callChunk: llm.ChatChunk | undefined;\n        // If we have a previous tool call and this is a new one, emit the previous\n        if (this.toolCallId && tool.id && tool.index !== this.toolIndex) {\n          callChunk = this.createRunningToolCallChunk(id, delta);\n          this.toolCallId = this.fncName = this.fncRawArguments = undefined;\n          // Note: We intentionally do NOT reset toolExtra here.\n          // For Gemini 3+, the thought_signature is only provided on the first tool call\n          // in a parallel batch, but must be applied to ALL tool calls in the batch.\n          // We preserve toolExtra so subsequent tool calls inherit the thought_signature.\n        }\n\n        // Start or continue building the current tool call\n        if (tool.function.name) {\n          this.toolIndex = tool.index;\n          this.toolCallId = tool.id;\n          this.fncName = tool.function.name;\n          this.fncRawArguments = tool.function.arguments || '';\n          // Extract extra from tool call (e.g., Google thought signatures)\n          // Only update toolExtra if this tool call has extra_content.\n          // Otherwise, inherit from previous tool call (for parallel Gemini tool calls).\n          const newToolExtra =\n            // eslint-disable-next-line @typescript-eslint/no-explicit-any\n            ((tool as any).extra_content as Record<string, unknown> | undefined) ?? undefined;\n          if (newToolExtra) {\n            this.toolExtra = newToolExtra;\n          }\n        } else if (tool.function.arguments) {\n          this.fncRawArguments = (this.fncRawArguments || '') + tool.function.arguments;\n        }\n\n        if (callChunk) {\n          return callChunk;\n        }\n      }\n    }\n\n    // If we're done with tool calls, emit the final one\n    if (\n      choice.finish_reason &&\n      ['tool_calls', 'stop'].includes(choice.finish_reason) &&\n      this.toolCallId !== undefined\n    ) {\n      const callChunk = this.createRunningToolCallChunk(id, delta);\n      this.toolCallId = this.fncName = this.fncRawArguments = undefined;\n      // Reset toolExtra at the end of the response (not between parallel tool calls)\n      this.toolExtra = undefined;\n      return callChunk;\n    }\n\n    // Extract extra from delta (e.g., Google thought signatures on text parts)\n    const deltaExtra =\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      ((delta as any).extra_content as Record<string, unknown> | undefined) ?? undefined;\n\n    // Regular content message\n    if (!delta.content && !deltaExtra) {\n      return undefined;\n    }\n\n    return {\n      id,\n      delta: {\n        role: 'assistant',\n        content: delta.content || undefined,\n        extra: deltaExtra,\n      },\n    };\n  }\n\n  private createRunningToolCallChunk(\n    id: string,\n    delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta,\n  ): llm.ChatChunk {\n    const toolExtra = this.toolExtra ? { ...this.toolExtra } : {};\n    const thoughtSignature = this.extractThoughtSignature(toolExtra);\n    const deltaExtra =\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      ((delta as any).extra_content as Record<string, unknown> | undefined) ?? undefined;\n\n    return {\n      id,\n      delta: {\n        role: 'assistant',\n        content: delta.content || undefined,\n        extra: deltaExtra,\n        toolCalls: [\n          llm.FunctionCall.create({\n            callId: this.toolCallId || '',\n            name: this.fncName || '',\n            args: this.fncRawArguments || '',\n            extra: toolExtra,\n            thoughtSignature,\n          }),\n        ],\n      },\n    };\n  }\n\n  private extractThoughtSignature(extra?: Record<string, unknown>): string | undefined {\n    const googleExtra = extra?.google;\n    if (googleExtra && typeof googleExtra === 'object') {\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      return (googleExtra as any).thoughtSignature || (googleExtra as any).thought_signature;\n    }\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAmB;AACnB,wBAAoE;AACpE,UAAqB;AACrB,mBAA4C;AAE5C,mBAAqC;AACrC,IAAAA,gBAA0E;AAwFnE,MAAM,YAAY,IAAI,IAAI;AAAA,EACvB;AAAA,EACA;AAAA,EAER,YAAY,MAQT;AACD,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,mBAAmB;AAAA,IACrB,IAAI;AAEJ,UAAM,YAAY,eAAW,sCAAuB;AACpD,UAAM,WAAW,UAAU,QAAQ,IAAI,6BAA6B,QAAQ,IAAI;AAChF,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,MAAM,wDAAwD;AAAA,IAC1E;AAEA,UAAM,cACJ,aAAa,QAAQ,IAAI,gCAAgC,QAAQ,IAAI;AACvE,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI,MAAM,iEAAiE;AAAA,IACnF;AAEA,SAAK,OAAO;AAAA,MACV;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,cAAc,gBAAgB,CAAC;AAAA,MAC/B;AAAA,IACF;AAEA,SAAK,SAAS,IAAI,cAAAC,QAAO;AAAA,MACvB,SAAS,KAAK,KAAK;AAAA,MACnB,QAAQ;AAAA;AAAA,IACV,CAAC;AAAA,EACH;AAAA,EAEA,QAAgB;AACd,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO;AAAA,EACT;AAAA,EAEA,OAAO,gBAAgB,aAA0B;AAC/C,WAAO,IAAI,IAAI,EAAE,OAAO,YAAY,CAAC;AAAA,EACvC;AAAA,EAEA,KAAK;AAAA,IACH;AAAA,IACA;AAAA,IACA,cAAc;AAAA,IACd;AAAA,IACA;AAAA;AAAA,IAEA;AAAA,EACF,GAQc;AACZ,QAAI,eAAwC,EAAE,GAAI,eAAe,CAAC,EAAG;AAErE,wBACE,sBAAsB,SAClB,oBACA,KAAK,KAAK,aAAa;AAE7B,QAAI,WAAW,OAAO,KAAK,OAAO,EAAE,SAAS,KAAK,sBAAsB,QAAW;AACjF,mBAAa,sBAAsB;AAAA,IACrC;AAEA,iBACE,eAAe,SACX,aACC,KAAK,KAAK,aAAa;AAE9B,QAAI,YAAY;AACd,mBAAa,cAAc;AAAA,IAC7B;AAIA,mBAAe,EAAE,GAAG,cAAc,GAAG,KAAK,KAAK,aAAa;AAE5D,WAAO,IAAI,UAAU,MAAM;AAAA,MACzB,OAAO,KAAK,KAAK;AAAA,MACjB,UAAU,KAAK,KAAK;AAAA,MACpB,QAAQ,KAAK;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,kBAAkB,KAAK,KAAK,oBAAoB;AAAA;AAAA,MAChD,gBAAgB;AAAA,QACd,QAAQ,KAAK,KAAK;AAAA,QAClB,WAAW,KAAK,KAAK;AAAA,MACvB;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEO,MAAM,kBAAkB,IAAI,UAAU;AAAA,EACnC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YACEC,MACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAYA;AACA,UAAMA,MAAK,EAAE,SAAS,SAAS,YAAY,CAAC;AAC5C,SAAK,SAAS;AACd,SAAK,iBAAiB;AACtB,SAAK,WAAW;AAChB,SAAK,cAAc,eAAe;AAClC,SAAK,eAAe;AACpB,SAAK,QAAQ;AACb,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,MAAgB,MAAqB;AAnRvC;AAsRI,QAAI,YAAY;AAChB,SAAK,aAAa,KAAK,UAAU,KAAK,kBAAkB,KAAK,YAAY;AACzE,SAAK,YAAY;AAEjB,QAAI;AACF,YAAM,WAAY,MAAM,KAAK,QAAQ;AAAA,QACnC,KAAK;AAAA,MACP;AAEA,YAAM,QAAQ,KAAK,UACf,OAAO,QAAQ,KAAK,OAAO,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,MAAM;AACjD,cAAM,YAAY;AAAA,UAChB,MAAM;AAAA,UACN,UAAU;AAAA,YACR;AAAA,YACA,aAAa,KAAK;AAAA,YAClB,YAAY,IAAI;AAAA,cACd,KAAK;AAAA,cACL;AAAA,cACA,KAAK;AAAA,YACP;AAAA,UACF;AAAA,QACF;AAEA,YAAI,KAAK,kBAAkB;AACzB,oBAAU,SAAS,SAAS;AAAA,QAC9B;AAEA,eAAO;AAAA,MACT,CAAC,IACD;AAEJ,YAAM,iBAA0C,EAAE,GAAG,KAAK,aAAa;AACvE,UAAI,CAAC,OAAO;AACV,eAAO,eAAe;AAAA,MACxB;AAGA,UAAI,KAAK,gBAAgB;AACvB,aAAK,OAAO,SAAS,UAAM;AAAA,UACzB,KAAK,eAAe;AAAA,UACpB,KAAK,eAAe;AAAA,QACtB;AAAA,MACF;AAEA,UAAI,KAAK,UAAU;AACjB,cAAM,eAAe,eAAe,gBAC/B,eAAe,gBAChB,CAAC;AACL,qBAAa,8BAA8B,IAAI,KAAK;AACpD,uBAAe,gBAAgB;AAAA,MACjC;AAEA,YAAM,SAAS,MAAM,KAAK,OAAO,KAAK,YAAY;AAAA,QAChD;AAAA,UACE,OAAO,KAAK;AAAA,UACZ;AAAA,UACA;AAAA,UACA,QAAQ;AAAA,UACR,gBAAgB,EAAE,eAAe,KAAK;AAAA,UACtC,GAAG;AAAA,QACL;AAAA,QACA;AAAA,UACE,SAAS,KAAK,YAAY;AAAA,QAC5B;AAAA,MACF;AAEA,uBAAiB,SAAS,QAAQ;AAChC,mBAAW,UAAU,MAAM,SAAS;AAClC,cAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,UACF;AACA,gBAAM,YAAY,KAAK,YAAY,MAAM,IAAI,MAAM;AACnD,cAAI,WAAW;AACb,wBAAY;AACZ,iBAAK,MAAM,IAAI,SAAS;AAAA,UAC1B;AAAA,QACF;AAEA,YAAI,MAAM,OAAO;AACf,gBAAM,QAAQ,MAAM;AACpB,sBAAY;AACZ,eAAK,MAAM,IAAI;AAAA,YACb,IAAI,MAAM;AAAA,YACV,OAAO;AAAA,cACL,kBAAkB,MAAM;AAAA,cACxB,cAAc,MAAM;AAAA,cACpB,sBAAoB,WAAM,0BAAN,mBAA6B,kBAAiB;AAAA,cAClE,aAAa,MAAM;AAAA,YACrB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,cAAAD,QAAO,2BAA2B;AACrD,cAAM,IAAI,kCAAgB,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;AAAA,MACtD,WAAW,iBAAiB,cAAAA,QAAO,UAAU;AAC3C,cAAM,IAAI,iCAAe;AAAA,UACvB,SAAS,MAAM;AAAA,UACf,SAAS;AAAA,YACP,YAAY,MAAM;AAAA,YAClB,MAAM,MAAM;AAAA,YACZ,WAAW,MAAM;AAAA,YACjB;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH,OAAO;AACL,cAAM,IAAI,qCAAmB;AAAA,UAC3B,aAAS,sBAAQ,KAAK,EAAE;AAAA,UACxB,SAAS,EAAE,UAAU;AAAA,QACvB,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,YACN,IACA,QAC2B;AAC3B,UAAM,QAAQ,OAAO;AAIrB,QAAI,UAAU,OAAW,QAAO;AAEhC,QAAI,MAAM,YAAY;AAEpB,iBAAW,QAAQ,MAAM,YAAY;AACnC,YAAI,CAAC,KAAK,UAAU;AAClB;AAAA,QACF;AAmBA,YAAI;AAEJ,YAAI,KAAK,cAAc,KAAK,MAAM,KAAK,UAAU,KAAK,WAAW;AAC/D,sBAAY,KAAK,2BAA2B,IAAI,KAAK;AACrD,eAAK,aAAa,KAAK,UAAU,KAAK,kBAAkB;AAAA,QAK1D;AAGA,YAAI,KAAK,SAAS,MAAM;AACtB,eAAK,YAAY,KAAK;AACtB,eAAK,aAAa,KAAK;AACvB,eAAK,UAAU,KAAK,SAAS;AAC7B,eAAK,kBAAkB,KAAK,SAAS,aAAa;AAIlD,gBAAM;AAAA;AAAA,YAEF,KAAa,iBAAyD;AAAA;AAC1E,cAAI,cAAc;AAChB,iBAAK,YAAY;AAAA,UACnB;AAAA,QACF,WAAW,KAAK,SAAS,WAAW;AAClC,eAAK,mBAAmB,KAAK,mBAAmB,MAAM,KAAK,SAAS;AAAA,QACtE;AAEA,YAAI,WAAW;AACb,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAGA,QACE,OAAO,iBACP,CAAC,cAAc,MAAM,EAAE,SAAS,OAAO,aAAa,KACpD,KAAK,eAAe,QACpB;AACA,YAAM,YAAY,KAAK,2BAA2B,IAAI,KAAK;AAC3D,WAAK,aAAa,KAAK,UAAU,KAAK,kBAAkB;AAExD,WAAK,YAAY;AACjB,aAAO;AAAA,IACT;AAGA,UAAM;AAAA;AAAA,MAEF,MAAc,iBAAyD;AAAA;AAG3E,QAAI,CAAC,MAAM,WAAW,CAAC,YAAY;AACjC,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,MACL;AAAA,MACA,OAAO;AAAA,QACL,MAAM;AAAA,QACN,SAAS,MAAM,WAAW;AAAA,QAC1B,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,2BACN,IACA,OACe;AACf,UAAM,YAAY,KAAK,YAAY,EAAE,GAAG,KAAK,UAAU,IAAI,CAAC;AAC5D,UAAM,mBAAmB,KAAK,wBAAwB,SAAS;AAC/D,UAAM;AAAA;AAAA,MAEF,MAAc,iBAAyD;AAAA;AAE3E,WAAO;AAAA,MACL;AAAA,MACA,OAAO;AAAA,QACL,MAAM;AAAA,QACN,SAAS,MAAM,WAAW;AAAA,QAC1B,OAAO;AAAA,QACP,WAAW;AAAA,UACT,IAAI,aAAa,OAAO;AAAA,YACtB,QAAQ,KAAK,cAAc;AAAA,YAC3B,MAAM,KAAK,WAAW;AAAA,YACtB,MAAM,KAAK,mBAAmB;AAAA,YAC9B,OAAO;AAAA,YACP;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,wBAAwB,OAAqD;AACnF,UAAM,cAAc,+BAAO;AAC3B,QAAI,eAAe,OAAO,gBAAgB,UAAU;AAElD,aAAQ,YAAoB,oBAAqB,YAAoB;AAAA,IACvE;AACA,WAAO;AAAA,EACT;AACF;","names":["import_utils","OpenAI","llm"]}

package/dist/inference/llm.d.cts

@@ -1,7 +1,7 @@
 import OpenAI from 'openai';
-import { type Expand } from '../index.js';
 import * as llm from '../llm/index.js';
 import type { APIConnectOptions } from '../types.js';
+import { type Expand } from '../utils.js';
 import { type AnyString } from './utils.js';
 export type OpenAIModels = 'openai/gpt-5.2' | 'openai/gpt-5.2-chat-latest' | 'openai/gpt-5.1' | 'openai/gpt-5.1-chat-latest' | 'openai/gpt-5' | 'openai/gpt-5-mini' | 'openai/gpt-5-nano' | 'openai/gpt-4.1' | 'openai/gpt-4.1-mini' | 'openai/gpt-4.1-nano' | 'openai/gpt-4o' | 'openai/gpt-4o-mini' | 'openai/gpt-oss-120b';
 export type GoogleModels = 'google/gemini-3-pro' | 'google/gemini-3-flash' | 'google/gemini-2.5-pro' | 'google/gemini-2.5-flash' | 'google/gemini-2.5-flash-lite' | 'google/gemini-2.0-flash' | 'google/gemini-2.0-flash-lite';
@@ -69,6 +69,7 @@ export declare class LLM extends llm.LLM {
     });
     label(): string;
     get model(): string;
+    get provider(): string;
     static fromModelString(modelString: string): LLM;
     chat({ chatCtx, toolCtx, connOptions, parallelToolCalls, toolChoice, extraKwargs, }: {
         chatCtx: llm.ChatContext;

package/dist/inference/llm.d.ts

@@ -1,7 +1,7 @@
 import OpenAI from 'openai';
-import { type Expand } from '../index.js';
 import * as llm from '../llm/index.js';
 import type { APIConnectOptions } from '../types.js';
+import { type Expand } from '../utils.js';
 import { type AnyString } from './utils.js';
 export type OpenAIModels = 'openai/gpt-5.2' | 'openai/gpt-5.2-chat-latest' | 'openai/gpt-5.1' | 'openai/gpt-5.1-chat-latest' | 'openai/gpt-5' | 'openai/gpt-5-mini' | 'openai/gpt-5-nano' | 'openai/gpt-4.1' | 'openai/gpt-4.1-mini' | 'openai/gpt-4.1-nano' | 'openai/gpt-4o' | 'openai/gpt-4o-mini' | 'openai/gpt-oss-120b';
 export type GoogleModels = 'google/gemini-3-pro' | 'google/gemini-3-flash' | 'google/gemini-2.5-pro' | 'google/gemini-2.5-flash' | 'google/gemini-2.5-flash-lite' | 'google/gemini-2.0-flash' | 'google/gemini-2.0-flash-lite';
@@ -69,6 +69,7 @@ export declare class LLM extends llm.LLM {
     });
     label(): string;
     get model(): string;
+    get provider(): string;
     static fromModelString(modelString: string): LLM;
     chat({ chatCtx, toolCtx, connOptions, parallelToolCalls, toolChoice, extraKwargs, }: {
         chatCtx: llm.ChatContext;

package/dist/inference/llm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/inference/llm.ts"],"names":[],"mappings":"AAGA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAKL,KAAK,MAAM,EAEZ,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,GAAG,MAAM,iBAAiB,CAAC;AACvC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,KAAK,SAAS,EAAqB,MAAM,YAAY,CAAC;AAI/D,MAAM,MAAM,YAAY,GACpB,gBAAgB,GAChB,4BAA4B,GAC5B,gBAAgB,GAChB,4BAA4B,GAC5B,cAAc,GACd,mBAAmB,GACnB,mBAAmB,GACnB,gBAAgB,GAChB,qBAAqB,GACrB,qBAAqB,GACrB,eAAe,GACf,oBAAoB,GACpB,qBAAqB,CAAC;AAE1B,MAAM,MAAM,YAAY,GACpB,qBAAqB,GACrB,uBAAuB,GACvB,uBAAuB,GACvB,yBAAyB,GACzB,8BAA8B,GAC9B,yBAAyB,GACzB,8BAA8B,CAAC;AAEnC,MAAM,MAAM,cAAc,GAAG,6BAA6B,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG,yBAAyB,GAAG,2BAA2B,CAAC;AAErF,KAAK,oCAAoC,GACvC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,+BAA+B,CAAC,CAAC;AAClE,KAAK,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,CAAC;AACpG,KAAK,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5F,KAAK,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE3C,MAAM,WAAW,qBAAsB,SAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACpE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC;IACrC,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,UAAU,CAAC,EAAE,oCAAoC,GAAG,IAAI,CAAC;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACzD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,GAAG,UAAU,CAAC;IAClE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IAGtC,WAAW,CAAC,EAAE,UAAU,CAAC;CAG1B;AAED,MAAM,MAAM,SAAS,GAAG,YAAY,GAAG,YAAY,GAAG,cAAc,GAAG,cAAc,GAAG,SAAS,CAAC;AAElG,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,SAAS,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,qBAAqB,CAAC;IACpC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;IAC9B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,IAAI,CAAsB;gBAEtB,IAAI,EAAE;QAChB,KAAK,EAAE,SAAS,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,mBAAmB,CAAC,cAAc,CAAC,CAAC;QACnD,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC5B;IAyCD,KAAK,IAAI,MAAM;IAIf,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,MAAM,CAAC,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,GAAG;IAIhD,IAAI,CAAC,EACH,OAAO,EACP,OAAO,EACP,WAAyC,EACzC,iBAAiB,EACjB,UAAU,EAEV,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;QAE5B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACvC,GAAG,SAAS;CAwCd;AAED,qBAAa,SAAU,SAAQ,GAAG,CAAC,SAAS;IAC1C,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAA0B;IAC9C,OAAO,CAAC,gBAAgB,CAAU;IAElC,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,UAAU,CAAC,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,OAAO,CAAC,CAAS;IACzB,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,SAAS,CAAC,CAA0B;gBAG1C,GAAG,EAAE,GAAG,EACR,EACE,KAAK,EACL,QAAQ,EACR,MAAM,EACN,OAAO,EACP,OAAO,EACP,cAAc,EACd,WAAW,EACX,YAAY,EACZ,WAAW,EACX,gBAAgB,GACjB,EAAE;QACD,KAAK,EAAE,SAAS,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,cAAc,CAAC,EAAE,cAAc,CAAC;QAChC,WAAW,EAAE,iBAAiB,CAAC;QAC/B,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,WAAW,CAAC,EAAE,GAAG,CAAC,cAAc,CAAC;QACjC,gBAAgB,EAAE,OAAO,CAAC;KAC3B;cAYa,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAsHpC,OAAO,CAAC,WAAW;IAuGnB,OAAO,CAAC,0BAA0B;IA6BlC,OAAO,CAAC,uBAAuB;CAQhC"}
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/inference/llm.ts"],"names":[],"mappings":"AAGA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,KAAK,GAAG,MAAM,iBAAiB,CAAC;AAEvC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,KAAK,MAAM,EAAW,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,KAAK,SAAS,EAA6C,MAAM,YAAY,CAAC;AAEvF,MAAM,MAAM,YAAY,GACpB,gBAAgB,GAChB,4BAA4B,GAC5B,gBAAgB,GAChB,4BAA4B,GAC5B,cAAc,GACd,mBAAmB,GACnB,mBAAmB,GACnB,gBAAgB,GAChB,qBAAqB,GACrB,qBAAqB,GACrB,eAAe,GACf,oBAAoB,GACpB,qBAAqB,CAAC;AAE1B,MAAM,MAAM,YAAY,GACpB,qBAAqB,GACrB,uBAAuB,GACvB,uBAAuB,GACvB,yBAAyB,GACzB,8BAA8B,GAC9B,yBAAyB,GACzB,8BAA8B,CAAC;AAEnC,MAAM,MAAM,cAAc,GAAG,6BAA6B,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG,yBAAyB,GAAG,2BAA2B,CAAC;AAErF,KAAK,oCAAoC,GACvC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,+BAA+B,CAAC,CAAC;AAClE,KAAK,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,CAAC;AACpG,KAAK,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,CAAC,aAAa,CAAC,CAAC,CAAC;AAC5F,KAAK,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE3C,MAAM,WAAW,qBAAsB,SAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACpE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC;IACrC,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,UAAU,CAAC,EAAE,oCAAoC,GAAG,IAAI,CAAC;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACzD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,GAAG,UAAU,CAAC;IAClE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IAGtC,WAAW,CAAC,EAAE,UAAU,CAAC;CAG1B;AAED,MAAM,MAAM,SAAS,GAAG,YAAY,GAAG,YAAY,GAAG,cAAc,GAAG,cAAc,GAAG,SAAS,CAAC;AAElG,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,SAAS,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,qBAAqB,CAAC;IACpC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;IAC9B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,IAAI,CAAsB;gBAEtB,IAAI,EAAE;QAChB,KAAK,EAAE,SAAS,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,mBAAmB,CAAC,cAAc,CAAC,CAAC;QACnD,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC5B;IAyCD,KAAK,IAAI,MAAM;IAIf,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,MAAM,CAAC,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,GAAG;IAIhD,IAAI,CAAC,EACH,OAAO,EACP,OAAO,EACP,WAAyC,EACzC,iBAAiB,EACjB,UAAU,EAEV,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC;QAE5B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACvC,GAAG,SAAS;CAwCd;AAED,qBAAa,SAAU,SAAQ,GAAG,CAAC,SAAS;IAC1C,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAA0B;IAC9C,OAAO,CAAC,gBAAgB,CAAU;IAElC,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,UAAU,CAAC,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,OAAO,CAAC,CAAS;IACzB,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,SAAS,CAAC,CAA0B;gBAG1C,GAAG,EAAE,GAAG,EACR,EACE,KAAK,EACL,QAAQ,EACR,MAAM,EACN,OAAO,EACP,OAAO,EACP,cAAc,EACd,WAAW,EACX,YAAY,EACZ,WAAW,EACX,gBAAgB,GACjB,EAAE;QACD,KAAK,EAAE,SAAS,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC;QACzB,OAAO,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC;QAC1B,cAAc,CAAC,EAAE,cAAc,CAAC;QAChC,WAAW,EAAE,iBAAiB,CAAC;QAC/B,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,WAAW,CAAC,EAAE,GAAG,CAAC,cAAc,CAAC;QACjC,gBAAgB,EAAE,OAAO,CAAC;KAC3B;cAYa,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAsHpC,OAAO,CAAC,WAAW;IAuGnB,OAAO,CAAC,0BAA0B;IA6BlC,OAAO,CAAC,uBAAuB;CAQhC"}

package/dist/inference/llm.js

@@ -1,14 +1,9 @@
 import OpenAI from "openai";
-import {
-  APIConnectionError,
-  APIStatusError,
-  APITimeoutError,
-  DEFAULT_API_CONNECT_OPTIONS,
-  toError
-} from "../index.js";
+import { APIConnectionError, APIStatusError, APITimeoutError } from "../_exceptions.js";
 import * as llm from "../llm/index.js";
-import { createAccessToken } from "./utils.js";
-const DEFAULT_BASE_URL = "https://agent-gateway.livekit.cloud/v1";
+import { DEFAULT_API_CONNECT_OPTIONS } from "../types.js";
+import { toError } from "../utils.js";
+import { createAccessToken, getDefaultInferenceUrl } from "./utils.js";
 class LLM extends llm.LLM {
   client;
   opts;
@@ -23,7 +18,7 @@ class LLM extends llm.LLM {
       modelOptions,
       strictToolSchema = false
     } = opts;
-    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkBaseURL = baseURL || getDefaultInferenceUrl();
     const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
     if (!lkApiKey) {
       throw new Error("apiKey is required: pass apiKey or set LIVEKIT_API_KEY");
@@ -53,6 +48,9 @@ class LLM extends llm.LLM {
   get model() {
     return this.opts.model;
   }
+  get provider() {
+    return "livekit";
+  }
   static fromModelString(modelString) {
     return new LLM({ model: modelString });
   }