@livekit/agents 1.0.16 → 1.0.18

package/dist/llm/llm.d.cts

@@ -53,7 +53,7 @@ export declare abstract class LLM extends LLM_base {
         connOptions?: APIConnectOptions;
         parallelToolCalls?: boolean;
         toolChoice?: ToolChoice;
-        extraKwargs?: Record<string, any>;
+        extraKwargs?: Record<string, unknown>;
     }): LLMStream;
     /**
      * Pre-warm connection to the LLM service

package/dist/llm/llm.d.ts

@@ -53,7 +53,7 @@ export declare abstract class LLM extends LLM_base {
         connOptions?: APIConnectOptions;
         parallelToolCalls?: boolean;
         toolChoice?: ToolChoice;
-        extraKwargs?: Record<string, any>;
+        extraKwargs?: Record<string, unknown>;
     }): LLMStream;
     /**
      * Pre-warm connection to the LLM service

package/dist/llm/llm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/llm/llm.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAIhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AAC5E,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACvF,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEjE,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAE2D,aAAa,YAAY,CAAC;AAAvF,8BAAsB,GAAI,SAAQ,QAAsD;;IAKtF,QAAQ,CAAC,KAAK,IAAI,MAAM;IAExB;;;;;;;OAOG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;OAEG;IACH,QAAQ,CAAC,IAAI,CAAC,EACZ,OAAO,EACP,OAAO,EACP,WAAW,EACX,iBAAiB,EACjB,UAAU,EACV,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KACnC,GAAG,SAAS;IAEb;;OAEG;IACH,OAAO,IAAI,IAAI;IAIT,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAG9B;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,SAAS,CAAC;;IACzE,SAAS,CAAC,MAAM,gCAAuC;IACvD,SAAS,CAAC,KAAK,gCAAuC;IACtD,SAAS,CAAC,MAAM,UAAS;IACzB,SAAS,CAAC,eAAe,kBAAyB;IAClD,SAAS,CAAC,YAAY,EAAE,iBAAiB,CAAC;IAC1C,SAAS,CAAC,MAAM,wBAAS;gBAOvB,GAAG,EAAE,GAAG,EACR,EACE,OAAO,EACP,OAAO,EACP,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,EAAE,iBAAiB,CAAC;KAChC;YAoBW,QAAQ;IAoCtB,OAAO,CAAC,SAAS;cAUD,cAAc;IA6C9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,2CAA2C;IAC3C,IAAI,OAAO,IAAI,WAAW,GAAG,SAAS,CAErC;IAED,+CAA+C;IAC/C,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,8CAA8C;IAC9C,IAAI,WAAW,IAAI,iBAAiB,CAEnC;IAED,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAI1C,KAAK;IAIL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/llm/llm.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAIhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AAC5E,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,QAAQ,EAAE,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACvF,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEjE,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAE2D,aAAa,YAAY,CAAC;AAAvF,8BAAsB,GAAI,SAAQ,QAAsD;;IAKtF,QAAQ,CAAC,KAAK,IAAI,MAAM;IAExB;;;;;;;OAOG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;OAEG;IACH,QAAQ,CAAC,IAAI,CAAC,EACZ,OAAO,EACP,OAAO,EACP,WAAW,EACX,iBAAiB,EACjB,UAAU,EACV,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,CAAC,EAAE,iBAAiB,CAAC;QAChC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACvC,GAAG,SAAS;IAEb;;OAEG;IACH,OAAO,IAAI,IAAI;IAIT,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAG9B;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,SAAS,CAAC;;IACzE,SAAS,CAAC,MAAM,gCAAuC;IACvD,SAAS,CAAC,KAAK,gCAAuC;IACtD,SAAS,CAAC,MAAM,UAAS;IACzB,SAAS,CAAC,eAAe,kBAAyB;IAClD,SAAS,CAAC,YAAY,EAAE,iBAAiB,CAAC;IAC1C,SAAS,CAAC,MAAM,wBAAS;gBAOvB,GAAG,EAAE,GAAG,EACR,EACE,OAAO,EACP,OAAO,EACP,WAAW,GACZ,EAAE;QACD,OAAO,EAAE,WAAW,CAAC;QACrB,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,WAAW,EAAE,iBAAiB,CAAC;KAChC;YAoBW,QAAQ;IAoCtB,OAAO,CAAC,SAAS;cAUD,cAAc;IA6C9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,2CAA2C;IAC3C,IAAI,OAAO,IAAI,WAAW,GAAG,SAAS,CAErC;IAED,+CAA+C;IAC/C,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,8CAA8C;IAC9C,IAAI,WAAW,IAAI,iBAAiB,CAEnC;IAED,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAI1C,KAAK;IAIL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}

package/dist/llm/llm.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, any>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n    };\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAC7B,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,WAAW;AAGpB,SAAS,oBAAoB,OAAO,WAAW,eAAe;AAC9D,eAAmE;AAmC5D,MAAe,YAAa,aAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,mBAA8B;AAAA,EAC3C,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,SAAS,IAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,cAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,IACL;AACA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}
+{"version":3,"sources":["../../src/llm/llm.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { log } from '../log.js';\nimport type { LLMMetrics } from '../metrics/base.js';\nimport type { APIConnectOptions } from '../types.js';\nimport { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';\nimport { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport interface ChoiceDelta {\n  role: ChatRole;\n  content?: string;\n  toolCalls?: FunctionCall[];\n}\n\nexport interface CompletionUsage {\n  completionTokens: number;\n  promptTokens: number;\n  promptCachedTokens: number;\n  totalTokens: number;\n}\n\nexport interface ChatChunk {\n  id: string;\n  delta?: ChoiceDelta;\n  usage?: CompletionUsage;\n}\n\nexport interface LLMError {\n  type: 'llm_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type LLMCallbacks = {\n  ['metrics_collected']: (metrics: LLMMetrics) => void;\n  ['error']: (error: LLMError) => void;\n};\n\nexport abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {\n  constructor() {\n    super();\n  }\n\n  abstract label(): string;\n\n  /**\n   * Get the model name/identifier for this LLM instance.\n   *\n   * @returns The model name if available, \"unknown\" otherwise.\n   *\n   * @remarks\n   * Plugins should override this property to provide their model information.\n   */\n  get model(): string {\n    return 'unknown';\n  }\n\n  /**\n   * Returns a {@link LLMStream} that can be used to push text and receive LLM responses.\n   */\n  abstract chat({\n    chatCtx,\n    toolCtx,\n    connOptions,\n    parallelToolCalls,\n    toolChoice,\n    extraKwargs,\n  }: {\n    chatCtx: ChatContext;\n    toolCtx?: ToolContext;\n    connOptions?: APIConnectOptions;\n    parallelToolCalls?: boolean;\n    toolChoice?: ToolChoice;\n    extraKwargs?: Record<string, unknown>;\n  }): LLMStream;\n\n  /**\n   * Pre-warm connection to the LLM service\n   */\n  prewarm(): void {\n    // Default implementation - subclasses can override\n  }\n\n  async aclose(): Promise<void> {\n    // Default implementation - subclasses can override\n  }\n}\n\nexport abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {\n  protected output = new AsyncIterableQueue<ChatChunk>();\n  protected queue = new AsyncIterableQueue<ChatChunk>();\n  protected closed = false;\n  protected abortController = new AbortController();\n  protected _connOptions: APIConnectOptions;\n  protected logger = log();\n\n  #llm: LLM;\n  #chatCtx: ChatContext;\n  #toolCtx?: ToolContext;\n\n  constructor(\n    llm: LLM,\n    {\n      chatCtx,\n      toolCtx,\n      connOptions,\n    }: {\n      chatCtx: ChatContext;\n      toolCtx?: ToolContext;\n      connOptions: APIConnectOptions;\n    },\n  ) {\n    this.#llm = llm;\n    this.#chatCtx = chatCtx;\n    this.#toolCtx = toolCtx;\n    this._connOptions = connOptions;\n    this.monitorMetrics();\n    this.abortController.signal.addEventListener('abort', () => {\n      // TODO (AJS-37) clean this up when we refactor with streams\n      this.output.close();\n      this.closed = true;\n    });\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to generate LLM completion after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { llm: this.#llm.label(), attempt: i + 1, error },\n              `failed to generate LLM completion, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#llm.emit('error', {\n      type: 'llm_error',\n      timestamp: Date.now(),\n      label: this.#llm.label(),\n      error,\n      recoverable,\n    });\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n    let ttft: bigint = BigInt(-1);\n    let requestId = '';\n    let usage: CompletionUsage | undefined;\n\n    for await (const ev of this.queue) {\n      if (this.abortController.signal.aborted) {\n        break;\n      }\n      this.output.put(ev);\n      requestId = ev.id;\n      if (ttft === BigInt(-1)) {\n        ttft = process.hrtime.bigint() - startTime;\n      }\n      if (ev.usage) {\n        usage = ev.usage;\n      }\n    }\n    this.output.close();\n\n    const duration = process.hrtime.bigint() - startTime;\n    const durationMs = Math.trunc(Number(duration / BigInt(1000000)));\n    const metrics: LLMMetrics = {\n      type: 'llm_metrics',\n      timestamp: Date.now(),\n      requestId,\n      ttftMs: ttft === BigInt(-1) ? -1 : Math.trunc(Number(ttft / BigInt(1000000))),\n      durationMs,\n      cancelled: this.abortController.signal.aborted,\n      label: this.#llm.label(),\n      completionTokens: usage?.completionTokens || 0,\n      promptTokens: usage?.promptTokens || 0,\n      promptCachedTokens: usage?.promptCachedTokens || 0,\n      totalTokens: usage?.totalTokens || 0,\n      tokensPerSecond: (() => {\n        if (durationMs <= 0) {\n          return 0;\n        }\n        return (usage?.completionTokens || 0) / (durationMs / 1000);\n      })(),\n    };\n    this.#llm.emit('metrics_collected', metrics);\n  }\n\n  protected abstract run(): Promise<void>;\n\n  /** The function context of this stream. */\n  get toolCtx(): ToolContext | undefined {\n    return this.#toolCtx;\n  }\n\n  /** The initial chat context of this stream. */\n  get chatCtx(): ChatContext {\n    return this.#chatCtx;\n  }\n\n  /** The connection options for this stream. */\n  get connOptions(): APIConnectOptions {\n    return this._connOptions;\n  }\n\n  next(): Promise<IteratorResult<ChatChunk>> {\n    return this.output.next();\n  }\n\n  close() {\n    this.abortController.abort();\n  }\n\n  [Symbol.asyncIterator](): LLMStream {\n    return this;\n  }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAC7B,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,WAAW;AAGpB,SAAS,oBAAoB,OAAO,WAAW,eAAe;AAC9D,eAAmE;AAmC5D,MAAe,YAAa,aAAsD;AAAA,EACvF,cAAc;AACZ,UAAM;AAAA,EACR;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAwBA,UAAgB;AAAA,EAEhB;AAAA,EAEA,MAAM,SAAwB;AAAA,EAE9B;AACF;AAEO,MAAe,UAAsD;AAAA,EAChE,SAAS,IAAI,mBAA8B;AAAA,EAC3C,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EACT,kBAAkB,IAAI,gBAAgB;AAAA,EACtC;AAAA,EACA,SAAS,IAAI;AAAA,EAEvB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,KACA;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKA;AACA,SAAK,OAAO;AACZ,SAAK,WAAW;AAChB,SAAK,WAAW;AAChB,SAAK,eAAe;AACpB,SAAK,eAAe;AACpB,SAAK,gBAAgB,OAAO,iBAAiB,SAAS,MAAM;AAE1D,WAAK,OAAO,MAAM;AAClB,WAAK,SAAS;AAAA,IAChB,CAAC;AAMD,cAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,2CAA2C,KAAK,aAAa,WAAW,CAAC;AAAA,cAClF,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,MAAM,GAAG,SAAS,IAAI,GAAG,MAAM;AAAA,cAChD,kDAAkD,aAAa;AAAA,YACjE;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,QAAI,OAAe,OAAO,EAAE;AAC5B,QAAI,YAAY;AAChB,QAAI;AAEJ,qBAAiB,MAAM,KAAK,OAAO;AACjC,UAAI,KAAK,gBAAgB,OAAO,SAAS;AACvC;AAAA,MACF;AACA,WAAK,OAAO,IAAI,EAAE;AAClB,kBAAY,GAAG;AACf,UAAI,SAAS,OAAO,EAAE,GAAG;AACvB,eAAO,QAAQ,OAAO,OAAO,IAAI;AAAA,MACnC;AACA,UAAI,GAAG,OAAO;AACZ,gBAAQ,GAAG;AAAA,MACb;AAAA,IACF;AACA,SAAK,OAAO,MAAM;AAElB,UAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,UAAM,aAAa,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAChE,UAAM,UAAsB;AAAA,MAC1B,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,QAAQ,SAAS,OAAO,EAAE,IAAI,KAAK,KAAK,MAAM,OAAO,OAAO,OAAO,GAAO,CAAC,CAAC;AAAA,MAC5E;AAAA,MACA,WAAW,KAAK,gBAAgB,OAAO;AAAA,MACvC,OAAO,KAAK,KAAK,MAAM;AAAA,MACvB,mBAAkB,+BAAO,qBAAoB;AAAA,MAC7C,eAAc,+BAAO,iBAAgB;AAAA,MACrC,qBAAoB,+BAAO,uBAAsB;AAAA,MACjD,cAAa,+BAAO,gBAAe;AAAA,MACnC,kBAAkB,MAAM;AACtB,YAAI,cAAc,GAAG;AACnB,iBAAO;AAAA,QACT;AACA,iBAAQ,+BAAO,qBAAoB,MAAM,aAAa;AAAA,MACxD,GAAG;AAAA,IACL;AACA,SAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,EAC7C;AAAA;AAAA,EAKA,IAAI,UAAmC;AACrC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,cAAiC;AACnC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,QAAQ;AACN,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/llm/provider_format/google.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/llm/provider_format/google.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChatContext, ChatItem, ImageContent } from '../chat_context.js';\nimport { type SerializedImage, serializeImage } from '../utils.js';\nimport { groupToolCalls } from './utils.js';\n\nexport interface GoogleFormatData {\n  systemMessages: string[] | null;\n}\n\nexport async function toChatCtx(\n  chatCtx: ChatContext,\n  injectDummyUserMessage: boolean = true,\n): Promise<[Record<string, any>[], GoogleFormatData]> {\n  const turns: Record<string, any>[] = [];\n  const systemMessages: string[] = [];\n  let currentRole: string | null = null;\n  let parts: Record<string, any>[] = [];\n\n  // Flatten all grouped tool calls to get individual messages\n  const itemGroups = groupToolCalls(chatCtx);\n  const flattenedItems: ChatItem[] = [];\n\n  for (const group of itemGroups) {\n    flattenedItems.push(...group.flatten());\n  }\n\n  for (const msg of flattenedItems) {\n    // Handle system messages separately\n    if (msg.type === 'message' && msg.role === 'system' && msg.textContent) {\n      systemMessages.push(msg.textContent);\n      continue;\n    }\n\n    let role: string;\n    if (msg.type === 'message') {\n      role = msg.role === 'assistant' ? 'model' : 'user';\n    } else if (msg.type === 'function_call') {\n      role = 'model';\n    } else if (msg.type === 'function_call_output') {\n      role = 'user';\n    } else {\n      continue; // Skip unknown message types\n    }\n\n    // If the effective role changed, finalize the previous turn\n    if (role !== currentRole) {\n      if (currentRole !== null && parts.length > 0) {\n        turns.push({ role: currentRole, parts: [...parts] });\n      }\n      parts = [];\n      currentRole = role;\n    }\n\n    if (msg.type === 'message') {\n      for (const content of msg.content) {\n        if (content && typeof content === 'string') {\n          parts.push({ text: content });\n        } else if (content && typeof content === 'object') {\n          if (content.type === 'image_content') {\n            parts.push(await toImagePart(content));\n          } else {\n            // Handle other content types as JSON\n            parts.push({ text: JSON.stringify(content) });\n          }\n        }\n      }\n    } else if (msg.type === 'function_call') {\n      parts.push({\n        functionCall: {\n          id: msg.callId,\n          name: msg.name,\n          args: JSON.parse(msg.args || '{}'),\n        },\n      });\n    } else if (msg.type === 'function_call_output') {\n      const response = msg.isError ? { error: msg.output } : { output: msg.output };\n      parts.push({\n        functionResponse: {\n          id: msg.callId,\n          name: msg.name,\n          response,\n        },\n      });\n    }\n  }\n\n  // Finalize the last turn\n  if (currentRole !== null && parts.length > 0) {\n    turns.push({ role: currentRole, parts });\n  }\n\n  // Gemini requires the last message to end with user's turn before they can generate\n  if (injectDummyUserMessage && currentRole !== 'user') {\n    turns.push({ role: 'user', parts: [{ text: '.' }] });\n  }\n\n  return [\n    turns,\n    {\n      systemMessages: systemMessages.length > 0 ? systemMessages : null,\n    },\n  ];\n}\n\nasync function toImagePart(image: ImageContent): Promise<Record<string, any>> {\n  const cacheKey = 'serialized_image';\n  if (!image._cache[cacheKey]) {\n    image._cache[cacheKey] = await serializeImage(image);\n  }\n  const img: SerializedImage = image._cache[cacheKey];\n\n  if (img.externalUrl) {\n    const mimeType = img.mimeType || 'image/jpeg';\n    return {\n      fileData: {\n        fileUri: img.externalUrl,\n        mimeType,\n      },\n    };\n  }\n\n  return {\n    inlineData: {\n      data: img.base64Data,\n      mimeType: img.mimeType,\n    },\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAqD;AACrD,IAAAA,gBAA+B;AAM/B,eAAsB,UACpB,SACA,yBAAkC,MACkB;AACpD,QAAM,QAA+B,CAAC;AACtC,QAAM,iBAA2B,CAAC;AAClC,MAAI,cAA6B;AACjC,MAAI,QAA+B,CAAC;AAGpC,QAAM,iBAAa,8BAAe,OAAO;AACzC,QAAM,iBAA6B,CAAC;AAEpC,aAAW,SAAS,YAAY;AAC9B,mBAAe,KAAK,GAAG,MAAM,QAAQ,CAAC;AAAA,EACxC;AAEA,aAAW,OAAO,gBAAgB;AAEhC,QAAI,IAAI,SAAS,aAAa,IAAI,SAAS,YAAY,IAAI,aAAa;AACtE,qBAAe,KAAK,IAAI,WAAW;AACnC;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,IAAI,SAAS,WAAW;AAC1B,aAAO,IAAI,SAAS,cAAc,UAAU;AAAA,IAC9C,WAAW,IAAI,SAAS,iBAAiB;AACvC,aAAO;AAAA,IACT,WAAW,IAAI,SAAS,wBAAwB;AAC9C,aAAO;AAAA,IACT,OAAO;AACL;AAAA,IACF;AAGA,QAAI,SAAS,aAAa;AACxB,UAAI,gBAAgB,QAAQ,MAAM,SAAS,GAAG;AAC5C,cAAM,KAAK,EAAE,MAAM,aAAa,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;AAAA,MACrD;AACA,cAAQ,CAAC;AACT,oBAAc;AAAA,IAChB;AAEA,QAAI,IAAI,SAAS,WAAW;AAC1B,iBAAW,WAAW,IAAI,SAAS;AACjC,YAAI,WAAW,OAAO,YAAY,UAAU;AAC1C,gBAAM,KAAK,EAAE,MAAM,QAAQ,CAAC;AAAA,QAC9B,WAAW,WAAW,OAAO,YAAY,UAAU;AACjD,cAAI,QAAQ,SAAS,iBAAiB;AACpC,kBAAM,KAAK,MAAM,YAAY,OAAO,CAAC;AAAA,UACvC,OAAO;AAEL,kBAAM,KAAK,EAAE,MAAM,KAAK,UAAU,OAAO,EAAE,CAAC;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AAAA,IACF,WAAW,IAAI,SAAS,iBAAiB;AACvC,YAAM,KAAK;AAAA,QACT,cAAc;AAAA,UACZ,IAAI,IAAI;AAAA,UACR,MAAM,IAAI;AAAA,UACV,MAAM,KAAK,MAAM,IAAI,QAAQ,IAAI;AAAA,QACnC;AAAA,MACF,CAAC;AAAA,IACH,WAAW,IAAI,SAAS,wBAAwB;AAC9C,YAAM,WAAW,IAAI,UAAU,EAAE,OAAO,IAAI,OAAO,IAAI,EAAE,QAAQ,IAAI,OAAO;AAC5E,YAAM,KAAK;AAAA,QACT,kBAAkB;AAAA,UAChB,IAAI,IAAI;AAAA,UACR,MAAM,IAAI;AAAA,UACV;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAGA,MAAI,gBAAgB,QAAQ,MAAM,SAAS,GAAG;AAC5C,UAAM,KAAK,EAAE,MAAM,aAAa,MAAM,CAAC;AAAA,EACzC;AAGA,MAAI,0BAA0B,gBAAgB,QAAQ;AACpD,UAAM,KAAK,EAAE,MAAM,QAAQ,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,EACrD;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,MACE,gBAAgB,eAAe,SAAS,IAAI,iBAAiB;AAAA,IAC/D;AAAA,EACF;AACF;AAEA,eAAe,YAAY,OAAmD;AAC5E,QAAM,WAAW;AACjB,MAAI,CAAC,MAAM,OAAO,QAAQ,GAAG;AAC3B,UAAM,OAAO,QAAQ,IAAI,UAAM,6BAAe,KAAK;AAAA,EACrD;AACA,QAAM,MAAuB,MAAM,OAAO,QAAQ;AAElD,MAAI,IAAI,aAAa;AACnB,UAAM,WAAW,IAAI,YAAY;AACjC,WAAO;AAAA,MACL,UAAU;AAAA,QACR,SAAS,IAAI;AAAA,QACb;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,YAAY;AAAA,MACV,MAAM,IAAI;AAAA,MACV,UAAU,IAAI;AAAA,IAChB;AAAA,EACF;AACF;","names":["import_utils"]}
+{"version":3,"sources":["../../../src/llm/provider_format/google.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChatContext, ChatItem, ImageContent } from '../chat_context.js';\nimport { type SerializedImage, serializeImage } from '../utils.js';\nimport { groupToolCalls } from './utils.js';\n\nexport interface GoogleFormatData {\n  systemMessages: string[] | null;\n}\n\nexport async function toChatCtx(\n  chatCtx: ChatContext,\n  injectDummyUserMessage: boolean = true,\n): Promise<[Record<string, unknown>[], GoogleFormatData]> {\n  const turns: Record<string, unknown>[] = [];\n  const systemMessages: string[] = [];\n  let currentRole: string | null = null;\n  let parts: Record<string, unknown>[] = [];\n\n  // Flatten all grouped tool calls to get individual messages\n  const itemGroups = groupToolCalls(chatCtx);\n  const flattenedItems: ChatItem[] = [];\n\n  for (const group of itemGroups) {\n    flattenedItems.push(...group.flatten());\n  }\n\n  for (const msg of flattenedItems) {\n    // Handle system messages separately\n    if (msg.type === 'message' && msg.role === 'system' && msg.textContent) {\n      systemMessages.push(msg.textContent);\n      continue;\n    }\n\n    let role: string;\n    if (msg.type === 'message') {\n      role = msg.role === 'assistant' ? 'model' : 'user';\n    } else if (msg.type === 'function_call') {\n      role = 'model';\n    } else if (msg.type === 'function_call_output') {\n      role = 'user';\n    } else {\n      continue; // Skip unknown message types\n    }\n\n    // If the effective role changed, finalize the previous turn\n    if (role !== currentRole) {\n      if (currentRole !== null && parts.length > 0) {\n        turns.push({ role: currentRole, parts: [...parts] });\n      }\n      parts = [];\n      currentRole = role;\n    }\n\n    if (msg.type === 'message') {\n      for (const content of msg.content) {\n        if (content && typeof content === 'string') {\n          parts.push({ text: content });\n        } else if (content && typeof content === 'object') {\n          if (content.type === 'image_content') {\n            parts.push(await toImagePart(content));\n          } else {\n            // Handle other content types as JSON\n            parts.push({ text: JSON.stringify(content) });\n          }\n        }\n      }\n    } else if (msg.type === 'function_call') {\n      parts.push({\n        functionCall: {\n          id: msg.callId,\n          name: msg.name,\n          args: JSON.parse(msg.args || '{}'),\n        },\n      });\n    } else if (msg.type === 'function_call_output') {\n      const response = msg.isError ? { error: msg.output } : { output: msg.output };\n      parts.push({\n        functionResponse: {\n          id: msg.callId,\n          name: msg.name,\n          response,\n        },\n      });\n    }\n  }\n\n  // Finalize the last turn\n  if (currentRole !== null && parts.length > 0) {\n    turns.push({ role: currentRole, parts });\n  }\n\n  // Gemini requires the last message to end with user's turn before they can generate\n  if (injectDummyUserMessage && currentRole !== 'user') {\n    turns.push({ role: 'user', parts: [{ text: '.' }] });\n  }\n\n  return [\n    turns,\n    {\n      systemMessages: systemMessages.length > 0 ? systemMessages : null,\n    },\n  ];\n}\n\nasync function toImagePart(image: ImageContent): Promise<Record<string, unknown>> {\n  const cacheKey = 'serialized_image';\n  if (!image._cache[cacheKey]) {\n    image._cache[cacheKey] = await serializeImage(image);\n  }\n  const img: SerializedImage = image._cache[cacheKey];\n\n  if (img.externalUrl) {\n    const mimeType = img.mimeType || 'image/jpeg';\n    return {\n      fileData: {\n        fileUri: img.externalUrl,\n        mimeType,\n      },\n    };\n  }\n\n  return {\n    inlineData: {\n      data: img.base64Data,\n      mimeType: img.mimeType,\n    },\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAqD;AACrD,IAAAA,gBAA+B;AAM/B,eAAsB,UACpB,SACA,yBAAkC,MACsB;AACxD,QAAM,QAAmC,CAAC;AAC1C,QAAM,iBAA2B,CAAC;AAClC,MAAI,cAA6B;AACjC,MAAI,QAAmC,CAAC;AAGxC,QAAM,iBAAa,8BAAe,OAAO;AACzC,QAAM,iBAA6B,CAAC;AAEpC,aAAW,SAAS,YAAY;AAC9B,mBAAe,KAAK,GAAG,MAAM,QAAQ,CAAC;AAAA,EACxC;AAEA,aAAW,OAAO,gBAAgB;AAEhC,QAAI,IAAI,SAAS,aAAa,IAAI,SAAS,YAAY,IAAI,aAAa;AACtE,qBAAe,KAAK,IAAI,WAAW;AACnC;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,IAAI,SAAS,WAAW;AAC1B,aAAO,IAAI,SAAS,cAAc,UAAU;AAAA,IAC9C,WAAW,IAAI,SAAS,iBAAiB;AACvC,aAAO;AAAA,IACT,WAAW,IAAI,SAAS,wBAAwB;AAC9C,aAAO;AAAA,IACT,OAAO;AACL;AAAA,IACF;AAGA,QAAI,SAAS,aAAa;AACxB,UAAI,gBAAgB,QAAQ,MAAM,SAAS,GAAG;AAC5C,cAAM,KAAK,EAAE,MAAM,aAAa,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;AAAA,MACrD;AACA,cAAQ,CAAC;AACT,oBAAc;AAAA,IAChB;AAEA,QAAI,IAAI,SAAS,WAAW;AAC1B,iBAAW,WAAW,IAAI,SAAS;AACjC,YAAI,WAAW,OAAO,YAAY,UAAU;AAC1C,gBAAM,KAAK,EAAE,MAAM,QAAQ,CAAC;AAAA,QAC9B,WAAW,WAAW,OAAO,YAAY,UAAU;AACjD,cAAI,QAAQ,SAAS,iBAAiB;AACpC,kBAAM,KAAK,MAAM,YAAY,OAAO,CAAC;AAAA,UACvC,OAAO;AAEL,kBAAM,KAAK,EAAE,MAAM,KAAK,UAAU,OAAO,EAAE,CAAC;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AAAA,IACF,WAAW,IAAI,SAAS,iBAAiB;AACvC,YAAM,KAAK;AAAA,QACT,cAAc;AAAA,UACZ,IAAI,IAAI;AAAA,UACR,MAAM,IAAI;AAAA,UACV,MAAM,KAAK,MAAM,IAAI,QAAQ,IAAI;AAAA,QACnC;AAAA,MACF,CAAC;AAAA,IACH,WAAW,IAAI,SAAS,wBAAwB;AAC9C,YAAM,WAAW,IAAI,UAAU,EAAE,OAAO,IAAI,OAAO,IAAI,EAAE,QAAQ,IAAI,OAAO;AAC5E,YAAM,KAAK;AAAA,QACT,kBAAkB;AAAA,UAChB,IAAI,IAAI;AAAA,UACR,MAAM,IAAI;AAAA,UACV;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAGA,MAAI,gBAAgB,QAAQ,MAAM,SAAS,GAAG;AAC5C,UAAM,KAAK,EAAE,MAAM,aAAa,MAAM,CAAC;AAAA,EACzC;AAGA,MAAI,0BAA0B,gBAAgB,QAAQ;AACpD,UAAM,KAAK,EAAE,MAAM,QAAQ,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,EACrD;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,MACE,gBAAgB,eAAe,SAAS,IAAI,iBAAiB;AAAA,IAC/D;AAAA,EACF;AACF;AAEA,eAAe,YAAY,OAAuD;AAChF,QAAM,WAAW;AACjB,MAAI,CAAC,MAAM,OAAO,QAAQ,GAAG;AAC3B,UAAM,OAAO,QAAQ,IAAI,UAAM,6BAAe,KAAK;AAAA,EACrD;AACA,QAAM,MAAuB,MAAM,OAAO,QAAQ;AAElD,MAAI,IAAI,aAAa;AACnB,UAAM,WAAW,IAAI,YAAY;AACjC,WAAO;AAAA,MACL,UAAU;AAAA,QACR,SAAS,IAAI;AAAA,QACb;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,YAAY;AAAA,MACV,MAAM,IAAI;AAAA,MACV,UAAU,IAAI;AAAA,IAChB;AAAA,EACF;AACF;","names":["import_utils"]}

package/dist/llm/provider_format/google.d.cts

@@ -2,5 +2,5 @@ import type { ChatContext } from '../chat_context.js';
 export interface GoogleFormatData {
     systemMessages: string[] | null;
 }
-export declare function toChatCtx(chatCtx: ChatContext, injectDummyUserMessage?: boolean): Promise<[Record<string, any>[], GoogleFormatData]>;
+export declare function toChatCtx(chatCtx: ChatContext, injectDummyUserMessage?: boolean): Promise<[Record<string, unknown>[], GoogleFormatData]>;
 //# sourceMappingURL=google.d.ts.map

package/dist/llm/provider_format/google.d.ts

@@ -2,5 +2,5 @@ import type { ChatContext } from '../chat_context.js';
 export interface GoogleFormatData {
     systemMessages: string[] | null;
 }
-export declare function toChatCtx(chatCtx: ChatContext, injectDummyUserMessage?: boolean): Promise<[Record<string, any>[], GoogleFormatData]>;
+export declare function toChatCtx(chatCtx: ChatContext, injectDummyUserMessage?: boolean): Promise<[Record<string, unknown>[], GoogleFormatData]>;
 //# sourceMappingURL=google.d.ts.map

package/dist/llm/provider_format/google.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"google.d.ts","sourceRoot":"","sources":["../../../src/llm/provider_format/google.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAA0B,MAAM,oBAAoB,CAAC;AAI9E,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;CACjC;AAED,wBAAsB,SAAS,CAC7B,OAAO,EAAE,WAAW,EACpB,sBAAsB,GAAE,OAAc,GACrC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CA0FpD"}
+{"version":3,"file":"google.d.ts","sourceRoot":"","sources":["../../../src/llm/provider_format/google.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAA0B,MAAM,oBAAoB,CAAC;AAI9E,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;CACjC;AAED,wBAAsB,SAAS,CAC7B,OAAO,EAAE,WAAW,EACpB,sBAAsB,GAAE,OAAc,GACrC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CA0FxD"}

package/dist/llm/provider_format/google.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/llm/provider_format/google.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChatContext, ChatItem, ImageContent } from '../chat_context.js';\nimport { type SerializedImage, serializeImage } from '../utils.js';\nimport { groupToolCalls } from './utils.js';\n\nexport interface GoogleFormatData {\n  systemMessages: string[] | null;\n}\n\nexport async function toChatCtx(\n  chatCtx: ChatContext,\n  injectDummyUserMessage: boolean = true,\n): Promise<[Record<string, any>[], GoogleFormatData]> {\n  const turns: Record<string, any>[] = [];\n  const systemMessages: string[] = [];\n  let currentRole: string | null = null;\n  let parts: Record<string, any>[] = [];\n\n  // Flatten all grouped tool calls to get individual messages\n  const itemGroups = groupToolCalls(chatCtx);\n  const flattenedItems: ChatItem[] = [];\n\n  for (const group of itemGroups) {\n    flattenedItems.push(...group.flatten());\n  }\n\n  for (const msg of flattenedItems) {\n    // Handle system messages separately\n    if (msg.type === 'message' && msg.role === 'system' && msg.textContent) {\n      systemMessages.push(msg.textContent);\n      continue;\n    }\n\n    let role: string;\n    if (msg.type === 'message') {\n      role = msg.role === 'assistant' ? 'model' : 'user';\n    } else if (msg.type === 'function_call') {\n      role = 'model';\n    } else if (msg.type === 'function_call_output') {\n      role = 'user';\n    } else {\n      continue; // Skip unknown message types\n    }\n\n    // If the effective role changed, finalize the previous turn\n    if (role !== currentRole) {\n      if (currentRole !== null && parts.length > 0) {\n        turns.push({ role: currentRole, parts: [...parts] });\n      }\n      parts = [];\n      currentRole = role;\n    }\n\n    if (msg.type === 'message') {\n      for (const content of msg.content) {\n        if (content && typeof content === 'string') {\n          parts.push({ text: content });\n        } else if (content && typeof content === 'object') {\n          if (content.type === 'image_content') {\n            parts.push(await toImagePart(content));\n          } else {\n            // Handle other content types as JSON\n            parts.push({ text: JSON.stringify(content) });\n          }\n        }\n      }\n    } else if (msg.type === 'function_call') {\n      parts.push({\n        functionCall: {\n          id: msg.callId,\n          name: msg.name,\n          args: JSON.parse(msg.args || '{}'),\n        },\n      });\n    } else if (msg.type === 'function_call_output') {\n      const response = msg.isError ? { error: msg.output } : { output: msg.output };\n      parts.push({\n        functionResponse: {\n          id: msg.callId,\n          name: msg.name,\n          response,\n        },\n      });\n    }\n  }\n\n  // Finalize the last turn\n  if (currentRole !== null && parts.length > 0) {\n    turns.push({ role: currentRole, parts });\n  }\n\n  // Gemini requires the last message to end with user's turn before they can generate\n  if (injectDummyUserMessage && currentRole !== 'user') {\n    turns.push({ role: 'user', parts: [{ text: '.' }] });\n  }\n\n  return [\n    turns,\n    {\n      systemMessages: systemMessages.length > 0 ? systemMessages : null,\n    },\n  ];\n}\n\nasync function toImagePart(image: ImageContent): Promise<Record<string, any>> {\n  const cacheKey = 'serialized_image';\n  if (!image._cache[cacheKey]) {\n    image._cache[cacheKey] = await serializeImage(image);\n  }\n  const img: SerializedImage = image._cache[cacheKey];\n\n  if (img.externalUrl) {\n    const mimeType = img.mimeType || 'image/jpeg';\n    return {\n      fileData: {\n        fileUri: img.externalUrl,\n        mimeType,\n      },\n    };\n  }\n\n  return {\n    inlineData: {\n      data: img.base64Data,\n      mimeType: img.mimeType,\n    },\n  };\n}\n"],"mappings":"AAIA,SAA+B,sBAAsB;AACrD,SAAS,sBAAsB;AAM/B,eAAsB,UACpB,SACA,yBAAkC,MACkB;AACpD,QAAM,QAA+B,CAAC;AACtC,QAAM,iBAA2B,CAAC;AAClC,MAAI,cAA6B;AACjC,MAAI,QAA+B,CAAC;AAGpC,QAAM,aAAa,eAAe,OAAO;AACzC,QAAM,iBAA6B,CAAC;AAEpC,aAAW,SAAS,YAAY;AAC9B,mBAAe,KAAK,GAAG,MAAM,QAAQ,CAAC;AAAA,EACxC;AAEA,aAAW,OAAO,gBAAgB;AAEhC,QAAI,IAAI,SAAS,aAAa,IAAI,SAAS,YAAY,IAAI,aAAa;AACtE,qBAAe,KAAK,IAAI,WAAW;AACnC;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,IAAI,SAAS,WAAW;AAC1B,aAAO,IAAI,SAAS,cAAc,UAAU;AAAA,IAC9C,WAAW,IAAI,SAAS,iBAAiB;AACvC,aAAO;AAAA,IACT,WAAW,IAAI,SAAS,wBAAwB;AAC9C,aAAO;AAAA,IACT,OAAO;AACL;AAAA,IACF;AAGA,QAAI,SAAS,aAAa;AACxB,UAAI,gBAAgB,QAAQ,MAAM,SAAS,GAAG;AAC5C,cAAM,KAAK,EAAE,MAAM,aAAa,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;AAAA,MACrD;AACA,cAAQ,CAAC;AACT,oBAAc;AAAA,IAChB;AAEA,QAAI,IAAI,SAAS,WAAW;AAC1B,iBAAW,WAAW,IAAI,SAAS;AACjC,YAAI,WAAW,OAAO,YAAY,UAAU;AAC1C,gBAAM,KAAK,EAAE,MAAM,QAAQ,CAAC;AAAA,QAC9B,WAAW,WAAW,OAAO,YAAY,UAAU;AACjD,cAAI,QAAQ,SAAS,iBAAiB;AACpC,kBAAM,KAAK,MAAM,YAAY,OAAO,CAAC;AAAA,UACvC,OAAO;AAEL,kBAAM,KAAK,EAAE,MAAM,KAAK,UAAU,OAAO,EAAE,CAAC;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AAAA,IACF,WAAW,IAAI,SAAS,iBAAiB;AACvC,YAAM,KAAK;AAAA,QACT,cAAc;AAAA,UACZ,IAAI,IAAI;AAAA,UACR,MAAM,IAAI;AAAA,UACV,MAAM,KAAK,MAAM,IAAI,QAAQ,IAAI;AAAA,QACnC;AAAA,MACF,CAAC;AAAA,IACH,WAAW,IAAI,SAAS,wBAAwB;AAC9C,YAAM,WAAW,IAAI,UAAU,EAAE,OAAO,IAAI,OAAO,IAAI,EAAE,QAAQ,IAAI,OAAO;AAC5E,YAAM,KAAK;AAAA,QACT,kBAAkB;AAAA,UAChB,IAAI,IAAI;AAAA,UACR,MAAM,IAAI;AAAA,UACV;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAGA,MAAI,gBAAgB,QAAQ,MAAM,SAAS,GAAG;AAC5C,UAAM,KAAK,EAAE,MAAM,aAAa,MAAM,CAAC;AAAA,EACzC;AAGA,MAAI,0BAA0B,gBAAgB,QAAQ;AACpD,UAAM,KAAK,EAAE,MAAM,QAAQ,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,EACrD;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,MACE,gBAAgB,eAAe,SAAS,IAAI,iBAAiB;AAAA,IAC/D;AAAA,EACF;AACF;AAEA,eAAe,YAAY,OAAmD;AAC5E,QAAM,WAAW;AACjB,MAAI,CAAC,MAAM,OAAO,QAAQ,GAAG;AAC3B,UAAM,OAAO,QAAQ,IAAI,MAAM,eAAe,KAAK;AAAA,EACrD;AACA,QAAM,MAAuB,MAAM,OAAO,QAAQ;AAElD,MAAI,IAAI,aAAa;AACnB,UAAM,WAAW,IAAI,YAAY;AACjC,WAAO;AAAA,MACL,UAAU;AAAA,QACR,SAAS,IAAI;AAAA,QACb;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,YAAY;AAAA,MACV,MAAM,IAAI;AAAA,MACV,UAAU,IAAI;AAAA,IAChB;AAAA,EACF;AACF;","names":[]}
+{"version":3,"sources":["../../../src/llm/provider_format/google.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChatContext, ChatItem, ImageContent } from '../chat_context.js';\nimport { type SerializedImage, serializeImage } from '../utils.js';\nimport { groupToolCalls } from './utils.js';\n\nexport interface GoogleFormatData {\n  systemMessages: string[] | null;\n}\n\nexport async function toChatCtx(\n  chatCtx: ChatContext,\n  injectDummyUserMessage: boolean = true,\n): Promise<[Record<string, unknown>[], GoogleFormatData]> {\n  const turns: Record<string, unknown>[] = [];\n  const systemMessages: string[] = [];\n  let currentRole: string | null = null;\n  let parts: Record<string, unknown>[] = [];\n\n  // Flatten all grouped tool calls to get individual messages\n  const itemGroups = groupToolCalls(chatCtx);\n  const flattenedItems: ChatItem[] = [];\n\n  for (const group of itemGroups) {\n    flattenedItems.push(...group.flatten());\n  }\n\n  for (const msg of flattenedItems) {\n    // Handle system messages separately\n    if (msg.type === 'message' && msg.role === 'system' && msg.textContent) {\n      systemMessages.push(msg.textContent);\n      continue;\n    }\n\n    let role: string;\n    if (msg.type === 'message') {\n      role = msg.role === 'assistant' ? 'model' : 'user';\n    } else if (msg.type === 'function_call') {\n      role = 'model';\n    } else if (msg.type === 'function_call_output') {\n      role = 'user';\n    } else {\n      continue; // Skip unknown message types\n    }\n\n    // If the effective role changed, finalize the previous turn\n    if (role !== currentRole) {\n      if (currentRole !== null && parts.length > 0) {\n        turns.push({ role: currentRole, parts: [...parts] });\n      }\n      parts = [];\n      currentRole = role;\n    }\n\n    if (msg.type === 'message') {\n      for (const content of msg.content) {\n        if (content && typeof content === 'string') {\n          parts.push({ text: content });\n        } else if (content && typeof content === 'object') {\n          if (content.type === 'image_content') {\n            parts.push(await toImagePart(content));\n          } else {\n            // Handle other content types as JSON\n            parts.push({ text: JSON.stringify(content) });\n          }\n        }\n      }\n    } else if (msg.type === 'function_call') {\n      parts.push({\n        functionCall: {\n          id: msg.callId,\n          name: msg.name,\n          args: JSON.parse(msg.args || '{}'),\n        },\n      });\n    } else if (msg.type === 'function_call_output') {\n      const response = msg.isError ? { error: msg.output } : { output: msg.output };\n      parts.push({\n        functionResponse: {\n          id: msg.callId,\n          name: msg.name,\n          response,\n        },\n      });\n    }\n  }\n\n  // Finalize the last turn\n  if (currentRole !== null && parts.length > 0) {\n    turns.push({ role: currentRole, parts });\n  }\n\n  // Gemini requires the last message to end with user's turn before they can generate\n  if (injectDummyUserMessage && currentRole !== 'user') {\n    turns.push({ role: 'user', parts: [{ text: '.' }] });\n  }\n\n  return [\n    turns,\n    {\n      systemMessages: systemMessages.length > 0 ? systemMessages : null,\n    },\n  ];\n}\n\nasync function toImagePart(image: ImageContent): Promise<Record<string, unknown>> {\n  const cacheKey = 'serialized_image';\n  if (!image._cache[cacheKey]) {\n    image._cache[cacheKey] = await serializeImage(image);\n  }\n  const img: SerializedImage = image._cache[cacheKey];\n\n  if (img.externalUrl) {\n    const mimeType = img.mimeType || 'image/jpeg';\n    return {\n      fileData: {\n        fileUri: img.externalUrl,\n        mimeType,\n      },\n    };\n  }\n\n  return {\n    inlineData: {\n      data: img.base64Data,\n      mimeType: img.mimeType,\n    },\n  };\n}\n"],"mappings":"AAIA,SAA+B,sBAAsB;AACrD,SAAS,sBAAsB;AAM/B,eAAsB,UACpB,SACA,yBAAkC,MACsB;AACxD,QAAM,QAAmC,CAAC;AAC1C,QAAM,iBAA2B,CAAC;AAClC,MAAI,cAA6B;AACjC,MAAI,QAAmC,CAAC;AAGxC,QAAM,aAAa,eAAe,OAAO;AACzC,QAAM,iBAA6B,CAAC;AAEpC,aAAW,SAAS,YAAY;AAC9B,mBAAe,KAAK,GAAG,MAAM,QAAQ,CAAC;AAAA,EACxC;AAEA,aAAW,OAAO,gBAAgB;AAEhC,QAAI,IAAI,SAAS,aAAa,IAAI,SAAS,YAAY,IAAI,aAAa;AACtE,qBAAe,KAAK,IAAI,WAAW;AACnC;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,IAAI,SAAS,WAAW;AAC1B,aAAO,IAAI,SAAS,cAAc,UAAU;AAAA,IAC9C,WAAW,IAAI,SAAS,iBAAiB;AACvC,aAAO;AAAA,IACT,WAAW,IAAI,SAAS,wBAAwB;AAC9C,aAAO;AAAA,IACT,OAAO;AACL;AAAA,IACF;AAGA,QAAI,SAAS,aAAa;AACxB,UAAI,gBAAgB,QAAQ,MAAM,SAAS,GAAG;AAC5C,cAAM,KAAK,EAAE,MAAM,aAAa,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;AAAA,MACrD;AACA,cAAQ,CAAC;AACT,oBAAc;AAAA,IAChB;AAEA,QAAI,IAAI,SAAS,WAAW;AAC1B,iBAAW,WAAW,IAAI,SAAS;AACjC,YAAI,WAAW,OAAO,YAAY,UAAU;AAC1C,gBAAM,KAAK,EAAE,MAAM,QAAQ,CAAC;AAAA,QAC9B,WAAW,WAAW,OAAO,YAAY,UAAU;AACjD,cAAI,QAAQ,SAAS,iBAAiB;AACpC,kBAAM,KAAK,MAAM,YAAY,OAAO,CAAC;AAAA,UACvC,OAAO;AAEL,kBAAM,KAAK,EAAE,MAAM,KAAK,UAAU,OAAO,EAAE,CAAC;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AAAA,IACF,WAAW,IAAI,SAAS,iBAAiB;AACvC,YAAM,KAAK;AAAA,QACT,cAAc;AAAA,UACZ,IAAI,IAAI;AAAA,UACR,MAAM,IAAI;AAAA,UACV,MAAM,KAAK,MAAM,IAAI,QAAQ,IAAI;AAAA,QACnC;AAAA,MACF,CAAC;AAAA,IACH,WAAW,IAAI,SAAS,wBAAwB;AAC9C,YAAM,WAAW,IAAI,UAAU,EAAE,OAAO,IAAI,OAAO,IAAI,EAAE,QAAQ,IAAI,OAAO;AAC5E,YAAM,KAAK;AAAA,QACT,kBAAkB;AAAA,UAChB,IAAI,IAAI;AAAA,UACR,MAAM,IAAI;AAAA,UACV;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAGA,MAAI,gBAAgB,QAAQ,MAAM,SAAS,GAAG;AAC5C,UAAM,KAAK,EAAE,MAAM,aAAa,MAAM,CAAC;AAAA,EACzC;AAGA,MAAI,0BAA0B,gBAAgB,QAAQ;AACpD,UAAM,KAAK,EAAE,MAAM,QAAQ,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,EACrD;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,MACE,gBAAgB,eAAe,SAAS,IAAI,iBAAiB;AAAA,IAC/D;AAAA,EACF;AACF;AAEA,eAAe,YAAY,OAAuD;AAChF,QAAM,WAAW;AACjB,MAAI,CAAC,MAAM,OAAO,QAAQ,GAAG;AAC3B,UAAM,OAAO,QAAQ,IAAI,MAAM,eAAe,KAAK;AAAA,EACrD;AACA,QAAM,MAAuB,MAAM,OAAO,QAAQ;AAElD,MAAI,IAAI,aAAa;AACnB,UAAM,WAAW,IAAI,YAAY;AACjC,WAAO;AAAA,MACL,UAAU;AAAA,QACR,SAAS,IAAI;AAAA,QACb;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,YAAY;AAAA,MACV,MAAM,IAAI;AAAA,MACV,UAAU,IAAI;AAAA,IAChB;AAAA,EACF;AACF;","names":[]}

package/dist/llm/provider_format/index.d.cts

@@ -1,4 +1,4 @@
 import type { ChatContext } from '../chat_context.js';
 export type ProviderFormat = 'openai' | 'google';
-export declare function toChatCtx(format: ProviderFormat, chatCtx: ChatContext, injectDummyUserMessage?: boolean): Promise<Record<string, any>[] | [Record<string, any>[], import("./google.js").GoogleFormatData]>;
+export declare function toChatCtx(format: ProviderFormat, chatCtx: ChatContext, injectDummyUserMessage?: boolean): Promise<Record<string, any>[] | [Record<string, unknown>[], import("./google.js").GoogleFormatData]>;
 //# sourceMappingURL=index.d.ts.map

package/dist/llm/provider_format/index.d.ts

@@ -1,4 +1,4 @@
 import type { ChatContext } from '../chat_context.js';
 export type ProviderFormat = 'openai' | 'google';
-export declare function toChatCtx(format: ProviderFormat, chatCtx: ChatContext, injectDummyUserMessage?: boolean): Promise<Record<string, any>[] | [Record<string, any>[], import("./google.js").GoogleFormatData]>;
+export declare function toChatCtx(format: ProviderFormat, chatCtx: ChatContext, injectDummyUserMessage?: boolean): Promise<Record<string, any>[] | [Record<string, unknown>[], import("./google.js").GoogleFormatData]>;
 //# sourceMappingURL=index.d.ts.map

package/dist/llm/provider_format/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/provider_format/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAItD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD,wBAAsB,SAAS,CAC7B,MAAM,EAAE,cAAc,EACtB,OAAO,EAAE,WAAW,EACpB,sBAAsB,GAAE,OAAc,oGAUvC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/provider_format/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAItD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD,wBAAsB,SAAS,CAC7B,MAAM,EAAE,cAAc,EACtB,OAAO,EAAE,WAAW,EACpB,sBAAsB,GAAE,OAAc,wGAUvC"}

package/dist/llm/realtime.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/realtime.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { Task } from '../utils.js';\nimport type { ChatContext, FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport type InputSpeechStartedEvent = object;\n\nexport interface InputSpeechStoppedEvent {\n  userTranscriptionEnabled: boolean;\n}\n\nexport interface MessageGeneration {\n  messageId: string;\n  textStream: ReadableStream<string>;\n  audioStream: ReadableStream<AudioFrame>;\n}\n\nexport interface GenerationCreatedEvent {\n  messageStream: ReadableStream<MessageGeneration>;\n  functionStream: ReadableStream<FunctionCall>;\n  userInitiated: boolean;\n}\n\nexport interface RealtimeModelError {\n  type: 'realtime_model_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport interface RealtimeCapabilities {\n  messageTruncation: boolean;\n  turnDetection: boolean;\n  userTranscription: boolean;\n  autoToolReplyGeneration: boolean;\n}\n\nexport interface InputTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n  isFinal: boolean;\n}\n\nexport interface RealtimeSessionReconnectedEvent {}\n\nexport abstract class RealtimeModel {\n  private _capabilities: RealtimeCapabilities;\n\n  constructor(capabilities: RealtimeCapabilities) {\n    this._capabilities = capabilities;\n  }\n\n  get capabilities() {\n    return this._capabilities;\n  }\n\n  abstract session(): RealtimeSession;\n\n  abstract close(): Promise<void>;\n}\n\nexport abstract class RealtimeSession extends EventEmitter {\n  protected _realtimeModel: RealtimeModel;\n  private deferredInputStream = new DeferredReadableStream<AudioFrame>();\n  private _mainTask: Task<void>;\n\n  constructor(realtimeModel: RealtimeModel) {\n    super();\n    this._realtimeModel = realtimeModel;\n    this._mainTask = Task.from((controller) => this._mainTaskImpl(controller.signal));\n  }\n\n  get realtimeModel() {\n    return this._realtimeModel;\n  }\n\n  abstract get chatCtx(): ChatContext;\n\n  abstract get tools(): ToolContext;\n\n  abstract updateInstructions(instructions: string): Promise<void>;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract updateChatCtx(chatCtx: ChatContext): Promise<void>;\n\n  abstract updateTools(tools: ToolContext): Promise<void>;\n\n  abstract updateOptions(options: { toolChoice?: ToolChoice | null }): void;\n\n  abstract pushAudio(frame: AudioFrame): void;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract generateReply(instructions?: string): Promise<GenerationCreatedEvent>;\n\n  /**\n   * Commit the input audio buffer to the server\n   */\n  abstract commitAudio(): Promise<void>;\n\n  /**\n   * Clear the input audio buffer to the server\n   */\n  abstract clearAudio(): Promise<void>;\n\n  /**\n   * Cancel the current generation (do nothing if no generation is in progress)\n   */\n  abstract interrupt(): Promise<void>;\n\n  /**\n   * Truncate the message at the given audio end time\n   */\n  abstract truncate(options: { messageId: string; audioEndMs: number }): Promise<void>;\n\n  async close(): Promise<void> {\n    this._mainTask.cancel();\n  }\n\n  /**\n   * Notifies the model that user activity has started\n   */\n  startUserActivity(): void {\n    return;\n  }\n\n  private async _mainTaskImpl(signal: AbortSignal): Promise<void> {\n    const reader = this.deferredInputStream.stream.getReader();\n    while (true) {\n      const { done, value } = await reader.read();\n      if (done || signal.aborted) {\n        break;\n      }\n      this.pushAudio(value);\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void {\n    this.deferredInputStream.setSource(audioStream);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,oBAA6B;AAE7B,6BAAuC;AACvC,mBAAqB;AA6Cd,MAAe,cAAc;AAAA,EAC1B;AAAA,EAER,YAAY,cAAoC;AAC9C,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAe;AACjB,WAAO,KAAK;AAAA,EACd;AAKF;AAEO,MAAe,wBAAwB,2BAAa;AAAA,EAC/C;AAAA,EACF,sBAAsB,IAAI,8CAAmC;AAAA,EAC7D;AAAA,EAER,YAAY,eAA8B;AACxC,UAAM;AACN,SAAK,iBAAiB;AACtB,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,cAAc,WAAW,MAAM,CAAC;AAAA,EAClF;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK;AAAA,EACd;AAAA,EA4CA,MAAM,QAAuB;AAC3B,SAAK,UAAU,OAAO;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,oBAA0B;AACxB;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,QAAoC;AAC9D,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,QAAQ,OAAO,SAAS;AAC1B;AAAA,MACF;AACA,WAAK,UAAU,KAAK;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,oBAAoB,aAA+C;AACjE,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AACF;","names":[]}
+{"version":3,"sources":["../../src/llm/realtime.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { Task } from '../utils.js';\nimport type { ChatContext, FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport type InputSpeechStartedEvent = object;\n\nexport interface InputSpeechStoppedEvent {\n  userTranscriptionEnabled: boolean;\n}\n\nexport interface MessageGeneration {\n  messageId: string;\n  textStream: ReadableStream<string>;\n  audioStream: ReadableStream<AudioFrame>;\n  modalities?: Promise<('text' | 'audio')[]>;\n}\n\nexport interface GenerationCreatedEvent {\n  messageStream: ReadableStream<MessageGeneration>;\n  functionStream: ReadableStream<FunctionCall>;\n  userInitiated: boolean;\n}\n\nexport interface RealtimeModelError {\n  type: 'realtime_model_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport interface RealtimeCapabilities {\n  messageTruncation: boolean;\n  turnDetection: boolean;\n  userTranscription: boolean;\n  autoToolReplyGeneration: boolean;\n  audioOutput: boolean;\n}\n\nexport interface InputTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n  isFinal: boolean;\n}\n\nexport interface RealtimeSessionReconnectedEvent {}\n\nexport abstract class RealtimeModel {\n  private _capabilities: RealtimeCapabilities;\n\n  constructor(capabilities: RealtimeCapabilities) {\n    this._capabilities = capabilities;\n  }\n\n  get capabilities() {\n    return this._capabilities;\n  }\n\n  abstract session(): RealtimeSession;\n\n  abstract close(): Promise<void>;\n}\n\nexport abstract class RealtimeSession extends EventEmitter {\n  protected _realtimeModel: RealtimeModel;\n  private deferredInputStream = new DeferredReadableStream<AudioFrame>();\n  private _mainTask: Task<void>;\n\n  constructor(realtimeModel: RealtimeModel) {\n    super();\n    this._realtimeModel = realtimeModel;\n    this._mainTask = Task.from((controller) => this._mainTaskImpl(controller.signal));\n  }\n\n  get realtimeModel() {\n    return this._realtimeModel;\n  }\n\n  abstract get chatCtx(): ChatContext;\n\n  abstract get tools(): ToolContext;\n\n  abstract updateInstructions(instructions: string): Promise<void>;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract updateChatCtx(chatCtx: ChatContext): Promise<void>;\n\n  abstract updateTools(tools: ToolContext): Promise<void>;\n\n  abstract updateOptions(options: { toolChoice?: ToolChoice | null }): void;\n\n  abstract pushAudio(frame: AudioFrame): void;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract generateReply(instructions?: string): Promise<GenerationCreatedEvent>;\n\n  /**\n   * Commit the input audio buffer to the server\n   */\n  abstract commitAudio(): Promise<void>;\n\n  /**\n   * Clear the input audio buffer to the server\n   */\n  abstract clearAudio(): Promise<void>;\n\n  /**\n   * Cancel the current generation (do nothing if no generation is in progress)\n   */\n  abstract interrupt(): Promise<void>;\n\n  /**\n   * Truncate the message at the given audio end time\n   */\n  abstract truncate(options: {\n    messageId: string;\n    audioEndMs: number;\n    modalities?: ('text' | 'audio')[];\n    audioTranscript?: string;\n  }): Promise<void>;\n\n  async close(): Promise<void> {\n    this._mainTask.cancel();\n  }\n\n  /**\n   * Notifies the model that user activity has started\n   */\n  startUserActivity(): void {\n    return;\n  }\n\n  private async _mainTaskImpl(signal: AbortSignal): Promise<void> {\n    const reader = this.deferredInputStream.stream.getReader();\n    while (true) {\n      const { done, value } = await reader.read();\n      if (done || signal.aborted) {\n        break;\n      }\n      this.pushAudio(value);\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void {\n    this.deferredInputStream.setSource(audioStream);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,oBAA6B;AAE7B,6BAAuC;AACvC,mBAAqB;AA+Cd,MAAe,cAAc;AAAA,EAC1B;AAAA,EAER,YAAY,cAAoC;AAC9C,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAe;AACjB,WAAO,KAAK;AAAA,EACd;AAKF;AAEO,MAAe,wBAAwB,2BAAa;AAAA,EAC/C;AAAA,EACF,sBAAsB,IAAI,8CAAmC;AAAA,EAC7D;AAAA,EAER,YAAY,eAA8B;AACxC,UAAM;AACN,SAAK,iBAAiB;AACtB,SAAK,YAAY,kBAAK,KAAK,CAAC,eAAe,KAAK,cAAc,WAAW,MAAM,CAAC;AAAA,EAClF;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK;AAAA,EACd;AAAA,EAiDA,MAAM,QAAuB;AAC3B,SAAK,UAAU,OAAO;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,oBAA0B;AACxB;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,QAAoC;AAC9D,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,QAAQ,OAAO,SAAS;AAC1B;AAAA,MACF;AACA,WAAK,UAAU,KAAK;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,oBAAoB,aAA+C;AACjE,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AACF;","names":[]}

package/dist/llm/realtime.d.cts

@@ -13,6 +13,7 @@ export interface MessageGeneration {
     messageId: string;
     textStream: ReadableStream<string>;
     audioStream: ReadableStream<AudioFrame>;
+    modalities?: Promise<('text' | 'audio')[]>;
 }
 export interface GenerationCreatedEvent {
     messageStream: ReadableStream<MessageGeneration>;
@@ -31,6 +32,7 @@ export interface RealtimeCapabilities {
     turnDetection: boolean;
     userTranscription: boolean;
     autoToolReplyGeneration: boolean;
+    audioOutput: boolean;
 }
 export interface InputTranscriptionCompleted {
     itemId: string;
@@ -86,6 +88,8 @@ export declare abstract class RealtimeSession extends EventEmitter {
     abstract truncate(options: {
         messageId: string;
         audioEndMs: number;
+        modalities?: ('text' | 'audio')[];
+        audioTranscript?: string;
     }): Promise<void>;
     close(): Promise<void>;
     /**

package/dist/llm/realtime.d.ts

@@ -13,6 +13,7 @@ export interface MessageGeneration {
     messageId: string;
     textStream: ReadableStream<string>;
     audioStream: ReadableStream<AudioFrame>;
+    modalities?: Promise<('text' | 'audio')[]>;
 }
 export interface GenerationCreatedEvent {
     messageStream: ReadableStream<MessageGeneration>;
@@ -31,6 +32,7 @@ export interface RealtimeCapabilities {
     turnDetection: boolean;
     userTranscription: boolean;
     autoToolReplyGeneration: boolean;
+    audioOutput: boolean;
 }
 export interface InputTranscriptionCompleted {
     itemId: string;
@@ -86,6 +88,8 @@ export declare abstract class RealtimeSession extends EventEmitter {
     abstract truncate(options: {
         messageId: string;
         audioEndMs: number;
+        modalities?: ('text' | 'audio')[];
+        audioTranscript?: string;
     }): Promise<void>;
     close(): Promise<void>;
     /**

package/dist/llm/realtime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"realtime.d.ts","sourceRoot":"","sources":["../../src/llm/realtime.ts"],"names":[],"mappings":";;AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACnE,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEjE,MAAM,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAE7C,MAAM,WAAW,uBAAuB;IACtC,wBAAwB,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IACnC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,cAAc,CAAC,iBAAiB,CAAC,CAAC;IACjD,cAAc,EAAE,cAAc,CAAC,YAAY,CAAC,CAAC;IAC7C,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,oBAAoB;IACnC,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,uBAAuB,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,+BAA+B;CAAG;AAEnD,8BAAsB,aAAa;IACjC,OAAO,CAAC,aAAa,CAAuB;gBAEhC,YAAY,EAAE,oBAAoB;IAI9C,IAAI,YAAY,yBAEf;IAED,QAAQ,CAAC,OAAO,IAAI,eAAe;IAEnC,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAChC;AAED,8BAAsB,eAAgB,SAAQ,YAAY;IACxD,SAAS,CAAC,cAAc,EAAE,aAAa,CAAC;IACxC,OAAO,CAAC,mBAAmB,CAA4C;IACvE,OAAO,CAAC,SAAS,CAAa;gBAElB,aAAa,EAAE,aAAa;IAMxC,IAAI,aAAa,kBAEhB;IAED,QAAQ,KAAK,OAAO,IAAI,WAAW,CAAC;IAEpC,QAAQ,KAAK,KAAK,IAAI,WAAW,CAAC;IAElC,QAAQ,CAAC,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAEhE;;OAEG;IACH,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAE3D,QAAQ,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAEvD,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAEzE,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAE3C;;OAEG;IACH,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAE9E;;OAEG;IACH,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAErC;;OAEG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAEpC;;OAEG;IACH,QAAQ,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAEnC;;OAEG;IACH,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAE9E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;OAEG;IACH,iBAAiB,IAAI,IAAI;YAIX,aAAa;IAW3B,mBAAmB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;CAGnE"}
+{"version":3,"file":"realtime.d.ts","sourceRoot":"","sources":["../../src/llm/realtime.ts"],"names":[],"mappings":";;AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGtD,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACnE,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEjE,MAAM,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAE7C,MAAM,WAAW,uBAAuB;IACtC,wBAAwB,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IACnC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IACxC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;CAC5C;AAED,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,cAAc,CAAC,iBAAiB,CAAC,CAAC;IACjD,cAAc,EAAE,cAAc,CAAC,YAAY,CAAC,CAAC;IAC7C,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,oBAAoB;IACnC,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,uBAAuB,EAAE,OAAO,CAAC;IACjC,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,+BAA+B;CAAG;AAEnD,8BAAsB,aAAa;IACjC,OAAO,CAAC,aAAa,CAAuB;gBAEhC,YAAY,EAAE,oBAAoB;IAI9C,IAAI,YAAY,yBAEf;IAED,QAAQ,CAAC,OAAO,IAAI,eAAe;IAEnC,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAChC;AAED,8BAAsB,eAAgB,SAAQ,YAAY;IACxD,SAAS,CAAC,cAAc,EAAE,aAAa,CAAC;IACxC,OAAO,CAAC,mBAAmB,CAA4C;IACvE,OAAO,CAAC,SAAS,CAAa;gBAElB,aAAa,EAAE,aAAa;IAMxC,IAAI,aAAa,kBAEhB;IAED,QAAQ,KAAK,OAAO,IAAI,WAAW,CAAC;IAEpC,QAAQ,KAAK,KAAK,IAAI,WAAW,CAAC;IAElC,QAAQ,CAAC,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAEhE;;OAEG;IACH,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAE3D,QAAQ,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAEvD,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAEzE,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAE3C;;OAEG;IACH,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAE9E;;OAEG;IACH,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAErC;;OAEG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAEpC;;OAEG;IACH,QAAQ,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAEnC;;OAEG;IACH,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE;QACzB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,CAAC,MAAM,GAAG,OAAO,CAAC,EAAE,CAAC;QAClC,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,GAAG,OAAO,CAAC,IAAI,CAAC;IAEX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;OAEG;IACH,iBAAiB,IAAI,IAAI;YAIX,aAAa;IAW3B,mBAAmB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;CAGnE"}

package/dist/llm/realtime.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/realtime.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { Task } from '../utils.js';\nimport type { ChatContext, FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport type InputSpeechStartedEvent = object;\n\nexport interface InputSpeechStoppedEvent {\n  userTranscriptionEnabled: boolean;\n}\n\nexport interface MessageGeneration {\n  messageId: string;\n  textStream: ReadableStream<string>;\n  audioStream: ReadableStream<AudioFrame>;\n}\n\nexport interface GenerationCreatedEvent {\n  messageStream: ReadableStream<MessageGeneration>;\n  functionStream: ReadableStream<FunctionCall>;\n  userInitiated: boolean;\n}\n\nexport interface RealtimeModelError {\n  type: 'realtime_model_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport interface RealtimeCapabilities {\n  messageTruncation: boolean;\n  turnDetection: boolean;\n  userTranscription: boolean;\n  autoToolReplyGeneration: boolean;\n}\n\nexport interface InputTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n  isFinal: boolean;\n}\n\nexport interface RealtimeSessionReconnectedEvent {}\n\nexport abstract class RealtimeModel {\n  private _capabilities: RealtimeCapabilities;\n\n  constructor(capabilities: RealtimeCapabilities) {\n    this._capabilities = capabilities;\n  }\n\n  get capabilities() {\n    return this._capabilities;\n  }\n\n  abstract session(): RealtimeSession;\n\n  abstract close(): Promise<void>;\n}\n\nexport abstract class RealtimeSession extends EventEmitter {\n  protected _realtimeModel: RealtimeModel;\n  private deferredInputStream = new DeferredReadableStream<AudioFrame>();\n  private _mainTask: Task<void>;\n\n  constructor(realtimeModel: RealtimeModel) {\n    super();\n    this._realtimeModel = realtimeModel;\n    this._mainTask = Task.from((controller) => this._mainTaskImpl(controller.signal));\n  }\n\n  get realtimeModel() {\n    return this._realtimeModel;\n  }\n\n  abstract get chatCtx(): ChatContext;\n\n  abstract get tools(): ToolContext;\n\n  abstract updateInstructions(instructions: string): Promise<void>;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract updateChatCtx(chatCtx: ChatContext): Promise<void>;\n\n  abstract updateTools(tools: ToolContext): Promise<void>;\n\n  abstract updateOptions(options: { toolChoice?: ToolChoice | null }): void;\n\n  abstract pushAudio(frame: AudioFrame): void;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract generateReply(instructions?: string): Promise<GenerationCreatedEvent>;\n\n  /**\n   * Commit the input audio buffer to the server\n   */\n  abstract commitAudio(): Promise<void>;\n\n  /**\n   * Clear the input audio buffer to the server\n   */\n  abstract clearAudio(): Promise<void>;\n\n  /**\n   * Cancel the current generation (do nothing if no generation is in progress)\n   */\n  abstract interrupt(): Promise<void>;\n\n  /**\n   * Truncate the message at the given audio end time\n   */\n  abstract truncate(options: { messageId: string; audioEndMs: number }): Promise<void>;\n\n  async close(): Promise<void> {\n    this._mainTask.cancel();\n  }\n\n  /**\n   * Notifies the model that user activity has started\n   */\n  startUserActivity(): void {\n    return;\n  }\n\n  private async _mainTaskImpl(signal: AbortSignal): Promise<void> {\n    const reader = this.deferredInputStream.stream.getReader();\n    while (true) {\n      const { done, value } = await reader.read();\n      if (done || signal.aborted) {\n        break;\n      }\n      this.pushAudio(value);\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void {\n    this.deferredInputStream.setSource(audioStream);\n  }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAE7B,SAAS,8BAA8B;AACvC,SAAS,YAAY;AA6Cd,MAAe,cAAc;AAAA,EAC1B;AAAA,EAER,YAAY,cAAoC;AAC9C,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAe;AACjB,WAAO,KAAK;AAAA,EACd;AAKF;AAEO,MAAe,wBAAwB,aAAa;AAAA,EAC/C;AAAA,EACF,sBAAsB,IAAI,uBAAmC;AAAA,EAC7D;AAAA,EAER,YAAY,eAA8B;AACxC,UAAM;AACN,SAAK,iBAAiB;AACtB,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,cAAc,WAAW,MAAM,CAAC;AAAA,EAClF;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK;AAAA,EACd;AAAA,EA4CA,MAAM,QAAuB;AAC3B,SAAK,UAAU,OAAO;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,oBAA0B;AACxB;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,QAAoC;AAC9D,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,QAAQ,OAAO,SAAS;AAC1B;AAAA,MACF;AACA,WAAK,UAAU,KAAK;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,oBAAoB,aAA+C;AACjE,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AACF;","names":[]}
+{"version":3,"sources":["../../src/llm/realtime.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { EventEmitter } from 'events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { Task } from '../utils.js';\nimport type { ChatContext, FunctionCall } from './chat_context.js';\nimport type { ToolChoice, ToolContext } from './tool_context.js';\n\nexport type InputSpeechStartedEvent = object;\n\nexport interface InputSpeechStoppedEvent {\n  userTranscriptionEnabled: boolean;\n}\n\nexport interface MessageGeneration {\n  messageId: string;\n  textStream: ReadableStream<string>;\n  audioStream: ReadableStream<AudioFrame>;\n  modalities?: Promise<('text' | 'audio')[]>;\n}\n\nexport interface GenerationCreatedEvent {\n  messageStream: ReadableStream<MessageGeneration>;\n  functionStream: ReadableStream<FunctionCall>;\n  userInitiated: boolean;\n}\n\nexport interface RealtimeModelError {\n  type: 'realtime_model_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport interface RealtimeCapabilities {\n  messageTruncation: boolean;\n  turnDetection: boolean;\n  userTranscription: boolean;\n  autoToolReplyGeneration: boolean;\n  audioOutput: boolean;\n}\n\nexport interface InputTranscriptionCompleted {\n  itemId: string;\n  transcript: string;\n  isFinal: boolean;\n}\n\nexport interface RealtimeSessionReconnectedEvent {}\n\nexport abstract class RealtimeModel {\n  private _capabilities: RealtimeCapabilities;\n\n  constructor(capabilities: RealtimeCapabilities) {\n    this._capabilities = capabilities;\n  }\n\n  get capabilities() {\n    return this._capabilities;\n  }\n\n  abstract session(): RealtimeSession;\n\n  abstract close(): Promise<void>;\n}\n\nexport abstract class RealtimeSession extends EventEmitter {\n  protected _realtimeModel: RealtimeModel;\n  private deferredInputStream = new DeferredReadableStream<AudioFrame>();\n  private _mainTask: Task<void>;\n\n  constructor(realtimeModel: RealtimeModel) {\n    super();\n    this._realtimeModel = realtimeModel;\n    this._mainTask = Task.from((controller) => this._mainTaskImpl(controller.signal));\n  }\n\n  get realtimeModel() {\n    return this._realtimeModel;\n  }\n\n  abstract get chatCtx(): ChatContext;\n\n  abstract get tools(): ToolContext;\n\n  abstract updateInstructions(instructions: string): Promise<void>;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract updateChatCtx(chatCtx: ChatContext): Promise<void>;\n\n  abstract updateTools(tools: ToolContext): Promise<void>;\n\n  abstract updateOptions(options: { toolChoice?: ToolChoice | null }): void;\n\n  abstract pushAudio(frame: AudioFrame): void;\n\n  /**\n   * @throws RealtimeError on Timeout\n   */\n  abstract generateReply(instructions?: string): Promise<GenerationCreatedEvent>;\n\n  /**\n   * Commit the input audio buffer to the server\n   */\n  abstract commitAudio(): Promise<void>;\n\n  /**\n   * Clear the input audio buffer to the server\n   */\n  abstract clearAudio(): Promise<void>;\n\n  /**\n   * Cancel the current generation (do nothing if no generation is in progress)\n   */\n  abstract interrupt(): Promise<void>;\n\n  /**\n   * Truncate the message at the given audio end time\n   */\n  abstract truncate(options: {\n    messageId: string;\n    audioEndMs: number;\n    modalities?: ('text' | 'audio')[];\n    audioTranscript?: string;\n  }): Promise<void>;\n\n  async close(): Promise<void> {\n    this._mainTask.cancel();\n  }\n\n  /**\n   * Notifies the model that user activity has started\n   */\n  startUserActivity(): void {\n    return;\n  }\n\n  private async _mainTaskImpl(signal: AbortSignal): Promise<void> {\n    const reader = this.deferredInputStream.stream.getReader();\n    while (true) {\n      const { done, value } = await reader.read();\n      if (done || signal.aborted) {\n        break;\n      }\n      this.pushAudio(value);\n    }\n  }\n\n  setInputAudioStream(audioStream: ReadableStream<AudioFrame>): void {\n    this.deferredInputStream.setSource(audioStream);\n  }\n}\n"],"mappings":"AAIA,SAAS,oBAAoB;AAE7B,SAAS,8BAA8B;AACvC,SAAS,YAAY;AA+Cd,MAAe,cAAc;AAAA,EAC1B;AAAA,EAER,YAAY,cAAoC;AAC9C,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAe;AACjB,WAAO,KAAK;AAAA,EACd;AAKF;AAEO,MAAe,wBAAwB,aAAa;AAAA,EAC/C;AAAA,EACF,sBAAsB,IAAI,uBAAmC;AAAA,EAC7D;AAAA,EAER,YAAY,eAA8B;AACxC,UAAM;AACN,SAAK,iBAAiB;AACtB,SAAK,YAAY,KAAK,KAAK,CAAC,eAAe,KAAK,cAAc,WAAW,MAAM,CAAC;AAAA,EAClF;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK;AAAA,EACd;AAAA,EAiDA,MAAM,QAAuB;AAC3B,SAAK,UAAU,OAAO;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,oBAA0B;AACxB;AAAA,EACF;AAAA,EAEA,MAAc,cAAc,QAAoC;AAC9D,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,QAAQ,OAAO,SAAS;AAC1B;AAAA,MACF;AACA,WAAK,UAAU,KAAK;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,oBAAoB,aAA+C;AACjE,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AACF;","names":[]}

package/dist/llm/utils.cjs

@@ -248,9 +248,9 @@ function computeChatCtxDiff(oldCtx, newCtx) {
     toCreate
   };
 }
-function toJsonSchema(schema, isOpenai = true) {
+function toJsonSchema(schema, isOpenai = true, strict = false) {
   if ((0, import_zod_utils.isZodSchema)(schema)) {
-    return (0, import_zod_utils.zodSchemaToJsonSchema)(schema, isOpenai);
+    return (0, import_zod_utils.zodSchemaToJsonSchema)(schema, isOpenai, strict);
   }
   return schema;
 }

package/dist/llm/utils.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { VideoBufferType, VideoFrame } from '@livekit/rtc-node';\nimport type { JSONSchema7 } from 'json-schema';\nimport sharp from 'sharp';\nimport type { UnknownUserData } from '../voice/run_context.js';\nimport type { ChatContext } from './chat_context.js';\nimport {\n  type ChatItem,\n  FunctionCall,\n  FunctionCallOutput,\n  type ImageContent,\n} from './chat_context.js';\nimport type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';\nimport { isZodSchema, parseZodSchema, zodSchemaToJsonSchema } from './zod-utils.js';\n\nexport interface SerializedImage {\n  inferenceDetail: 'auto' | 'high' | 'low';\n  mimeType?: string;\n  base64Data?: string;\n  externalUrl?: string;\n}\n\nfunction getChannelsFromVideoBufferType(type: VideoBufferType): 3 | 4 {\n  switch (type) {\n    case VideoBufferType.RGBA:\n    case VideoBufferType.ABGR:\n    case VideoBufferType.ARGB:\n    case VideoBufferType.BGRA:\n      return 4;\n    case VideoBufferType.RGB24:\n      return 3;\n    default:\n      // YUV formats (I420, I420A, I422, I444, I010, NV12) need conversion\n      throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);\n  }\n}\n\nfunction ensureRGBCompatible(frame: VideoFrame): VideoFrame {\n  // If the frame is already in an RGB/RGBA-compatible format, return it directly\n  if (\n    frame.type === VideoBufferType.RGBA ||\n    frame.type === VideoBufferType.BGRA ||\n    frame.type === VideoBufferType.ARGB ||\n    frame.type === VideoBufferType.ABGR ||\n    frame.type === VideoBufferType.RGB24\n  ) {\n    return frame;\n  }\n\n  // Otherwise, attempt conversion for other formats (like YUV)\n  try {\n    return frame.convert(VideoBufferType.RGBA);\n  } catch (error) {\n    throw new Error(\n      `Failed to convert format ${frame.type} to RGB: ${error}. ` +\n        `Consider using RGB/RGBA formats or converting on the client side.`,\n    );\n  }\n}\n\nexport async function serializeImage(image: ImageContent): Promise<SerializedImage> {\n  if (typeof image.image === 'string') {\n    if (image.image.startsWith('data:')) {\n      const [header, base64Data] = image.image.split(',', 2) as [string, string];\n      const headerParts = header.split(';');\n      const mimeParts = headerParts[0]?.split(':');\n      const headerMime = mimeParts?.[1];\n\n      if (!headerMime) {\n        throw new Error('Invalid data URL format');\n      }\n\n      let mimeType: string;\n      if (image.mimeType && image.mimeType !== headerMime) {\n        console.warn(\n          `Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`,\n        );\n        mimeType = image.mimeType;\n      } else {\n        mimeType = headerMime;\n      }\n\n      const supportedTypes = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);\n      if (!supportedTypes.has(mimeType)) {\n        throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);\n      }\n\n      return {\n        base64Data,\n        mimeType: mimeType,\n        inferenceDetail: image.inferenceDetail,\n      };\n    }\n\n    // External URL\n    return {\n      mimeType: image.mimeType,\n      inferenceDetail: image.inferenceDetail,\n      externalUrl: image.image,\n    };\n  } else if (image.image instanceof VideoFrame) {\n    const frame = ensureRGBCompatible(image.image);\n    const channels = getChannelsFromVideoBufferType(frame.type);\n\n    // Sharp needs to know the format of raw pixel data\n    let encoded = sharp(frame.data, {\n      raw: {\n        width: frame.width,\n        height: frame.height,\n        channels,\n      },\n    });\n\n    if (image.inferenceWidth && image.inferenceHeight) {\n      encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n    }\n\n    const base64Data = await encoded\n      .png()\n      .toBuffer()\n      .then((buffer) => buffer.toString('base64'));\n\n    return {\n      base64Data,\n      mimeType: 'image/png',\n      inferenceDetail: image.inferenceDetail,\n    };\n  } else {\n    throw new Error('Unsupported image type');\n  }\n}\n\n/** Raw OpenAI-adherent function parameters. */\nexport type OpenAIFunctionParameters = {\n  type: 'object';\n  properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any\n  required: string[];\n  additionalProperties?: boolean;\n};\n\n// TODO(brian): remove this helper once we have the real RunContext user data\nexport const createToolOptions = <UserData extends UnknownUserData>(\n  toolCallId: string,\n  userData: UserData = {} as UserData,\n): ToolOptions<UserData> => {\n  return { ctx: { userData }, toolCallId } as unknown as ToolOptions<UserData>;\n};\n\n/** @internal */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const oaiParams = (schema: any, isOpenai: boolean = true): OpenAIFunctionParameters => {\n  // Adapted from https://github.com/vercel/ai/blob/56eb0ee9/packages/provider-utils/src/zod-schema.ts\n  const jsonSchema = zodSchemaToJsonSchema(schema, isOpenai);\n  const { properties, required, additionalProperties } = jsonSchema as OpenAIFunctionParameters;\n\n  return {\n    type: 'object',\n    properties,\n    required,\n    additionalProperties,\n  };\n};\n\n/** @internal */\nexport const oaiBuildFunctionInfo = (\n  toolCtx: ToolContext,\n  toolCallId: string,\n  toolName: string,\n  rawArgs: string,\n): FunctionCall => {\n  const tool = toolCtx[toolName];\n  if (!tool) {\n    throw new Error(`AI tool ${toolName} not found`);\n  }\n\n  return FunctionCall.create({\n    callId: toolCallId,\n    name: toolName,\n    args: rawArgs,\n  });\n};\n\nexport async function executeToolCall(\n  toolCall: FunctionCall,\n  toolCtx: ToolContext,\n): Promise<FunctionCallOutput> {\n  const tool = toolCtx[toolCall.name]!;\n  let args: object | undefined;\n  let params: object | undefined;\n\n  // Ensure valid JSON\n  try {\n    args = JSON.parse(toolCall.args);\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Invalid JSON: ${error}`,\n      isError: true,\n    });\n  }\n\n  // Ensure valid arguments schema\n  try {\n    if (isZodSchema(tool.parameters)) {\n      const result = await parseZodSchema<object>(tool.parameters, args);\n      if (result.success) {\n        params = result.data;\n      } else {\n        return FunctionCallOutput.create({\n          callId: toolCall.callId,\n          output: `Arguments parsing failed: ${result.error}`,\n          isError: true,\n        });\n      }\n    } else {\n      params = args;\n    }\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Arguments parsing failed: ${error}`,\n      isError: true,\n    });\n  }\n\n  try {\n    const result = await tool.execute(params, createToolOptions(toolCall.callId));\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: JSON.stringify(result),\n      isError: false,\n    });\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Tool execution failed: ${error}`,\n      isError: true,\n    });\n  }\n}\n\n/**\n * Standard dynamic-programming LCS to get the common subsequence\n * of IDs (in order) that appear in both old_ids and new_ids.\n *\n * @param oldIds - The old list of IDs.\n * @param newIds - The new list of IDs.\n * @returns The longest common subsequence of the two lists of IDs.\n */\nfunction computeLCS(oldIds: string[], newIds: string[]): string[] {\n  const n = oldIds.length;\n  const m = newIds.length;\n  const dp: number[][] = Array(n + 1)\n    .fill(null)\n    .map(() => Array(m + 1).fill(0));\n\n  // Fill DP table\n  for (let i = 1; i <= n; i++) {\n    for (let j = 1; j <= m; j++) {\n      if (oldIds[i - 1] === newIds[j - 1]) {\n        dp[i]![j] = dp[i - 1]![j - 1]! + 1;\n      } else {\n        dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n      }\n    }\n  }\n\n  // Backtrack to find the actual LCS sequence\n  const lcsIds: string[] = [];\n  let i = n;\n  let j = m;\n  while (i > 0 && j > 0) {\n    if (oldIds[i - 1] === newIds[j - 1]) {\n      lcsIds.push(oldIds[i - 1]!);\n      i--;\n      j--;\n    } else if (dp[i - 1]![j]! > dp[i]![j - 1]!) {\n      i--;\n    } else {\n      j--;\n    }\n  }\n\n  return lcsIds.reverse();\n}\n\ninterface DiffOps {\n  toRemove: string[];\n  toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root\n}\n\n/**\n * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.\n *\n * @param oldCtx - The old chat context.\n * @param newCtx - The new chat context.\n * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.\n */\nexport function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps {\n  const oldIds = oldCtx.items.map((item: ChatItem) => item.id);\n  const newIds = newCtx.items.map((item: ChatItem) => item.id);\n  const lcsIds = new Set(computeLCS(oldIds, newIds));\n\n  const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);\n  const toCreate: Array<[string | null, string]> = [];\n\n  let lastIdInSequence: string | null = null;\n  for (const newItem of newCtx.items) {\n    if (lcsIds.has(newItem.id)) {\n      lastIdInSequence = newItem.id;\n    } else {\n      const prevId = lastIdInSequence; // null if root\n      toCreate.push([prevId, newItem.id]);\n      lastIdInSequence = newItem.id;\n    }\n  }\n\n  return {\n    toRemove,\n    toCreate,\n  };\n}\n\nexport function toJsonSchema(schema: ToolInputSchema<any>, isOpenai: boolean = true): JSONSchema7 {\n  if (isZodSchema(schema)) {\n    return zodSchemaToJsonSchema(schema, isOpenai);\n  }\n  return schema as JSONSchema7;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAA4C;AAE5C,mBAAkB;AAGlB,0BAKO;AAEP,uBAAmE;AASnE,SAAS,+BAA+B,MAA8B;AACpE,UAAQ,MAAM;AAAA,IACZ,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AACnB,aAAO;AAAA,IACT,KAAK,gCAAgB;AACnB,aAAO;AAAA,IACT;AAEE,YAAM,IAAI,MAAM,gCAAgC,IAAI,wCAAwC;AAAA,EAChG;AACF;AAEA,SAAS,oBAAoB,OAA+B;AAE1D,MACE,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,OAC/B;AACA,WAAO;AAAA,EACT;AAGA,MAAI;AACF,WAAO,MAAM,QAAQ,gCAAgB,IAAI;AAAA,EAC3C,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,IAAI,YAAY,KAAK;AAAA,IAEzD;AAAA,EACF;AACF;AAEA,eAAsB,eAAe,OAA+C;AA9DpF;AA+DE,MAAI,OAAO,MAAM,UAAU,UAAU;AACnC,QAAI,MAAM,MAAM,WAAW,OAAO,GAAG;AACnC,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,MAAM,KAAK,CAAC;AACrD,YAAM,cAAc,OAAO,MAAM,GAAG;AACpC,YAAM,aAAY,iBAAY,CAAC,MAAb,mBAAgB,MAAM;AACxC,YAAM,aAAa,uCAAY;AAE/B,UAAI,CAAC,YAAY;AACf,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAEA,UAAI;AACJ,UAAI,MAAM,YAAY,MAAM,aAAa,YAAY;AACnD,gBAAQ;AAAA,UACN,sBAAsB,MAAM,QAAQ,wCAAwC,UAAU;AAAA,QACxF;AACA,mBAAW,MAAM;AAAA,MACnB,OAAO;AACL,mBAAW;AAAA,MACb;AAEA,YAAM,iBAAiB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,WAAW,CAAC;AACrF,UAAI,CAAC,eAAe,IAAI,QAAQ,GAAG;AACjC,cAAM,IAAI,MAAM,wBAAwB,QAAQ,mCAAmC;AAAA,MACrF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,iBAAiB,MAAM;AAAA,MACzB;AAAA,IACF;AAGA,WAAO;AAAA,MACL,UAAU,MAAM;AAAA,MAChB,iBAAiB,MAAM;AAAA,MACvB,aAAa,MAAM;AAAA,IACrB;AAAA,EACF,WAAW,MAAM,iBAAiB,4BAAY;AAC5C,UAAM,QAAQ,oBAAoB,MAAM,KAAK;AAC7C,UAAM,WAAW,+BAA+B,MAAM,IAAI;AAG1D,QAAI,cAAU,aAAAA,SAAM,MAAM,MAAM;AAAA,MAC9B,KAAK;AAAA,QACH,OAAO,MAAM;AAAA,QACb,QAAQ,MAAM;AAAA,QACd;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,MAAM,kBAAkB,MAAM,iBAAiB;AACjD,gBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,QACtB,IAAI,EACJ,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,QAAQ,CAAC;AAE7C,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,iBAAiB,MAAM;AAAA,IACzB;AAAA,EACF,OAAO;AACL,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AACF;AAWO,MAAM,oBAAoB,CAC/B,YACA,WAAqB,CAAC,MACI;AAC1B,SAAO,EAAE,KAAK,EAAE,SAAS,GAAG,WAAW;AACzC;AAIO,MAAM,YAAY,CAAC,QAAa,WAAoB,SAAmC;AAE5F,QAAM,iBAAa,wCAAsB,QAAQ,QAAQ;AACzD,QAAM,EAAE,YAAY,UAAU,qBAAqB,IAAI;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGO,MAAM,uBAAuB,CAClC,SACA,YACA,UACA,YACiB;AACjB,QAAM,OAAO,QAAQ,QAAQ;AAC7B,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,MAAM,WAAW,QAAQ,YAAY;AAAA,EACjD;AAEA,SAAO,iCAAa,OAAO;AAAA,IACzB,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACH;AAEA,eAAsB,gBACpB,UACA,SAC6B;AAC7B,QAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,MAAI;AACJ,MAAI;AAGJ,MAAI;AACF,WAAO,KAAK,MAAM,SAAS,IAAI;AAAA,EACjC,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,iBAAiB,KAAK;AAAA,MAC9B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,MAAI;AACF,YAAI,8BAAY,KAAK,UAAU,GAAG;AAChC,YAAM,SAAS,UAAM,iCAAuB,KAAK,YAAY,IAAI;AACjE,UAAI,OAAO,SAAS;AAClB,iBAAS,OAAO;AAAA,MAClB,OAAO;AACL,eAAO,uCAAmB,OAAO;AAAA,UAC/B,QAAQ,SAAS;AAAA,UACjB,QAAQ,6BAA6B,OAAO,KAAK;AAAA,UACjD,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAAA,IACF,OAAO;AACL,eAAS;AAAA,IACX;AAAA,EACF,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,6BAA6B,KAAK;AAAA,MAC1C,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,QAAQ,QAAQ,kBAAkB,SAAS,MAAM,CAAC;AAC5E,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,KAAK,UAAU,MAAM;AAAA,MAC7B,SAAS;AAAA,IACX,CAAC;AAAA,EACH,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,0BAA0B,KAAK;AAAA,MACvC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAUA,SAAS,WAAW,QAAkB,QAA4B;AAChE,QAAM,IAAI,OAAO;AACjB,QAAM,IAAI,OAAO;AACjB,QAAM,KAAiB,MAAM,IAAI,CAAC,EAC/B,KAAK,IAAI,EACT,IAAI,MAAM,MAAM,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAGjC,WAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,aAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,UAAI,OAAOD,KAAI,CAAC,MAAM,OAAOC,KAAI,CAAC,GAAG;AACnC,WAAGD,EAAC,EAAGC,EAAC,IAAI,GAAGD,KAAI,CAAC,EAAGC,KAAI,CAAC,IAAK;AAAA,MACnC,OAAO;AACL,WAAGD,EAAC,EAAGC,EAAC,IAAI,KAAK,IAAI,GAAGD,KAAI,CAAC,EAAGC,EAAC,GAAI,GAAGD,EAAC,EAAGC,KAAI,CAAC,CAAE;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AAGA,QAAM,SAAmB,CAAC;AAC1B,MAAI,IAAI;AACR,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,IAAI,GAAG;AACrB,QAAI,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,GAAG;AACnC,aAAO,KAAK,OAAO,IAAI,CAAC,CAAE;AAC1B;AACA;AAAA,IACF,WAAW,GAAG,IAAI,CAAC,EAAG,CAAC,IAAK,GAAG,CAAC,EAAG,IAAI,CAAC,GAAI;AAC1C;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,OAAO,QAAQ;AACxB;AAcO,SAAS,mBAAmB,QAAqB,QAA8B;AACpF,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,IAAI,IAAI,WAAW,QAAQ,MAAM,CAAC;AAEjD,QAAM,WAAW,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtF,QAAM,WAA2C,CAAC;AAElD,MAAI,mBAAkC;AACtC,aAAW,WAAW,OAAO,OAAO;AAClC,QAAI,OAAO,IAAI,QAAQ,EAAE,GAAG;AAC1B,yBAAmB,QAAQ;AAAA,IAC7B,OAAO;AACL,YAAM,SAAS;AACf,eAAS,KAAK,CAAC,QAAQ,QAAQ,EAAE,CAAC;AAClC,yBAAmB,QAAQ;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,aAAa,QAA8B,WAAoB,MAAmB;AAChG,UAAI,8BAAY,MAAM,GAAG;AACvB,eAAO,wCAAsB,QAAQ,QAAQ;AAAA,EAC/C;AACA,SAAO;AACT;","names":["sharp","i","j"]}
+{"version":3,"sources":["../../src/llm/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { VideoBufferType, VideoFrame } from '@livekit/rtc-node';\nimport type { JSONSchema7 } from 'json-schema';\nimport sharp from 'sharp';\nimport type { UnknownUserData } from '../voice/run_context.js';\nimport type { ChatContext } from './chat_context.js';\nimport {\n  type ChatItem,\n  FunctionCall,\n  FunctionCallOutput,\n  type ImageContent,\n} from './chat_context.js';\nimport type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';\nimport { isZodSchema, parseZodSchema, zodSchemaToJsonSchema } from './zod-utils.js';\n\nexport interface SerializedImage {\n  inferenceDetail: 'auto' | 'high' | 'low';\n  mimeType?: string;\n  base64Data?: string;\n  externalUrl?: string;\n}\n\nfunction getChannelsFromVideoBufferType(type: VideoBufferType): 3 | 4 {\n  switch (type) {\n    case VideoBufferType.RGBA:\n    case VideoBufferType.ABGR:\n    case VideoBufferType.ARGB:\n    case VideoBufferType.BGRA:\n      return 4;\n    case VideoBufferType.RGB24:\n      return 3;\n    default:\n      // YUV formats (I420, I420A, I422, I444, I010, NV12) need conversion\n      throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);\n  }\n}\n\nfunction ensureRGBCompatible(frame: VideoFrame): VideoFrame {\n  // If the frame is already in an RGB/RGBA-compatible format, return it directly\n  if (\n    frame.type === VideoBufferType.RGBA ||\n    frame.type === VideoBufferType.BGRA ||\n    frame.type === VideoBufferType.ARGB ||\n    frame.type === VideoBufferType.ABGR ||\n    frame.type === VideoBufferType.RGB24\n  ) {\n    return frame;\n  }\n\n  // Otherwise, attempt conversion for other formats (like YUV)\n  try {\n    return frame.convert(VideoBufferType.RGBA);\n  } catch (error) {\n    throw new Error(\n      `Failed to convert format ${frame.type} to RGB: ${error}. ` +\n        `Consider using RGB/RGBA formats or converting on the client side.`,\n    );\n  }\n}\n\nexport async function serializeImage(image: ImageContent): Promise<SerializedImage> {\n  if (typeof image.image === 'string') {\n    if (image.image.startsWith('data:')) {\n      const [header, base64Data] = image.image.split(',', 2) as [string, string];\n      const headerParts = header.split(';');\n      const mimeParts = headerParts[0]?.split(':');\n      const headerMime = mimeParts?.[1];\n\n      if (!headerMime) {\n        throw new Error('Invalid data URL format');\n      }\n\n      let mimeType: string;\n      if (image.mimeType && image.mimeType !== headerMime) {\n        console.warn(\n          `Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`,\n        );\n        mimeType = image.mimeType;\n      } else {\n        mimeType = headerMime;\n      }\n\n      const supportedTypes = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);\n      if (!supportedTypes.has(mimeType)) {\n        throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);\n      }\n\n      return {\n        base64Data,\n        mimeType: mimeType,\n        inferenceDetail: image.inferenceDetail,\n      };\n    }\n\n    // External URL\n    return {\n      mimeType: image.mimeType,\n      inferenceDetail: image.inferenceDetail,\n      externalUrl: image.image,\n    };\n  } else if (image.image instanceof VideoFrame) {\n    const frame = ensureRGBCompatible(image.image);\n    const channels = getChannelsFromVideoBufferType(frame.type);\n\n    // Sharp needs to know the format of raw pixel data\n    let encoded = sharp(frame.data, {\n      raw: {\n        width: frame.width,\n        height: frame.height,\n        channels,\n      },\n    });\n\n    if (image.inferenceWidth && image.inferenceHeight) {\n      encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n    }\n\n    const base64Data = await encoded\n      .png()\n      .toBuffer()\n      .then((buffer) => buffer.toString('base64'));\n\n    return {\n      base64Data,\n      mimeType: 'image/png',\n      inferenceDetail: image.inferenceDetail,\n    };\n  } else {\n    throw new Error('Unsupported image type');\n  }\n}\n\n/** Raw OpenAI-adherent function parameters. */\nexport type OpenAIFunctionParameters = {\n  type: 'object';\n  properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any\n  required: string[];\n  additionalProperties?: boolean;\n};\n\n// TODO(brian): remove this helper once we have the real RunContext user data\nexport const createToolOptions = <UserData extends UnknownUserData>(\n  toolCallId: string,\n  userData: UserData = {} as UserData,\n): ToolOptions<UserData> => {\n  return { ctx: { userData }, toolCallId } as unknown as ToolOptions<UserData>;\n};\n\n/** @internal */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const oaiParams = (schema: any, isOpenai: boolean = true): OpenAIFunctionParameters => {\n  // Adapted from https://github.com/vercel/ai/blob/56eb0ee9/packages/provider-utils/src/zod-schema.ts\n  const jsonSchema = zodSchemaToJsonSchema(schema, isOpenai);\n  const { properties, required, additionalProperties } = jsonSchema as OpenAIFunctionParameters;\n\n  return {\n    type: 'object',\n    properties,\n    required,\n    additionalProperties,\n  };\n};\n\n/** @internal */\nexport const oaiBuildFunctionInfo = (\n  toolCtx: ToolContext,\n  toolCallId: string,\n  toolName: string,\n  rawArgs: string,\n): FunctionCall => {\n  const tool = toolCtx[toolName];\n  if (!tool) {\n    throw new Error(`AI tool ${toolName} not found`);\n  }\n\n  return FunctionCall.create({\n    callId: toolCallId,\n    name: toolName,\n    args: rawArgs,\n  });\n};\n\nexport async function executeToolCall(\n  toolCall: FunctionCall,\n  toolCtx: ToolContext,\n): Promise<FunctionCallOutput> {\n  const tool = toolCtx[toolCall.name]!;\n  let args: object | undefined;\n  let params: object | undefined;\n\n  // Ensure valid JSON\n  try {\n    args = JSON.parse(toolCall.args);\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Invalid JSON: ${error}`,\n      isError: true,\n    });\n  }\n\n  // Ensure valid arguments schema\n  try {\n    if (isZodSchema(tool.parameters)) {\n      const result = await parseZodSchema<object>(tool.parameters, args);\n      if (result.success) {\n        params = result.data;\n      } else {\n        return FunctionCallOutput.create({\n          callId: toolCall.callId,\n          output: `Arguments parsing failed: ${result.error}`,\n          isError: true,\n        });\n      }\n    } else {\n      params = args;\n    }\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Arguments parsing failed: ${error}`,\n      isError: true,\n    });\n  }\n\n  try {\n    const result = await tool.execute(params, createToolOptions(toolCall.callId));\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: JSON.stringify(result),\n      isError: false,\n    });\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Tool execution failed: ${error}`,\n      isError: true,\n    });\n  }\n}\n\n/**\n * Standard dynamic-programming LCS to get the common subsequence\n * of IDs (in order) that appear in both old_ids and new_ids.\n *\n * @param oldIds - The old list of IDs.\n * @param newIds - The new list of IDs.\n * @returns The longest common subsequence of the two lists of IDs.\n */\nfunction computeLCS(oldIds: string[], newIds: string[]): string[] {\n  const n = oldIds.length;\n  const m = newIds.length;\n  const dp: number[][] = Array(n + 1)\n    .fill(null)\n    .map(() => Array(m + 1).fill(0));\n\n  // Fill DP table\n  for (let i = 1; i <= n; i++) {\n    for (let j = 1; j <= m; j++) {\n      if (oldIds[i - 1] === newIds[j - 1]) {\n        dp[i]![j] = dp[i - 1]![j - 1]! + 1;\n      } else {\n        dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n      }\n    }\n  }\n\n  // Backtrack to find the actual LCS sequence\n  const lcsIds: string[] = [];\n  let i = n;\n  let j = m;\n  while (i > 0 && j > 0) {\n    if (oldIds[i - 1] === newIds[j - 1]) {\n      lcsIds.push(oldIds[i - 1]!);\n      i--;\n      j--;\n    } else if (dp[i - 1]![j]! > dp[i]![j - 1]!) {\n      i--;\n    } else {\n      j--;\n    }\n  }\n\n  return lcsIds.reverse();\n}\n\ninterface DiffOps {\n  toRemove: string[];\n  toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root\n}\n\n/**\n * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.\n *\n * @param oldCtx - The old chat context.\n * @param newCtx - The new chat context.\n * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.\n */\nexport function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps {\n  const oldIds = oldCtx.items.map((item: ChatItem) => item.id);\n  const newIds = newCtx.items.map((item: ChatItem) => item.id);\n  const lcsIds = new Set(computeLCS(oldIds, newIds));\n\n  const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);\n  const toCreate: Array<[string | null, string]> = [];\n\n  let lastIdInSequence: string | null = null;\n  for (const newItem of newCtx.items) {\n    if (lcsIds.has(newItem.id)) {\n      lastIdInSequence = newItem.id;\n    } else {\n      const prevId = lastIdInSequence; // null if root\n      toCreate.push([prevId, newItem.id]);\n      lastIdInSequence = newItem.id;\n    }\n  }\n\n  return {\n    toRemove,\n    toCreate,\n  };\n}\n\nexport function toJsonSchema(\n  schema: ToolInputSchema<any>,\n  isOpenai: boolean = true,\n  strict: boolean = false,\n): JSONSchema7 {\n  if (isZodSchema(schema)) {\n    return zodSchemaToJsonSchema(schema, isOpenai, strict);\n  }\n\n  return schema as JSONSchema7;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAA4C;AAE5C,mBAAkB;AAGlB,0BAKO;AAEP,uBAAmE;AASnE,SAAS,+BAA+B,MAA8B;AACpE,UAAQ,MAAM;AAAA,IACZ,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AACnB,aAAO;AAAA,IACT,KAAK,gCAAgB;AACnB,aAAO;AAAA,IACT;AAEE,YAAM,IAAI,MAAM,gCAAgC,IAAI,wCAAwC;AAAA,EAChG;AACF;AAEA,SAAS,oBAAoB,OAA+B;AAE1D,MACE,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,OAC/B;AACA,WAAO;AAAA,EACT;AAGA,MAAI;AACF,WAAO,MAAM,QAAQ,gCAAgB,IAAI;AAAA,EAC3C,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,IAAI,YAAY,KAAK;AAAA,IAEzD;AAAA,EACF;AACF;AAEA,eAAsB,eAAe,OAA+C;AA9DpF;AA+DE,MAAI,OAAO,MAAM,UAAU,UAAU;AACnC,QAAI,MAAM,MAAM,WAAW,OAAO,GAAG;AACnC,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,MAAM,KAAK,CAAC;AACrD,YAAM,cAAc,OAAO,MAAM,GAAG;AACpC,YAAM,aAAY,iBAAY,CAAC,MAAb,mBAAgB,MAAM;AACxC,YAAM,aAAa,uCAAY;AAE/B,UAAI,CAAC,YAAY;AACf,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAEA,UAAI;AACJ,UAAI,MAAM,YAAY,MAAM,aAAa,YAAY;AACnD,gBAAQ;AAAA,UACN,sBAAsB,MAAM,QAAQ,wCAAwC,UAAU;AAAA,QACxF;AACA,mBAAW,MAAM;AAAA,MACnB,OAAO;AACL,mBAAW;AAAA,MACb;AAEA,YAAM,iBAAiB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,WAAW,CAAC;AACrF,UAAI,CAAC,eAAe,IAAI,QAAQ,GAAG;AACjC,cAAM,IAAI,MAAM,wBAAwB,QAAQ,mCAAmC;AAAA,MACrF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,iBAAiB,MAAM;AAAA,MACzB;AAAA,IACF;AAGA,WAAO;AAAA,MACL,UAAU,MAAM;AAAA,MAChB,iBAAiB,MAAM;AAAA,MACvB,aAAa,MAAM;AAAA,IACrB;AAAA,EACF,WAAW,MAAM,iBAAiB,4BAAY;AAC5C,UAAM,QAAQ,oBAAoB,MAAM,KAAK;AAC7C,UAAM,WAAW,+BAA+B,MAAM,IAAI;AAG1D,QAAI,cAAU,aAAAA,SAAM,MAAM,MAAM;AAAA,MAC9B,KAAK;AAAA,QACH,OAAO,MAAM;AAAA,QACb,QAAQ,MAAM;AAAA,QACd;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,MAAM,kBAAkB,MAAM,iBAAiB;AACjD,gBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,QACtB,IAAI,EACJ,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,QAAQ,CAAC;AAE7C,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,iBAAiB,MAAM;AAAA,IACzB;AAAA,EACF,OAAO;AACL,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AACF;AAWO,MAAM,oBAAoB,CAC/B,YACA,WAAqB,CAAC,MACI;AAC1B,SAAO,EAAE,KAAK,EAAE,SAAS,GAAG,WAAW;AACzC;AAIO,MAAM,YAAY,CAAC,QAAa,WAAoB,SAAmC;AAE5F,QAAM,iBAAa,wCAAsB,QAAQ,QAAQ;AACzD,QAAM,EAAE,YAAY,UAAU,qBAAqB,IAAI;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGO,MAAM,uBAAuB,CAClC,SACA,YACA,UACA,YACiB;AACjB,QAAM,OAAO,QAAQ,QAAQ;AAC7B,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,MAAM,WAAW,QAAQ,YAAY;AAAA,EACjD;AAEA,SAAO,iCAAa,OAAO;AAAA,IACzB,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACH;AAEA,eAAsB,gBACpB,UACA,SAC6B;AAC7B,QAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,MAAI;AACJ,MAAI;AAGJ,MAAI;AACF,WAAO,KAAK,MAAM,SAAS,IAAI;AAAA,EACjC,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,iBAAiB,KAAK;AAAA,MAC9B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,MAAI;AACF,YAAI,8BAAY,KAAK,UAAU,GAAG;AAChC,YAAM,SAAS,UAAM,iCAAuB,KAAK,YAAY,IAAI;AACjE,UAAI,OAAO,SAAS;AAClB,iBAAS,OAAO;AAAA,MAClB,OAAO;AACL,eAAO,uCAAmB,OAAO;AAAA,UAC/B,QAAQ,SAAS;AAAA,UACjB,QAAQ,6BAA6B,OAAO,KAAK;AAAA,UACjD,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAAA,IACF,OAAO;AACL,eAAS;AAAA,IACX;AAAA,EACF,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,6BAA6B,KAAK;AAAA,MAC1C,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,QAAQ,QAAQ,kBAAkB,SAAS,MAAM,CAAC;AAC5E,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,KAAK,UAAU,MAAM;AAAA,MAC7B,SAAS;AAAA,IACX,CAAC;AAAA,EACH,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,0BAA0B,KAAK;AAAA,MACvC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAUA,SAAS,WAAW,QAAkB,QAA4B;AAChE,QAAM,IAAI,OAAO;AACjB,QAAM,IAAI,OAAO;AACjB,QAAM,KAAiB,MAAM,IAAI,CAAC,EAC/B,KAAK,IAAI,EACT,IAAI,MAAM,MAAM,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAGjC,WAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,aAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,UAAI,OAAOD,KAAI,CAAC,MAAM,OAAOC,KAAI,CAAC,GAAG;AACnC,WAAGD,EAAC,EAAGC,EAAC,IAAI,GAAGD,KAAI,CAAC,EAAGC,KAAI,CAAC,IAAK;AAAA,MACnC,OAAO;AACL,WAAGD,EAAC,EAAGC,EAAC,IAAI,KAAK,IAAI,GAAGD,KAAI,CAAC,EAAGC,EAAC,GAAI,GAAGD,EAAC,EAAGC,KAAI,CAAC,CAAE;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AAGA,QAAM,SAAmB,CAAC;AAC1B,MAAI,IAAI;AACR,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,IAAI,GAAG;AACrB,QAAI,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,GAAG;AACnC,aAAO,KAAK,OAAO,IAAI,CAAC,CAAE;AAC1B;AACA;AAAA,IACF,WAAW,GAAG,IAAI,CAAC,EAAG,CAAC,IAAK,GAAG,CAAC,EAAG,IAAI,CAAC,GAAI;AAC1C;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,OAAO,QAAQ;AACxB;AAcO,SAAS,mBAAmB,QAAqB,QAA8B;AACpF,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,IAAI,IAAI,WAAW,QAAQ,MAAM,CAAC;AAEjD,QAAM,WAAW,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtF,QAAM,WAA2C,CAAC;AAElD,MAAI,mBAAkC;AACtC,aAAW,WAAW,OAAO,OAAO;AAClC,QAAI,OAAO,IAAI,QAAQ,EAAE,GAAG;AAC1B,yBAAmB,QAAQ;AAAA,IAC7B,OAAO;AACL,YAAM,SAAS;AACf,eAAS,KAAK,CAAC,QAAQ,QAAQ,EAAE,CAAC;AAClC,yBAAmB,QAAQ;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,aACd,QACA,WAAoB,MACpB,SAAkB,OACL;AACb,UAAI,8BAAY,MAAM,GAAG;AACvB,eAAO,wCAAsB,QAAQ,UAAU,MAAM;AAAA,EACvD;AAEA,SAAO;AACT;","names":["sharp","i","j"]}

package/dist/llm/utils.d.cts

@@ -36,6 +36,6 @@ interface DiffOps {
  * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.
  */
 export declare function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps;
-export declare function toJsonSchema(schema: ToolInputSchema<any>, isOpenai?: boolean): JSONSchema7;
+export declare function toJsonSchema(schema: ToolInputSchema<any>, isOpenai?: boolean, strict?: boolean): JSONSchema7;
 export {};
 //# sourceMappingURL=utils.d.ts.map

package/dist/llm/utils.d.ts

@@ -36,6 +36,6 @@ interface DiffOps {
  * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.
  */
 export declare function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps;
-export declare function toJsonSchema(schema: ToolInputSchema<any>, isOpenai?: boolean): JSONSchema7;
+export declare function toJsonSchema(schema: ToolInputSchema<any>, isOpenai?: boolean, strict?: boolean): JSONSchema7;
 export {};
 //# sourceMappingURL=utils.d.ts.map

package/dist/llm/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/llm/utils.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAEL,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGnF,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAwCD,wBAAsB,cAAc,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAsElF;AAED,+CAA+C;AAC/C,MAAM,MAAM,wBAAwB,GAAG;IACrC,IAAI,EAAE,QAAQ,CAAC;IACf,UAAU,EAAE;QAAE,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;IAClC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,CAAC;AAGF,eAAO,MAAM,iBAAiB,yCAChB,MAAM,aACR,QAAQ,KACjB,YAAY,QAAQ,CAEtB,CAAC;AAEF,gBAAgB;AAEhB,eAAO,MAAM,SAAS,WAAY,GAAG,aAAY,OAAO,KAAU,wBAWjE,CAAC;AAEF,gBAAgB;AAChB,eAAO,MAAM,oBAAoB,YACtB,WAAW,cACR,MAAM,YACR,MAAM,WACP,MAAM,KACd,YAWF,CAAC;AAEF,wBAAsB,eAAe,CACnC,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,kBAAkB,CAAC,CAsD7B;AA+CD,UAAU,OAAO;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;CAC1C;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAuBpF;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE,QAAQ,GAAE,OAAc,GAAG,WAAW,CAKhG"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/llm/utils.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAEL,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGnF,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAwCD,wBAAsB,cAAc,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAsElF;AAED,+CAA+C;AAC/C,MAAM,MAAM,wBAAwB,GAAG;IACrC,IAAI,EAAE,QAAQ,CAAC;IACf,UAAU,EAAE;QAAE,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;IAClC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,CAAC;AAGF,eAAO,MAAM,iBAAiB,yCAChB,MAAM,aACR,QAAQ,KACjB,YAAY,QAAQ,CAEtB,CAAC;AAEF,gBAAgB;AAEhB,eAAO,MAAM,SAAS,WAAY,GAAG,aAAY,OAAO,KAAU,wBAWjE,CAAC;AAEF,gBAAgB;AAChB,eAAO,MAAM,oBAAoB,YACtB,WAAW,cACR,MAAM,YACR,MAAM,WACP,MAAM,KACd,YAWF,CAAC;AAEF,wBAAsB,eAAe,CACnC,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,kBAAkB,CAAC,CAsD7B;AA+CD,UAAU,OAAO;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;CAC1C;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAuBpF;AAED,wBAAgB,YAAY,CAC1B,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,EAC5B,QAAQ,GAAE,OAAc,EACxB,MAAM,GAAE,OAAe,GACtB,WAAW,CAMb"}

package/dist/llm/utils.js

@@ -212,9 +212,9 @@ function computeChatCtxDiff(oldCtx, newCtx) {
     toCreate
   };
 }
-function toJsonSchema(schema, isOpenai = true) {
+function toJsonSchema(schema, isOpenai = true, strict = false) {
   if (isZodSchema(schema)) {
-    return zodSchemaToJsonSchema(schema, isOpenai);
+    return zodSchemaToJsonSchema(schema, isOpenai, strict);
   }
   return schema;
 }

package/dist/llm/utils.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { VideoBufferType, VideoFrame } from '@livekit/rtc-node';\nimport type { JSONSchema7 } from 'json-schema';\nimport sharp from 'sharp';\nimport type { UnknownUserData } from '../voice/run_context.js';\nimport type { ChatContext } from './chat_context.js';\nimport {\n  type ChatItem,\n  FunctionCall,\n  FunctionCallOutput,\n  type ImageContent,\n} from './chat_context.js';\nimport type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';\nimport { isZodSchema, parseZodSchema, zodSchemaToJsonSchema } from './zod-utils.js';\n\nexport interface SerializedImage {\n  inferenceDetail: 'auto' | 'high' | 'low';\n  mimeType?: string;\n  base64Data?: string;\n  externalUrl?: string;\n}\n\nfunction getChannelsFromVideoBufferType(type: VideoBufferType): 3 | 4 {\n  switch (type) {\n    case VideoBufferType.RGBA:\n    case VideoBufferType.ABGR:\n    case VideoBufferType.ARGB:\n    case VideoBufferType.BGRA:\n      return 4;\n    case VideoBufferType.RGB24:\n      return 3;\n    default:\n      // YUV formats (I420, I420A, I422, I444, I010, NV12) need conversion\n      throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);\n  }\n}\n\nfunction ensureRGBCompatible(frame: VideoFrame): VideoFrame {\n  // If the frame is already in an RGB/RGBA-compatible format, return it directly\n  if (\n    frame.type === VideoBufferType.RGBA ||\n    frame.type === VideoBufferType.BGRA ||\n    frame.type === VideoBufferType.ARGB ||\n    frame.type === VideoBufferType.ABGR ||\n    frame.type === VideoBufferType.RGB24\n  ) {\n    return frame;\n  }\n\n  // Otherwise, attempt conversion for other formats (like YUV)\n  try {\n    return frame.convert(VideoBufferType.RGBA);\n  } catch (error) {\n    throw new Error(\n      `Failed to convert format ${frame.type} to RGB: ${error}. ` +\n        `Consider using RGB/RGBA formats or converting on the client side.`,\n    );\n  }\n}\n\nexport async function serializeImage(image: ImageContent): Promise<SerializedImage> {\n  if (typeof image.image === 'string') {\n    if (image.image.startsWith('data:')) {\n      const [header, base64Data] = image.image.split(',', 2) as [string, string];\n      const headerParts = header.split(';');\n      const mimeParts = headerParts[0]?.split(':');\n      const headerMime = mimeParts?.[1];\n\n      if (!headerMime) {\n        throw new Error('Invalid data URL format');\n      }\n\n      let mimeType: string;\n      if (image.mimeType && image.mimeType !== headerMime) {\n        console.warn(\n          `Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`,\n        );\n        mimeType = image.mimeType;\n      } else {\n        mimeType = headerMime;\n      }\n\n      const supportedTypes = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);\n      if (!supportedTypes.has(mimeType)) {\n        throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);\n      }\n\n      return {\n        base64Data,\n        mimeType: mimeType,\n        inferenceDetail: image.inferenceDetail,\n      };\n    }\n\n    // External URL\n    return {\n      mimeType: image.mimeType,\n      inferenceDetail: image.inferenceDetail,\n      externalUrl: image.image,\n    };\n  } else if (image.image instanceof VideoFrame) {\n    const frame = ensureRGBCompatible(image.image);\n    const channels = getChannelsFromVideoBufferType(frame.type);\n\n    // Sharp needs to know the format of raw pixel data\n    let encoded = sharp(frame.data, {\n      raw: {\n        width: frame.width,\n        height: frame.height,\n        channels,\n      },\n    });\n\n    if (image.inferenceWidth && image.inferenceHeight) {\n      encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n    }\n\n    const base64Data = await encoded\n      .png()\n      .toBuffer()\n      .then((buffer) => buffer.toString('base64'));\n\n    return {\n      base64Data,\n      mimeType: 'image/png',\n      inferenceDetail: image.inferenceDetail,\n    };\n  } else {\n    throw new Error('Unsupported image type');\n  }\n}\n\n/** Raw OpenAI-adherent function parameters. */\nexport type OpenAIFunctionParameters = {\n  type: 'object';\n  properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any\n  required: string[];\n  additionalProperties?: boolean;\n};\n\n// TODO(brian): remove this helper once we have the real RunContext user data\nexport const createToolOptions = <UserData extends UnknownUserData>(\n  toolCallId: string,\n  userData: UserData = {} as UserData,\n): ToolOptions<UserData> => {\n  return { ctx: { userData }, toolCallId } as unknown as ToolOptions<UserData>;\n};\n\n/** @internal */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const oaiParams = (schema: any, isOpenai: boolean = true): OpenAIFunctionParameters => {\n  // Adapted from https://github.com/vercel/ai/blob/56eb0ee9/packages/provider-utils/src/zod-schema.ts\n  const jsonSchema = zodSchemaToJsonSchema(schema, isOpenai);\n  const { properties, required, additionalProperties } = jsonSchema as OpenAIFunctionParameters;\n\n  return {\n    type: 'object',\n    properties,\n    required,\n    additionalProperties,\n  };\n};\n\n/** @internal */\nexport const oaiBuildFunctionInfo = (\n  toolCtx: ToolContext,\n  toolCallId: string,\n  toolName: string,\n  rawArgs: string,\n): FunctionCall => {\n  const tool = toolCtx[toolName];\n  if (!tool) {\n    throw new Error(`AI tool ${toolName} not found`);\n  }\n\n  return FunctionCall.create({\n    callId: toolCallId,\n    name: toolName,\n    args: rawArgs,\n  });\n};\n\nexport async function executeToolCall(\n  toolCall: FunctionCall,\n  toolCtx: ToolContext,\n): Promise<FunctionCallOutput> {\n  const tool = toolCtx[toolCall.name]!;\n  let args: object | undefined;\n  let params: object | undefined;\n\n  // Ensure valid JSON\n  try {\n    args = JSON.parse(toolCall.args);\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Invalid JSON: ${error}`,\n      isError: true,\n    });\n  }\n\n  // Ensure valid arguments schema\n  try {\n    if (isZodSchema(tool.parameters)) {\n      const result = await parseZodSchema<object>(tool.parameters, args);\n      if (result.success) {\n        params = result.data;\n      } else {\n        return FunctionCallOutput.create({\n          callId: toolCall.callId,\n          output: `Arguments parsing failed: ${result.error}`,\n          isError: true,\n        });\n      }\n    } else {\n      params = args;\n    }\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Arguments parsing failed: ${error}`,\n      isError: true,\n    });\n  }\n\n  try {\n    const result = await tool.execute(params, createToolOptions(toolCall.callId));\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: JSON.stringify(result),\n      isError: false,\n    });\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Tool execution failed: ${error}`,\n      isError: true,\n    });\n  }\n}\n\n/**\n * Standard dynamic-programming LCS to get the common subsequence\n * of IDs (in order) that appear in both old_ids and new_ids.\n *\n * @param oldIds - The old list of IDs.\n * @param newIds - The new list of IDs.\n * @returns The longest common subsequence of the two lists of IDs.\n */\nfunction computeLCS(oldIds: string[], newIds: string[]): string[] {\n  const n = oldIds.length;\n  const m = newIds.length;\n  const dp: number[][] = Array(n + 1)\n    .fill(null)\n    .map(() => Array(m + 1).fill(0));\n\n  // Fill DP table\n  for (let i = 1; i <= n; i++) {\n    for (let j = 1; j <= m; j++) {\n      if (oldIds[i - 1] === newIds[j - 1]) {\n        dp[i]![j] = dp[i - 1]![j - 1]! + 1;\n      } else {\n        dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n      }\n    }\n  }\n\n  // Backtrack to find the actual LCS sequence\n  const lcsIds: string[] = [];\n  let i = n;\n  let j = m;\n  while (i > 0 && j > 0) {\n    if (oldIds[i - 1] === newIds[j - 1]) {\n      lcsIds.push(oldIds[i - 1]!);\n      i--;\n      j--;\n    } else if (dp[i - 1]![j]! > dp[i]![j - 1]!) {\n      i--;\n    } else {\n      j--;\n    }\n  }\n\n  return lcsIds.reverse();\n}\n\ninterface DiffOps {\n  toRemove: string[];\n  toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root\n}\n\n/**\n * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.\n *\n * @param oldCtx - The old chat context.\n * @param newCtx - The new chat context.\n * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.\n */\nexport function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps {\n  const oldIds = oldCtx.items.map((item: ChatItem) => item.id);\n  const newIds = newCtx.items.map((item: ChatItem) => item.id);\n  const lcsIds = new Set(computeLCS(oldIds, newIds));\n\n  const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);\n  const toCreate: Array<[string | null, string]> = [];\n\n  let lastIdInSequence: string | null = null;\n  for (const newItem of newCtx.items) {\n    if (lcsIds.has(newItem.id)) {\n      lastIdInSequence = newItem.id;\n    } else {\n      const prevId = lastIdInSequence; // null if root\n      toCreate.push([prevId, newItem.id]);\n      lastIdInSequence = newItem.id;\n    }\n  }\n\n  return {\n    toRemove,\n    toCreate,\n  };\n}\n\nexport function toJsonSchema(schema: ToolInputSchema<any>, isOpenai: boolean = true): JSONSchema7 {\n  if (isZodSchema(schema)) {\n    return zodSchemaToJsonSchema(schema, isOpenai);\n  }\n  return schema as JSONSchema7;\n}\n"],"mappings":"AAGA,SAAS,iBAAiB,kBAAkB;AAE5C,OAAO,WAAW;AAGlB;AAAA,EAEE;AAAA,EACA;AAAA,OAEK;AAEP,SAAS,aAAa,gBAAgB,6BAA6B;AASnE,SAAS,+BAA+B,MAA8B;AACpE,UAAQ,MAAM;AAAA,IACZ,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AACnB,aAAO;AAAA,IACT,KAAK,gBAAgB;AACnB,aAAO;AAAA,IACT;AAEE,YAAM,IAAI,MAAM,gCAAgC,IAAI,wCAAwC;AAAA,EAChG;AACF;AAEA,SAAS,oBAAoB,OAA+B;AAE1D,MACE,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,OAC/B;AACA,WAAO;AAAA,EACT;AAGA,MAAI;AACF,WAAO,MAAM,QAAQ,gBAAgB,IAAI;AAAA,EAC3C,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,IAAI,YAAY,KAAK;AAAA,IAEzD;AAAA,EACF;AACF;AAEA,eAAsB,eAAe,OAA+C;AA9DpF;AA+DE,MAAI,OAAO,MAAM,UAAU,UAAU;AACnC,QAAI,MAAM,MAAM,WAAW,OAAO,GAAG;AACnC,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,MAAM,KAAK,CAAC;AACrD,YAAM,cAAc,OAAO,MAAM,GAAG;AACpC,YAAM,aAAY,iBAAY,CAAC,MAAb,mBAAgB,MAAM;AACxC,YAAM,aAAa,uCAAY;AAE/B,UAAI,CAAC,YAAY;AACf,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAEA,UAAI;AACJ,UAAI,MAAM,YAAY,MAAM,aAAa,YAAY;AACnD,gBAAQ;AAAA,UACN,sBAAsB,MAAM,QAAQ,wCAAwC,UAAU;AAAA,QACxF;AACA,mBAAW,MAAM;AAAA,MACnB,OAAO;AACL,mBAAW;AAAA,MACb;AAEA,YAAM,iBAAiB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,WAAW,CAAC;AACrF,UAAI,CAAC,eAAe,IAAI,QAAQ,GAAG;AACjC,cAAM,IAAI,MAAM,wBAAwB,QAAQ,mCAAmC;AAAA,MACrF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,iBAAiB,MAAM;AAAA,MACzB;AAAA,IACF;AAGA,WAAO;AAAA,MACL,UAAU,MAAM;AAAA,MAChB,iBAAiB,MAAM;AAAA,MACvB,aAAa,MAAM;AAAA,IACrB;AAAA,EACF,WAAW,MAAM,iBAAiB,YAAY;AAC5C,UAAM,QAAQ,oBAAoB,MAAM,KAAK;AAC7C,UAAM,WAAW,+BAA+B,MAAM,IAAI;AAG1D,QAAI,UAAU,MAAM,MAAM,MAAM;AAAA,MAC9B,KAAK;AAAA,QACH,OAAO,MAAM;AAAA,QACb,QAAQ,MAAM;AAAA,QACd;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,MAAM,kBAAkB,MAAM,iBAAiB;AACjD,gBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,QACtB,IAAI,EACJ,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,QAAQ,CAAC;AAE7C,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,iBAAiB,MAAM;AAAA,IACzB;AAAA,EACF,OAAO;AACL,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AACF;AAWO,MAAM,oBAAoB,CAC/B,YACA,WAAqB,CAAC,MACI;AAC1B,SAAO,EAAE,KAAK,EAAE,SAAS,GAAG,WAAW;AACzC;AAIO,MAAM,YAAY,CAAC,QAAa,WAAoB,SAAmC;AAE5F,QAAM,aAAa,sBAAsB,QAAQ,QAAQ;AACzD,QAAM,EAAE,YAAY,UAAU,qBAAqB,IAAI;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGO,MAAM,uBAAuB,CAClC,SACA,YACA,UACA,YACiB;AACjB,QAAM,OAAO,QAAQ,QAAQ;AAC7B,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,MAAM,WAAW,QAAQ,YAAY;AAAA,EACjD;AAEA,SAAO,aAAa,OAAO;AAAA,IACzB,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACH;AAEA,eAAsB,gBACpB,UACA,SAC6B;AAC7B,QAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,MAAI;AACJ,MAAI;AAGJ,MAAI;AACF,WAAO,KAAK,MAAM,SAAS,IAAI;AAAA,EACjC,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,iBAAiB,KAAK;AAAA,MAC9B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,MAAI;AACF,QAAI,YAAY,KAAK,UAAU,GAAG;AAChC,YAAM,SAAS,MAAM,eAAuB,KAAK,YAAY,IAAI;AACjE,UAAI,OAAO,SAAS;AAClB,iBAAS,OAAO;AAAA,MAClB,OAAO;AACL,eAAO,mBAAmB,OAAO;AAAA,UAC/B,QAAQ,SAAS;AAAA,UACjB,QAAQ,6BAA6B,OAAO,KAAK;AAAA,UACjD,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAAA,IACF,OAAO;AACL,eAAS;AAAA,IACX;AAAA,EACF,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,6BAA6B,KAAK;AAAA,MAC1C,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,QAAQ,QAAQ,kBAAkB,SAAS,MAAM,CAAC;AAC5E,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,KAAK,UAAU,MAAM;AAAA,MAC7B,SAAS;AAAA,IACX,CAAC;AAAA,EACH,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,0BAA0B,KAAK;AAAA,MACvC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAUA,SAAS,WAAW,QAAkB,QAA4B;AAChE,QAAM,IAAI,OAAO;AACjB,QAAM,IAAI,OAAO;AACjB,QAAM,KAAiB,MAAM,IAAI,CAAC,EAC/B,KAAK,IAAI,EACT,IAAI,MAAM,MAAM,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAGjC,WAASA,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,aAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,UAAI,OAAOD,KAAI,CAAC,MAAM,OAAOC,KAAI,CAAC,GAAG;AACnC,WAAGD,EAAC,EAAGC,EAAC,IAAI,GAAGD,KAAI,CAAC,EAAGC,KAAI,CAAC,IAAK;AAAA,MACnC,OAAO;AACL,WAAGD,EAAC,EAAGC,EAAC,IAAI,KAAK,IAAI,GAAGD,KAAI,CAAC,EAAGC,EAAC,GAAI,GAAGD,EAAC,EAAGC,KAAI,CAAC,CAAE;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AAGA,QAAM,SAAmB,CAAC;AAC1B,MAAI,IAAI;AACR,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,IAAI,GAAG;AACrB,QAAI,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,GAAG;AACnC,aAAO,KAAK,OAAO,IAAI,CAAC,CAAE;AAC1B;AACA;AAAA,IACF,WAAW,GAAG,IAAI,CAAC,EAAG,CAAC,IAAK,GAAG,CAAC,EAAG,IAAI,CAAC,GAAI;AAC1C;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,OAAO,QAAQ;AACxB;AAcO,SAAS,mBAAmB,QAAqB,QAA8B;AACpF,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,IAAI,IAAI,WAAW,QAAQ,MAAM,CAAC;AAEjD,QAAM,WAAW,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtF,QAAM,WAA2C,CAAC;AAElD,MAAI,mBAAkC;AACtC,aAAW,WAAW,OAAO,OAAO;AAClC,QAAI,OAAO,IAAI,QAAQ,EAAE,GAAG;AAC1B,yBAAmB,QAAQ;AAAA,IAC7B,OAAO;AACL,YAAM,SAAS;AACf,eAAS,KAAK,CAAC,QAAQ,QAAQ,EAAE,CAAC;AAClC,yBAAmB,QAAQ;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,aAAa,QAA8B,WAAoB,MAAmB;AAChG,MAAI,YAAY,MAAM,GAAG;AACvB,WAAO,sBAAsB,QAAQ,QAAQ;AAAA,EAC/C;AACA,SAAO;AACT;","names":["i","j"]}
+{"version":3,"sources":["../../src/llm/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { VideoBufferType, VideoFrame } from '@livekit/rtc-node';\nimport type { JSONSchema7 } from 'json-schema';\nimport sharp from 'sharp';\nimport type { UnknownUserData } from '../voice/run_context.js';\nimport type { ChatContext } from './chat_context.js';\nimport {\n  type ChatItem,\n  FunctionCall,\n  FunctionCallOutput,\n  type ImageContent,\n} from './chat_context.js';\nimport type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';\nimport { isZodSchema, parseZodSchema, zodSchemaToJsonSchema } from './zod-utils.js';\n\nexport interface SerializedImage {\n  inferenceDetail: 'auto' | 'high' | 'low';\n  mimeType?: string;\n  base64Data?: string;\n  externalUrl?: string;\n}\n\nfunction getChannelsFromVideoBufferType(type: VideoBufferType): 3 | 4 {\n  switch (type) {\n    case VideoBufferType.RGBA:\n    case VideoBufferType.ABGR:\n    case VideoBufferType.ARGB:\n    case VideoBufferType.BGRA:\n      return 4;\n    case VideoBufferType.RGB24:\n      return 3;\n    default:\n      // YUV formats (I420, I420A, I422, I444, I010, NV12) need conversion\n      throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);\n  }\n}\n\nfunction ensureRGBCompatible(frame: VideoFrame): VideoFrame {\n  // If the frame is already in an RGB/RGBA-compatible format, return it directly\n  if (\n    frame.type === VideoBufferType.RGBA ||\n    frame.type === VideoBufferType.BGRA ||\n    frame.type === VideoBufferType.ARGB ||\n    frame.type === VideoBufferType.ABGR ||\n    frame.type === VideoBufferType.RGB24\n  ) {\n    return frame;\n  }\n\n  // Otherwise, attempt conversion for other formats (like YUV)\n  try {\n    return frame.convert(VideoBufferType.RGBA);\n  } catch (error) {\n    throw new Error(\n      `Failed to convert format ${frame.type} to RGB: ${error}. ` +\n        `Consider using RGB/RGBA formats or converting on the client side.`,\n    );\n  }\n}\n\nexport async function serializeImage(image: ImageContent): Promise<SerializedImage> {\n  if (typeof image.image === 'string') {\n    if (image.image.startsWith('data:')) {\n      const [header, base64Data] = image.image.split(',', 2) as [string, string];\n      const headerParts = header.split(';');\n      const mimeParts = headerParts[0]?.split(':');\n      const headerMime = mimeParts?.[1];\n\n      if (!headerMime) {\n        throw new Error('Invalid data URL format');\n      }\n\n      let mimeType: string;\n      if (image.mimeType && image.mimeType !== headerMime) {\n        console.warn(\n          `Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`,\n        );\n        mimeType = image.mimeType;\n      } else {\n        mimeType = headerMime;\n      }\n\n      const supportedTypes = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);\n      if (!supportedTypes.has(mimeType)) {\n        throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);\n      }\n\n      return {\n        base64Data,\n        mimeType: mimeType,\n        inferenceDetail: image.inferenceDetail,\n      };\n    }\n\n    // External URL\n    return {\n      mimeType: image.mimeType,\n      inferenceDetail: image.inferenceDetail,\n      externalUrl: image.image,\n    };\n  } else if (image.image instanceof VideoFrame) {\n    const frame = ensureRGBCompatible(image.image);\n    const channels = getChannelsFromVideoBufferType(frame.type);\n\n    // Sharp needs to know the format of raw pixel data\n    let encoded = sharp(frame.data, {\n      raw: {\n        width: frame.width,\n        height: frame.height,\n        channels,\n      },\n    });\n\n    if (image.inferenceWidth && image.inferenceHeight) {\n      encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n    }\n\n    const base64Data = await encoded\n      .png()\n      .toBuffer()\n      .then((buffer) => buffer.toString('base64'));\n\n    return {\n      base64Data,\n      mimeType: 'image/png',\n      inferenceDetail: image.inferenceDetail,\n    };\n  } else {\n    throw new Error('Unsupported image type');\n  }\n}\n\n/** Raw OpenAI-adherent function parameters. */\nexport type OpenAIFunctionParameters = {\n  type: 'object';\n  properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any\n  required: string[];\n  additionalProperties?: boolean;\n};\n\n// TODO(brian): remove this helper once we have the real RunContext user data\nexport const createToolOptions = <UserData extends UnknownUserData>(\n  toolCallId: string,\n  userData: UserData = {} as UserData,\n): ToolOptions<UserData> => {\n  return { ctx: { userData }, toolCallId } as unknown as ToolOptions<UserData>;\n};\n\n/** @internal */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const oaiParams = (schema: any, isOpenai: boolean = true): OpenAIFunctionParameters => {\n  // Adapted from https://github.com/vercel/ai/blob/56eb0ee9/packages/provider-utils/src/zod-schema.ts\n  const jsonSchema = zodSchemaToJsonSchema(schema, isOpenai);\n  const { properties, required, additionalProperties } = jsonSchema as OpenAIFunctionParameters;\n\n  return {\n    type: 'object',\n    properties,\n    required,\n    additionalProperties,\n  };\n};\n\n/** @internal */\nexport const oaiBuildFunctionInfo = (\n  toolCtx: ToolContext,\n  toolCallId: string,\n  toolName: string,\n  rawArgs: string,\n): FunctionCall => {\n  const tool = toolCtx[toolName];\n  if (!tool) {\n    throw new Error(`AI tool ${toolName} not found`);\n  }\n\n  return FunctionCall.create({\n    callId: toolCallId,\n    name: toolName,\n    args: rawArgs,\n  });\n};\n\nexport async function executeToolCall(\n  toolCall: FunctionCall,\n  toolCtx: ToolContext,\n): Promise<FunctionCallOutput> {\n  const tool = toolCtx[toolCall.name]!;\n  let args: object | undefined;\n  let params: object | undefined;\n\n  // Ensure valid JSON\n  try {\n    args = JSON.parse(toolCall.args);\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Invalid JSON: ${error}`,\n      isError: true,\n    });\n  }\n\n  // Ensure valid arguments schema\n  try {\n    if (isZodSchema(tool.parameters)) {\n      const result = await parseZodSchema<object>(tool.parameters, args);\n      if (result.success) {\n        params = result.data;\n      } else {\n        return FunctionCallOutput.create({\n          callId: toolCall.callId,\n          output: `Arguments parsing failed: ${result.error}`,\n          isError: true,\n        });\n      }\n    } else {\n      params = args;\n    }\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Arguments parsing failed: ${error}`,\n      isError: true,\n    });\n  }\n\n  try {\n    const result = await tool.execute(params, createToolOptions(toolCall.callId));\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: JSON.stringify(result),\n      isError: false,\n    });\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Tool execution failed: ${error}`,\n      isError: true,\n    });\n  }\n}\n\n/**\n * Standard dynamic-programming LCS to get the common subsequence\n * of IDs (in order) that appear in both old_ids and new_ids.\n *\n * @param oldIds - The old list of IDs.\n * @param newIds - The new list of IDs.\n * @returns The longest common subsequence of the two lists of IDs.\n */\nfunction computeLCS(oldIds: string[], newIds: string[]): string[] {\n  const n = oldIds.length;\n  const m = newIds.length;\n  const dp: number[][] = Array(n + 1)\n    .fill(null)\n    .map(() => Array(m + 1).fill(0));\n\n  // Fill DP table\n  for (let i = 1; i <= n; i++) {\n    for (let j = 1; j <= m; j++) {\n      if (oldIds[i - 1] === newIds[j - 1]) {\n        dp[i]![j] = dp[i - 1]![j - 1]! + 1;\n      } else {\n        dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n      }\n    }\n  }\n\n  // Backtrack to find the actual LCS sequence\n  const lcsIds: string[] = [];\n  let i = n;\n  let j = m;\n  while (i > 0 && j > 0) {\n    if (oldIds[i - 1] === newIds[j - 1]) {\n      lcsIds.push(oldIds[i - 1]!);\n      i--;\n      j--;\n    } else if (dp[i - 1]![j]! > dp[i]![j - 1]!) {\n      i--;\n    } else {\n      j--;\n    }\n  }\n\n  return lcsIds.reverse();\n}\n\ninterface DiffOps {\n  toRemove: string[];\n  toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root\n}\n\n/**\n * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.\n *\n * @param oldCtx - The old chat context.\n * @param newCtx - The new chat context.\n * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.\n */\nexport function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps {\n  const oldIds = oldCtx.items.map((item: ChatItem) => item.id);\n  const newIds = newCtx.items.map((item: ChatItem) => item.id);\n  const lcsIds = new Set(computeLCS(oldIds, newIds));\n\n  const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);\n  const toCreate: Array<[string | null, string]> = [];\n\n  let lastIdInSequence: string | null = null;\n  for (const newItem of newCtx.items) {\n    if (lcsIds.has(newItem.id)) {\n      lastIdInSequence = newItem.id;\n    } else {\n      const prevId = lastIdInSequence; // null if root\n      toCreate.push([prevId, newItem.id]);\n      lastIdInSequence = newItem.id;\n    }\n  }\n\n  return {\n    toRemove,\n    toCreate,\n  };\n}\n\nexport function toJsonSchema(\n  schema: ToolInputSchema<any>,\n  isOpenai: boolean = true,\n  strict: boolean = false,\n): JSONSchema7 {\n  if (isZodSchema(schema)) {\n    return zodSchemaToJsonSchema(schema, isOpenai, strict);\n  }\n\n  return schema as JSONSchema7;\n}\n"],"mappings":"AAGA,SAAS,iBAAiB,kBAAkB;AAE5C,OAAO,WAAW;AAGlB;AAAA,EAEE;AAAA,EACA;AAAA,OAEK;AAEP,SAAS,aAAa,gBAAgB,6BAA6B;AASnE,SAAS,+BAA+B,MAA8B;AACpE,UAAQ,MAAM;AAAA,IACZ,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AACnB,aAAO;AAAA,IACT,KAAK,gBAAgB;AACnB,aAAO;AAAA,IACT;AAEE,YAAM,IAAI,MAAM,gCAAgC,IAAI,wCAAwC;AAAA,EAChG;AACF;AAEA,SAAS,oBAAoB,OAA+B;AAE1D,MACE,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,OAC/B;AACA,WAAO;AAAA,EACT;AAGA,MAAI;AACF,WAAO,MAAM,QAAQ,gBAAgB,IAAI;AAAA,EAC3C,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,IAAI,YAAY,KAAK;AAAA,IAEzD;AAAA,EACF;AACF;AAEA,eAAsB,eAAe,OAA+C;AA9DpF;AA+DE,MAAI,OAAO,MAAM,UAAU,UAAU;AACnC,QAAI,MAAM,MAAM,WAAW,OAAO,GAAG;AACnC,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,MAAM,KAAK,CAAC;AACrD,YAAM,cAAc,OAAO,MAAM,GAAG;AACpC,YAAM,aAAY,iBAAY,CAAC,MAAb,mBAAgB,MAAM;AACxC,YAAM,aAAa,uCAAY;AAE/B,UAAI,CAAC,YAAY;AACf,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAEA,UAAI;AACJ,UAAI,MAAM,YAAY,MAAM,aAAa,YAAY;AACnD,gBAAQ;AAAA,UACN,sBAAsB,MAAM,QAAQ,wCAAwC,UAAU;AAAA,QACxF;AACA,mBAAW,MAAM;AAAA,MACnB,OAAO;AACL,mBAAW;AAAA,MACb;AAEA,YAAM,iBAAiB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,WAAW,CAAC;AACrF,UAAI,CAAC,eAAe,IAAI,QAAQ,GAAG;AACjC,cAAM,IAAI,MAAM,wBAAwB,QAAQ,mCAAmC;AAAA,MACrF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,iBAAiB,MAAM;AAAA,MACzB;AAAA,IACF;AAGA,WAAO;AAAA,MACL,UAAU,MAAM;AAAA,MAChB,iBAAiB,MAAM;AAAA,MACvB,aAAa,MAAM;AAAA,IACrB;AAAA,EACF,WAAW,MAAM,iBAAiB,YAAY;AAC5C,UAAM,QAAQ,oBAAoB,MAAM,KAAK;AAC7C,UAAM,WAAW,+BAA+B,MAAM,IAAI;AAG1D,QAAI,UAAU,MAAM,MAAM,MAAM;AAAA,MAC9B,KAAK;AAAA,QACH,OAAO,MAAM;AAAA,QACb,QAAQ,MAAM;AAAA,QACd;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,MAAM,kBAAkB,MAAM,iBAAiB;AACjD,gBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,QACtB,IAAI,EACJ,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,QAAQ,CAAC;AAE7C,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,iBAAiB,MAAM;AAAA,IACzB;AAAA,EACF,OAAO;AACL,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AACF;AAWO,MAAM,oBAAoB,CAC/B,YACA,WAAqB,CAAC,MACI;AAC1B,SAAO,EAAE,KAAK,EAAE,SAAS,GAAG,WAAW;AACzC;AAIO,MAAM,YAAY,CAAC,QAAa,WAAoB,SAAmC;AAE5F,QAAM,aAAa,sBAAsB,QAAQ,QAAQ;AACzD,QAAM,EAAE,YAAY,UAAU,qBAAqB,IAAI;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGO,MAAM,uBAAuB,CAClC,SACA,YACA,UACA,YACiB;AACjB,QAAM,OAAO,QAAQ,QAAQ;AAC7B,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,MAAM,WAAW,QAAQ,YAAY;AAAA,EACjD;AAEA,SAAO,aAAa,OAAO;AAAA,IACzB,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACH;AAEA,eAAsB,gBACpB,UACA,SAC6B;AAC7B,QAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,MAAI;AACJ,MAAI;AAGJ,MAAI;AACF,WAAO,KAAK,MAAM,SAAS,IAAI;AAAA,EACjC,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,iBAAiB,KAAK;AAAA,MAC9B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,MAAI;AACF,QAAI,YAAY,KAAK,UAAU,GAAG;AAChC,YAAM,SAAS,MAAM,eAAuB,KAAK,YAAY,IAAI;AACjE,UAAI,OAAO,SAAS;AAClB,iBAAS,OAAO;AAAA,MAClB,OAAO;AACL,eAAO,mBAAmB,OAAO;AAAA,UAC/B,QAAQ,SAAS;AAAA,UACjB,QAAQ,6BAA6B,OAAO,KAAK;AAAA,UACjD,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAAA,IACF,OAAO;AACL,eAAS;AAAA,IACX;AAAA,EACF,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,6BAA6B,KAAK;AAAA,MAC1C,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,QAAQ,QAAQ,kBAAkB,SAAS,MAAM,CAAC;AAC5E,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,KAAK,UAAU,MAAM;AAAA,MAC7B,SAAS;AAAA,IACX,CAAC;AAAA,EACH,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,0BAA0B,KAAK;AAAA,MACvC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAUA,SAAS,WAAW,QAAkB,QAA4B;AAChE,QAAM,IAAI,OAAO;AACjB,QAAM,IAAI,OAAO;AACjB,QAAM,KAAiB,MAAM,IAAI,CAAC,EAC/B,KAAK,IAAI,EACT,IAAI,MAAM,MAAM,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAGjC,WAASA,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,aAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,UAAI,OAAOD,KAAI,CAAC,MAAM,OAAOC,KAAI,CAAC,GAAG;AACnC,WAAGD,EAAC,EAAGC,EAAC,IAAI,GAAGD,KAAI,CAAC,EAAGC,KAAI,CAAC,IAAK;AAAA,MACnC,OAAO;AACL,WAAGD,EAAC,EAAGC,EAAC,IAAI,KAAK,IAAI,GAAGD,KAAI,CAAC,EAAGC,EAAC,GAAI,GAAGD,EAAC,EAAGC,KAAI,CAAC,CAAE;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AAGA,QAAM,SAAmB,CAAC;AAC1B,MAAI,IAAI;AACR,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,IAAI,GAAG;AACrB,QAAI,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,GAAG;AACnC,aAAO,KAAK,OAAO,IAAI,CAAC,CAAE;AAC1B;AACA;AAAA,IACF,WAAW,GAAG,IAAI,CAAC,EAAG,CAAC,IAAK,GAAG,CAAC,EAAG,IAAI,CAAC,GAAI;AAC1C;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,OAAO,QAAQ;AACxB;AAcO,SAAS,mBAAmB,QAAqB,QAA8B;AACpF,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,IAAI,IAAI,WAAW,QAAQ,MAAM,CAAC;AAEjD,QAAM,WAAW,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtF,QAAM,WAA2C,CAAC;AAElD,MAAI,mBAAkC;AACtC,aAAW,WAAW,OAAO,OAAO;AAClC,QAAI,OAAO,IAAI,QAAQ,EAAE,GAAG;AAC1B,yBAAmB,QAAQ;AAAA,IAC7B,OAAO;AACL,YAAM,SAAS;AACf,eAAS,KAAK,CAAC,QAAQ,QAAQ,EAAE,CAAC;AAClC,yBAAmB,QAAQ;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,aACd,QACA,WAAoB,MACpB,SAAkB,OACL;AACb,MAAI,YAAY,MAAM,GAAG;AACvB,WAAO,sBAAsB,QAAQ,UAAU,MAAM;AAAA,EACvD;AAEA,SAAO;AACT;","names":["i","j"]}

package/dist/llm/zod-utils.cjs

@@ -35,6 +35,7 @@ __export(zod_utils_exports, {
   zodSchemaToJsonSchema: () => zodSchemaToJsonSchema
 });
 module.exports = __toCommonJS(zod_utils_exports);
+var import_transform = require("openai/lib/transform");
 var import_zod_to_json_schema = require("zod-to-json-schema");
 var z4 = __toESM(require("zod/v4"), 1);
 function isZod4Schema(schema) {
@@ -63,21 +64,23 @@ function isZodObjectSchema(schema) {
   }
   return ((_d = schemaWithInternals._def) == null ? void 0 : _d.typeName) === "ZodObject";
 }
-function zodSchemaToJsonSchema(schema, isOpenai = true) {
+function zodSchemaToJsonSchema(schema, isOpenai = true, strict = false) {
+  let result;
   if (isZod4Schema(schema)) {
-    return z4.toJSONSchema(schema, {
+    result = z4.toJSONSchema(schema, {
       target: "draft-7",
       io: "output",
       reused: "inline"
       // Don't use references by default (to support openapi conversion for google)
     });
   } else {
-    return (0, import_zod_to_json_schema.zodToJsonSchema)(schema, {
+    result = (0, import_zod_to_json_schema.zodToJsonSchema)(schema, {
       target: isOpenai ? "openAi" : "jsonSchema7",
       $refStrategy: "none"
       // Don't use references by default (to support openapi conversion for google)
     });
   }
+  return strict ? (0, import_transform.toStrictJsonSchema)(result) : result;
 }
 async function parseZodSchema(schema, value) {
   if (isZod4Schema(schema)) {