@livekit/agents 1.0.0-next.1 → 1.0.0-next.3

package/dist/llm/utils.cjs

@@ -128,9 +128,10 @@ async function serializeImage(image) {
 const createToolOptions = (toolCallId, userData = {}) => {
   return { ctx: { userData }, toolCallId };
 };
-const oaiParams = (p) => {
+const oaiParams = (p, isOpenai = true) => {
   const { properties, required, additionalProperties } = (0, import_zod_to_json_schema.zodToJsonSchema)(p, {
-    target: "openAi"
+    // note: openai mode breaks various gemini conversions
+    target: isOpenai ? "openAi" : "jsonSchema7"
   });
   return {
     type: "object",
@@ -241,9 +242,9 @@ function computeChatCtxDiff(oldCtx, newCtx) {
     toCreate
   };
 }
-function toJsonSchema(schema) {
+function toJsonSchema(schema, isOpenai = true) {
   if (schema instanceof import_zod.ZodObject) {
-    return oaiParams(schema);
+    return oaiParams(schema, isOpenai);
   }
   return schema;
 }

package/dist/llm/utils.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { VideoBufferType, VideoFrame } from '@livekit/rtc-node';\nimport type { JSONSchema7 } from 'json-schema';\nimport sharp from 'sharp';\nimport { ZodObject } from 'zod';\nimport { zodToJsonSchema } from 'zod-to-json-schema';\nimport type { UnknownUserData } from '../voice/run_context.js';\nimport type { ChatContext } from './chat_context.js';\nimport {\n  type ChatItem,\n  FunctionCall,\n  FunctionCallOutput,\n  type ImageContent,\n} from './chat_context.js';\nimport type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';\n\nexport interface SerializedImage {\n  inferenceDetail: 'auto' | 'high' | 'low';\n  mimeType?: string;\n  base64Data?: string;\n  externalUrl?: string;\n}\n\nfunction getChannelsFromVideoBufferType(type: VideoBufferType): 3 | 4 {\n  switch (type) {\n    case VideoBufferType.RGBA:\n    case VideoBufferType.ABGR:\n    case VideoBufferType.ARGB:\n    case VideoBufferType.BGRA:\n      return 4;\n    case VideoBufferType.RGB24:\n      return 3;\n    default:\n      // YUV formats (I420, I420A, I422, I444, I010, NV12) need conversion\n      throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);\n  }\n}\n\nfunction ensureRGBCompatible(frame: VideoFrame): VideoFrame {\n  // If the frame is already in an RGB/RGBA-compatible format, return it directly\n  if (\n    frame.type === VideoBufferType.RGBA ||\n    frame.type === VideoBufferType.BGRA ||\n    frame.type === VideoBufferType.ARGB ||\n    frame.type === VideoBufferType.ABGR ||\n    frame.type === VideoBufferType.RGB24\n  ) {\n    return frame;\n  }\n\n  // Otherwise, attempt conversion for other formats (like YUV)\n  try {\n    return frame.convert(VideoBufferType.RGBA);\n  } catch (error) {\n    throw new Error(\n      `Failed to convert format ${frame.type} to RGB: ${error}. ` +\n        `Consider using RGB/RGBA formats or converting on the client side.`,\n    );\n  }\n}\n\nexport async function serializeImage(image: ImageContent): Promise<SerializedImage> {\n  if (typeof image.image === 'string') {\n    if (image.image.startsWith('data:')) {\n      const [header, base64Data] = image.image.split(',', 2) as [string, string];\n      const headerParts = header.split(';');\n      const mimeParts = headerParts[0]?.split(':');\n      const headerMime = mimeParts?.[1];\n\n      if (!headerMime) {\n        throw new Error('Invalid data URL format');\n      }\n\n      let mimeType: string;\n      if (image.mimeType && image.mimeType !== headerMime) {\n        console.warn(\n          `Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`,\n        );\n        mimeType = image.mimeType;\n      } else {\n        mimeType = headerMime;\n      }\n\n      const supportedTypes = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);\n      if (!supportedTypes.has(mimeType)) {\n        throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);\n      }\n\n      return {\n        base64Data,\n        mimeType: mimeType,\n        inferenceDetail: image.inferenceDetail,\n      };\n    }\n\n    // External URL\n    return {\n      mimeType: image.mimeType,\n      inferenceDetail: image.inferenceDetail,\n      externalUrl: image.image,\n    };\n  } else if (image.image instanceof VideoFrame) {\n    const frame = ensureRGBCompatible(image.image);\n    const channels = getChannelsFromVideoBufferType(frame.type);\n\n    // Sharp needs to know the format of raw pixel data\n    let encoded = sharp(frame.data, {\n      raw: {\n        width: frame.width,\n        height: frame.height,\n        channels,\n      },\n    });\n\n    if (image.inferenceWidth && image.inferenceHeight) {\n      encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n    }\n\n    const base64Data = await encoded\n      .png()\n      .toBuffer()\n      .then((buffer) => buffer.toString('base64'));\n\n    return {\n      base64Data,\n      mimeType: 'image/png',\n      inferenceDetail: image.inferenceDetail,\n    };\n  } else {\n    throw new Error('Unsupported image type');\n  }\n}\n\n/** Raw OpenAI-adherent function parameters. */\nexport type OpenAIFunctionParameters = {\n  type: 'object';\n  properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any\n  required: string[];\n  additionalProperties?: boolean;\n};\n\n// TODO(brian): remove this helper once we have the real RunContext user data\nexport const createToolOptions = <UserData extends UnknownUserData>(\n  toolCallId: string,\n  userData: UserData = {} as UserData,\n): ToolOptions<UserData> => {\n  return { ctx: { userData }, toolCallId } as unknown as ToolOptions<UserData>;\n};\n\n/** @internal */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const oaiParams = (p: ZodObject<any>): OpenAIFunctionParameters => {\n  // TODO(AJS-162): make zod to JSON parsing able to handle z.optional(v.field())\n  const { properties, required, additionalProperties } = zodToJsonSchema(p, {\n    target: 'openAi',\n  }) as OpenAIFunctionParameters;\n\n  return {\n    type: 'object',\n    properties,\n    required,\n    additionalProperties,\n  };\n};\n\n/** @internal */\nexport const oaiBuildFunctionInfo = (\n  toolCtx: ToolContext,\n  toolCallId: string,\n  toolName: string,\n  rawArgs: string,\n): FunctionCall => {\n  const tool = toolCtx[toolName];\n  if (!tool) {\n    throw new Error(`AI tool ${toolName} not found`);\n  }\n\n  return FunctionCall.create({\n    callId: toolCallId,\n    name: toolName,\n    args: rawArgs,\n  });\n};\n\nexport async function executeToolCall(\n  toolCall: FunctionCall,\n  toolCtx: ToolContext,\n): Promise<FunctionCallOutput> {\n  const tool = toolCtx[toolCall.name]!;\n  let args: object | undefined;\n  let params: object | undefined;\n\n  // Ensure valid JSON\n  try {\n    args = JSON.parse(toolCall.args);\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Invalid JSON: ${error}`,\n      isError: true,\n    });\n  }\n\n  // Ensure valid arguments schema\n  try {\n    if (tool.parameters instanceof ZodObject) {\n      params = tool.parameters.parse(args);\n    } else {\n      params = args;\n    }\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Arguments parsing failed: ${error}`,\n      isError: true,\n    });\n  }\n\n  try {\n    const result = await tool.execute(params, createToolOptions(toolCall.callId));\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: JSON.stringify(result),\n      isError: false,\n    });\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Tool execution failed: ${error}`,\n      isError: true,\n    });\n  }\n}\n\n/**\n * Standard dynamic-programming LCS to get the common subsequence\n * of IDs (in order) that appear in both old_ids and new_ids.\n *\n * @param oldIds - The old list of IDs.\n * @param newIds - The new list of IDs.\n * @returns The longest common subsequence of the two lists of IDs.\n */\nfunction computeLCS(oldIds: string[], newIds: string[]): string[] {\n  const n = oldIds.length;\n  const m = newIds.length;\n  const dp: number[][] = Array(n + 1)\n    .fill(null)\n    .map(() => Array(m + 1).fill(0));\n\n  // Fill DP table\n  for (let i = 1; i <= n; i++) {\n    for (let j = 1; j <= m; j++) {\n      if (oldIds[i - 1] === newIds[j - 1]) {\n        dp[i]![j] = dp[i - 1]![j - 1]! + 1;\n      } else {\n        dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n      }\n    }\n  }\n\n  // Backtrack to find the actual LCS sequence\n  const lcsIds: string[] = [];\n  let i = n;\n  let j = m;\n  while (i > 0 && j > 0) {\n    if (oldIds[i - 1] === newIds[j - 1]) {\n      lcsIds.push(oldIds[i - 1]!);\n      i--;\n      j--;\n    } else if (dp[i - 1]![j]! > dp[i]![j - 1]!) {\n      i--;\n    } else {\n      j--;\n    }\n  }\n\n  return lcsIds.reverse();\n}\n\ninterface DiffOps {\n  toRemove: string[];\n  toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root\n}\n\n/**\n * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.\n *\n * @param oldCtx - The old chat context.\n * @param newCtx - The new chat context.\n * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.\n */\nexport function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps {\n  const oldIds = oldCtx.items.map((item: ChatItem) => item.id);\n  const newIds = newCtx.items.map((item: ChatItem) => item.id);\n  const lcsIds = new Set(computeLCS(oldIds, newIds));\n\n  const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);\n  const toCreate: Array<[string | null, string]> = [];\n\n  let lastIdInSequence: string | null = null;\n  for (const newItem of newCtx.items) {\n    if (lcsIds.has(newItem.id)) {\n      lastIdInSequence = newItem.id;\n    } else {\n      const prevId = lastIdInSequence; // null if root\n      toCreate.push([prevId, newItem.id]);\n      lastIdInSequence = newItem.id;\n    }\n  }\n\n  return {\n    toRemove,\n    toCreate,\n  };\n}\n\nexport function toJsonSchema(schema: ToolInputSchema<any>): JSONSchema7 {\n  if (schema instanceof ZodObject) {\n    return oaiParams(schema);\n  }\n  return schema;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAA4C;AAE5C,mBAAkB;AAClB,iBAA0B;AAC1B,gCAAgC;AAGhC,0BAKO;AAUP,SAAS,+BAA+B,MAA8B;AACpE,UAAQ,MAAM;AAAA,IACZ,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AACnB,aAAO;AAAA,IACT,KAAK,gCAAgB;AACnB,aAAO;AAAA,IACT;AAEE,YAAM,IAAI,MAAM,gCAAgC,IAAI,wCAAwC;AAAA,EAChG;AACF;AAEA,SAAS,oBAAoB,OAA+B;AAE1D,MACE,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,OAC/B;AACA,WAAO;AAAA,EACT;AAGA,MAAI;AACF,WAAO,MAAM,QAAQ,gCAAgB,IAAI;AAAA,EAC3C,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,IAAI,YAAY,KAAK;AAAA,IAEzD;AAAA,EACF;AACF;AAEA,eAAsB,eAAe,OAA+C;AA/DpF;AAgEE,MAAI,OAAO,MAAM,UAAU,UAAU;AACnC,QAAI,MAAM,MAAM,WAAW,OAAO,GAAG;AACnC,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,MAAM,KAAK,CAAC;AACrD,YAAM,cAAc,OAAO,MAAM,GAAG;AACpC,YAAM,aAAY,iBAAY,CAAC,MAAb,mBAAgB,MAAM;AACxC,YAAM,aAAa,uCAAY;AAE/B,UAAI,CAAC,YAAY;AACf,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAEA,UAAI;AACJ,UAAI,MAAM,YAAY,MAAM,aAAa,YAAY;AACnD,gBAAQ;AAAA,UACN,sBAAsB,MAAM,QAAQ,wCAAwC,UAAU;AAAA,QACxF;AACA,mBAAW,MAAM;AAAA,MACnB,OAAO;AACL,mBAAW;AAAA,MACb;AAEA,YAAM,iBAAiB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,WAAW,CAAC;AACrF,UAAI,CAAC,eAAe,IAAI,QAAQ,GAAG;AACjC,cAAM,IAAI,MAAM,wBAAwB,QAAQ,mCAAmC;AAAA,MACrF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,iBAAiB,MAAM;AAAA,MACzB;AAAA,IACF;AAGA,WAAO;AAAA,MACL,UAAU,MAAM;AAAA,MAChB,iBAAiB,MAAM;AAAA,MACvB,aAAa,MAAM;AAAA,IACrB;AAAA,EACF,WAAW,MAAM,iBAAiB,4BAAY;AAC5C,UAAM,QAAQ,oBAAoB,MAAM,KAAK;AAC7C,UAAM,WAAW,+BAA+B,MAAM,IAAI;AAG1D,QAAI,cAAU,aAAAA,SAAM,MAAM,MAAM;AAAA,MAC9B,KAAK;AAAA,QACH,OAAO,MAAM;AAAA,QACb,QAAQ,MAAM;AAAA,QACd;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,MAAM,kBAAkB,MAAM,iBAAiB;AACjD,gBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,QACtB,IAAI,EACJ,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,QAAQ,CAAC;AAE7C,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,iBAAiB,MAAM;AAAA,IACzB;AAAA,EACF,OAAO;AACL,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AACF;AAWO,MAAM,oBAAoB,CAC/B,YACA,WAAqB,CAAC,MACI;AAC1B,SAAO,EAAE,KAAK,EAAE,SAAS,GAAG,WAAW;AACzC;AAIO,MAAM,YAAY,CAAC,MAAgD;AAExE,QAAM,EAAE,YAAY,UAAU,qBAAqB,QAAI,2CAAgB,GAAG;AAAA,IACxE,QAAQ;AAAA,EACV,CAAC;AAED,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGO,MAAM,uBAAuB,CAClC,SACA,YACA,UACA,YACiB;AACjB,QAAM,OAAO,QAAQ,QAAQ;AAC7B,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,MAAM,WAAW,QAAQ,YAAY;AAAA,EACjD;AAEA,SAAO,iCAAa,OAAO;AAAA,IACzB,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACH;AAEA,eAAsB,gBACpB,UACA,SAC6B;AAC7B,QAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,MAAI;AACJ,MAAI;AAGJ,MAAI;AACF,WAAO,KAAK,MAAM,SAAS,IAAI;AAAA,EACjC,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,iBAAiB,KAAK;AAAA,MAC9B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,MAAI;AACF,QAAI,KAAK,sBAAsB,sBAAW;AACxC,eAAS,KAAK,WAAW,MAAM,IAAI;AAAA,IACrC,OAAO;AACL,eAAS;AAAA,IACX;AAAA,EACF,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,6BAA6B,KAAK;AAAA,MAC1C,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,QAAQ,QAAQ,kBAAkB,SAAS,MAAM,CAAC;AAC5E,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,KAAK,UAAU,MAAM;AAAA,MAC7B,SAAS;AAAA,IACX,CAAC;AAAA,EACH,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,0BAA0B,KAAK;AAAA,MACvC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAUA,SAAS,WAAW,QAAkB,QAA4B;AAChE,QAAM,IAAI,OAAO;AACjB,QAAM,IAAI,OAAO;AACjB,QAAM,KAAiB,MAAM,IAAI,CAAC,EAC/B,KAAK,IAAI,EACT,IAAI,MAAM,MAAM,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAGjC,WAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,aAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,UAAI,OAAOD,KAAI,CAAC,MAAM,OAAOC,KAAI,CAAC,GAAG;AACnC,WAAGD,EAAC,EAAGC,EAAC,IAAI,GAAGD,KAAI,CAAC,EAAGC,KAAI,CAAC,IAAK;AAAA,MACnC,OAAO;AACL,WAAGD,EAAC,EAAGC,EAAC,IAAI,KAAK,IAAI,GAAGD,KAAI,CAAC,EAAGC,EAAC,GAAI,GAAGD,EAAC,EAAGC,KAAI,CAAC,CAAE;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AAGA,QAAM,SAAmB,CAAC;AAC1B,MAAI,IAAI;AACR,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,IAAI,GAAG;AACrB,QAAI,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,GAAG;AACnC,aAAO,KAAK,OAAO,IAAI,CAAC,CAAE;AAC1B;AACA;AAAA,IACF,WAAW,GAAG,IAAI,CAAC,EAAG,CAAC,IAAK,GAAG,CAAC,EAAG,IAAI,CAAC,GAAI;AAC1C;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,OAAO,QAAQ;AACxB;AAcO,SAAS,mBAAmB,QAAqB,QAA8B;AACpF,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,IAAI,IAAI,WAAW,QAAQ,MAAM,CAAC;AAEjD,QAAM,WAAW,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtF,QAAM,WAA2C,CAAC;AAElD,MAAI,mBAAkC;AACtC,aAAW,WAAW,OAAO,OAAO;AAClC,QAAI,OAAO,IAAI,QAAQ,EAAE,GAAG;AAC1B,yBAAmB,QAAQ;AAAA,IAC7B,OAAO;AACL,YAAM,SAAS;AACf,eAAS,KAAK,CAAC,QAAQ,QAAQ,EAAE,CAAC;AAClC,yBAAmB,QAAQ;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,aAAa,QAA2C;AACtE,MAAI,kBAAkB,sBAAW;AAC/B,WAAO,UAAU,MAAM;AAAA,EACzB;AACA,SAAO;AACT;","names":["sharp","i","j"]}
+{"version":3,"sources":["../../src/llm/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { VideoBufferType, VideoFrame } from '@livekit/rtc-node';\nimport type { JSONSchema7 } from 'json-schema';\nimport sharp from 'sharp';\nimport { ZodObject } from 'zod';\nimport { zodToJsonSchema } from 'zod-to-json-schema';\nimport type { UnknownUserData } from '../voice/run_context.js';\nimport type { ChatContext } from './chat_context.js';\nimport {\n  type ChatItem,\n  FunctionCall,\n  FunctionCallOutput,\n  type ImageContent,\n} from './chat_context.js';\nimport type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';\n\nexport interface SerializedImage {\n  inferenceDetail: 'auto' | 'high' | 'low';\n  mimeType?: string;\n  base64Data?: string;\n  externalUrl?: string;\n}\n\nfunction getChannelsFromVideoBufferType(type: VideoBufferType): 3 | 4 {\n  switch (type) {\n    case VideoBufferType.RGBA:\n    case VideoBufferType.ABGR:\n    case VideoBufferType.ARGB:\n    case VideoBufferType.BGRA:\n      return 4;\n    case VideoBufferType.RGB24:\n      return 3;\n    default:\n      // YUV formats (I420, I420A, I422, I444, I010, NV12) need conversion\n      throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);\n  }\n}\n\nfunction ensureRGBCompatible(frame: VideoFrame): VideoFrame {\n  // If the frame is already in an RGB/RGBA-compatible format, return it directly\n  if (\n    frame.type === VideoBufferType.RGBA ||\n    frame.type === VideoBufferType.BGRA ||\n    frame.type === VideoBufferType.ARGB ||\n    frame.type === VideoBufferType.ABGR ||\n    frame.type === VideoBufferType.RGB24\n  ) {\n    return frame;\n  }\n\n  // Otherwise, attempt conversion for other formats (like YUV)\n  try {\n    return frame.convert(VideoBufferType.RGBA);\n  } catch (error) {\n    throw new Error(\n      `Failed to convert format ${frame.type} to RGB: ${error}. ` +\n        `Consider using RGB/RGBA formats or converting on the client side.`,\n    );\n  }\n}\n\nexport async function serializeImage(image: ImageContent): Promise<SerializedImage> {\n  if (typeof image.image === 'string') {\n    if (image.image.startsWith('data:')) {\n      const [header, base64Data] = image.image.split(',', 2) as [string, string];\n      const headerParts = header.split(';');\n      const mimeParts = headerParts[0]?.split(':');\n      const headerMime = mimeParts?.[1];\n\n      if (!headerMime) {\n        throw new Error('Invalid data URL format');\n      }\n\n      let mimeType: string;\n      if (image.mimeType && image.mimeType !== headerMime) {\n        console.warn(\n          `Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`,\n        );\n        mimeType = image.mimeType;\n      } else {\n        mimeType = headerMime;\n      }\n\n      const supportedTypes = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);\n      if (!supportedTypes.has(mimeType)) {\n        throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);\n      }\n\n      return {\n        base64Data,\n        mimeType: mimeType,\n        inferenceDetail: image.inferenceDetail,\n      };\n    }\n\n    // External URL\n    return {\n      mimeType: image.mimeType,\n      inferenceDetail: image.inferenceDetail,\n      externalUrl: image.image,\n    };\n  } else if (image.image instanceof VideoFrame) {\n    const frame = ensureRGBCompatible(image.image);\n    const channels = getChannelsFromVideoBufferType(frame.type);\n\n    // Sharp needs to know the format of raw pixel data\n    let encoded = sharp(frame.data, {\n      raw: {\n        width: frame.width,\n        height: frame.height,\n        channels,\n      },\n    });\n\n    if (image.inferenceWidth && image.inferenceHeight) {\n      encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n    }\n\n    const base64Data = await encoded\n      .png()\n      .toBuffer()\n      .then((buffer) => buffer.toString('base64'));\n\n    return {\n      base64Data,\n      mimeType: 'image/png',\n      inferenceDetail: image.inferenceDetail,\n    };\n  } else {\n    throw new Error('Unsupported image type');\n  }\n}\n\n/** Raw OpenAI-adherent function parameters. */\nexport type OpenAIFunctionParameters = {\n  type: 'object';\n  properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any\n  required: string[];\n  additionalProperties?: boolean;\n};\n\n// TODO(brian): remove this helper once we have the real RunContext user data\nexport const createToolOptions = <UserData extends UnknownUserData>(\n  toolCallId: string,\n  userData: UserData = {} as UserData,\n): ToolOptions<UserData> => {\n  return { ctx: { userData }, toolCallId } as unknown as ToolOptions<UserData>;\n};\n\n/** @internal */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const oaiParams = (\n  p: ZodObject<any>,\n  isOpenai: boolean = true,\n): OpenAIFunctionParameters => {\n  // Adapted from https://github.com/vercel/ai/blob/56eb0ee9/packages/provider-utils/src/zod-schema.ts\n  const { properties, required, additionalProperties } = zodToJsonSchema(p, {\n    // note: openai mode breaks various gemini conversions\n    target: isOpenai ? 'openAi' : 'jsonSchema7',\n  }) as OpenAIFunctionParameters;\n\n  return {\n    type: 'object',\n    properties,\n    required,\n    additionalProperties,\n  };\n};\n\n/** @internal */\nexport const oaiBuildFunctionInfo = (\n  toolCtx: ToolContext,\n  toolCallId: string,\n  toolName: string,\n  rawArgs: string,\n): FunctionCall => {\n  const tool = toolCtx[toolName];\n  if (!tool) {\n    throw new Error(`AI tool ${toolName} not found`);\n  }\n\n  return FunctionCall.create({\n    callId: toolCallId,\n    name: toolName,\n    args: rawArgs,\n  });\n};\n\nexport async function executeToolCall(\n  toolCall: FunctionCall,\n  toolCtx: ToolContext,\n): Promise<FunctionCallOutput> {\n  const tool = toolCtx[toolCall.name]!;\n  let args: object | undefined;\n  let params: object | undefined;\n\n  // Ensure valid JSON\n  try {\n    args = JSON.parse(toolCall.args);\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Invalid JSON: ${error}`,\n      isError: true,\n    });\n  }\n\n  // Ensure valid arguments schema\n  try {\n    if (tool.parameters instanceof ZodObject) {\n      params = tool.parameters.parse(args);\n    } else {\n      params = args;\n    }\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Arguments parsing failed: ${error}`,\n      isError: true,\n    });\n  }\n\n  try {\n    const result = await tool.execute(params, createToolOptions(toolCall.callId));\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: JSON.stringify(result),\n      isError: false,\n    });\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Tool execution failed: ${error}`,\n      isError: true,\n    });\n  }\n}\n\n/**\n * Standard dynamic-programming LCS to get the common subsequence\n * of IDs (in order) that appear in both old_ids and new_ids.\n *\n * @param oldIds - The old list of IDs.\n * @param newIds - The new list of IDs.\n * @returns The longest common subsequence of the two lists of IDs.\n */\nfunction computeLCS(oldIds: string[], newIds: string[]): string[] {\n  const n = oldIds.length;\n  const m = newIds.length;\n  const dp: number[][] = Array(n + 1)\n    .fill(null)\n    .map(() => Array(m + 1).fill(0));\n\n  // Fill DP table\n  for (let i = 1; i <= n; i++) {\n    for (let j = 1; j <= m; j++) {\n      if (oldIds[i - 1] === newIds[j - 1]) {\n        dp[i]![j] = dp[i - 1]![j - 1]! + 1;\n      } else {\n        dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n      }\n    }\n  }\n\n  // Backtrack to find the actual LCS sequence\n  const lcsIds: string[] = [];\n  let i = n;\n  let j = m;\n  while (i > 0 && j > 0) {\n    if (oldIds[i - 1] === newIds[j - 1]) {\n      lcsIds.push(oldIds[i - 1]!);\n      i--;\n      j--;\n    } else if (dp[i - 1]![j]! > dp[i]![j - 1]!) {\n      i--;\n    } else {\n      j--;\n    }\n  }\n\n  return lcsIds.reverse();\n}\n\ninterface DiffOps {\n  toRemove: string[];\n  toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root\n}\n\n/**\n * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.\n *\n * @param oldCtx - The old chat context.\n * @param newCtx - The new chat context.\n * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.\n */\nexport function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps {\n  const oldIds = oldCtx.items.map((item: ChatItem) => item.id);\n  const newIds = newCtx.items.map((item: ChatItem) => item.id);\n  const lcsIds = new Set(computeLCS(oldIds, newIds));\n\n  const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);\n  const toCreate: Array<[string | null, string]> = [];\n\n  let lastIdInSequence: string | null = null;\n  for (const newItem of newCtx.items) {\n    if (lcsIds.has(newItem.id)) {\n      lastIdInSequence = newItem.id;\n    } else {\n      const prevId = lastIdInSequence; // null if root\n      toCreate.push([prevId, newItem.id]);\n      lastIdInSequence = newItem.id;\n    }\n  }\n\n  return {\n    toRemove,\n    toCreate,\n  };\n}\n\nexport function toJsonSchema(schema: ToolInputSchema<any>, isOpenai: boolean = true): JSONSchema7 {\n  if (schema instanceof ZodObject) {\n    return oaiParams(schema, isOpenai);\n  }\n  return schema;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAA4C;AAE5C,mBAAkB;AAClB,iBAA0B;AAC1B,gCAAgC;AAGhC,0BAKO;AAUP,SAAS,+BAA+B,MAA8B;AACpE,UAAQ,MAAM;AAAA,IACZ,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AAAA,IACrB,KAAK,gCAAgB;AACnB,aAAO;AAAA,IACT,KAAK,gCAAgB;AACnB,aAAO;AAAA,IACT;AAEE,YAAM,IAAI,MAAM,gCAAgC,IAAI,wCAAwC;AAAA,EAChG;AACF;AAEA,SAAS,oBAAoB,OAA+B;AAE1D,MACE,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,QAC/B,MAAM,SAAS,gCAAgB,OAC/B;AACA,WAAO;AAAA,EACT;AAGA,MAAI;AACF,WAAO,MAAM,QAAQ,gCAAgB,IAAI;AAAA,EAC3C,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,IAAI,YAAY,KAAK;AAAA,IAEzD;AAAA,EACF;AACF;AAEA,eAAsB,eAAe,OAA+C;AA/DpF;AAgEE,MAAI,OAAO,MAAM,UAAU,UAAU;AACnC,QAAI,MAAM,MAAM,WAAW,OAAO,GAAG;AACnC,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,MAAM,KAAK,CAAC;AACrD,YAAM,cAAc,OAAO,MAAM,GAAG;AACpC,YAAM,aAAY,iBAAY,CAAC,MAAb,mBAAgB,MAAM;AACxC,YAAM,aAAa,uCAAY;AAE/B,UAAI,CAAC,YAAY;AACf,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAEA,UAAI;AACJ,UAAI,MAAM,YAAY,MAAM,aAAa,YAAY;AACnD,gBAAQ;AAAA,UACN,sBAAsB,MAAM,QAAQ,wCAAwC,UAAU;AAAA,QACxF;AACA,mBAAW,MAAM;AAAA,MACnB,OAAO;AACL,mBAAW;AAAA,MACb;AAEA,YAAM,iBAAiB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,WAAW,CAAC;AACrF,UAAI,CAAC,eAAe,IAAI,QAAQ,GAAG;AACjC,cAAM,IAAI,MAAM,wBAAwB,QAAQ,mCAAmC;AAAA,MACrF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,iBAAiB,MAAM;AAAA,MACzB;AAAA,IACF;AAGA,WAAO;AAAA,MACL,UAAU,MAAM;AAAA,MAChB,iBAAiB,MAAM;AAAA,MACvB,aAAa,MAAM;AAAA,IACrB;AAAA,EACF,WAAW,MAAM,iBAAiB,4BAAY;AAC5C,UAAM,QAAQ,oBAAoB,MAAM,KAAK;AAC7C,UAAM,WAAW,+BAA+B,MAAM,IAAI;AAG1D,QAAI,cAAU,aAAAA,SAAM,MAAM,MAAM;AAAA,MAC9B,KAAK;AAAA,QACH,OAAO,MAAM;AAAA,QACb,QAAQ,MAAM;AAAA,QACd;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,MAAM,kBAAkB,MAAM,iBAAiB;AACjD,gBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,QACtB,IAAI,EACJ,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,QAAQ,CAAC;AAE7C,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,iBAAiB,MAAM;AAAA,IACzB;AAAA,EACF,OAAO;AACL,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AACF;AAWO,MAAM,oBAAoB,CAC/B,YACA,WAAqB,CAAC,MACI;AAC1B,SAAO,EAAE,KAAK,EAAE,SAAS,GAAG,WAAW;AACzC;AAIO,MAAM,YAAY,CACvB,GACA,WAAoB,SACS;AAE7B,QAAM,EAAE,YAAY,UAAU,qBAAqB,QAAI,2CAAgB,GAAG;AAAA;AAAA,IAExE,QAAQ,WAAW,WAAW;AAAA,EAChC,CAAC;AAED,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGO,MAAM,uBAAuB,CAClC,SACA,YACA,UACA,YACiB;AACjB,QAAM,OAAO,QAAQ,QAAQ;AAC7B,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,MAAM,WAAW,QAAQ,YAAY;AAAA,EACjD;AAEA,SAAO,iCAAa,OAAO;AAAA,IACzB,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACH;AAEA,eAAsB,gBACpB,UACA,SAC6B;AAC7B,QAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,MAAI;AACJ,MAAI;AAGJ,MAAI;AACF,WAAO,KAAK,MAAM,SAAS,IAAI;AAAA,EACjC,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,iBAAiB,KAAK;AAAA,MAC9B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,MAAI;AACF,QAAI,KAAK,sBAAsB,sBAAW;AACxC,eAAS,KAAK,WAAW,MAAM,IAAI;AAAA,IACrC,OAAO;AACL,eAAS;AAAA,IACX;AAAA,EACF,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,6BAA6B,KAAK;AAAA,MAC1C,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,QAAQ,QAAQ,kBAAkB,SAAS,MAAM,CAAC;AAC5E,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,KAAK,UAAU,MAAM;AAAA,MAC7B,SAAS;AAAA,IACX,CAAC;AAAA,EACH,SAAS,OAAO;AACd,WAAO,uCAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,0BAA0B,KAAK;AAAA,MACvC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAUA,SAAS,WAAW,QAAkB,QAA4B;AAChE,QAAM,IAAI,OAAO;AACjB,QAAM,IAAI,OAAO;AACjB,QAAM,KAAiB,MAAM,IAAI,CAAC,EAC/B,KAAK,IAAI,EACT,IAAI,MAAM,MAAM,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAGjC,WAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,aAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,UAAI,OAAOD,KAAI,CAAC,MAAM,OAAOC,KAAI,CAAC,GAAG;AACnC,WAAGD,EAAC,EAAGC,EAAC,IAAI,GAAGD,KAAI,CAAC,EAAGC,KAAI,CAAC,IAAK;AAAA,MACnC,OAAO;AACL,WAAGD,EAAC,EAAGC,EAAC,IAAI,KAAK,IAAI,GAAGD,KAAI,CAAC,EAAGC,EAAC,GAAI,GAAGD,EAAC,EAAGC,KAAI,CAAC,CAAE;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AAGA,QAAM,SAAmB,CAAC;AAC1B,MAAI,IAAI;AACR,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,IAAI,GAAG;AACrB,QAAI,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,GAAG;AACnC,aAAO,KAAK,OAAO,IAAI,CAAC,CAAE;AAC1B;AACA;AAAA,IACF,WAAW,GAAG,IAAI,CAAC,EAAG,CAAC,IAAK,GAAG,CAAC,EAAG,IAAI,CAAC,GAAI;AAC1C;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,OAAO,QAAQ;AACxB;AAcO,SAAS,mBAAmB,QAAqB,QAA8B;AACpF,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,IAAI,IAAI,WAAW,QAAQ,MAAM,CAAC;AAEjD,QAAM,WAAW,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtF,QAAM,WAA2C,CAAC;AAElD,MAAI,mBAAkC;AACtC,aAAW,WAAW,OAAO,OAAO;AAClC,QAAI,OAAO,IAAI,QAAQ,EAAE,GAAG;AAC1B,yBAAmB,QAAQ;AAAA,IAC7B,OAAO;AACL,YAAM,SAAS;AACf,eAAS,KAAK,CAAC,QAAQ,QAAQ,EAAE,CAAC;AAClC,yBAAmB,QAAQ;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,aAAa,QAA8B,WAAoB,MAAmB;AAChG,MAAI,kBAAkB,sBAAW;AAC/B,WAAO,UAAU,QAAQ,QAAQ;AAAA,EACnC;AACA,SAAO;AACT;","names":["sharp","i","j"]}

package/dist/llm/utils.d.cts

@@ -21,7 +21,7 @@ export type OpenAIFunctionParameters = {
 };
 export declare const createToolOptions: <UserData extends unknown>(toolCallId: string, userData?: UserData) => ToolOptions<UserData>;
 /** @internal */
-export declare const oaiParams: (p: ZodObject<any>) => OpenAIFunctionParameters;
+export declare const oaiParams: (p: ZodObject<any>, isOpenai?: boolean) => OpenAIFunctionParameters;
 /** @internal */
 export declare const oaiBuildFunctionInfo: (toolCtx: ToolContext, toolCallId: string, toolName: string, rawArgs: string) => FunctionCall;
 export declare function executeToolCall(toolCall: FunctionCall, toolCtx: ToolContext): Promise<FunctionCallOutput>;
@@ -37,6 +37,6 @@ interface DiffOps {
  * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.
  */
 export declare function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps;
-export declare function toJsonSchema(schema: ToolInputSchema<any>): JSONSchema7;
+export declare function toJsonSchema(schema: ToolInputSchema<any>, isOpenai?: boolean): JSONSchema7;
 export {};
 //# sourceMappingURL=utils.d.ts.map

package/dist/llm/utils.d.ts

@@ -21,7 +21,7 @@ export type OpenAIFunctionParameters = {
 };
 export declare const createToolOptions: <UserData extends unknown>(toolCallId: string, userData?: UserData) => ToolOptions<UserData>;
 /** @internal */
-export declare const oaiParams: (p: ZodObject<any>) => OpenAIFunctionParameters;
+export declare const oaiParams: (p: ZodObject<any>, isOpenai?: boolean) => OpenAIFunctionParameters;
 /** @internal */
 export declare const oaiBuildFunctionInfo: (toolCtx: ToolContext, toolCallId: string, toolName: string, rawArgs: string) => FunctionCall;
 export declare function executeToolCall(toolCall: FunctionCall, toolCtx: ToolContext): Promise<FunctionCallOutput>;
@@ -37,6 +37,6 @@ interface DiffOps {
  * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.
  */
 export declare function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps;
-export declare function toJsonSchema(schema: ToolInputSchema<any>): JSONSchema7;
+export declare function toJsonSchema(schema: ToolInputSchema<any>, isOpenai?: boolean): JSONSchema7;
 export {};
 //# sourceMappingURL=utils.d.ts.map

package/dist/llm/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/llm/utils.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AAGhC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAEL,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEnF,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAwCD,wBAAsB,cAAc,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAsElF;AAED,+CAA+C;AAC/C,MAAM,MAAM,wBAAwB,GAAG;IACrC,IAAI,EAAE,QAAQ,CAAC;IACf,UAAU,EAAE;QAAE,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;IAClC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,CAAC;AAGF,eAAO,MAAM,iBAAiB,yCAChB,MAAM,aACR,QAAQ,KACjB,YAAY,QAAQ,CAEtB,CAAC;AAEF,gBAAgB;AAEhB,eAAO,MAAM,SAAS,MAAO,UAAU,GAAG,CAAC,KAAG,wBAY7C,CAAC;AAEF,gBAAgB;AAChB,eAAO,MAAM,oBAAoB,YACtB,WAAW,cACR,MAAM,YACR,MAAM,WACP,MAAM,KACd,YAWF,CAAC;AAEF,wBAAsB,eAAe,CACnC,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,kBAAkB,CAAC,CA6C7B;AA+CD,UAAU,OAAO;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;CAC1C;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAuBpF;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,GAAG,WAAW,CAKtE"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/llm/utils.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AAGhC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAEL,YAAY,EACZ,kBAAkB,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEnF,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAwCD,wBAAsB,cAAc,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAsElF;AAED,+CAA+C;AAC/C,MAAM,MAAM,wBAAwB,GAAG;IACrC,IAAI,EAAE,QAAQ,CAAC;IACf,UAAU,EAAE;QAAE,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC;IAClC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,CAAC;AAGF,eAAO,MAAM,iBAAiB,yCAChB,MAAM,aACR,QAAQ,KACjB,YAAY,QAAQ,CAEtB,CAAC;AAEF,gBAAgB;AAEhB,eAAO,MAAM,SAAS,MACjB,UAAU,GAAG,CAAC,aACP,OAAO,KAChB,wBAaF,CAAC;AAEF,gBAAgB;AAChB,eAAO,MAAM,oBAAoB,YACtB,WAAW,cACR,MAAM,YACR,MAAM,WACP,MAAM,KACd,YAWF,CAAC;AAEF,wBAAsB,eAAe,CACnC,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,kBAAkB,CAAC,CA6C7B;AA+CD,UAAU,OAAO;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;CAC1C;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAuBpF;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE,QAAQ,GAAE,OAAc,GAAG,WAAW,CAKhG"}

package/dist/llm/utils.js

@@ -92,9 +92,10 @@ async function serializeImage(image) {
 const createToolOptions = (toolCallId, userData = {}) => {
   return { ctx: { userData }, toolCallId };
 };
-const oaiParams = (p) => {
+const oaiParams = (p, isOpenai = true) => {
   const { properties, required, additionalProperties } = zodToJsonSchema(p, {
-    target: "openAi"
+    // note: openai mode breaks various gemini conversions
+    target: isOpenai ? "openAi" : "jsonSchema7"
   });
   return {
     type: "object",
@@ -205,9 +206,9 @@ function computeChatCtxDiff(oldCtx, newCtx) {
     toCreate
   };
 }
-function toJsonSchema(schema) {
+function toJsonSchema(schema, isOpenai = true) {
   if (schema instanceof ZodObject) {
-    return oaiParams(schema);
+    return oaiParams(schema, isOpenai);
   }
   return schema;
 }

package/dist/llm/utils.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/llm/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { VideoBufferType, VideoFrame } from '@livekit/rtc-node';\nimport type { JSONSchema7 } from 'json-schema';\nimport sharp from 'sharp';\nimport { ZodObject } from 'zod';\nimport { zodToJsonSchema } from 'zod-to-json-schema';\nimport type { UnknownUserData } from '../voice/run_context.js';\nimport type { ChatContext } from './chat_context.js';\nimport {\n  type ChatItem,\n  FunctionCall,\n  FunctionCallOutput,\n  type ImageContent,\n} from './chat_context.js';\nimport type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';\n\nexport interface SerializedImage {\n  inferenceDetail: 'auto' | 'high' | 'low';\n  mimeType?: string;\n  base64Data?: string;\n  externalUrl?: string;\n}\n\nfunction getChannelsFromVideoBufferType(type: VideoBufferType): 3 | 4 {\n  switch (type) {\n    case VideoBufferType.RGBA:\n    case VideoBufferType.ABGR:\n    case VideoBufferType.ARGB:\n    case VideoBufferType.BGRA:\n      return 4;\n    case VideoBufferType.RGB24:\n      return 3;\n    default:\n      // YUV formats (I420, I420A, I422, I444, I010, NV12) need conversion\n      throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);\n  }\n}\n\nfunction ensureRGBCompatible(frame: VideoFrame): VideoFrame {\n  // If the frame is already in an RGB/RGBA-compatible format, return it directly\n  if (\n    frame.type === VideoBufferType.RGBA ||\n    frame.type === VideoBufferType.BGRA ||\n    frame.type === VideoBufferType.ARGB ||\n    frame.type === VideoBufferType.ABGR ||\n    frame.type === VideoBufferType.RGB24\n  ) {\n    return frame;\n  }\n\n  // Otherwise, attempt conversion for other formats (like YUV)\n  try {\n    return frame.convert(VideoBufferType.RGBA);\n  } catch (error) {\n    throw new Error(\n      `Failed to convert format ${frame.type} to RGB: ${error}. ` +\n        `Consider using RGB/RGBA formats or converting on the client side.`,\n    );\n  }\n}\n\nexport async function serializeImage(image: ImageContent): Promise<SerializedImage> {\n  if (typeof image.image === 'string') {\n    if (image.image.startsWith('data:')) {\n      const [header, base64Data] = image.image.split(',', 2) as [string, string];\n      const headerParts = header.split(';');\n      const mimeParts = headerParts[0]?.split(':');\n      const headerMime = mimeParts?.[1];\n\n      if (!headerMime) {\n        throw new Error('Invalid data URL format');\n      }\n\n      let mimeType: string;\n      if (image.mimeType && image.mimeType !== headerMime) {\n        console.warn(\n          `Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`,\n        );\n        mimeType = image.mimeType;\n      } else {\n        mimeType = headerMime;\n      }\n\n      const supportedTypes = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);\n      if (!supportedTypes.has(mimeType)) {\n        throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);\n      }\n\n      return {\n        base64Data,\n        mimeType: mimeType,\n        inferenceDetail: image.inferenceDetail,\n      };\n    }\n\n    // External URL\n    return {\n      mimeType: image.mimeType,\n      inferenceDetail: image.inferenceDetail,\n      externalUrl: image.image,\n    };\n  } else if (image.image instanceof VideoFrame) {\n    const frame = ensureRGBCompatible(image.image);\n    const channels = getChannelsFromVideoBufferType(frame.type);\n\n    // Sharp needs to know the format of raw pixel data\n    let encoded = sharp(frame.data, {\n      raw: {\n        width: frame.width,\n        height: frame.height,\n        channels,\n      },\n    });\n\n    if (image.inferenceWidth && image.inferenceHeight) {\n      encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n    }\n\n    const base64Data = await encoded\n      .png()\n      .toBuffer()\n      .then((buffer) => buffer.toString('base64'));\n\n    return {\n      base64Data,\n      mimeType: 'image/png',\n      inferenceDetail: image.inferenceDetail,\n    };\n  } else {\n    throw new Error('Unsupported image type');\n  }\n}\n\n/** Raw OpenAI-adherent function parameters. */\nexport type OpenAIFunctionParameters = {\n  type: 'object';\n  properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any\n  required: string[];\n  additionalProperties?: boolean;\n};\n\n// TODO(brian): remove this helper once we have the real RunContext user data\nexport const createToolOptions = <UserData extends UnknownUserData>(\n  toolCallId: string,\n  userData: UserData = {} as UserData,\n): ToolOptions<UserData> => {\n  return { ctx: { userData }, toolCallId } as unknown as ToolOptions<UserData>;\n};\n\n/** @internal */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const oaiParams = (p: ZodObject<any>): OpenAIFunctionParameters => {\n  // TODO(AJS-162): make zod to JSON parsing able to handle z.optional(v.field())\n  const { properties, required, additionalProperties } = zodToJsonSchema(p, {\n    target: 'openAi',\n  }) as OpenAIFunctionParameters;\n\n  return {\n    type: 'object',\n    properties,\n    required,\n    additionalProperties,\n  };\n};\n\n/** @internal */\nexport const oaiBuildFunctionInfo = (\n  toolCtx: ToolContext,\n  toolCallId: string,\n  toolName: string,\n  rawArgs: string,\n): FunctionCall => {\n  const tool = toolCtx[toolName];\n  if (!tool) {\n    throw new Error(`AI tool ${toolName} not found`);\n  }\n\n  return FunctionCall.create({\n    callId: toolCallId,\n    name: toolName,\n    args: rawArgs,\n  });\n};\n\nexport async function executeToolCall(\n  toolCall: FunctionCall,\n  toolCtx: ToolContext,\n): Promise<FunctionCallOutput> {\n  const tool = toolCtx[toolCall.name]!;\n  let args: object | undefined;\n  let params: object | undefined;\n\n  // Ensure valid JSON\n  try {\n    args = JSON.parse(toolCall.args);\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Invalid JSON: ${error}`,\n      isError: true,\n    });\n  }\n\n  // Ensure valid arguments schema\n  try {\n    if (tool.parameters instanceof ZodObject) {\n      params = tool.parameters.parse(args);\n    } else {\n      params = args;\n    }\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Arguments parsing failed: ${error}`,\n      isError: true,\n    });\n  }\n\n  try {\n    const result = await tool.execute(params, createToolOptions(toolCall.callId));\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: JSON.stringify(result),\n      isError: false,\n    });\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Tool execution failed: ${error}`,\n      isError: true,\n    });\n  }\n}\n\n/**\n * Standard dynamic-programming LCS to get the common subsequence\n * of IDs (in order) that appear in both old_ids and new_ids.\n *\n * @param oldIds - The old list of IDs.\n * @param newIds - The new list of IDs.\n * @returns The longest common subsequence of the two lists of IDs.\n */\nfunction computeLCS(oldIds: string[], newIds: string[]): string[] {\n  const n = oldIds.length;\n  const m = newIds.length;\n  const dp: number[][] = Array(n + 1)\n    .fill(null)\n    .map(() => Array(m + 1).fill(0));\n\n  // Fill DP table\n  for (let i = 1; i <= n; i++) {\n    for (let j = 1; j <= m; j++) {\n      if (oldIds[i - 1] === newIds[j - 1]) {\n        dp[i]![j] = dp[i - 1]![j - 1]! + 1;\n      } else {\n        dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n      }\n    }\n  }\n\n  // Backtrack to find the actual LCS sequence\n  const lcsIds: string[] = [];\n  let i = n;\n  let j = m;\n  while (i > 0 && j > 0) {\n    if (oldIds[i - 1] === newIds[j - 1]) {\n      lcsIds.push(oldIds[i - 1]!);\n      i--;\n      j--;\n    } else if (dp[i - 1]![j]! > dp[i]![j - 1]!) {\n      i--;\n    } else {\n      j--;\n    }\n  }\n\n  return lcsIds.reverse();\n}\n\ninterface DiffOps {\n  toRemove: string[];\n  toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root\n}\n\n/**\n * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.\n *\n * @param oldCtx - The old chat context.\n * @param newCtx - The new chat context.\n * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.\n */\nexport function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps {\n  const oldIds = oldCtx.items.map((item: ChatItem) => item.id);\n  const newIds = newCtx.items.map((item: ChatItem) => item.id);\n  const lcsIds = new Set(computeLCS(oldIds, newIds));\n\n  const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);\n  const toCreate: Array<[string | null, string]> = [];\n\n  let lastIdInSequence: string | null = null;\n  for (const newItem of newCtx.items) {\n    if (lcsIds.has(newItem.id)) {\n      lastIdInSequence = newItem.id;\n    } else {\n      const prevId = lastIdInSequence; // null if root\n      toCreate.push([prevId, newItem.id]);\n      lastIdInSequence = newItem.id;\n    }\n  }\n\n  return {\n    toRemove,\n    toCreate,\n  };\n}\n\nexport function toJsonSchema(schema: ToolInputSchema<any>): JSONSchema7 {\n  if (schema instanceof ZodObject) {\n    return oaiParams(schema);\n  }\n  return schema;\n}\n"],"mappings":"AAGA,SAAS,iBAAiB,kBAAkB;AAE5C,OAAO,WAAW;AAClB,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC;AAAA,EAEE;AAAA,EACA;AAAA,OAEK;AAUP,SAAS,+BAA+B,MAA8B;AACpE,UAAQ,MAAM;AAAA,IACZ,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AACnB,aAAO;AAAA,IACT,KAAK,gBAAgB;AACnB,aAAO;AAAA,IACT;AAEE,YAAM,IAAI,MAAM,gCAAgC,IAAI,wCAAwC;AAAA,EAChG;AACF;AAEA,SAAS,oBAAoB,OAA+B;AAE1D,MACE,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,OAC/B;AACA,WAAO;AAAA,EACT;AAGA,MAAI;AACF,WAAO,MAAM,QAAQ,gBAAgB,IAAI;AAAA,EAC3C,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,IAAI,YAAY,KAAK;AAAA,IAEzD;AAAA,EACF;AACF;AAEA,eAAsB,eAAe,OAA+C;AA/DpF;AAgEE,MAAI,OAAO,MAAM,UAAU,UAAU;AACnC,QAAI,MAAM,MAAM,WAAW,OAAO,GAAG;AACnC,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,MAAM,KAAK,CAAC;AACrD,YAAM,cAAc,OAAO,MAAM,GAAG;AACpC,YAAM,aAAY,iBAAY,CAAC,MAAb,mBAAgB,MAAM;AACxC,YAAM,aAAa,uCAAY;AAE/B,UAAI,CAAC,YAAY;AACf,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAEA,UAAI;AACJ,UAAI,MAAM,YAAY,MAAM,aAAa,YAAY;AACnD,gBAAQ;AAAA,UACN,sBAAsB,MAAM,QAAQ,wCAAwC,UAAU;AAAA,QACxF;AACA,mBAAW,MAAM;AAAA,MACnB,OAAO;AACL,mBAAW;AAAA,MACb;AAEA,YAAM,iBAAiB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,WAAW,CAAC;AACrF,UAAI,CAAC,eAAe,IAAI,QAAQ,GAAG;AACjC,cAAM,IAAI,MAAM,wBAAwB,QAAQ,mCAAmC;AAAA,MACrF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,iBAAiB,MAAM;AAAA,MACzB;AAAA,IACF;AAGA,WAAO;AAAA,MACL,UAAU,MAAM;AAAA,MAChB,iBAAiB,MAAM;AAAA,MACvB,aAAa,MAAM;AAAA,IACrB;AAAA,EACF,WAAW,MAAM,iBAAiB,YAAY;AAC5C,UAAM,QAAQ,oBAAoB,MAAM,KAAK;AAC7C,UAAM,WAAW,+BAA+B,MAAM,IAAI;AAG1D,QAAI,UAAU,MAAM,MAAM,MAAM;AAAA,MAC9B,KAAK;AAAA,QACH,OAAO,MAAM;AAAA,QACb,QAAQ,MAAM;AAAA,QACd;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,MAAM,kBAAkB,MAAM,iBAAiB;AACjD,gBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,QACtB,IAAI,EACJ,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,QAAQ,CAAC;AAE7C,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,iBAAiB,MAAM;AAAA,IACzB;AAAA,EACF,OAAO;AACL,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AACF;AAWO,MAAM,oBAAoB,CAC/B,YACA,WAAqB,CAAC,MACI;AAC1B,SAAO,EAAE,KAAK,EAAE,SAAS,GAAG,WAAW;AACzC;AAIO,MAAM,YAAY,CAAC,MAAgD;AAExE,QAAM,EAAE,YAAY,UAAU,qBAAqB,IAAI,gBAAgB,GAAG;AAAA,IACxE,QAAQ;AAAA,EACV,CAAC;AAED,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGO,MAAM,uBAAuB,CAClC,SACA,YACA,UACA,YACiB;AACjB,QAAM,OAAO,QAAQ,QAAQ;AAC7B,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,MAAM,WAAW,QAAQ,YAAY;AAAA,EACjD;AAEA,SAAO,aAAa,OAAO;AAAA,IACzB,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACH;AAEA,eAAsB,gBACpB,UACA,SAC6B;AAC7B,QAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,MAAI;AACJ,MAAI;AAGJ,MAAI;AACF,WAAO,KAAK,MAAM,SAAS,IAAI;AAAA,EACjC,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,iBAAiB,KAAK;AAAA,MAC9B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,MAAI;AACF,QAAI,KAAK,sBAAsB,WAAW;AACxC,eAAS,KAAK,WAAW,MAAM,IAAI;AAAA,IACrC,OAAO;AACL,eAAS;AAAA,IACX;AAAA,EACF,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,6BAA6B,KAAK;AAAA,MAC1C,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,QAAQ,QAAQ,kBAAkB,SAAS,MAAM,CAAC;AAC5E,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,KAAK,UAAU,MAAM;AAAA,MAC7B,SAAS;AAAA,IACX,CAAC;AAAA,EACH,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,0BAA0B,KAAK;AAAA,MACvC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAUA,SAAS,WAAW,QAAkB,QAA4B;AAChE,QAAM,IAAI,OAAO;AACjB,QAAM,IAAI,OAAO;AACjB,QAAM,KAAiB,MAAM,IAAI,CAAC,EAC/B,KAAK,IAAI,EACT,IAAI,MAAM,MAAM,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAGjC,WAASA,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,aAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,UAAI,OAAOD,KAAI,CAAC,MAAM,OAAOC,KAAI,CAAC,GAAG;AACnC,WAAGD,EAAC,EAAGC,EAAC,IAAI,GAAGD,KAAI,CAAC,EAAGC,KAAI,CAAC,IAAK;AAAA,MACnC,OAAO;AACL,WAAGD,EAAC,EAAGC,EAAC,IAAI,KAAK,IAAI,GAAGD,KAAI,CAAC,EAAGC,EAAC,GAAI,GAAGD,EAAC,EAAGC,KAAI,CAAC,CAAE;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AAGA,QAAM,SAAmB,CAAC;AAC1B,MAAI,IAAI;AACR,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,IAAI,GAAG;AACrB,QAAI,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,GAAG;AACnC,aAAO,KAAK,OAAO,IAAI,CAAC,CAAE;AAC1B;AACA;AAAA,IACF,WAAW,GAAG,IAAI,CAAC,EAAG,CAAC,IAAK,GAAG,CAAC,EAAG,IAAI,CAAC,GAAI;AAC1C;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,OAAO,QAAQ;AACxB;AAcO,SAAS,mBAAmB,QAAqB,QAA8B;AACpF,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,IAAI,IAAI,WAAW,QAAQ,MAAM,CAAC;AAEjD,QAAM,WAAW,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtF,QAAM,WAA2C,CAAC;AAElD,MAAI,mBAAkC;AACtC,aAAW,WAAW,OAAO,OAAO;AAClC,QAAI,OAAO,IAAI,QAAQ,EAAE,GAAG;AAC1B,yBAAmB,QAAQ;AAAA,IAC7B,OAAO;AACL,YAAM,SAAS;AACf,eAAS,KAAK,CAAC,QAAQ,QAAQ,EAAE,CAAC;AAClC,yBAAmB,QAAQ;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,aAAa,QAA2C;AACtE,MAAI,kBAAkB,WAAW;AAC/B,WAAO,UAAU,MAAM;AAAA,EACzB;AACA,SAAO;AACT;","names":["i","j"]}
+{"version":3,"sources":["../../src/llm/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { VideoBufferType, VideoFrame } from '@livekit/rtc-node';\nimport type { JSONSchema7 } from 'json-schema';\nimport sharp from 'sharp';\nimport { ZodObject } from 'zod';\nimport { zodToJsonSchema } from 'zod-to-json-schema';\nimport type { UnknownUserData } from '../voice/run_context.js';\nimport type { ChatContext } from './chat_context.js';\nimport {\n  type ChatItem,\n  FunctionCall,\n  FunctionCallOutput,\n  type ImageContent,\n} from './chat_context.js';\nimport type { ToolContext, ToolInputSchema, ToolOptions } from './tool_context.js';\n\nexport interface SerializedImage {\n  inferenceDetail: 'auto' | 'high' | 'low';\n  mimeType?: string;\n  base64Data?: string;\n  externalUrl?: string;\n}\n\nfunction getChannelsFromVideoBufferType(type: VideoBufferType): 3 | 4 {\n  switch (type) {\n    case VideoBufferType.RGBA:\n    case VideoBufferType.ABGR:\n    case VideoBufferType.ARGB:\n    case VideoBufferType.BGRA:\n      return 4;\n    case VideoBufferType.RGB24:\n      return 3;\n    default:\n      // YUV formats (I420, I420A, I422, I444, I010, NV12) need conversion\n      throw new Error(`Unsupported VideoBufferType: ${type}. Only RGB/RGBA formats are supported.`);\n  }\n}\n\nfunction ensureRGBCompatible(frame: VideoFrame): VideoFrame {\n  // If the frame is already in an RGB/RGBA-compatible format, return it directly\n  if (\n    frame.type === VideoBufferType.RGBA ||\n    frame.type === VideoBufferType.BGRA ||\n    frame.type === VideoBufferType.ARGB ||\n    frame.type === VideoBufferType.ABGR ||\n    frame.type === VideoBufferType.RGB24\n  ) {\n    return frame;\n  }\n\n  // Otherwise, attempt conversion for other formats (like YUV)\n  try {\n    return frame.convert(VideoBufferType.RGBA);\n  } catch (error) {\n    throw new Error(\n      `Failed to convert format ${frame.type} to RGB: ${error}. ` +\n        `Consider using RGB/RGBA formats or converting on the client side.`,\n    );\n  }\n}\n\nexport async function serializeImage(image: ImageContent): Promise<SerializedImage> {\n  if (typeof image.image === 'string') {\n    if (image.image.startsWith('data:')) {\n      const [header, base64Data] = image.image.split(',', 2) as [string, string];\n      const headerParts = header.split(';');\n      const mimeParts = headerParts[0]?.split(':');\n      const headerMime = mimeParts?.[1];\n\n      if (!headerMime) {\n        throw new Error('Invalid data URL format');\n      }\n\n      let mimeType: string;\n      if (image.mimeType && image.mimeType !== headerMime) {\n        console.warn(\n          `Provided mimeType '${image.mimeType}' does not match data URL mime type '${headerMime}'. Using provided mimeType.`,\n        );\n        mimeType = image.mimeType;\n      } else {\n        mimeType = headerMime;\n      }\n\n      const supportedTypes = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);\n      if (!supportedTypes.has(mimeType)) {\n        throw new Error(`Unsupported mimeType ${mimeType}. Must be jpeg, png, webp, or gif`);\n      }\n\n      return {\n        base64Data,\n        mimeType: mimeType,\n        inferenceDetail: image.inferenceDetail,\n      };\n    }\n\n    // External URL\n    return {\n      mimeType: image.mimeType,\n      inferenceDetail: image.inferenceDetail,\n      externalUrl: image.image,\n    };\n  } else if (image.image instanceof VideoFrame) {\n    const frame = ensureRGBCompatible(image.image);\n    const channels = getChannelsFromVideoBufferType(frame.type);\n\n    // Sharp needs to know the format of raw pixel data\n    let encoded = sharp(frame.data, {\n      raw: {\n        width: frame.width,\n        height: frame.height,\n        channels,\n      },\n    });\n\n    if (image.inferenceWidth && image.inferenceHeight) {\n      encoded = encoded.resize(image.inferenceWidth, image.inferenceHeight);\n    }\n\n    const base64Data = await encoded\n      .png()\n      .toBuffer()\n      .then((buffer) => buffer.toString('base64'));\n\n    return {\n      base64Data,\n      mimeType: 'image/png',\n      inferenceDetail: image.inferenceDetail,\n    };\n  } else {\n    throw new Error('Unsupported image type');\n  }\n}\n\n/** Raw OpenAI-adherent function parameters. */\nexport type OpenAIFunctionParameters = {\n  type: 'object';\n  properties: { [id: string]: any }; // eslint-disable-line @typescript-eslint/no-explicit-any\n  required: string[];\n  additionalProperties?: boolean;\n};\n\n// TODO(brian): remove this helper once we have the real RunContext user data\nexport const createToolOptions = <UserData extends UnknownUserData>(\n  toolCallId: string,\n  userData: UserData = {} as UserData,\n): ToolOptions<UserData> => {\n  return { ctx: { userData }, toolCallId } as unknown as ToolOptions<UserData>;\n};\n\n/** @internal */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const oaiParams = (\n  p: ZodObject<any>,\n  isOpenai: boolean = true,\n): OpenAIFunctionParameters => {\n  // Adapted from https://github.com/vercel/ai/blob/56eb0ee9/packages/provider-utils/src/zod-schema.ts\n  const { properties, required, additionalProperties } = zodToJsonSchema(p, {\n    // note: openai mode breaks various gemini conversions\n    target: isOpenai ? 'openAi' : 'jsonSchema7',\n  }) as OpenAIFunctionParameters;\n\n  return {\n    type: 'object',\n    properties,\n    required,\n    additionalProperties,\n  };\n};\n\n/** @internal */\nexport const oaiBuildFunctionInfo = (\n  toolCtx: ToolContext,\n  toolCallId: string,\n  toolName: string,\n  rawArgs: string,\n): FunctionCall => {\n  const tool = toolCtx[toolName];\n  if (!tool) {\n    throw new Error(`AI tool ${toolName} not found`);\n  }\n\n  return FunctionCall.create({\n    callId: toolCallId,\n    name: toolName,\n    args: rawArgs,\n  });\n};\n\nexport async function executeToolCall(\n  toolCall: FunctionCall,\n  toolCtx: ToolContext,\n): Promise<FunctionCallOutput> {\n  const tool = toolCtx[toolCall.name]!;\n  let args: object | undefined;\n  let params: object | undefined;\n\n  // Ensure valid JSON\n  try {\n    args = JSON.parse(toolCall.args);\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Invalid JSON: ${error}`,\n      isError: true,\n    });\n  }\n\n  // Ensure valid arguments schema\n  try {\n    if (tool.parameters instanceof ZodObject) {\n      params = tool.parameters.parse(args);\n    } else {\n      params = args;\n    }\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Arguments parsing failed: ${error}`,\n      isError: true,\n    });\n  }\n\n  try {\n    const result = await tool.execute(params, createToolOptions(toolCall.callId));\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: JSON.stringify(result),\n      isError: false,\n    });\n  } catch (error) {\n    return FunctionCallOutput.create({\n      callId: toolCall.callId,\n      output: `Tool execution failed: ${error}`,\n      isError: true,\n    });\n  }\n}\n\n/**\n * Standard dynamic-programming LCS to get the common subsequence\n * of IDs (in order) that appear in both old_ids and new_ids.\n *\n * @param oldIds - The old list of IDs.\n * @param newIds - The new list of IDs.\n * @returns The longest common subsequence of the two lists of IDs.\n */\nfunction computeLCS(oldIds: string[], newIds: string[]): string[] {\n  const n = oldIds.length;\n  const m = newIds.length;\n  const dp: number[][] = Array(n + 1)\n    .fill(null)\n    .map(() => Array(m + 1).fill(0));\n\n  // Fill DP table\n  for (let i = 1; i <= n; i++) {\n    for (let j = 1; j <= m; j++) {\n      if (oldIds[i - 1] === newIds[j - 1]) {\n        dp[i]![j] = dp[i - 1]![j - 1]! + 1;\n      } else {\n        dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);\n      }\n    }\n  }\n\n  // Backtrack to find the actual LCS sequence\n  const lcsIds: string[] = [];\n  let i = n;\n  let j = m;\n  while (i > 0 && j > 0) {\n    if (oldIds[i - 1] === newIds[j - 1]) {\n      lcsIds.push(oldIds[i - 1]!);\n      i--;\n      j--;\n    } else if (dp[i - 1]![j]! > dp[i]![j - 1]!) {\n      i--;\n    } else {\n      j--;\n    }\n  }\n\n  return lcsIds.reverse();\n}\n\ninterface DiffOps {\n  toRemove: string[];\n  toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root\n}\n\n/**\n * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.\n *\n * @param oldCtx - The old chat context.\n * @param newCtx - The new chat context.\n * @returns The minimal list of create/remove operations to transform oldCtx into newCtx.\n */\nexport function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): DiffOps {\n  const oldIds = oldCtx.items.map((item: ChatItem) => item.id);\n  const newIds = newCtx.items.map((item: ChatItem) => item.id);\n  const lcsIds = new Set(computeLCS(oldIds, newIds));\n\n  const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);\n  const toCreate: Array<[string | null, string]> = [];\n\n  let lastIdInSequence: string | null = null;\n  for (const newItem of newCtx.items) {\n    if (lcsIds.has(newItem.id)) {\n      lastIdInSequence = newItem.id;\n    } else {\n      const prevId = lastIdInSequence; // null if root\n      toCreate.push([prevId, newItem.id]);\n      lastIdInSequence = newItem.id;\n    }\n  }\n\n  return {\n    toRemove,\n    toCreate,\n  };\n}\n\nexport function toJsonSchema(schema: ToolInputSchema<any>, isOpenai: boolean = true): JSONSchema7 {\n  if (schema instanceof ZodObject) {\n    return oaiParams(schema, isOpenai);\n  }\n  return schema;\n}\n"],"mappings":"AAGA,SAAS,iBAAiB,kBAAkB;AAE5C,OAAO,WAAW;AAClB,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAGhC;AAAA,EAEE;AAAA,EACA;AAAA,OAEK;AAUP,SAAS,+BAA+B,MAA8B;AACpE,UAAQ,MAAM;AAAA,IACZ,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AAAA,IACrB,KAAK,gBAAgB;AACnB,aAAO;AAAA,IACT,KAAK,gBAAgB;AACnB,aAAO;AAAA,IACT;AAEE,YAAM,IAAI,MAAM,gCAAgC,IAAI,wCAAwC;AAAA,EAChG;AACF;AAEA,SAAS,oBAAoB,OAA+B;AAE1D,MACE,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,QAC/B,MAAM,SAAS,gBAAgB,OAC/B;AACA,WAAO;AAAA,EACT;AAGA,MAAI;AACF,WAAO,MAAM,QAAQ,gBAAgB,IAAI;AAAA,EAC3C,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR,4BAA4B,MAAM,IAAI,YAAY,KAAK;AAAA,IAEzD;AAAA,EACF;AACF;AAEA,eAAsB,eAAe,OAA+C;AA/DpF;AAgEE,MAAI,OAAO,MAAM,UAAU,UAAU;AACnC,QAAI,MAAM,MAAM,WAAW,OAAO,GAAG;AACnC,YAAM,CAAC,QAAQ,UAAU,IAAI,MAAM,MAAM,MAAM,KAAK,CAAC;AACrD,YAAM,cAAc,OAAO,MAAM,GAAG;AACpC,YAAM,aAAY,iBAAY,CAAC,MAAb,mBAAgB,MAAM;AACxC,YAAM,aAAa,uCAAY;AAE/B,UAAI,CAAC,YAAY;AACf,cAAM,IAAI,MAAM,yBAAyB;AAAA,MAC3C;AAEA,UAAI;AACJ,UAAI,MAAM,YAAY,MAAM,aAAa,YAAY;AACnD,gBAAQ;AAAA,UACN,sBAAsB,MAAM,QAAQ,wCAAwC,UAAU;AAAA,QACxF;AACA,mBAAW,MAAM;AAAA,MACnB,OAAO;AACL,mBAAW;AAAA,MACb;AAEA,YAAM,iBAAiB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,WAAW,CAAC;AACrF,UAAI,CAAC,eAAe,IAAI,QAAQ,GAAG;AACjC,cAAM,IAAI,MAAM,wBAAwB,QAAQ,mCAAmC;AAAA,MACrF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,iBAAiB,MAAM;AAAA,MACzB;AAAA,IACF;AAGA,WAAO;AAAA,MACL,UAAU,MAAM;AAAA,MAChB,iBAAiB,MAAM;AAAA,MACvB,aAAa,MAAM;AAAA,IACrB;AAAA,EACF,WAAW,MAAM,iBAAiB,YAAY;AAC5C,UAAM,QAAQ,oBAAoB,MAAM,KAAK;AAC7C,UAAM,WAAW,+BAA+B,MAAM,IAAI;AAG1D,QAAI,UAAU,MAAM,MAAM,MAAM;AAAA,MAC9B,KAAK;AAAA,QACH,OAAO,MAAM;AAAA,QACb,QAAQ,MAAM;AAAA,QACd;AAAA,MACF;AAAA,IACF,CAAC;AAED,QAAI,MAAM,kBAAkB,MAAM,iBAAiB;AACjD,gBAAU,QAAQ,OAAO,MAAM,gBAAgB,MAAM,eAAe;AAAA,IACtE;AAEA,UAAM,aAAa,MAAM,QACtB,IAAI,EACJ,SAAS,EACT,KAAK,CAAC,WAAW,OAAO,SAAS,QAAQ,CAAC;AAE7C,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,iBAAiB,MAAM;AAAA,IACzB;AAAA,EACF,OAAO;AACL,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AACF;AAWO,MAAM,oBAAoB,CAC/B,YACA,WAAqB,CAAC,MACI;AAC1B,SAAO,EAAE,KAAK,EAAE,SAAS,GAAG,WAAW;AACzC;AAIO,MAAM,YAAY,CACvB,GACA,WAAoB,SACS;AAE7B,QAAM,EAAE,YAAY,UAAU,qBAAqB,IAAI,gBAAgB,GAAG;AAAA;AAAA,IAExE,QAAQ,WAAW,WAAW;AAAA,EAChC,CAAC;AAED,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAGO,MAAM,uBAAuB,CAClC,SACA,YACA,UACA,YACiB;AACjB,QAAM,OAAO,QAAQ,QAAQ;AAC7B,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,MAAM,WAAW,QAAQ,YAAY;AAAA,EACjD;AAEA,SAAO,aAAa,OAAO;AAAA,IACzB,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACH;AAEA,eAAsB,gBACpB,UACA,SAC6B;AAC7B,QAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,MAAI;AACJ,MAAI;AAGJ,MAAI;AACF,WAAO,KAAK,MAAM,SAAS,IAAI;AAAA,EACjC,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,iBAAiB,KAAK;AAAA,MAC9B,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAGA,MAAI;AACF,QAAI,KAAK,sBAAsB,WAAW;AACxC,eAAS,KAAK,WAAW,MAAM,IAAI;AAAA,IACrC,OAAO;AACL,eAAS;AAAA,IACX;AAAA,EACF,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,6BAA6B,KAAK;AAAA,MAC1C,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,QAAQ,QAAQ,kBAAkB,SAAS,MAAM,CAAC;AAC5E,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,KAAK,UAAU,MAAM;AAAA,MAC7B,SAAS;AAAA,IACX,CAAC;AAAA,EACH,SAAS,OAAO;AACd,WAAO,mBAAmB,OAAO;AAAA,MAC/B,QAAQ,SAAS;AAAA,MACjB,QAAQ,0BAA0B,KAAK;AAAA,MACvC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAUA,SAAS,WAAW,QAAkB,QAA4B;AAChE,QAAM,IAAI,OAAO;AACjB,QAAM,IAAI,OAAO;AACjB,QAAM,KAAiB,MAAM,IAAI,CAAC,EAC/B,KAAK,IAAI,EACT,IAAI,MAAM,MAAM,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;AAGjC,WAASA,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,aAASC,KAAI,GAAGA,MAAK,GAAGA,MAAK;AAC3B,UAAI,OAAOD,KAAI,CAAC,MAAM,OAAOC,KAAI,CAAC,GAAG;AACnC,WAAGD,EAAC,EAAGC,EAAC,IAAI,GAAGD,KAAI,CAAC,EAAGC,KAAI,CAAC,IAAK;AAAA,MACnC,OAAO;AACL,WAAGD,EAAC,EAAGC,EAAC,IAAI,KAAK,IAAI,GAAGD,KAAI,CAAC,EAAGC,EAAC,GAAI,GAAGD,EAAC,EAAGC,KAAI,CAAC,CAAE;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AAGA,QAAM,SAAmB,CAAC;AAC1B,MAAI,IAAI;AACR,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,IAAI,GAAG;AACrB,QAAI,OAAO,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,GAAG;AACnC,aAAO,KAAK,OAAO,IAAI,CAAC,CAAE;AAC1B;AACA;AAAA,IACF,WAAW,GAAG,IAAI,CAAC,EAAG,CAAC,IAAK,GAAG,CAAC,EAAG,IAAI,CAAC,GAAI;AAC1C;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,OAAO,QAAQ;AACxB;AAcO,SAAS,mBAAmB,QAAqB,QAA8B;AACpF,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,OAAO,MAAM,IAAI,CAAC,SAAmB,KAAK,EAAE;AAC3D,QAAM,SAAS,IAAI,IAAI,WAAW,QAAQ,MAAM,CAAC;AAEjD,QAAM,WAAW,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtF,QAAM,WAA2C,CAAC;AAElD,MAAI,mBAAkC;AACtC,aAAW,WAAW,OAAO,OAAO;AAClC,QAAI,OAAO,IAAI,QAAQ,EAAE,GAAG;AAC1B,yBAAmB,QAAQ;AAAA,IAC7B,OAAO;AACL,YAAM,SAAS;AACf,eAAS,KAAK,CAAC,QAAQ,QAAQ,EAAE,CAAC;AAClC,yBAAmB,QAAQ;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,aAAa,QAA8B,WAAoB,MAAmB;AAChG,MAAI,kBAAkB,WAAW;AAC/B,WAAO,UAAU,QAAQ,QAAQ;AAAA,EACnC;AACA,SAAO;AACT;","names":["i","j"]}

package/dist/voice/agent_session.cjs

@@ -91,6 +91,9 @@ class AgentSession extends import_node_events.EventEmitter {
     }
     return this._userData;
   }
+  get history() {
+    return this._chatCtx;
+  }
   set userData(value) {
     this._userData = value;
   }

package/dist/voice/agent_session.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/agent_session.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, Room } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { ChatContext, ChatMessage } from '../llm/chat_context.js';\nimport type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport { log } from '../log.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS, TTSError } from '../tts/tts.js';\nimport type { VAD } from '../vad.js';\nimport type { Agent } from './agent.js';\nimport { AgentActivity } from './agent_activity.js';\nimport type { _TurnDetector } from './audio_recognition.js';\nimport {\n  AgentSessionEventTypes,\n  type AgentState,\n  type AgentStateChangedEvent,\n  type CloseEvent,\n  CloseReason,\n  type ConversationItemAddedEvent,\n  type ErrorEvent,\n  type FunctionToolsExecutedEvent,\n  type MetricsCollectedEvent,\n  type SpeechCreatedEvent,\n  type UserInputTranscribedEvent,\n  type UserState,\n  type UserStateChangedEvent,\n  createAgentStateChangedEvent,\n  createCloseEvent,\n  createConversationItemAddedEvent,\n  createUserStateChangedEvent,\n} from './events.js';\nimport { AgentInput, AgentOutput } from './io.js';\nimport { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';\nimport type { UnknownUserData } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport interface VoiceOptions {\n  allowInterruptions: boolean;\n  discardAudioIfUninterruptible: boolean;\n  minInterruptionDuration: number;\n  minInterruptionWords: number;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n  maxToolSteps: number;\n}\n\nconst defaultVoiceOptions: VoiceOptions = {\n  allowInterruptions: true,\n  discardAudioIfUninterruptible: true,\n  minInterruptionDuration: 500,\n  minInterruptionWords: 0,\n  minEndpointingDelay: 500,\n  maxEndpointingDelay: 6000,\n  maxToolSteps: 3,\n} as const;\n\nexport type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;\n\nexport type AgentSessionCallbacks = {\n  [AgentSessionEventTypes.UserInputTranscribed]: (ev: UserInputTranscribedEvent) => void;\n  [AgentSessionEventTypes.AgentStateChanged]: (ev: AgentStateChangedEvent) => void;\n  [AgentSessionEventTypes.UserStateChanged]: (ev: UserStateChangedEvent) => void;\n  [AgentSessionEventTypes.ConversationItemAdded]: (ev: ConversationItemAddedEvent) => void;\n  [AgentSessionEventTypes.FunctionToolsExecuted]: (ev: FunctionToolsExecutedEvent) => void;\n  [AgentSessionEventTypes.MetricsCollected]: (ev: MetricsCollectedEvent) => void;\n  [AgentSessionEventTypes.SpeechCreated]: (ev: SpeechCreatedEvent) => void;\n  [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void;\n  [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void;\n};\n\nexport type AgentSessionOptions<UserData = UnknownUserData> = {\n  turnDetection?: TurnDetectionMode;\n  stt?: STT;\n  vad?: VAD;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  userData?: UserData;\n  voiceOptions?: Partial<VoiceOptions>;\n};\n\nexport class AgentSession<\n  UserData = UnknownUserData,\n> extends (EventEmitter as new () => TypedEmitter<AgentSessionCallbacks>) {\n  vad?: VAD;\n  stt?: STT;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  turnDetection?: TurnDetectionMode;\n\n  readonly options: VoiceOptions;\n\n  private agent?: Agent;\n  private activity?: AgentActivity;\n  private nextActivity?: AgentActivity;\n  private started = false;\n  private userState: UserState = 'listening';\n\n  private roomIO?: RoomIO;\n  private logger = log();\n\n  private _chatCtx: ChatContext;\n  private _userData: UserData | undefined;\n  private _agentState: AgentState = 'initializing';\n\n  private _input: AgentInput;\n  private _output: AgentOutput;\n\n  private closingTask: Promise<void> | null = null;\n\n  constructor(opts: AgentSessionOptions<UserData>) {\n    super();\n\n    const {\n      vad,\n      stt,\n      llm,\n      tts,\n      turnDetection,\n      userData,\n      voiceOptions = defaultVoiceOptions,\n    } = opts;\n\n    this.vad = vad;\n    this.stt = stt;\n    this.llm = llm;\n    this.tts = tts;\n    this.turnDetection = turnDetection;\n    this._userData = userData;\n\n    // configurable IO\n    this._input = new AgentInput(this.onAudioInputChanged);\n    this._output = new AgentOutput(this.onAudioOutputChanged, this.onTextOutputChanged);\n\n    // This is the \"global\" chat context, it holds the entire conversation history\n    this._chatCtx = ChatContext.empty();\n    this.options = { ...defaultVoiceOptions, ...voiceOptions };\n  }\n\n  get input(): AgentInput {\n    return this._input;\n  }\n\n  get output(): AgentOutput {\n    return this._output;\n  }\n\n  get userData(): UserData {\n    if (this._userData === undefined) {\n      throw new Error('Voice agent userData is not set');\n    }\n\n    return this._userData;\n  }\n\n  set userData(value: UserData) {\n    this._userData = value;\n  }\n\n  async start({\n    agent,\n    room,\n    inputOptions,\n    outputOptions,\n  }: {\n    agent: Agent;\n    room: Room;\n    inputOptions?: Partial<RoomInputOptions>;\n    outputOptions?: Partial<RoomOutputOptions>;\n  }): Promise<void> {\n    if (this.started) {\n      return;\n    }\n\n    this.agent = agent;\n    this._updateAgentState('initializing');\n\n    // Check for existing input/output configuration and warn if needed\n    if (this.input.audio && inputOptions?.audioEnabled !== false) {\n      this.logger.warn('RoomIO audio input is enabled but input.audio is already set, ignoring..');\n    }\n\n    if (this.output.audio && outputOptions?.audioEnabled !== false) {\n      this.logger.warn(\n        'RoomIO audio output is enabled but output.audio is already set, ignoring..',\n      );\n    }\n\n    if (this.output.transcription && outputOptions?.transcriptionEnabled !== false) {\n      this.logger.warn(\n        'RoomIO transcription output is enabled but output.transcription is already set, ignoring..',\n      );\n    }\n\n    this.roomIO = new RoomIO({\n      agentSession: this,\n      room,\n      inputOptions,\n      outputOptions,\n    });\n    this.roomIO.start();\n\n    this.updateActivity(this.agent);\n\n    // Log used IO configuration\n    this.logger.debug(\n      `using audio io: ${this.input.audio ? '`' + this.input.audio.constructor.name + '`' : '(none)'} -> \\`AgentSession\\` -> ${this.output.audio ? '`' + this.output.audio.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug(\n      `using transcript io: \\`AgentSession\\` -> ${this.output.transcription ? '`' + this.output.transcription.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug('AgentSession started');\n    this.started = true;\n    this._updateAgentState('listening');\n  }\n\n  updateAgent(agent: Agent): void {\n    this.agent = agent;\n\n    if (this.started) {\n      this.updateActivity(agent);\n    }\n  }\n\n  commitUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    this.activity.commitUserTurn();\n  }\n\n  clearUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    this.activity.clearUserTurn();\n  }\n\n  say(\n    text: string | ReadableStream<string>,\n    options?: {\n      audio?: ReadableStream<AudioFrame>;\n      allowInterruptions?: boolean;\n      addToChatCtx?: boolean;\n    },\n  ): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.activity.say(text, options);\n  }\n\n  interrupt() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    return this.activity.interrupt();\n  }\n\n  generateReply(options?: {\n    userInput?: string;\n    instructions?: string;\n    toolChoice?: ToolChoice;\n    allowInterruptions?: boolean;\n  }): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    const userMessage = options?.userInput\n      ? new ChatMessage({\n          role: 'user',\n          content: options.userInput,\n        })\n      : undefined;\n\n    if (this.activity.draining) {\n      if (!this.nextActivity) {\n        throw new Error('AgentSession is closing, cannot use generateReply()');\n      }\n      return this.nextActivity.generateReply({ userMessage, ...options });\n    }\n\n    return this.activity.generateReply({ userMessage, ...options });\n  }\n\n  private async updateActivity(agent: Agent): Promise<void> {\n    // TODO(AJS-129): add lock to agent activity core lifecycle\n    this.nextActivity = new AgentActivity(agent, this);\n\n    if (this.activity) {\n      await this.activity.drain();\n      await this.activity.close();\n    }\n\n    this.activity = this.nextActivity;\n    this.nextActivity = undefined;\n\n    await this.activity.start();\n\n    if (this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  get chatCtx(): ChatContext {\n    return this._chatCtx.copy();\n  }\n\n  get agentState(): AgentState {\n    return this._agentState;\n  }\n\n  get currentAgent(): Agent {\n    if (!this.agent) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.agent;\n  }\n\n  async close(): Promise<void> {\n    await this.closeImpl(CloseReason.USER_INITIATED);\n  }\n\n  /** @internal */\n  _closeSoon({\n    reason,\n    drain = false,\n    error = null,\n  }: {\n    reason: CloseReason;\n    drain?: boolean;\n    error?: RealtimeModelError | STTError | TTSError | LLMError | null;\n  }): void {\n    if (this.closingTask) {\n      return;\n    }\n    this.closeImpl(reason, error, drain);\n  }\n\n  /** @internal */\n  _onError(error: RealtimeModelError | STTError | TTSError | LLMError): void {\n    if (this.closingTask || error.recoverable) {\n      return;\n    }\n\n    this.logger.error(error, 'AgentSession is closing due to unrecoverable error');\n\n    this.closingTask = (async () => {\n      await this.closeImpl(CloseReason.ERROR, error);\n    })().then(() => {\n      this.closingTask = null;\n    });\n  }\n\n  /** @internal */\n  _conversationItemAdded(item: ChatMessage): void {\n    this._chatCtx.insert(item);\n    this.emit(AgentSessionEventTypes.ConversationItemAdded, createConversationItemAddedEvent(item));\n  }\n\n  /** @internal */\n  _updateAgentState(state: AgentState) {\n    if (this._agentState === state) {\n      return;\n    }\n\n    const oldState = this._agentState;\n    this._agentState = state;\n    this.emit(\n      AgentSessionEventTypes.AgentStateChanged,\n      createAgentStateChangedEvent(oldState, state),\n    );\n  }\n\n  /** @internal */\n  _updateUserState(state: UserState) {\n    if (this.userState === state) {\n      return;\n    }\n\n    const oldState = this.userState;\n    this.userState = state;\n    this.emit(\n      AgentSessionEventTypes.UserStateChanged,\n      createUserStateChangedEvent(oldState, state),\n    );\n  }\n\n  // -- User changed input/output streams/sinks --\n  private onAudioInputChanged(): void {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity && this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  private onAudioOutputChanged(): void {}\n\n  private onTextOutputChanged(): void {}\n\n  private async closeImpl(\n    reason: CloseReason,\n    error: RealtimeModelError | LLMError | TTSError | STTError | null = null,\n    drain: boolean = false,\n  ): Promise<void> {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity) {\n      if (!drain) {\n        try {\n          this.activity.interrupt();\n        } catch (error) {\n          // uninterruptible speech [copied from python]\n          // TODO(shubhra): force interrupt or wait for it to finish?\n          // it might be an audio played from the error callback\n        }\n      }\n      await this.activity.drain();\n      // wait any uninterruptible speech to finish\n      await this.activity.currentSpeech?.waitForPlayout();\n      this.activity.detachAudioInput();\n    }\n\n    // detach the inputs and outputs\n    this.input.audio = null;\n    this.output.audio = null;\n    this.output.transcription = null;\n\n    await this.roomIO?.close();\n    this.roomIO = undefined;\n\n    await this.activity?.close();\n    this.activity = undefined;\n\n    this.started = false;\n\n    this.emit(AgentSessionEventTypes.Close, createCloseEvent(reason, error));\n\n    this.userState = 'listening';\n    this._agentState = 'initializing';\n\n    this.logger.info({ reason, error }, 'AgentSession closed');\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAE7B,0BAAyC;AAGzC,iBAAoB;AAMpB,4BAA8B;AAE9B,oBAkBO;AACP,gBAAwC;AACxC,qBAAsE;AActE,MAAM,sBAAoC;AAAA,EACxC,oBAAoB;AAAA,EACpB,+BAA+B;AAAA,EAC/B,yBAAyB;AAAA,EACzB,sBAAsB;AAAA,EACtB,qBAAqB;AAAA,EACrB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AA0BO,MAAM,qBAEF,gCAA+D;AAAA,EACxE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAES;AAAA,EAED;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV,YAAuB;AAAA,EAEvB;AAAA,EACA,aAAS,gBAAI;AAAA,EAEb;AAAA,EACA;AAAA,EACA,cAA0B;AAAA,EAE1B;AAAA,EACA;AAAA,EAEA,cAAoC;AAAA,EAE5C,YAAY,MAAqC;AAC/C,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IACjB,IAAI;AAEJ,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAGjB,SAAK,SAAS,IAAI,qBAAW,KAAK,mBAAmB;AACrD,SAAK,UAAU,IAAI,sBAAY,KAAK,sBAAsB,KAAK,mBAAmB;AAGlF,SAAK,WAAW,gCAAY,MAAM;AAClC,SAAK,UAAU,EAAE,GAAG,qBAAqB,GAAG,aAAa;AAAA,EAC3D;AAAA,EAEA,IAAI,QAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,WAAqB;AACvB,QAAI,KAAK,cAAc,QAAW;AAChC,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,OAAiB;AAC5B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,MAAM;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKkB;AAChB,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AAEA,SAAK,QAAQ;AACb,SAAK,kBAAkB,cAAc;AAGrC,QAAI,KAAK,MAAM,UAAS,6CAAc,kBAAiB,OAAO;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAAA,IAC7F;AAEA,QAAI,KAAK,OAAO,UAAS,+CAAe,kBAAiB,OAAO;AAC9D,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,kBAAiB,+CAAe,0BAAyB,OAAO;AAC9E,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,IAAI,sBAAO;AAAA,MACvB,cAAc;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,SAAK,OAAO,MAAM;AAElB,SAAK,eAAe,KAAK,KAAK;AAG9B,SAAK,OAAO;AAAA,MACV,mBAAmB,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,YAAY,OAAO,MAAM,QAAQ,2BAA2B,KAAK,OAAO,QAAQ,MAAM,KAAK,OAAO,MAAM,YAAY,OAAO,MAAM,QAAQ;AAAA,IACxM;AAEA,SAAK,OAAO;AAAA,MACV,4CAA4C,KAAK,OAAO,gBAAgB,MAAM,KAAK,OAAO,cAAc,YAAY,OAAO,MAAM,QAAQ;AAAA,IAC3I;AAEA,SAAK,OAAO,MAAM,sBAAsB;AACxC,SAAK,UAAU;AACf,SAAK,kBAAkB,WAAW;AAAA,EACpC;AAAA,EAEA,YAAY,OAAoB;AAC9B,SAAK,QAAQ;AAEb,QAAI,KAAK,SAAS;AAChB,WAAK,eAAe,KAAK;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,iBAAiB;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,SAAK,SAAS,eAAe;AAAA,EAC/B;AAAA,EAEA,gBAAgB;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,SAAK,SAAS,cAAc;AAAA,EAC9B;AAAA,EAEA,IACE,MACA,SAKc;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK,SAAS,IAAI,MAAM,OAAO;AAAA,EACxC;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,WAAO,KAAK,SAAS,UAAU;AAAA,EACjC;AAAA,EAEA,cAAc,SAKG;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,UAAM,eAAc,mCAAS,aACzB,IAAI,gCAAY;AAAA,MACd,MAAM;AAAA,MACN,SAAS,QAAQ;AAAA,IACnB,CAAC,IACD;AAEJ,QAAI,KAAK,SAAS,UAAU;AAC1B,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AACA,aAAO,KAAK,aAAa,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,IACpE;AAEA,WAAO,KAAK,SAAS,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,eAAe,OAA6B;AAExD,SAAK,eAAe,IAAI,oCAAc,OAAO,IAAI;AAEjD,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,SAAS,MAAM;AAC1B,YAAM,KAAK,SAAS,MAAM;AAAA,IAC5B;AAEA,SAAK,WAAW,KAAK;AACrB,SAAK,eAAe;AAEpB,UAAM,KAAK,SAAS,MAAM;AAE1B,QAAI,KAAK,OAAO,OAAO;AACrB,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,aAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAsB;AACxB,QAAI,CAAC,KAAK,OAAO;AACf,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,UAAU,0BAAY,cAAc;AAAA,EACjD;AAAA;AAAA,EAGA,WAAW;AAAA,IACT;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV,GAIS;AACP,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AACA,SAAK,UAAU,QAAQ,OAAO,KAAK;AAAA,EACrC;AAAA;AAAA,EAGA,SAAS,OAAkE;AACzE,QAAI,KAAK,eAAe,MAAM,aAAa;AACzC;AAAA,IACF;AAEA,SAAK,OAAO,MAAM,OAAO,oDAAoD;AAE7E,SAAK,eAAe,YAAY;AAC9B,YAAM,KAAK,UAAU,0BAAY,OAAO,KAAK;AAAA,IAC/C,GAAG,EAAE,KAAK,MAAM;AACd,WAAK,cAAc;AAAA,IACrB,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,uBAAuB,MAAyB;AAC9C,SAAK,SAAS,OAAO,IAAI;AACzB,SAAK,KAAK,qCAAuB,2BAAuB,gDAAiC,IAAI,CAAC;AAAA,EAChG;AAAA;AAAA,EAGA,kBAAkB,OAAmB;AACnC,QAAI,KAAK,gBAAgB,OAAO;AAC9B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,cAAc;AACnB,SAAK;AAAA,MACH,qCAAuB;AAAA,UACvB,4CAA6B,UAAU,KAAK;AAAA,IAC9C;AAAA,EACF;AAAA;AAAA,EAGA,iBAAiB,OAAkB;AACjC,QAAI,KAAK,cAAc,OAAO;AAC5B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,YAAY;AACjB,SAAK;AAAA,MACH,qCAAuB;AAAA,UACvB,2CAA4B,UAAU,KAAK;AAAA,IAC7C;AAAA,EACF;AAAA;AAAA,EAGQ,sBAA4B;AAClC,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,YAAY,KAAK,OAAO,OAAO;AACtC,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,uBAA6B;AAAA,EAAC;AAAA,EAE9B,sBAA4B;AAAA,EAAC;AAAA,EAErC,MAAc,UACZ,QACA,QAAoE,MACpE,QAAiB,OACF;AAlanB;AAmaI,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,UAAU;AACjB,UAAI,CAAC,OAAO;AACV,YAAI;AACF,eAAK,SAAS,UAAU;AAAA,QAC1B,SAASA,QAAO;AAAA,QAIhB;AAAA,MACF;AACA,YAAM,KAAK,SAAS,MAAM;AAE1B,cAAM,UAAK,SAAS,kBAAd,mBAA6B;AACnC,WAAK,SAAS,iBAAiB;AAAA,IACjC;AAGA,SAAK,MAAM,QAAQ;AACnB,SAAK,OAAO,QAAQ;AACpB,SAAK,OAAO,gBAAgB;AAE5B,YAAM,UAAK,WAAL,mBAAa;AACnB,SAAK,SAAS;AAEd,YAAM,UAAK,aAAL,mBAAe;AACrB,SAAK,WAAW;AAEhB,SAAK,UAAU;AAEf,SAAK,KAAK,qCAAuB,WAAO,gCAAiB,QAAQ,KAAK,CAAC;AAEvE,SAAK,YAAY;AACjB,SAAK,cAAc;AAEnB,SAAK,OAAO,KAAK,EAAE,QAAQ,MAAM,GAAG,qBAAqB;AAAA,EAC3D;AACF;","names":["error"]}
+{"version":3,"sources":["../../src/voice/agent_session.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, Room } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { ChatContext, ChatMessage } from '../llm/chat_context.js';\nimport type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport { log } from '../log.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS, TTSError } from '../tts/tts.js';\nimport type { VAD } from '../vad.js';\nimport type { Agent } from './agent.js';\nimport { AgentActivity } from './agent_activity.js';\nimport type { _TurnDetector } from './audio_recognition.js';\nimport {\n  AgentSessionEventTypes,\n  type AgentState,\n  type AgentStateChangedEvent,\n  type CloseEvent,\n  CloseReason,\n  type ConversationItemAddedEvent,\n  type ErrorEvent,\n  type FunctionToolsExecutedEvent,\n  type MetricsCollectedEvent,\n  type SpeechCreatedEvent,\n  type UserInputTranscribedEvent,\n  type UserState,\n  type UserStateChangedEvent,\n  createAgentStateChangedEvent,\n  createCloseEvent,\n  createConversationItemAddedEvent,\n  createUserStateChangedEvent,\n} from './events.js';\nimport { AgentInput, AgentOutput } from './io.js';\nimport { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';\nimport type { UnknownUserData } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport interface VoiceOptions {\n  allowInterruptions: boolean;\n  discardAudioIfUninterruptible: boolean;\n  minInterruptionDuration: number;\n  minInterruptionWords: number;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n  maxToolSteps: number;\n}\n\nconst defaultVoiceOptions: VoiceOptions = {\n  allowInterruptions: true,\n  discardAudioIfUninterruptible: true,\n  minInterruptionDuration: 500,\n  minInterruptionWords: 0,\n  minEndpointingDelay: 500,\n  maxEndpointingDelay: 6000,\n  maxToolSteps: 3,\n} as const;\n\nexport type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;\n\nexport type AgentSessionCallbacks = {\n  [AgentSessionEventTypes.UserInputTranscribed]: (ev: UserInputTranscribedEvent) => void;\n  [AgentSessionEventTypes.AgentStateChanged]: (ev: AgentStateChangedEvent) => void;\n  [AgentSessionEventTypes.UserStateChanged]: (ev: UserStateChangedEvent) => void;\n  [AgentSessionEventTypes.ConversationItemAdded]: (ev: ConversationItemAddedEvent) => void;\n  [AgentSessionEventTypes.FunctionToolsExecuted]: (ev: FunctionToolsExecutedEvent) => void;\n  [AgentSessionEventTypes.MetricsCollected]: (ev: MetricsCollectedEvent) => void;\n  [AgentSessionEventTypes.SpeechCreated]: (ev: SpeechCreatedEvent) => void;\n  [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void;\n  [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void;\n};\n\nexport type AgentSessionOptions<UserData = UnknownUserData> = {\n  turnDetection?: TurnDetectionMode;\n  stt?: STT;\n  vad?: VAD;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  userData?: UserData;\n  voiceOptions?: Partial<VoiceOptions>;\n};\n\nexport class AgentSession<\n  UserData = UnknownUserData,\n> extends (EventEmitter as new () => TypedEmitter<AgentSessionCallbacks>) {\n  vad?: VAD;\n  stt?: STT;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  turnDetection?: TurnDetectionMode;\n\n  readonly options: VoiceOptions;\n\n  private agent?: Agent;\n  private activity?: AgentActivity;\n  private nextActivity?: AgentActivity;\n  private started = false;\n  private userState: UserState = 'listening';\n\n  private roomIO?: RoomIO;\n  private logger = log();\n\n  private _chatCtx: ChatContext;\n  private _userData: UserData | undefined;\n  private _agentState: AgentState = 'initializing';\n\n  private _input: AgentInput;\n  private _output: AgentOutput;\n\n  private closingTask: Promise<void> | null = null;\n\n  constructor(opts: AgentSessionOptions<UserData>) {\n    super();\n\n    const {\n      vad,\n      stt,\n      llm,\n      tts,\n      turnDetection,\n      userData,\n      voiceOptions = defaultVoiceOptions,\n    } = opts;\n\n    this.vad = vad;\n    this.stt = stt;\n    this.llm = llm;\n    this.tts = tts;\n    this.turnDetection = turnDetection;\n    this._userData = userData;\n\n    // configurable IO\n    this._input = new AgentInput(this.onAudioInputChanged);\n    this._output = new AgentOutput(this.onAudioOutputChanged, this.onTextOutputChanged);\n\n    // This is the \"global\" chat context, it holds the entire conversation history\n    this._chatCtx = ChatContext.empty();\n    this.options = { ...defaultVoiceOptions, ...voiceOptions };\n  }\n\n  get input(): AgentInput {\n    return this._input;\n  }\n\n  get output(): AgentOutput {\n    return this._output;\n  }\n\n  get userData(): UserData {\n    if (this._userData === undefined) {\n      throw new Error('Voice agent userData is not set');\n    }\n\n    return this._userData;\n  }\n\n  get history(): ChatContext {\n    return this._chatCtx;\n  }\n\n  set userData(value: UserData) {\n    this._userData = value;\n  }\n\n  async start({\n    agent,\n    room,\n    inputOptions,\n    outputOptions,\n  }: {\n    agent: Agent;\n    room: Room;\n    inputOptions?: Partial<RoomInputOptions>;\n    outputOptions?: Partial<RoomOutputOptions>;\n  }): Promise<void> {\n    if (this.started) {\n      return;\n    }\n\n    this.agent = agent;\n    this._updateAgentState('initializing');\n\n    // Check for existing input/output configuration and warn if needed\n    if (this.input.audio && inputOptions?.audioEnabled !== false) {\n      this.logger.warn('RoomIO audio input is enabled but input.audio is already set, ignoring..');\n    }\n\n    if (this.output.audio && outputOptions?.audioEnabled !== false) {\n      this.logger.warn(\n        'RoomIO audio output is enabled but output.audio is already set, ignoring..',\n      );\n    }\n\n    if (this.output.transcription && outputOptions?.transcriptionEnabled !== false) {\n      this.logger.warn(\n        'RoomIO transcription output is enabled but output.transcription is already set, ignoring..',\n      );\n    }\n\n    this.roomIO = new RoomIO({\n      agentSession: this,\n      room,\n      inputOptions,\n      outputOptions,\n    });\n    this.roomIO.start();\n\n    this.updateActivity(this.agent);\n\n    // Log used IO configuration\n    this.logger.debug(\n      `using audio io: ${this.input.audio ? '`' + this.input.audio.constructor.name + '`' : '(none)'} -> \\`AgentSession\\` -> ${this.output.audio ? '`' + this.output.audio.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug(\n      `using transcript io: \\`AgentSession\\` -> ${this.output.transcription ? '`' + this.output.transcription.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug('AgentSession started');\n    this.started = true;\n    this._updateAgentState('listening');\n  }\n\n  updateAgent(agent: Agent): void {\n    this.agent = agent;\n\n    if (this.started) {\n      this.updateActivity(agent);\n    }\n  }\n\n  commitUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    this.activity.commitUserTurn();\n  }\n\n  clearUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    this.activity.clearUserTurn();\n  }\n\n  say(\n    text: string | ReadableStream<string>,\n    options?: {\n      audio?: ReadableStream<AudioFrame>;\n      allowInterruptions?: boolean;\n      addToChatCtx?: boolean;\n    },\n  ): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.activity.say(text, options);\n  }\n\n  interrupt() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    return this.activity.interrupt();\n  }\n\n  generateReply(options?: {\n    userInput?: string;\n    instructions?: string;\n    toolChoice?: ToolChoice;\n    allowInterruptions?: boolean;\n  }): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    const userMessage = options?.userInput\n      ? new ChatMessage({\n          role: 'user',\n          content: options.userInput,\n        })\n      : undefined;\n\n    if (this.activity.draining) {\n      if (!this.nextActivity) {\n        throw new Error('AgentSession is closing, cannot use generateReply()');\n      }\n      return this.nextActivity.generateReply({ userMessage, ...options });\n    }\n\n    return this.activity.generateReply({ userMessage, ...options });\n  }\n\n  private async updateActivity(agent: Agent): Promise<void> {\n    // TODO(AJS-129): add lock to agent activity core lifecycle\n    this.nextActivity = new AgentActivity(agent, this);\n\n    if (this.activity) {\n      await this.activity.drain();\n      await this.activity.close();\n    }\n\n    this.activity = this.nextActivity;\n    this.nextActivity = undefined;\n\n    await this.activity.start();\n\n    if (this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  get chatCtx(): ChatContext {\n    return this._chatCtx.copy();\n  }\n\n  get agentState(): AgentState {\n    return this._agentState;\n  }\n\n  get currentAgent(): Agent {\n    if (!this.agent) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.agent;\n  }\n\n  async close(): Promise<void> {\n    await this.closeImpl(CloseReason.USER_INITIATED);\n  }\n\n  /** @internal */\n  _closeSoon({\n    reason,\n    drain = false,\n    error = null,\n  }: {\n    reason: CloseReason;\n    drain?: boolean;\n    error?: RealtimeModelError | STTError | TTSError | LLMError | null;\n  }): void {\n    if (this.closingTask) {\n      return;\n    }\n    this.closeImpl(reason, error, drain);\n  }\n\n  /** @internal */\n  _onError(error: RealtimeModelError | STTError | TTSError | LLMError): void {\n    if (this.closingTask || error.recoverable) {\n      return;\n    }\n\n    this.logger.error(error, 'AgentSession is closing due to unrecoverable error');\n\n    this.closingTask = (async () => {\n      await this.closeImpl(CloseReason.ERROR, error);\n    })().then(() => {\n      this.closingTask = null;\n    });\n  }\n\n  /** @internal */\n  _conversationItemAdded(item: ChatMessage): void {\n    this._chatCtx.insert(item);\n    this.emit(AgentSessionEventTypes.ConversationItemAdded, createConversationItemAddedEvent(item));\n  }\n\n  /** @internal */\n  _updateAgentState(state: AgentState) {\n    if (this._agentState === state) {\n      return;\n    }\n\n    const oldState = this._agentState;\n    this._agentState = state;\n    this.emit(\n      AgentSessionEventTypes.AgentStateChanged,\n      createAgentStateChangedEvent(oldState, state),\n    );\n  }\n\n  /** @internal */\n  _updateUserState(state: UserState) {\n    if (this.userState === state) {\n      return;\n    }\n\n    const oldState = this.userState;\n    this.userState = state;\n    this.emit(\n      AgentSessionEventTypes.UserStateChanged,\n      createUserStateChangedEvent(oldState, state),\n    );\n  }\n\n  // -- User changed input/output streams/sinks --\n  private onAudioInputChanged(): void {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity && this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  private onAudioOutputChanged(): void {}\n\n  private onTextOutputChanged(): void {}\n\n  private async closeImpl(\n    reason: CloseReason,\n    error: RealtimeModelError | LLMError | TTSError | STTError | null = null,\n    drain: boolean = false,\n  ): Promise<void> {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity) {\n      if (!drain) {\n        try {\n          this.activity.interrupt();\n        } catch (error) {\n          // uninterruptible speech [copied from python]\n          // TODO(shubhra): force interrupt or wait for it to finish?\n          // it might be an audio played from the error callback\n        }\n      }\n      await this.activity.drain();\n      // wait any uninterruptible speech to finish\n      await this.activity.currentSpeech?.waitForPlayout();\n      this.activity.detachAudioInput();\n    }\n\n    // detach the inputs and outputs\n    this.input.audio = null;\n    this.output.audio = null;\n    this.output.transcription = null;\n\n    await this.roomIO?.close();\n    this.roomIO = undefined;\n\n    await this.activity?.close();\n    this.activity = undefined;\n\n    this.started = false;\n\n    this.emit(AgentSessionEventTypes.Close, createCloseEvent(reason, error));\n\n    this.userState = 'listening';\n    this._agentState = 'initializing';\n\n    this.logger.info({ reason, error }, 'AgentSession closed');\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAE7B,0BAAyC;AAGzC,iBAAoB;AAMpB,4BAA8B;AAE9B,oBAkBO;AACP,gBAAwC;AACxC,qBAAsE;AActE,MAAM,sBAAoC;AAAA,EACxC,oBAAoB;AAAA,EACpB,+BAA+B;AAAA,EAC/B,yBAAyB;AAAA,EACzB,sBAAsB;AAAA,EACtB,qBAAqB;AAAA,EACrB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AA0BO,MAAM,qBAEF,gCAA+D;AAAA,EACxE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAES;AAAA,EAED;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV,YAAuB;AAAA,EAEvB;AAAA,EACA,aAAS,gBAAI;AAAA,EAEb;AAAA,EACA;AAAA,EACA,cAA0B;AAAA,EAE1B;AAAA,EACA;AAAA,EAEA,cAAoC;AAAA,EAE5C,YAAY,MAAqC;AAC/C,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IACjB,IAAI;AAEJ,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAGjB,SAAK,SAAS,IAAI,qBAAW,KAAK,mBAAmB;AACrD,SAAK,UAAU,IAAI,sBAAY,KAAK,sBAAsB,KAAK,mBAAmB;AAGlF,SAAK,WAAW,gCAAY,MAAM;AAClC,SAAK,UAAU,EAAE,GAAG,qBAAqB,GAAG,aAAa;AAAA,EAC3D;AAAA,EAEA,IAAI,QAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,WAAqB;AACvB,QAAI,KAAK,cAAc,QAAW;AAChC,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,OAAiB;AAC5B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,MAAM;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKkB;AAChB,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AAEA,SAAK,QAAQ;AACb,SAAK,kBAAkB,cAAc;AAGrC,QAAI,KAAK,MAAM,UAAS,6CAAc,kBAAiB,OAAO;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAAA,IAC7F;AAEA,QAAI,KAAK,OAAO,UAAS,+CAAe,kBAAiB,OAAO;AAC9D,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,kBAAiB,+CAAe,0BAAyB,OAAO;AAC9E,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,IAAI,sBAAO;AAAA,MACvB,cAAc;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,SAAK,OAAO,MAAM;AAElB,SAAK,eAAe,KAAK,KAAK;AAG9B,SAAK,OAAO;AAAA,MACV,mBAAmB,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,YAAY,OAAO,MAAM,QAAQ,2BAA2B,KAAK,OAAO,QAAQ,MAAM,KAAK,OAAO,MAAM,YAAY,OAAO,MAAM,QAAQ;AAAA,IACxM;AAEA,SAAK,OAAO;AAAA,MACV,4CAA4C,KAAK,OAAO,gBAAgB,MAAM,KAAK,OAAO,cAAc,YAAY,OAAO,MAAM,QAAQ;AAAA,IAC3I;AAEA,SAAK,OAAO,MAAM,sBAAsB;AACxC,SAAK,UAAU;AACf,SAAK,kBAAkB,WAAW;AAAA,EACpC;AAAA,EAEA,YAAY,OAAoB;AAC9B,SAAK,QAAQ;AAEb,QAAI,KAAK,SAAS;AAChB,WAAK,eAAe,KAAK;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,iBAAiB;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,SAAK,SAAS,eAAe;AAAA,EAC/B;AAAA,EAEA,gBAAgB;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,SAAK,SAAS,cAAc;AAAA,EAC9B;AAAA,EAEA,IACE,MACA,SAKc;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK,SAAS,IAAI,MAAM,OAAO;AAAA,EACxC;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,WAAO,KAAK,SAAS,UAAU;AAAA,EACjC;AAAA,EAEA,cAAc,SAKG;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,UAAM,eAAc,mCAAS,aACzB,IAAI,gCAAY;AAAA,MACd,MAAM;AAAA,MACN,SAAS,QAAQ;AAAA,IACnB,CAAC,IACD;AAEJ,QAAI,KAAK,SAAS,UAAU;AAC1B,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AACA,aAAO,KAAK,aAAa,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,IACpE;AAEA,WAAO,KAAK,SAAS,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,eAAe,OAA6B;AAExD,SAAK,eAAe,IAAI,oCAAc,OAAO,IAAI;AAEjD,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,SAAS,MAAM;AAC1B,YAAM,KAAK,SAAS,MAAM;AAAA,IAC5B;AAEA,SAAK,WAAW,KAAK;AACrB,SAAK,eAAe;AAEpB,UAAM,KAAK,SAAS,MAAM;AAE1B,QAAI,KAAK,OAAO,OAAO;AACrB,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,aAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAsB;AACxB,QAAI,CAAC,KAAK,OAAO;AACf,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,UAAU,0BAAY,cAAc;AAAA,EACjD;AAAA;AAAA,EAGA,WAAW;AAAA,IACT;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV,GAIS;AACP,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AACA,SAAK,UAAU,QAAQ,OAAO,KAAK;AAAA,EACrC;AAAA;AAAA,EAGA,SAAS,OAAkE;AACzE,QAAI,KAAK,eAAe,MAAM,aAAa;AACzC;AAAA,IACF;AAEA,SAAK,OAAO,MAAM,OAAO,oDAAoD;AAE7E,SAAK,eAAe,YAAY;AAC9B,YAAM,KAAK,UAAU,0BAAY,OAAO,KAAK;AAAA,IAC/C,GAAG,EAAE,KAAK,MAAM;AACd,WAAK,cAAc;AAAA,IACrB,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,uBAAuB,MAAyB;AAC9C,SAAK,SAAS,OAAO,IAAI;AACzB,SAAK,KAAK,qCAAuB,2BAAuB,gDAAiC,IAAI,CAAC;AAAA,EAChG;AAAA;AAAA,EAGA,kBAAkB,OAAmB;AACnC,QAAI,KAAK,gBAAgB,OAAO;AAC9B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,cAAc;AACnB,SAAK;AAAA,MACH,qCAAuB;AAAA,UACvB,4CAA6B,UAAU,KAAK;AAAA,IAC9C;AAAA,EACF;AAAA;AAAA,EAGA,iBAAiB,OAAkB;AACjC,QAAI,KAAK,cAAc,OAAO;AAC5B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,YAAY;AACjB,SAAK;AAAA,MACH,qCAAuB;AAAA,UACvB,2CAA4B,UAAU,KAAK;AAAA,IAC7C;AAAA,EACF;AAAA;AAAA,EAGQ,sBAA4B;AAClC,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,YAAY,KAAK,OAAO,OAAO;AACtC,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,uBAA6B;AAAA,EAAC;AAAA,EAE9B,sBAA4B;AAAA,EAAC;AAAA,EAErC,MAAc,UACZ,QACA,QAAoE,MACpE,QAAiB,OACF;AAtanB;AAuaI,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,UAAU;AACjB,UAAI,CAAC,OAAO;AACV,YAAI;AACF,eAAK,SAAS,UAAU;AAAA,QAC1B,SAASA,QAAO;AAAA,QAIhB;AAAA,MACF;AACA,YAAM,KAAK,SAAS,MAAM;AAE1B,cAAM,UAAK,SAAS,kBAAd,mBAA6B;AACnC,WAAK,SAAS,iBAAiB;AAAA,IACjC;AAGA,SAAK,MAAM,QAAQ;AACnB,SAAK,OAAO,QAAQ;AACpB,SAAK,OAAO,gBAAgB;AAE5B,YAAM,UAAK,WAAL,mBAAa;AACnB,SAAK,SAAS;AAEd,YAAM,UAAK,aAAL,mBAAe;AACrB,SAAK,WAAW;AAEhB,SAAK,UAAU;AAEf,SAAK,KAAK,qCAAuB,WAAO,gCAAiB,QAAQ,KAAK,CAAC;AAEvE,SAAK,YAAY;AACjB,SAAK,cAAc;AAEnB,SAAK,OAAO,KAAK,EAAE,QAAQ,MAAM,GAAG,qBAAqB;AAAA,EAC3D;AACF;","names":["error"]}

package/dist/voice/agent_session.d.cts

@@ -71,6 +71,7 @@ export declare class AgentSession<UserData = UnknownUserData> extends AgentSessi
     get input(): AgentInput;
     get output(): AgentOutput;
     get userData(): UserData;
+    get history(): ChatContext;
     set userData(value: UserData);
     start({ agent, room, inputOptions, outputOptions, }: {
         agent: Agent;

package/dist/voice/agent_session.d.ts

@@ -71,6 +71,7 @@ export declare class AgentSession<UserData = UnknownUserData> extends AgentSessi
     get input(): AgentInput;
     get output(): AgentOutput;
     get userData(): UserData;
+    get history(): ChatContext;
     set userData(value: UserData);
     start({ agent, room, inputOptions, outputOptions, }: {
         agent: Agent;

package/dist/voice/agent_session.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent_session.d.ts","sourceRoot":"","sources":["../../src/voice/agent_session.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EACL,sBAAsB,EACtB,KAAK,UAAU,EACf,KAAK,sBAAsB,EAC3B,KAAK,UAAU,EACf,WAAW,EACX,KAAK,0BAA0B,EAC/B,KAAK,UAAU,EACf,KAAK,0BAA0B,EAC/B,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,SAAS,EACd,KAAK,qBAAqB,EAK3B,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAU,KAAK,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,YAAY;IAC3B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,6BAA6B,EAAE,OAAO,CAAC;IACvC,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;CACtB;AAYD,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,cAAc,GAAG,QAAQ,GAAG,aAAa,CAAC;AAE1F,MAAM,MAAM,qBAAqB,GAAG;IAClC,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAAE,yBAAyB,KAAK,IAAI,CAAC;IACvF,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,EAAE,CAAC,EAAE,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACjF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACzE,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;IACzD,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;CAC1D,CAAC;AAEF,MAAM,MAAM,mBAAmB,CAAC,QAAQ,GAAG,eAAe,IAAI;IAC5D,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,CAAC;IAC1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;CACtC,CAAC;2CAImC,aAAa,qBAAqB,CAAC;AAFxE,qBAAa,YAAY,CACvB,QAAQ,GAAG,eAAe,CAC1B,SAAQ,iBAA+D;IACvE,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,CAAC;IAC1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAElC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAE/B,OAAO,CAAC,KAAK,CAAC,CAAQ;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAgB;IACjC,OAAO,CAAC,YAAY,CAAC,CAAgB;IACrC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,SAAS,CAA0B;IAE3C,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IAEvB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,WAAW,CAA8B;IAEjD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,WAAW,CAA8B;gBAErC,IAAI,EAAE,mBAAmB,CAAC,QAAQ,CAAC;IA6B/C,IAAI,KAAK,IAAI,UAAU,CAEtB;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,IAAI,QAAQ,IAAI,QAAQ,CAMvB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAEK,KAAK,CAAC,EACV,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,aAAa,GACd,EAAE;QACD,KAAK,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,IAAI,CAAC;QACX,YAAY,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACzC,aAAa,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAC5C,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDjB,WAAW,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAQ/B,cAAc;IAQd,aAAa;IAOb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAQf,SAAS;IAOT,aAAa,CAAC,OAAO,CAAC,EAAE;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,kBAAkB,CAAC,EAAE,OAAO,CAAC;KAC9B,GAAG,YAAY;YAsBF,cAAc;IAmB5B,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,UAAU,IAAI,UAAU,CAE3B;IAED,IAAI,YAAY,IAAI,KAAK,CAMxB;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,gBAAgB;IAChB,UAAU,CAAC,EACT,MAAM,EACN,KAAa,EACb,KAAY,GACb,EAAE;QACD,MAAM,EAAE,WAAW,CAAC;QACpB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,KAAK,CAAC,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;KACpE,GAAG,IAAI;IAOR,gBAAgB;IAChB,QAAQ,CAAC,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI;IAc1E,gBAAgB;IAChB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI;IAK/C,gBAAgB;IAChB,iBAAiB,CAAC,KAAK,EAAE,UAAU;IAanC,gBAAgB;IAChB,gBAAgB,CAAC,KAAK,EAAE,SAAS;IAcjC,OAAO,CAAC,mBAAmB;IAU3B,OAAO,CAAC,oBAAoB;IAE5B,OAAO,CAAC,mBAAmB;YAEb,SAAS;CA6CxB"}
+{"version":3,"file":"agent_session.d.ts","sourceRoot":"","sources":["../../src/voice/agent_session.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EACL,sBAAsB,EACtB,KAAK,UAAU,EACf,KAAK,sBAAsB,EAC3B,KAAK,UAAU,EACf,WAAW,EACX,KAAK,0BAA0B,EAC/B,KAAK,UAAU,EACf,KAAK,0BAA0B,EAC/B,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,SAAS,EACd,KAAK,qBAAqB,EAK3B,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAU,KAAK,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,YAAY;IAC3B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,6BAA6B,EAAE,OAAO,CAAC;IACvC,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;CACtB;AAYD,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,cAAc,GAAG,QAAQ,GAAG,aAAa,CAAC;AAE1F,MAAM,MAAM,qBAAqB,GAAG;IAClC,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAAE,yBAAyB,KAAK,IAAI,CAAC;IACvF,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,EAAE,CAAC,EAAE,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACjF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACzE,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;IACzD,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;CAC1D,CAAC;AAEF,MAAM,MAAM,mBAAmB,CAAC,QAAQ,GAAG,eAAe,IAAI;IAC5D,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,CAAC;IAC1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;CACtC,CAAC;2CAImC,aAAa,qBAAqB,CAAC;AAFxE,qBAAa,YAAY,CACvB,QAAQ,GAAG,eAAe,CAC1B,SAAQ,iBAA+D;IACvE,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,CAAC;IAC1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAElC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAE/B,OAAO,CAAC,KAAK,CAAC,CAAQ;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAgB;IACjC,OAAO,CAAC,YAAY,CAAC,CAAgB;IACrC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,SAAS,CAA0B;IAE3C,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IAEvB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,WAAW,CAA8B;IAEjD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,WAAW,CAA8B;gBAErC,IAAI,EAAE,mBAAmB,CAAC,QAAQ,CAAC;IA6B/C,IAAI,KAAK,IAAI,UAAU,CAEtB;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,IAAI,QAAQ,IAAI,QAAQ,CAMvB;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAEK,KAAK,CAAC,EACV,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,aAAa,GACd,EAAE;QACD,KAAK,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,IAAI,CAAC;QACX,YAAY,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACzC,aAAa,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAC5C,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDjB,WAAW,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAQ/B,cAAc;IAQd,aAAa;IAOb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAQf,SAAS;IAOT,aAAa,CAAC,OAAO,CAAC,EAAE;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,kBAAkB,CAAC,EAAE,OAAO,CAAC;KAC9B,GAAG,YAAY;YAsBF,cAAc;IAmB5B,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,UAAU,IAAI,UAAU,CAE3B;IAED,IAAI,YAAY,IAAI,KAAK,CAMxB;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,gBAAgB;IAChB,UAAU,CAAC,EACT,MAAM,EACN,KAAa,EACb,KAAY,GACb,EAAE;QACD,MAAM,EAAE,WAAW,CAAC;QACpB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,KAAK,CAAC,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;KACpE,GAAG,IAAI;IAOR,gBAAgB;IAChB,QAAQ,CAAC,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI;IAc1E,gBAAgB;IAChB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI;IAK/C,gBAAgB;IAChB,iBAAiB,CAAC,KAAK,EAAE,UAAU;IAanC,gBAAgB;IAChB,gBAAgB,CAAC,KAAK,EAAE,SAAS;IAcjC,OAAO,CAAC,mBAAmB;IAU3B,OAAO,CAAC,oBAAoB;IAE5B,OAAO,CAAC,mBAAmB;YAEb,SAAS;CA6CxB"}

package/dist/voice/agent_session.js

@@ -75,6 +75,9 @@ class AgentSession extends EventEmitter {
     }
     return this._userData;
   }
+  get history() {
+    return this._chatCtx;
+  }
   set userData(value) {
     this._userData = value;
   }

package/dist/voice/agent_session.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/agent_session.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, Room } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { ChatContext, ChatMessage } from '../llm/chat_context.js';\nimport type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport { log } from '../log.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS, TTSError } from '../tts/tts.js';\nimport type { VAD } from '../vad.js';\nimport type { Agent } from './agent.js';\nimport { AgentActivity } from './agent_activity.js';\nimport type { _TurnDetector } from './audio_recognition.js';\nimport {\n  AgentSessionEventTypes,\n  type AgentState,\n  type AgentStateChangedEvent,\n  type CloseEvent,\n  CloseReason,\n  type ConversationItemAddedEvent,\n  type ErrorEvent,\n  type FunctionToolsExecutedEvent,\n  type MetricsCollectedEvent,\n  type SpeechCreatedEvent,\n  type UserInputTranscribedEvent,\n  type UserState,\n  type UserStateChangedEvent,\n  createAgentStateChangedEvent,\n  createCloseEvent,\n  createConversationItemAddedEvent,\n  createUserStateChangedEvent,\n} from './events.js';\nimport { AgentInput, AgentOutput } from './io.js';\nimport { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';\nimport type { UnknownUserData } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport interface VoiceOptions {\n  allowInterruptions: boolean;\n  discardAudioIfUninterruptible: boolean;\n  minInterruptionDuration: number;\n  minInterruptionWords: number;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n  maxToolSteps: number;\n}\n\nconst defaultVoiceOptions: VoiceOptions = {\n  allowInterruptions: true,\n  discardAudioIfUninterruptible: true,\n  minInterruptionDuration: 500,\n  minInterruptionWords: 0,\n  minEndpointingDelay: 500,\n  maxEndpointingDelay: 6000,\n  maxToolSteps: 3,\n} as const;\n\nexport type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;\n\nexport type AgentSessionCallbacks = {\n  [AgentSessionEventTypes.UserInputTranscribed]: (ev: UserInputTranscribedEvent) => void;\n  [AgentSessionEventTypes.AgentStateChanged]: (ev: AgentStateChangedEvent) => void;\n  [AgentSessionEventTypes.UserStateChanged]: (ev: UserStateChangedEvent) => void;\n  [AgentSessionEventTypes.ConversationItemAdded]: (ev: ConversationItemAddedEvent) => void;\n  [AgentSessionEventTypes.FunctionToolsExecuted]: (ev: FunctionToolsExecutedEvent) => void;\n  [AgentSessionEventTypes.MetricsCollected]: (ev: MetricsCollectedEvent) => void;\n  [AgentSessionEventTypes.SpeechCreated]: (ev: SpeechCreatedEvent) => void;\n  [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void;\n  [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void;\n};\n\nexport type AgentSessionOptions<UserData = UnknownUserData> = {\n  turnDetection?: TurnDetectionMode;\n  stt?: STT;\n  vad?: VAD;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  userData?: UserData;\n  voiceOptions?: Partial<VoiceOptions>;\n};\n\nexport class AgentSession<\n  UserData = UnknownUserData,\n> extends (EventEmitter as new () => TypedEmitter<AgentSessionCallbacks>) {\n  vad?: VAD;\n  stt?: STT;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  turnDetection?: TurnDetectionMode;\n\n  readonly options: VoiceOptions;\n\n  private agent?: Agent;\n  private activity?: AgentActivity;\n  private nextActivity?: AgentActivity;\n  private started = false;\n  private userState: UserState = 'listening';\n\n  private roomIO?: RoomIO;\n  private logger = log();\n\n  private _chatCtx: ChatContext;\n  private _userData: UserData | undefined;\n  private _agentState: AgentState = 'initializing';\n\n  private _input: AgentInput;\n  private _output: AgentOutput;\n\n  private closingTask: Promise<void> | null = null;\n\n  constructor(opts: AgentSessionOptions<UserData>) {\n    super();\n\n    const {\n      vad,\n      stt,\n      llm,\n      tts,\n      turnDetection,\n      userData,\n      voiceOptions = defaultVoiceOptions,\n    } = opts;\n\n    this.vad = vad;\n    this.stt = stt;\n    this.llm = llm;\n    this.tts = tts;\n    this.turnDetection = turnDetection;\n    this._userData = userData;\n\n    // configurable IO\n    this._input = new AgentInput(this.onAudioInputChanged);\n    this._output = new AgentOutput(this.onAudioOutputChanged, this.onTextOutputChanged);\n\n    // This is the \"global\" chat context, it holds the entire conversation history\n    this._chatCtx = ChatContext.empty();\n    this.options = { ...defaultVoiceOptions, ...voiceOptions };\n  }\n\n  get input(): AgentInput {\n    return this._input;\n  }\n\n  get output(): AgentOutput {\n    return this._output;\n  }\n\n  get userData(): UserData {\n    if (this._userData === undefined) {\n      throw new Error('Voice agent userData is not set');\n    }\n\n    return this._userData;\n  }\n\n  set userData(value: UserData) {\n    this._userData = value;\n  }\n\n  async start({\n    agent,\n    room,\n    inputOptions,\n    outputOptions,\n  }: {\n    agent: Agent;\n    room: Room;\n    inputOptions?: Partial<RoomInputOptions>;\n    outputOptions?: Partial<RoomOutputOptions>;\n  }): Promise<void> {\n    if (this.started) {\n      return;\n    }\n\n    this.agent = agent;\n    this._updateAgentState('initializing');\n\n    // Check for existing input/output configuration and warn if needed\n    if (this.input.audio && inputOptions?.audioEnabled !== false) {\n      this.logger.warn('RoomIO audio input is enabled but input.audio is already set, ignoring..');\n    }\n\n    if (this.output.audio && outputOptions?.audioEnabled !== false) {\n      this.logger.warn(\n        'RoomIO audio output is enabled but output.audio is already set, ignoring..',\n      );\n    }\n\n    if (this.output.transcription && outputOptions?.transcriptionEnabled !== false) {\n      this.logger.warn(\n        'RoomIO transcription output is enabled but output.transcription is already set, ignoring..',\n      );\n    }\n\n    this.roomIO = new RoomIO({\n      agentSession: this,\n      room,\n      inputOptions,\n      outputOptions,\n    });\n    this.roomIO.start();\n\n    this.updateActivity(this.agent);\n\n    // Log used IO configuration\n    this.logger.debug(\n      `using audio io: ${this.input.audio ? '`' + this.input.audio.constructor.name + '`' : '(none)'} -> \\`AgentSession\\` -> ${this.output.audio ? '`' + this.output.audio.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug(\n      `using transcript io: \\`AgentSession\\` -> ${this.output.transcription ? '`' + this.output.transcription.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug('AgentSession started');\n    this.started = true;\n    this._updateAgentState('listening');\n  }\n\n  updateAgent(agent: Agent): void {\n    this.agent = agent;\n\n    if (this.started) {\n      this.updateActivity(agent);\n    }\n  }\n\n  commitUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    this.activity.commitUserTurn();\n  }\n\n  clearUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    this.activity.clearUserTurn();\n  }\n\n  say(\n    text: string | ReadableStream<string>,\n    options?: {\n      audio?: ReadableStream<AudioFrame>;\n      allowInterruptions?: boolean;\n      addToChatCtx?: boolean;\n    },\n  ): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.activity.say(text, options);\n  }\n\n  interrupt() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    return this.activity.interrupt();\n  }\n\n  generateReply(options?: {\n    userInput?: string;\n    instructions?: string;\n    toolChoice?: ToolChoice;\n    allowInterruptions?: boolean;\n  }): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    const userMessage = options?.userInput\n      ? new ChatMessage({\n          role: 'user',\n          content: options.userInput,\n        })\n      : undefined;\n\n    if (this.activity.draining) {\n      if (!this.nextActivity) {\n        throw new Error('AgentSession is closing, cannot use generateReply()');\n      }\n      return this.nextActivity.generateReply({ userMessage, ...options });\n    }\n\n    return this.activity.generateReply({ userMessage, ...options });\n  }\n\n  private async updateActivity(agent: Agent): Promise<void> {\n    // TODO(AJS-129): add lock to agent activity core lifecycle\n    this.nextActivity = new AgentActivity(agent, this);\n\n    if (this.activity) {\n      await this.activity.drain();\n      await this.activity.close();\n    }\n\n    this.activity = this.nextActivity;\n    this.nextActivity = undefined;\n\n    await this.activity.start();\n\n    if (this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  get chatCtx(): ChatContext {\n    return this._chatCtx.copy();\n  }\n\n  get agentState(): AgentState {\n    return this._agentState;\n  }\n\n  get currentAgent(): Agent {\n    if (!this.agent) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.agent;\n  }\n\n  async close(): Promise<void> {\n    await this.closeImpl(CloseReason.USER_INITIATED);\n  }\n\n  /** @internal */\n  _closeSoon({\n    reason,\n    drain = false,\n    error = null,\n  }: {\n    reason: CloseReason;\n    drain?: boolean;\n    error?: RealtimeModelError | STTError | TTSError | LLMError | null;\n  }): void {\n    if (this.closingTask) {\n      return;\n    }\n    this.closeImpl(reason, error, drain);\n  }\n\n  /** @internal */\n  _onError(error: RealtimeModelError | STTError | TTSError | LLMError): void {\n    if (this.closingTask || error.recoverable) {\n      return;\n    }\n\n    this.logger.error(error, 'AgentSession is closing due to unrecoverable error');\n\n    this.closingTask = (async () => {\n      await this.closeImpl(CloseReason.ERROR, error);\n    })().then(() => {\n      this.closingTask = null;\n    });\n  }\n\n  /** @internal */\n  _conversationItemAdded(item: ChatMessage): void {\n    this._chatCtx.insert(item);\n    this.emit(AgentSessionEventTypes.ConversationItemAdded, createConversationItemAddedEvent(item));\n  }\n\n  /** @internal */\n  _updateAgentState(state: AgentState) {\n    if (this._agentState === state) {\n      return;\n    }\n\n    const oldState = this._agentState;\n    this._agentState = state;\n    this.emit(\n      AgentSessionEventTypes.AgentStateChanged,\n      createAgentStateChangedEvent(oldState, state),\n    );\n  }\n\n  /** @internal */\n  _updateUserState(state: UserState) {\n    if (this.userState === state) {\n      return;\n    }\n\n    const oldState = this.userState;\n    this.userState = state;\n    this.emit(\n      AgentSessionEventTypes.UserStateChanged,\n      createUserStateChangedEvent(oldState, state),\n    );\n  }\n\n  // -- User changed input/output streams/sinks --\n  private onAudioInputChanged(): void {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity && this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  private onAudioOutputChanged(): void {}\n\n  private onTextOutputChanged(): void {}\n\n  private async closeImpl(\n    reason: CloseReason,\n    error: RealtimeModelError | LLMError | TTSError | STTError | null = null,\n    drain: boolean = false,\n  ): Promise<void> {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity) {\n      if (!drain) {\n        try {\n          this.activity.interrupt();\n        } catch (error) {\n          // uninterruptible speech [copied from python]\n          // TODO(shubhra): force interrupt or wait for it to finish?\n          // it might be an audio played from the error callback\n        }\n      }\n      await this.activity.drain();\n      // wait any uninterruptible speech to finish\n      await this.activity.currentSpeech?.waitForPlayout();\n      this.activity.detachAudioInput();\n    }\n\n    // detach the inputs and outputs\n    this.input.audio = null;\n    this.output.audio = null;\n    this.output.transcription = null;\n\n    await this.roomIO?.close();\n    this.roomIO = undefined;\n\n    await this.activity?.close();\n    this.activity = undefined;\n\n    this.started = false;\n\n    this.emit(AgentSessionEventTypes.Close, createCloseEvent(reason, error));\n\n    this.userState = 'listening';\n    this._agentState = 'initializing';\n\n    this.logger.info({ reason, error }, 'AgentSession closed');\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,aAAa,mBAAmB;AAGzC,SAAS,WAAW;AAMpB,SAAS,qBAAqB;AAE9B;AAAA,EACE;AAAA,EAIA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,YAAY,mBAAmB;AACxC,SAAS,cAA6D;AActE,MAAM,sBAAoC;AAAA,EACxC,oBAAoB;AAAA,EACpB,+BAA+B;AAAA,EAC/B,yBAAyB;AAAA,EACzB,sBAAsB;AAAA,EACtB,qBAAqB;AAAA,EACrB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AA0BO,MAAM,qBAEF,aAA+D;AAAA,EACxE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAES;AAAA,EAED;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV,YAAuB;AAAA,EAEvB;AAAA,EACA,SAAS,IAAI;AAAA,EAEb;AAAA,EACA;AAAA,EACA,cAA0B;AAAA,EAE1B;AAAA,EACA;AAAA,EAEA,cAAoC;AAAA,EAE5C,YAAY,MAAqC;AAC/C,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IACjB,IAAI;AAEJ,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAGjB,SAAK,SAAS,IAAI,WAAW,KAAK,mBAAmB;AACrD,SAAK,UAAU,IAAI,YAAY,KAAK,sBAAsB,KAAK,mBAAmB;AAGlF,SAAK,WAAW,YAAY,MAAM;AAClC,SAAK,UAAU,EAAE,GAAG,qBAAqB,GAAG,aAAa;AAAA,EAC3D;AAAA,EAEA,IAAI,QAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,WAAqB;AACvB,QAAI,KAAK,cAAc,QAAW;AAChC,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,OAAiB;AAC5B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,MAAM;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKkB;AAChB,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AAEA,SAAK,QAAQ;AACb,SAAK,kBAAkB,cAAc;AAGrC,QAAI,KAAK,MAAM,UAAS,6CAAc,kBAAiB,OAAO;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAAA,IAC7F;AAEA,QAAI,KAAK,OAAO,UAAS,+CAAe,kBAAiB,OAAO;AAC9D,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,kBAAiB,+CAAe,0BAAyB,OAAO;AAC9E,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,cAAc;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,SAAK,OAAO,MAAM;AAElB,SAAK,eAAe,KAAK,KAAK;AAG9B,SAAK,OAAO;AAAA,MACV,mBAAmB,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,YAAY,OAAO,MAAM,QAAQ,2BAA2B,KAAK,OAAO,QAAQ,MAAM,KAAK,OAAO,MAAM,YAAY,OAAO,MAAM,QAAQ;AAAA,IACxM;AAEA,SAAK,OAAO;AAAA,MACV,4CAA4C,KAAK,OAAO,gBAAgB,MAAM,KAAK,OAAO,cAAc,YAAY,OAAO,MAAM,QAAQ;AAAA,IAC3I;AAEA,SAAK,OAAO,MAAM,sBAAsB;AACxC,SAAK,UAAU;AACf,SAAK,kBAAkB,WAAW;AAAA,EACpC;AAAA,EAEA,YAAY,OAAoB;AAC9B,SAAK,QAAQ;AAEb,QAAI,KAAK,SAAS;AAChB,WAAK,eAAe,KAAK;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,iBAAiB;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,SAAK,SAAS,eAAe;AAAA,EAC/B;AAAA,EAEA,gBAAgB;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,SAAK,SAAS,cAAc;AAAA,EAC9B;AAAA,EAEA,IACE,MACA,SAKc;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK,SAAS,IAAI,MAAM,OAAO;AAAA,EACxC;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,WAAO,KAAK,SAAS,UAAU;AAAA,EACjC;AAAA,EAEA,cAAc,SAKG;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,UAAM,eAAc,mCAAS,aACzB,IAAI,YAAY;AAAA,MACd,MAAM;AAAA,MACN,SAAS,QAAQ;AAAA,IACnB,CAAC,IACD;AAEJ,QAAI,KAAK,SAAS,UAAU;AAC1B,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AACA,aAAO,KAAK,aAAa,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,IACpE;AAEA,WAAO,KAAK,SAAS,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,eAAe,OAA6B;AAExD,SAAK,eAAe,IAAI,cAAc,OAAO,IAAI;AAEjD,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,SAAS,MAAM;AAC1B,YAAM,KAAK,SAAS,MAAM;AAAA,IAC5B;AAEA,SAAK,WAAW,KAAK;AACrB,SAAK,eAAe;AAEpB,UAAM,KAAK,SAAS,MAAM;AAE1B,QAAI,KAAK,OAAO,OAAO;AACrB,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,aAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAsB;AACxB,QAAI,CAAC,KAAK,OAAO;AACf,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,UAAU,YAAY,cAAc;AAAA,EACjD;AAAA;AAAA,EAGA,WAAW;AAAA,IACT;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV,GAIS;AACP,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AACA,SAAK,UAAU,QAAQ,OAAO,KAAK;AAAA,EACrC;AAAA;AAAA,EAGA,SAAS,OAAkE;AACzE,QAAI,KAAK,eAAe,MAAM,aAAa;AACzC;AAAA,IACF;AAEA,SAAK,OAAO,MAAM,OAAO,oDAAoD;AAE7E,SAAK,eAAe,YAAY;AAC9B,YAAM,KAAK,UAAU,YAAY,OAAO,KAAK;AAAA,IAC/C,GAAG,EAAE,KAAK,MAAM;AACd,WAAK,cAAc;AAAA,IACrB,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,uBAAuB,MAAyB;AAC9C,SAAK,SAAS,OAAO,IAAI;AACzB,SAAK,KAAK,uBAAuB,uBAAuB,iCAAiC,IAAI,CAAC;AAAA,EAChG;AAAA;AAAA,EAGA,kBAAkB,OAAmB;AACnC,QAAI,KAAK,gBAAgB,OAAO;AAC9B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,cAAc;AACnB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,6BAA6B,UAAU,KAAK;AAAA,IAC9C;AAAA,EACF;AAAA;AAAA,EAGA,iBAAiB,OAAkB;AACjC,QAAI,KAAK,cAAc,OAAO;AAC5B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,YAAY;AACjB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,4BAA4B,UAAU,KAAK;AAAA,IAC7C;AAAA,EACF;AAAA;AAAA,EAGQ,sBAA4B;AAClC,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,YAAY,KAAK,OAAO,OAAO;AACtC,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,uBAA6B;AAAA,EAAC;AAAA,EAE9B,sBAA4B;AAAA,EAAC;AAAA,EAErC,MAAc,UACZ,QACA,QAAoE,MACpE,QAAiB,OACF;AAlanB;AAmaI,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,UAAU;AACjB,UAAI,CAAC,OAAO;AACV,YAAI;AACF,eAAK,SAAS,UAAU;AAAA,QAC1B,SAASA,QAAO;AAAA,QAIhB;AAAA,MACF;AACA,YAAM,KAAK,SAAS,MAAM;AAE1B,cAAM,UAAK,SAAS,kBAAd,mBAA6B;AACnC,WAAK,SAAS,iBAAiB;AAAA,IACjC;AAGA,SAAK,MAAM,QAAQ;AACnB,SAAK,OAAO,QAAQ;AACpB,SAAK,OAAO,gBAAgB;AAE5B,YAAM,UAAK,WAAL,mBAAa;AACnB,SAAK,SAAS;AAEd,YAAM,UAAK,aAAL,mBAAe;AACrB,SAAK,WAAW;AAEhB,SAAK,UAAU;AAEf,SAAK,KAAK,uBAAuB,OAAO,iBAAiB,QAAQ,KAAK,CAAC;AAEvE,SAAK,YAAY;AACjB,SAAK,cAAc;AAEnB,SAAK,OAAO,KAAK,EAAE,QAAQ,MAAM,GAAG,qBAAqB;AAAA,EAC3D;AACF;","names":["error"]}
+{"version":3,"sources":["../../src/voice/agent_session.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, Room } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { ChatContext, ChatMessage } from '../llm/chat_context.js';\nimport type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport { log } from '../log.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS, TTSError } from '../tts/tts.js';\nimport type { VAD } from '../vad.js';\nimport type { Agent } from './agent.js';\nimport { AgentActivity } from './agent_activity.js';\nimport type { _TurnDetector } from './audio_recognition.js';\nimport {\n  AgentSessionEventTypes,\n  type AgentState,\n  type AgentStateChangedEvent,\n  type CloseEvent,\n  CloseReason,\n  type ConversationItemAddedEvent,\n  type ErrorEvent,\n  type FunctionToolsExecutedEvent,\n  type MetricsCollectedEvent,\n  type SpeechCreatedEvent,\n  type UserInputTranscribedEvent,\n  type UserState,\n  type UserStateChangedEvent,\n  createAgentStateChangedEvent,\n  createCloseEvent,\n  createConversationItemAddedEvent,\n  createUserStateChangedEvent,\n} from './events.js';\nimport { AgentInput, AgentOutput } from './io.js';\nimport { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';\nimport type { UnknownUserData } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport interface VoiceOptions {\n  allowInterruptions: boolean;\n  discardAudioIfUninterruptible: boolean;\n  minInterruptionDuration: number;\n  minInterruptionWords: number;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n  maxToolSteps: number;\n}\n\nconst defaultVoiceOptions: VoiceOptions = {\n  allowInterruptions: true,\n  discardAudioIfUninterruptible: true,\n  minInterruptionDuration: 500,\n  minInterruptionWords: 0,\n  minEndpointingDelay: 500,\n  maxEndpointingDelay: 6000,\n  maxToolSteps: 3,\n} as const;\n\nexport type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;\n\nexport type AgentSessionCallbacks = {\n  [AgentSessionEventTypes.UserInputTranscribed]: (ev: UserInputTranscribedEvent) => void;\n  [AgentSessionEventTypes.AgentStateChanged]: (ev: AgentStateChangedEvent) => void;\n  [AgentSessionEventTypes.UserStateChanged]: (ev: UserStateChangedEvent) => void;\n  [AgentSessionEventTypes.ConversationItemAdded]: (ev: ConversationItemAddedEvent) => void;\n  [AgentSessionEventTypes.FunctionToolsExecuted]: (ev: FunctionToolsExecutedEvent) => void;\n  [AgentSessionEventTypes.MetricsCollected]: (ev: MetricsCollectedEvent) => void;\n  [AgentSessionEventTypes.SpeechCreated]: (ev: SpeechCreatedEvent) => void;\n  [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void;\n  [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void;\n};\n\nexport type AgentSessionOptions<UserData = UnknownUserData> = {\n  turnDetection?: TurnDetectionMode;\n  stt?: STT;\n  vad?: VAD;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  userData?: UserData;\n  voiceOptions?: Partial<VoiceOptions>;\n};\n\nexport class AgentSession<\n  UserData = UnknownUserData,\n> extends (EventEmitter as new () => TypedEmitter<AgentSessionCallbacks>) {\n  vad?: VAD;\n  stt?: STT;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  turnDetection?: TurnDetectionMode;\n\n  readonly options: VoiceOptions;\n\n  private agent?: Agent;\n  private activity?: AgentActivity;\n  private nextActivity?: AgentActivity;\n  private started = false;\n  private userState: UserState = 'listening';\n\n  private roomIO?: RoomIO;\n  private logger = log();\n\n  private _chatCtx: ChatContext;\n  private _userData: UserData | undefined;\n  private _agentState: AgentState = 'initializing';\n\n  private _input: AgentInput;\n  private _output: AgentOutput;\n\n  private closingTask: Promise<void> | null = null;\n\n  constructor(opts: AgentSessionOptions<UserData>) {\n    super();\n\n    const {\n      vad,\n      stt,\n      llm,\n      tts,\n      turnDetection,\n      userData,\n      voiceOptions = defaultVoiceOptions,\n    } = opts;\n\n    this.vad = vad;\n    this.stt = stt;\n    this.llm = llm;\n    this.tts = tts;\n    this.turnDetection = turnDetection;\n    this._userData = userData;\n\n    // configurable IO\n    this._input = new AgentInput(this.onAudioInputChanged);\n    this._output = new AgentOutput(this.onAudioOutputChanged, this.onTextOutputChanged);\n\n    // This is the \"global\" chat context, it holds the entire conversation history\n    this._chatCtx = ChatContext.empty();\n    this.options = { ...defaultVoiceOptions, ...voiceOptions };\n  }\n\n  get input(): AgentInput {\n    return this._input;\n  }\n\n  get output(): AgentOutput {\n    return this._output;\n  }\n\n  get userData(): UserData {\n    if (this._userData === undefined) {\n      throw new Error('Voice agent userData is not set');\n    }\n\n    return this._userData;\n  }\n\n  get history(): ChatContext {\n    return this._chatCtx;\n  }\n\n  set userData(value: UserData) {\n    this._userData = value;\n  }\n\n  async start({\n    agent,\n    room,\n    inputOptions,\n    outputOptions,\n  }: {\n    agent: Agent;\n    room: Room;\n    inputOptions?: Partial<RoomInputOptions>;\n    outputOptions?: Partial<RoomOutputOptions>;\n  }): Promise<void> {\n    if (this.started) {\n      return;\n    }\n\n    this.agent = agent;\n    this._updateAgentState('initializing');\n\n    // Check for existing input/output configuration and warn if needed\n    if (this.input.audio && inputOptions?.audioEnabled !== false) {\n      this.logger.warn('RoomIO audio input is enabled but input.audio is already set, ignoring..');\n    }\n\n    if (this.output.audio && outputOptions?.audioEnabled !== false) {\n      this.logger.warn(\n        'RoomIO audio output is enabled but output.audio is already set, ignoring..',\n      );\n    }\n\n    if (this.output.transcription && outputOptions?.transcriptionEnabled !== false) {\n      this.logger.warn(\n        'RoomIO transcription output is enabled but output.transcription is already set, ignoring..',\n      );\n    }\n\n    this.roomIO = new RoomIO({\n      agentSession: this,\n      room,\n      inputOptions,\n      outputOptions,\n    });\n    this.roomIO.start();\n\n    this.updateActivity(this.agent);\n\n    // Log used IO configuration\n    this.logger.debug(\n      `using audio io: ${this.input.audio ? '`' + this.input.audio.constructor.name + '`' : '(none)'} -> \\`AgentSession\\` -> ${this.output.audio ? '`' + this.output.audio.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug(\n      `using transcript io: \\`AgentSession\\` -> ${this.output.transcription ? '`' + this.output.transcription.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug('AgentSession started');\n    this.started = true;\n    this._updateAgentState('listening');\n  }\n\n  updateAgent(agent: Agent): void {\n    this.agent = agent;\n\n    if (this.started) {\n      this.updateActivity(agent);\n    }\n  }\n\n  commitUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    this.activity.commitUserTurn();\n  }\n\n  clearUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    this.activity.clearUserTurn();\n  }\n\n  say(\n    text: string | ReadableStream<string>,\n    options?: {\n      audio?: ReadableStream<AudioFrame>;\n      allowInterruptions?: boolean;\n      addToChatCtx?: boolean;\n    },\n  ): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.activity.say(text, options);\n  }\n\n  interrupt() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    return this.activity.interrupt();\n  }\n\n  generateReply(options?: {\n    userInput?: string;\n    instructions?: string;\n    toolChoice?: ToolChoice;\n    allowInterruptions?: boolean;\n  }): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    const userMessage = options?.userInput\n      ? new ChatMessage({\n          role: 'user',\n          content: options.userInput,\n        })\n      : undefined;\n\n    if (this.activity.draining) {\n      if (!this.nextActivity) {\n        throw new Error('AgentSession is closing, cannot use generateReply()');\n      }\n      return this.nextActivity.generateReply({ userMessage, ...options });\n    }\n\n    return this.activity.generateReply({ userMessage, ...options });\n  }\n\n  private async updateActivity(agent: Agent): Promise<void> {\n    // TODO(AJS-129): add lock to agent activity core lifecycle\n    this.nextActivity = new AgentActivity(agent, this);\n\n    if (this.activity) {\n      await this.activity.drain();\n      await this.activity.close();\n    }\n\n    this.activity = this.nextActivity;\n    this.nextActivity = undefined;\n\n    await this.activity.start();\n\n    if (this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  get chatCtx(): ChatContext {\n    return this._chatCtx.copy();\n  }\n\n  get agentState(): AgentState {\n    return this._agentState;\n  }\n\n  get currentAgent(): Agent {\n    if (!this.agent) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.agent;\n  }\n\n  async close(): Promise<void> {\n    await this.closeImpl(CloseReason.USER_INITIATED);\n  }\n\n  /** @internal */\n  _closeSoon({\n    reason,\n    drain = false,\n    error = null,\n  }: {\n    reason: CloseReason;\n    drain?: boolean;\n    error?: RealtimeModelError | STTError | TTSError | LLMError | null;\n  }): void {\n    if (this.closingTask) {\n      return;\n    }\n    this.closeImpl(reason, error, drain);\n  }\n\n  /** @internal */\n  _onError(error: RealtimeModelError | STTError | TTSError | LLMError): void {\n    if (this.closingTask || error.recoverable) {\n      return;\n    }\n\n    this.logger.error(error, 'AgentSession is closing due to unrecoverable error');\n\n    this.closingTask = (async () => {\n      await this.closeImpl(CloseReason.ERROR, error);\n    })().then(() => {\n      this.closingTask = null;\n    });\n  }\n\n  /** @internal */\n  _conversationItemAdded(item: ChatMessage): void {\n    this._chatCtx.insert(item);\n    this.emit(AgentSessionEventTypes.ConversationItemAdded, createConversationItemAddedEvent(item));\n  }\n\n  /** @internal */\n  _updateAgentState(state: AgentState) {\n    if (this._agentState === state) {\n      return;\n    }\n\n    const oldState = this._agentState;\n    this._agentState = state;\n    this.emit(\n      AgentSessionEventTypes.AgentStateChanged,\n      createAgentStateChangedEvent(oldState, state),\n    );\n  }\n\n  /** @internal */\n  _updateUserState(state: UserState) {\n    if (this.userState === state) {\n      return;\n    }\n\n    const oldState = this.userState;\n    this.userState = state;\n    this.emit(\n      AgentSessionEventTypes.UserStateChanged,\n      createUserStateChangedEvent(oldState, state),\n    );\n  }\n\n  // -- User changed input/output streams/sinks --\n  private onAudioInputChanged(): void {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity && this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  private onAudioOutputChanged(): void {}\n\n  private onTextOutputChanged(): void {}\n\n  private async closeImpl(\n    reason: CloseReason,\n    error: RealtimeModelError | LLMError | TTSError | STTError | null = null,\n    drain: boolean = false,\n  ): Promise<void> {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity) {\n      if (!drain) {\n        try {\n          this.activity.interrupt();\n        } catch (error) {\n          // uninterruptible speech [copied from python]\n          // TODO(shubhra): force interrupt or wait for it to finish?\n          // it might be an audio played from the error callback\n        }\n      }\n      await this.activity.drain();\n      // wait any uninterruptible speech to finish\n      await this.activity.currentSpeech?.waitForPlayout();\n      this.activity.detachAudioInput();\n    }\n\n    // detach the inputs and outputs\n    this.input.audio = null;\n    this.output.audio = null;\n    this.output.transcription = null;\n\n    await this.roomIO?.close();\n    this.roomIO = undefined;\n\n    await this.activity?.close();\n    this.activity = undefined;\n\n    this.started = false;\n\n    this.emit(AgentSessionEventTypes.Close, createCloseEvent(reason, error));\n\n    this.userState = 'listening';\n    this._agentState = 'initializing';\n\n    this.logger.info({ reason, error }, 'AgentSession closed');\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,aAAa,mBAAmB;AAGzC,SAAS,WAAW;AAMpB,SAAS,qBAAqB;AAE9B;AAAA,EACE;AAAA,EAIA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,YAAY,mBAAmB;AACxC,SAAS,cAA6D;AActE,MAAM,sBAAoC;AAAA,EACxC,oBAAoB;AAAA,EACpB,+BAA+B;AAAA,EAC/B,yBAAyB;AAAA,EACzB,sBAAsB;AAAA,EACtB,qBAAqB;AAAA,EACrB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AA0BO,MAAM,qBAEF,aAA+D;AAAA,EACxE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAES;AAAA,EAED;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV,YAAuB;AAAA,EAEvB;AAAA,EACA,SAAS,IAAI;AAAA,EAEb;AAAA,EACA;AAAA,EACA,cAA0B;AAAA,EAE1B;AAAA,EACA;AAAA,EAEA,cAAoC;AAAA,EAE5C,YAAY,MAAqC;AAC/C,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IACjB,IAAI;AAEJ,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAGjB,SAAK,SAAS,IAAI,WAAW,KAAK,mBAAmB;AACrD,SAAK,UAAU,IAAI,YAAY,KAAK,sBAAsB,KAAK,mBAAmB;AAGlF,SAAK,WAAW,YAAY,MAAM;AAClC,SAAK,UAAU,EAAE,GAAG,qBAAqB,GAAG,aAAa;AAAA,EAC3D;AAAA,EAEA,IAAI,QAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,WAAqB;AACvB,QAAI,KAAK,cAAc,QAAW;AAChC,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,OAAiB;AAC5B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,MAAM;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKkB;AAChB,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AAEA,SAAK,QAAQ;AACb,SAAK,kBAAkB,cAAc;AAGrC,QAAI,KAAK,MAAM,UAAS,6CAAc,kBAAiB,OAAO;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAAA,IAC7F;AAEA,QAAI,KAAK,OAAO,UAAS,+CAAe,kBAAiB,OAAO;AAC9D,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,kBAAiB,+CAAe,0BAAyB,OAAO;AAC9E,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,cAAc;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,SAAK,OAAO,MAAM;AAElB,SAAK,eAAe,KAAK,KAAK;AAG9B,SAAK,OAAO;AAAA,MACV,mBAAmB,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,YAAY,OAAO,MAAM,QAAQ,2BAA2B,KAAK,OAAO,QAAQ,MAAM,KAAK,OAAO,MAAM,YAAY,OAAO,MAAM,QAAQ;AAAA,IACxM;AAEA,SAAK,OAAO;AAAA,MACV,4CAA4C,KAAK,OAAO,gBAAgB,MAAM,KAAK,OAAO,cAAc,YAAY,OAAO,MAAM,QAAQ;AAAA,IAC3I;AAEA,SAAK,OAAO,MAAM,sBAAsB;AACxC,SAAK,UAAU;AACf,SAAK,kBAAkB,WAAW;AAAA,EACpC;AAAA,EAEA,YAAY,OAAoB;AAC9B,SAAK,QAAQ;AAEb,QAAI,KAAK,SAAS;AAChB,WAAK,eAAe,KAAK;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,iBAAiB;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,SAAK,SAAS,eAAe;AAAA,EAC/B;AAAA,EAEA,gBAAgB;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,SAAK,SAAS,cAAc;AAAA,EAC9B;AAAA,EAEA,IACE,MACA,SAKc;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK,SAAS,IAAI,MAAM,OAAO;AAAA,EACxC;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,WAAO,KAAK,SAAS,UAAU;AAAA,EACjC;AAAA,EAEA,cAAc,SAKG;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,UAAM,eAAc,mCAAS,aACzB,IAAI,YAAY;AAAA,MACd,MAAM;AAAA,MACN,SAAS,QAAQ;AAAA,IACnB,CAAC,IACD;AAEJ,QAAI,KAAK,SAAS,UAAU;AAC1B,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AACA,aAAO,KAAK,aAAa,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,IACpE;AAEA,WAAO,KAAK,SAAS,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,eAAe,OAA6B;AAExD,SAAK,eAAe,IAAI,cAAc,OAAO,IAAI;AAEjD,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,SAAS,MAAM;AAC1B,YAAM,KAAK,SAAS,MAAM;AAAA,IAC5B;AAEA,SAAK,WAAW,KAAK;AACrB,SAAK,eAAe;AAEpB,UAAM,KAAK,SAAS,MAAM;AAE1B,QAAI,KAAK,OAAO,OAAO;AACrB,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,aAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAsB;AACxB,QAAI,CAAC,KAAK,OAAO;AACf,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,UAAU,YAAY,cAAc;AAAA,EACjD;AAAA;AAAA,EAGA,WAAW;AAAA,IACT;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV,GAIS;AACP,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AACA,SAAK,UAAU,QAAQ,OAAO,KAAK;AAAA,EACrC;AAAA;AAAA,EAGA,SAAS,OAAkE;AACzE,QAAI,KAAK,eAAe,MAAM,aAAa;AACzC;AAAA,IACF;AAEA,SAAK,OAAO,MAAM,OAAO,oDAAoD;AAE7E,SAAK,eAAe,YAAY;AAC9B,YAAM,KAAK,UAAU,YAAY,OAAO,KAAK;AAAA,IAC/C,GAAG,EAAE,KAAK,MAAM;AACd,WAAK,cAAc;AAAA,IACrB,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,uBAAuB,MAAyB;AAC9C,SAAK,SAAS,OAAO,IAAI;AACzB,SAAK,KAAK,uBAAuB,uBAAuB,iCAAiC,IAAI,CAAC;AAAA,EAChG;AAAA;AAAA,EAGA,kBAAkB,OAAmB;AACnC,QAAI,KAAK,gBAAgB,OAAO;AAC9B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,cAAc;AACnB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,6BAA6B,UAAU,KAAK;AAAA,IAC9C;AAAA,EACF;AAAA;AAAA,EAGA,iBAAiB,OAAkB;AACjC,QAAI,KAAK,cAAc,OAAO;AAC5B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,YAAY;AACjB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,4BAA4B,UAAU,KAAK;AAAA,IAC7C;AAAA,EACF;AAAA;AAAA,EAGQ,sBAA4B;AAClC,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,YAAY,KAAK,OAAO,OAAO;AACtC,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,uBAA6B;AAAA,EAAC;AAAA,EAE9B,sBAA4B;AAAA,EAAC;AAAA,EAErC,MAAc,UACZ,QACA,QAAoE,MACpE,QAAiB,OACF;AAtanB;AAuaI,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,UAAU;AACjB,UAAI,CAAC,OAAO;AACV,YAAI;AACF,eAAK,SAAS,UAAU;AAAA,QAC1B,SAASA,QAAO;AAAA,QAIhB;AAAA,MACF;AACA,YAAM,KAAK,SAAS,MAAM;AAE1B,cAAM,UAAK,SAAS,kBAAd,mBAA6B;AACnC,WAAK,SAAS,iBAAiB;AAAA,IACjC;AAGA,SAAK,MAAM,QAAQ;AACnB,SAAK,OAAO,QAAQ;AACpB,SAAK,OAAO,gBAAgB;AAE5B,YAAM,UAAK,WAAL,mBAAa;AACnB,SAAK,SAAS;AAEd,YAAM,UAAK,aAAL,mBAAe;AACrB,SAAK,WAAW;AAEhB,SAAK,UAAU;AAEf,SAAK,KAAK,uBAAuB,OAAO,iBAAiB,QAAQ,KAAK,CAAC;AAEvE,SAAK,YAAY;AACjB,SAAK,cAAc;AAEnB,SAAK,OAAO,KAAK,EAAE,QAAQ,MAAM,GAAG,qBAAqB;AAAA,EAC3D;AACF;","names":["error"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents",
-  "version": "1.0.0-next.1",
+  "version": "1.0.0-next.3",
   "description": "LiveKit Agents - Node.js",
   "main": "dist/index.js",
   "require": "dist/index.cjs",

package/src/llm/utils.ts

@@ -151,10 +151,14 @@ export const createToolOptions = <UserData extends UnknownUserData>(
 
 /** @internal */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export const oaiParams = (p: ZodObject<any>): OpenAIFunctionParameters => {
-  // TODO(AJS-162): make zod to JSON parsing able to handle z.optional(v.field())
+export const oaiParams = (
+  p: ZodObject<any>,
+  isOpenai: boolean = true,
+): OpenAIFunctionParameters => {
+  // Adapted from https://github.com/vercel/ai/blob/56eb0ee9/packages/provider-utils/src/zod-schema.ts
   const { properties, required, additionalProperties } = zodToJsonSchema(p, {
-    target: 'openAi',
+    // note: openai mode breaks various gemini conversions
+    target: isOpenai ? 'openAi' : 'jsonSchema7',
   }) as OpenAIFunctionParameters;
 
   return {
@@ -316,9 +320,9 @@ export function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): Di
   };
 }
 
-export function toJsonSchema(schema: ToolInputSchema<any>): JSONSchema7 {
+export function toJsonSchema(schema: ToolInputSchema<any>, isOpenai: boolean = true): JSONSchema7 {
   if (schema instanceof ZodObject) {
-    return oaiParams(schema);
+    return oaiParams(schema, isOpenai);
   }
   return schema;
 }

package/src/voice/agent_session.ts

@@ -158,6 +158,10 @@ export class AgentSession<
     return this._userData;
   }
 
+  get history(): ChatContext {
+    return this._chatCtx;
+  }
+
   set userData(value: UserData) {
     this._userData = value;
   }