@librechat/agents 2.4.84 → 2.4.85

package/dist/esm/llm/google/utils/common.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"common.mjs","sources":["../../../../../src/llm/google/utils/common.ts"],"sourcesContent":["import {\n  POSSIBLE_ROLES,\n  type Part,\n  type Content,\n  type TextPart,\n  type FileDataPart,\n  type InlineDataPart,\n  type FunctionCallPart,\n  type GenerateContentCandidate,\n  type EnhancedGenerateContentResponse,\n  type FunctionDeclaration as GenerativeAIFunctionDeclaration,\n  type FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,\n} from '@google/generative-ai';\nimport {\n  AIMessageChunk,\n  BaseMessage,\n  ChatMessage,\n  ToolMessage,\n  ToolMessageChunk,\n  MessageContent,\n  MessageContentComplex,\n  UsageMetadata,\n  isAIMessage,\n  isBaseMessage,\n  isToolMessage,\n  StandardContentBlockConverter,\n  parseBase64DataUrl,\n  convertToProviderContentBlock,\n  isDataContentBlock,\n} from '@langchain/core/messages';\nimport { ChatGenerationChunk } from '@langchain/core/outputs';\nimport type { ChatGeneration } from '@langchain/core/outputs';\nimport { isLangChainTool } from '@langchain/core/utils/function_calling';\nimport { isOpenAITool } from '@langchain/core/language_models/base';\nimport { ToolCallChunk } from '@langchain/core/messages/tool';\nimport { v4 as uuidv4 } from 'uuid';\nimport {\n  jsonSchemaToGeminiParameters,\n  schemaToGenerativeAIParameters,\n} from './zod_to_genai_parameters';\nimport { GoogleGenerativeAIToolType } from '../types';\n\nexport function getMessageAuthor(message: BaseMessage): string {\n  const type = message._getType();\n  if (ChatMessage.isInstance(message)) {\n    return message.role;\n  }\n  if (type === 'tool') {\n    return type;\n  }\n  return message.name ?? type;\n}\n\n/**\n * Maps a message type to a Google Generative AI chat author.\n * @param message The message to map.\n * @param model The model to use for mapping.\n * @returns The message type mapped to a Google Generative AI chat author.\n */\nexport function convertAuthorToRole(\n  author: string\n): (typeof POSSIBLE_ROLES)[number] {\n  switch (author) {\n  /**\n     *  Note: Gemini currently is not supporting system messages\n     *  we will convert them to human messages and merge with following\n     * */\n  case 'supervisor':\n  case 'ai':\n  case 'model': // getMessageAuthor returns message.name. code ex.: return message.name ?? type;\n    return 'model';\n  case 'system':\n    return 'system';\n  case 'human':\n    return 'user';\n  case 'tool':\n  case 'function':\n    return 'function';\n  default:\n    throw new Error(`Unknown / unsupported author: ${author}`);\n  }\n}\n\nfunction messageContentMedia(content: MessageContentComplex): Part {\n  if ('mimeType' in content && 'data' in content) {\n    return {\n      inlineData: {\n        mimeType: content.mimeType,\n        data: content.data,\n      },\n    };\n  }\n  if ('mimeType' in content && 'fileUri' in content) {\n    return {\n      fileData: {\n        mimeType: content.mimeType,\n        fileUri: content.fileUri,\n      },\n    };\n  }\n\n  throw new Error('Invalid media content');\n}\n\nfunction inferToolNameFromPreviousMessages(\n  message: ToolMessage | ToolMessageChunk,\n  previousMessages: BaseMessage[]\n): string | undefined {\n  return previousMessages\n    .map((msg) => {\n      if (isAIMessage(msg)) {\n        return msg.tool_calls ?? [];\n      }\n      return [];\n    })\n    .flat()\n    .find((toolCall) => {\n      return toolCall.id === message.tool_call_id;\n    })?.name;\n}\n\nfunction _getStandardContentBlockConverter(\n  isMultimodalModel: boolean\n): StandardContentBlockConverter<{\n  text: TextPart;\n  image: FileDataPart | InlineDataPart;\n  audio: FileDataPart | InlineDataPart;\n  file: FileDataPart | InlineDataPart | TextPart;\n}> {\n  const standardContentBlockConverter: StandardContentBlockConverter<{\n    text: TextPart;\n    image: FileDataPart | InlineDataPart;\n    audio: FileDataPart | InlineDataPart;\n    file: FileDataPart | InlineDataPart | TextPart;\n  }> = {\n    providerName: 'Google Gemini',\n\n    fromStandardTextBlock(block) {\n      return {\n        text: block.text,\n      };\n    },\n\n    fromStandardImageBlock(block): FileDataPart | InlineDataPart {\n      if (!isMultimodalModel) {\n        throw new Error('This model does not support images');\n      }\n      if (block.source_type === 'url') {\n        const data = parseBase64DataUrl({ dataUrl: block.url });\n        if (data) {\n          return {\n            inlineData: {\n              mimeType: data.mime_type,\n              data: data.data,\n            },\n          };\n        } else {\n          return {\n            fileData: {\n              mimeType: block.mime_type ?? '',\n              fileUri: block.url,\n            },\n          };\n        }\n      }\n\n      if (block.source_type === 'base64') {\n        return {\n          inlineData: {\n            mimeType: block.mime_type ?? '',\n            data: block.data,\n          },\n        };\n      }\n\n      throw new Error(`Unsupported source type: ${block.source_type}`);\n    },\n\n    fromStandardAudioBlock(block): FileDataPart | InlineDataPart {\n      if (!isMultimodalModel) {\n        throw new Error('This model does not support audio');\n      }\n      if (block.source_type === 'url') {\n        const data = parseBase64DataUrl({ dataUrl: block.url });\n        if (data) {\n          return {\n            inlineData: {\n              mimeType: data.mime_type,\n              data: data.data,\n            },\n          };\n        } else {\n          return {\n            fileData: {\n              mimeType: block.mime_type ?? '',\n              fileUri: block.url,\n            },\n          };\n        }\n      }\n\n      if (block.source_type === 'base64') {\n        return {\n          inlineData: {\n            mimeType: block.mime_type ?? '',\n            data: block.data,\n          },\n        };\n      }\n\n      throw new Error(`Unsupported source type: ${block.source_type}`);\n    },\n\n    fromStandardFileBlock(block): FileDataPart | InlineDataPart | TextPart {\n      if (!isMultimodalModel) {\n        throw new Error('This model does not support files');\n      }\n      if (block.source_type === 'text') {\n        return {\n          text: block.text,\n        };\n      }\n      if (block.source_type === 'url') {\n        const data = parseBase64DataUrl({ dataUrl: block.url });\n        if (data) {\n          return {\n            inlineData: {\n              mimeType: data.mime_type,\n              data: data.data,\n            },\n          };\n        } else {\n          return {\n            fileData: {\n              mimeType: block.mime_type ?? '',\n              fileUri: block.url,\n            },\n          };\n        }\n      }\n\n      if (block.source_type === 'base64') {\n        return {\n          inlineData: {\n            mimeType: block.mime_type ?? '',\n            data: block.data,\n          },\n        };\n      }\n      throw new Error(`Unsupported source type: ${block.source_type}`);\n    },\n  };\n  return standardContentBlockConverter;\n}\n\nfunction _convertLangChainContentToPart(\n  content: MessageContentComplex,\n  isMultimodalModel: boolean\n): Part | undefined {\n  if (isDataContentBlock(content)) {\n    return convertToProviderContentBlock(\n      content,\n      _getStandardContentBlockConverter(isMultimodalModel)\n    );\n  }\n\n  if (content.type === 'text') {\n    return { text: content.text };\n  } else if (content.type === 'executableCode') {\n    return { executableCode: content.executableCode };\n  } else if (content.type === 'codeExecutionResult') {\n    return { codeExecutionResult: content.codeExecutionResult };\n  } else if (content.type === 'image_url') {\n    if (!isMultimodalModel) {\n      throw new Error('This model does not support images');\n    }\n    let source: string;\n    if (typeof content.image_url === 'string') {\n      source = content.image_url;\n    } else if (\n      typeof content.image_url === 'object' &&\n      'url' in content.image_url\n    ) {\n      source = content.image_url.url;\n    } else {\n      throw new Error('Please provide image as base64 encoded data URL');\n    }\n    const [dm, data] = source.split(',');\n    if (!dm.startsWith('data:')) {\n      throw new Error('Please provide image as base64 encoded data URL');\n    }\n\n    const [mimeType, encoding] = dm.replace(/^data:/, '').split(';');\n    if (encoding !== 'base64') {\n      throw new Error('Please provide image as base64 encoded data URL');\n    }\n\n    return {\n      inlineData: {\n        data,\n        mimeType,\n      },\n    };\n  } else if (content.type === 'media') {\n    return messageContentMedia(content);\n  } else if (content.type === 'tool_use') {\n    return {\n      functionCall: {\n        name: content.name,\n        args: content.input,\n      },\n    };\n  } else if (\n    content.type?.includes('/') === true &&\n    // Ensure it's a single slash.\n    content.type.split('/').length === 2 &&\n    'data' in content &&\n    typeof content.data === 'string'\n  ) {\n    return {\n      inlineData: {\n        mimeType: content.type,\n        data: content.data,\n      },\n    };\n  } else if ('functionCall' in content) {\n    // No action needed here — function calls will be added later from message.tool_calls\n    return undefined;\n  } else {\n    if ('type' in content) {\n      throw new Error(`Unknown content type ${content.type}`);\n    } else {\n      throw new Error(`Unknown content ${JSON.stringify(content)}`);\n    }\n  }\n}\n\nexport function convertMessageContentToParts(\n  message: BaseMessage,\n  isMultimodalModel: boolean,\n  previousMessages: BaseMessage[]\n): Part[] {\n  if (isToolMessage(message)) {\n    const messageName =\n      message.name ??\n      inferToolNameFromPreviousMessages(message, previousMessages);\n    if (messageName === undefined) {\n      throw new Error(\n        `Google requires a tool name for each tool call response, and we could not infer a called tool name for ToolMessage \"${message.id}\" from your passed messages. Please populate a \"name\" field on that ToolMessage explicitly.`\n      );\n    }\n\n    const result = Array.isArray(message.content)\n      ? (message.content\n        .map((c) => _convertLangChainContentToPart(c, isMultimodalModel))\n        .filter((p) => p !== undefined) as Part[])\n      : message.content;\n\n    if (message.status === 'error') {\n      return [\n        {\n          functionResponse: {\n            name: messageName,\n            // The API expects an object with an `error` field if the function call fails.\n            // `error` must be a valid object (not a string or array), so we wrap `message.content` here\n            response: { error: { details: result } },\n          },\n        },\n      ];\n    }\n\n    return [\n      {\n        functionResponse: {\n          name: messageName,\n          // again, can't have a string or array value for `response`, so we wrap it as an object here\n          response: { result },\n        },\n      },\n    ];\n  }\n\n  let functionCalls: FunctionCallPart[] = [];\n  const messageParts: Part[] = [];\n\n  if (typeof message.content === 'string' && message.content) {\n    messageParts.push({ text: message.content });\n  }\n\n  if (Array.isArray(message.content)) {\n    messageParts.push(\n      ...(message.content\n        .map((c) => _convertLangChainContentToPart(c, isMultimodalModel))\n        .filter((p) => p !== undefined) as Part[])\n    );\n  }\n\n  if (isAIMessage(message) && message.tool_calls?.length != null) {\n    functionCalls = message.tool_calls.map((tc) => {\n      return {\n        functionCall: {\n          name: tc.name,\n          args: tc.args,\n        },\n      };\n    });\n  }\n\n  return [...messageParts, ...functionCalls];\n}\n\nexport function convertBaseMessagesToContent(\n  messages: BaseMessage[],\n  isMultimodalModel: boolean,\n  convertSystemMessageToHumanContent: boolean = false\n): Content[] | undefined {\n  return messages.reduce<{\n    content: Content[] | undefined;\n    mergeWithPreviousContent: boolean;\n  }>(\n    (acc, message, index) => {\n      if (!isBaseMessage(message)) {\n        throw new Error('Unsupported message input');\n      }\n      const author = getMessageAuthor(message);\n      if (author === 'system' && index !== 0) {\n        throw new Error('System message should be the first one');\n      }\n      const role = convertAuthorToRole(author);\n\n      const prevContent = acc.content?.[acc.content.length];\n      if (\n        !acc.mergeWithPreviousContent &&\n        prevContent &&\n        prevContent.role === role\n      ) {\n        throw new Error(\n          'Google Generative AI requires alternate messages between authors'\n        );\n      }\n\n      const parts = convertMessageContentToParts(\n        message,\n        isMultimodalModel,\n        messages.slice(0, index)\n      );\n\n      if (acc.mergeWithPreviousContent) {\n        const prevContent = acc.content?.[acc.content.length - 1];\n        if (!prevContent) {\n          throw new Error(\n            'There was a problem parsing your system message. Please try a prompt without one.'\n          );\n        }\n        prevContent.parts.push(...parts);\n\n        return {\n          mergeWithPreviousContent: false,\n          content: acc.content,\n        };\n      }\n      let actualRole = role;\n      if (\n        actualRole === 'function' ||\n        (actualRole === 'system' && !convertSystemMessageToHumanContent)\n      ) {\n        // GenerativeAI API will throw an error if the role is not \"user\" or \"model.\"\n        actualRole = 'user';\n      }\n      const content: Content = {\n        role: actualRole,\n        parts,\n      };\n      return {\n        mergeWithPreviousContent:\n          author === 'system' && !convertSystemMessageToHumanContent,\n        content: [...(acc.content ?? []), content],\n      };\n    },\n    { content: [], mergeWithPreviousContent: false }\n  ).content;\n}\n\nexport function convertResponseContentToChatGenerationChunk(\n  response: EnhancedGenerateContentResponse,\n  extra: {\n    usageMetadata?: UsageMetadata | undefined;\n    index: number;\n  }\n): ChatGenerationChunk | null {\n  if (!response.candidates || response.candidates.length === 0) {\n    return null;\n  }\n  const functionCalls = response.functionCalls();\n  const [candidate] = response.candidates as [\n    Partial<GenerateContentCandidate> | undefined,\n  ];\n  const { content: candidateContent, ...generationInfo } = candidate ?? {};\n  let content: MessageContent | undefined;\n  // Checks if some parts do not have text. If false, it means that the content is a string.\n  const reasoningParts: string[] = [];\n  if (\n    candidateContent != null &&\n    Array.isArray(candidateContent.parts) &&\n    candidateContent.parts.every((p) => 'text' in p)\n  ) {\n    // content = candidateContent.parts.map((p) => p.text).join('');\n    const textParts: string[] = [];\n    for (const part of candidateContent.parts) {\n      if ('thought' in part && part.thought === true) {\n        reasoningParts.push(part.text ?? '');\n        continue;\n      }\n      textParts.push(part.text ?? '');\n    }\n    content = textParts.join('');\n  } else if (candidateContent && Array.isArray(candidateContent.parts)) {\n    content = candidateContent.parts.map((p) => {\n      if ('text' in p && 'thought' in p && p.thought === true) {\n        reasoningParts.push(p.text ?? '');\n      } else if ('text' in p) {\n        return {\n          type: 'text',\n          text: p.text,\n        };\n      } else if ('executableCode' in p) {\n        return {\n          type: 'executableCode',\n          executableCode: p.executableCode,\n        };\n      } else if ('codeExecutionResult' in p) {\n        return {\n          type: 'codeExecutionResult',\n          codeExecutionResult: p.codeExecutionResult,\n        };\n      }\n      return p;\n    });\n  } else {\n    // no content returned - likely due to abnormal stop reason, e.g. malformed function call\n    content = [];\n  }\n\n  let text = '';\n  if (typeof content === 'string' && content) {\n    text = content;\n  } else if (Array.isArray(content)) {\n    const block = content.find((b) => 'text' in b) as\n      | { text: string }\n      | undefined;\n    text = block?.text ?? '';\n  }\n\n  const toolCallChunks: ToolCallChunk[] = [];\n  if (functionCalls) {\n    toolCallChunks.push(\n      ...functionCalls.map((fc) => ({\n        ...fc,\n        args: JSON.stringify(fc.args),\n        // Un-commenting this causes LangChain to incorrectly merge tool calls together\n        // index: extra.index,\n        type: 'tool_call_chunk' as const,\n        id: 'id' in fc && typeof fc.id === 'string' ? fc.id : uuidv4(),\n      }))\n    );\n  }\n\n  const additional_kwargs: ChatGeneration['message']['additional_kwargs'] = {};\n  if (reasoningParts.length > 0) {\n    additional_kwargs.reasoning = reasoningParts.join('');\n  }\n\n  if (candidate?.groundingMetadata) {\n    additional_kwargs.groundingMetadata = candidate.groundingMetadata;\n  }\n\n  const isFinalChunk =\n    response.candidates[0]?.finishReason === 'STOP' ||\n    response.candidates[0]?.finishReason === 'MAX_TOKENS' ||\n    response.candidates[0]?.finishReason === 'SAFETY';\n\n  return new ChatGenerationChunk({\n    text,\n    message: new AIMessageChunk({\n      content: content,\n      name: !candidateContent ? undefined : candidateContent.role,\n      tool_call_chunks: toolCallChunks,\n      // Each chunk can have unique \"generationInfo\", and merging strategy is unclear,\n      // so leave blank for now.\n      additional_kwargs,\n      usage_metadata: isFinalChunk ? extra.usageMetadata : undefined,\n    }),\n    generationInfo,\n  });\n}\n\nexport function convertToGenerativeAITools(\n  tools: GoogleGenerativeAIToolType[]\n): GoogleGenerativeAIFunctionDeclarationsTool[] {\n  if (\n    tools.every(\n      (tool) =>\n        'functionDeclarations' in tool &&\n        Array.isArray(tool.functionDeclarations)\n    )\n  ) {\n    return tools as GoogleGenerativeAIFunctionDeclarationsTool[];\n  }\n  return [\n    {\n      functionDeclarations: tools.map(\n        (tool): GenerativeAIFunctionDeclaration => {\n          if (isLangChainTool(tool)) {\n            const jsonSchema = schemaToGenerativeAIParameters(tool.schema);\n            if (\n              jsonSchema.type === 'object' &&\n              'properties' in jsonSchema &&\n              Object.keys(jsonSchema.properties).length === 0\n            ) {\n              return {\n                name: tool.name,\n                description: tool.description,\n              };\n            }\n            return {\n              name: tool.name,\n              description: tool.description,\n              parameters: jsonSchema,\n            };\n          }\n          if (isOpenAITool(tool)) {\n            return {\n              name: tool.function.name,\n              description:\n                tool.function.description ?? 'A function available to call.',\n              parameters: jsonSchemaToGeminiParameters(\n                tool.function.parameters\n              ),\n            };\n          }\n          return tool as unknown as GenerativeAIFunctionDeclaration;\n        }\n      ),\n    },\n  ];\n}\n"],"names":["uuidv4"],"mappings":";;;;;;;;AA0CM,SAAU,gBAAgB,CAAC,OAAoB,EAAA;AACnD,IAAA,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE;AAC/B,IAAA,IAAI,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QACnC,OAAO,OAAO,CAAC,IAAI;;AAErB,IAAA,IAAI,IAAI,KAAK,MAAM,EAAE;AACnB,QAAA,OAAO,IAAI;;AAEb,IAAA,OAAO,OAAO,CAAC,IAAI,IAAI,IAAI;AAC7B;AAEA;;;;;AAKG;AACG,SAAU,mBAAmB,CACjC,MAAc,EAAA;IAEd,QAAQ,MAAM;AACd;;;AAGO;AACP,QAAA,KAAK,YAAY;AACjB,QAAA,KAAK,IAAI;QACT,KAAK,OAAO;AACV,YAAA,OAAO,OAAO;AAChB,QAAA,KAAK,QAAQ;AACX,YAAA,OAAO,QAAQ;AACjB,QAAA,KAAK,OAAO;AACV,YAAA,OAAO,MAAM;AACf,QAAA,KAAK,MAAM;AACX,QAAA,KAAK,UAAU;AACb,YAAA,OAAO,UAAU;AACnB,QAAA;AACE,YAAA,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,CAAA,CAAE,CAAC;;AAE9D;AAEA,SAAS,mBAAmB,CAAC,OAA8B,EAAA;IACzD,IAAI,UAAU,IAAI,OAAO,IAAI,MAAM,IAAI,OAAO,EAAE;QAC9C,OAAO;AACL,YAAA,UAAU,EAAE;gBACV,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;AACnB,aAAA;SACF;;IAEH,IAAI,UAAU,IAAI,OAAO,IAAI,SAAS,IAAI,OAAO,EAAE;QACjD,OAAO;AACL,YAAA,QAAQ,EAAE;gBACR,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,OAAO,EAAE,OAAO,CAAC,OAAO;AACzB,aAAA;SACF;;AAGH,IAAA,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC;AAC1C;AAEA,SAAS,iCAAiC,CACxC,OAAuC,EACvC,gBAA+B,EAAA;AAE/B,IAAA,OAAO;AACJ,SAAA,GAAG,CAAC,CAAC,GAAG,KAAI;AACX,QAAA,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE;AACpB,YAAA,OAAO,GAAG,CAAC,UAAU,IAAI,EAAE;;AAE7B,QAAA,OAAO,EAAE;AACX,KAAC;AACA,SAAA,IAAI;AACJ,SAAA,IAAI,CAAC,CAAC,QAAQ,KAAI;AACjB,QAAA,OAAO,QAAQ,CAAC,EAAE,KAAK,OAAO,CAAC,YAAY;KAC5C,CAAC,EAAE,IAAI;AACZ;AAEA,SAAS,iCAAiC,CACxC,iBAA0B,EAAA;AAO1B,IAAA,MAAM,6BAA6B,GAK9B;AACH,QAAA,YAAY,EAAE,eAAe;AAE7B,QAAA,qBAAqB,CAAC,KAAK,EAAA;YACzB,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;aACjB;SACF;AAED,QAAA,sBAAsB,CAAC,KAAK,EAAA;YAC1B,IAAI,CAAC,iBAAiB,EAAE;AACtB,gBAAA,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC;;AAEvD,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,EAAE;AAC/B,gBAAA,MAAM,IAAI,GAAG,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC;gBACvD,IAAI,IAAI,EAAE;oBACR,OAAO;AACL,wBAAA,UAAU,EAAE;4BACV,QAAQ,EAAE,IAAI,CAAC,SAAS;4BACxB,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,yBAAA;qBACF;;qBACI;oBACL,OAAO;AACL,wBAAA,QAAQ,EAAE;AACR,4BAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;4BAC/B,OAAO,EAAE,KAAK,CAAC,GAAG;AACnB,yBAAA;qBACF;;;AAIL,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE;gBAClC,OAAO;AACL,oBAAA,UAAU,EAAE;AACV,wBAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;wBAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;AACjB,qBAAA;iBACF;;YAGH,MAAM,IAAI,KAAK,CAAC,CAAA,yBAAA,EAA4B,KAAK,CAAC,WAAW,CAAE,CAAA,CAAC;SACjE;AAED,QAAA,sBAAsB,CAAC,KAAK,EAAA;YAC1B,IAAI,CAAC,iBAAiB,EAAE;AACtB,gBAAA,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC;;AAEtD,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,EAAE;AAC/B,gBAAA,MAAM,IAAI,GAAG,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC;gBACvD,IAAI,IAAI,EAAE;oBACR,OAAO;AACL,wBAAA,UAAU,EAAE;4BACV,QAAQ,EAAE,IAAI,CAAC,SAAS;4BACxB,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,yBAAA;qBACF;;qBACI;oBACL,OAAO;AACL,wBAAA,QAAQ,EAAE;AACR,4BAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;4BAC/B,OAAO,EAAE,KAAK,CAAC,GAAG;AACnB,yBAAA;qBACF;;;AAIL,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE;gBAClC,OAAO;AACL,oBAAA,UAAU,EAAE;AACV,wBAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;wBAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;AACjB,qBAAA;iBACF;;YAGH,MAAM,IAAI,KAAK,CAAC,CAAA,yBAAA,EAA4B,KAAK,CAAC,WAAW,CAAE,CAAA,CAAC;SACjE;AAED,QAAA,qBAAqB,CAAC,KAAK,EAAA;YACzB,IAAI,CAAC,iBAAiB,EAAE;AACtB,gBAAA,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC;;AAEtD,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,MAAM,EAAE;gBAChC,OAAO;oBACL,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB;;AAEH,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,EAAE;AAC/B,gBAAA,MAAM,IAAI,GAAG,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC;gBACvD,IAAI,IAAI,EAAE;oBACR,OAAO;AACL,wBAAA,UAAU,EAAE;4BACV,QAAQ,EAAE,IAAI,CAAC,SAAS;4BACxB,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,yBAAA;qBACF;;qBACI;oBACL,OAAO;AACL,wBAAA,QAAQ,EAAE;AACR,4BAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;4BAC/B,OAAO,EAAE,KAAK,CAAC,GAAG;AACnB,yBAAA;qBACF;;;AAIL,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE;gBAClC,OAAO;AACL,oBAAA,UAAU,EAAE;AACV,wBAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;wBAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;AACjB,qBAAA;iBACF;;YAEH,MAAM,IAAI,KAAK,CAAC,CAAA,yBAAA,EAA4B,KAAK,CAAC,WAAW,CAAE,CAAA,CAAC;SACjE;KACF;AACD,IAAA,OAAO,6BAA6B;AACtC;AAEA,SAAS,8BAA8B,CACrC,OAA8B,EAC9B,iBAA0B,EAAA;AAE1B,IAAA,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAC/B,OAAO,6BAA6B,CAClC,OAAO,EACP,iCAAiC,CAAC,iBAAiB,CAAC,CACrD;;AAGH,IAAA,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE;AAC3B,QAAA,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;;AACxB,SAAA,IAAI,OAAO,CAAC,IAAI,KAAK,gBAAgB,EAAE;AAC5C,QAAA,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE;;AAC5C,SAAA,IAAI,OAAO,CAAC,IAAI,KAAK,qBAAqB,EAAE;AACjD,QAAA,OAAO,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE;;AACtD,SAAA,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE;QACvC,IAAI,CAAC,iBAAiB,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC;;AAEvD,QAAA,IAAI,MAAc;AAClB,QAAA,IAAI,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,EAAE;AACzC,YAAA,MAAM,GAAG,OAAO,CAAC,SAAS;;AACrB,aAAA,IACL,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ;AACrC,YAAA,KAAK,IAAI,OAAO,CAAC,SAAS,EAC1B;AACA,YAAA,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG;;aACzB;AACL,YAAA,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC;;AAEpE,QAAA,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;QACpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;AAC3B,YAAA,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC;;AAGpE,QAAA,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;AAChE,QAAA,IAAI,QAAQ,KAAK,QAAQ,EAAE;AACzB,YAAA,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC;;QAGpE,OAAO;AACL,YAAA,UAAU,EAAE;gBACV,IAAI;gBACJ,QAAQ;AACT,aAAA;SACF;;AACI,SAAA,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE;AACnC,QAAA,OAAO,mBAAmB,CAAC,OAAO,CAAC;;AAC9B,SAAA,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE;QACtC,OAAO;AACL,YAAA,YAAY,EAAE;gBACZ,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,OAAO,CAAC,KAAK;AACpB,aAAA;SACF;;SACI,IACL,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI;;QAEpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC;AACpC,QAAA,MAAM,IAAI,OAAO;AACjB,QAAA,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,EAChC;QACA,OAAO;AACL,YAAA,UAAU,EAAE;gBACV,QAAQ,EAAE,OAAO,CAAC,IAAI;gBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;AACnB,aAAA;SACF;;AACI,SAAA,IAAI,cAAc,IAAI,OAAO,EAAE;;AAEpC,QAAA,OAAO,SAAS;;SACX;AACL,QAAA,IAAI,MAAM,IAAI,OAAO,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,CAAA,qBAAA,EAAwB,OAAO,CAAC,IAAI,CAAE,CAAA,CAAC;;aAClD;AACL,YAAA,MAAM,IAAI,KAAK,CAAC,CAAA,gBAAA,EAAmB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAE,CAAA,CAAC;;;AAGnE;SAEgB,4BAA4B,CAC1C,OAAoB,EACpB,iBAA0B,EAC1B,gBAA+B,EAAA;AAE/B,IAAA,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE;AAC1B,QAAA,MAAM,WAAW,GACf,OAAO,CAAC,IAAI;AACZ,YAAA,iCAAiC,CAAC,OAAO,EAAE,gBAAgB,CAAC;AAC9D,QAAA,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,MAAM,IAAI,KAAK,CACb,CAAA,oHAAA,EAAuH,OAAO,CAAC,EAAE,CAA6F,2FAAA,CAAA,CAC/N;;QAGH,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO;cACvC,OAAO,CAAC;AACR,iBAAA,GAAG,CAAC,CAAC,CAAC,KAAK,8BAA8B,CAAC,CAAC,EAAE,iBAAiB,CAAC;iBAC/D,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,SAAS;AAChC,cAAE,OAAO,CAAC,OAAO;AAEnB,QAAA,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE;YAC9B,OAAO;AACL,gBAAA;AACE,oBAAA,gBAAgB,EAAE;AAChB,wBAAA,IAAI,EAAE,WAAW;;;wBAGjB,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;AACzC,qBAAA;AACF,iBAAA;aACF;;QAGH,OAAO;AACL,YAAA;AACE,gBAAA,gBAAgB,EAAE;AAChB,oBAAA,IAAI,EAAE,WAAW;;oBAEjB,QAAQ,EAAE,EAAE,MAAM,EAAE;AACrB,iBAAA;AACF,aAAA;SACF;;IAGH,IAAI,aAAa,GAAuB,EAAE;IAC1C,MAAM,YAAY,GAAW,EAAE;IAE/B,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE;QAC1D,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;;IAG9C,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;AAClC,QAAA,YAAY,CAAC,IAAI,CACf,GAAI,OAAO,CAAC;AACT,aAAA,GAAG,CAAC,CAAC,CAAC,KAAK,8BAA8B,CAAC,CAAC,EAAE,iBAAiB,CAAC;aAC/D,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,SAAS,CAAY,CAC7C;;AAGH,IAAA,IAAI,WAAW,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,UAAU,EAAE,MAAM,IAAI,IAAI,EAAE;QAC9D,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,KAAI;YAC5C,OAAO;AACL,gBAAA,YAAY,EAAE;oBACZ,IAAI,EAAE,EAAE,CAAC,IAAI;oBACb,IAAI,EAAE,EAAE,CAAC,IAAI;AACd,iBAAA;aACF;AACH,SAAC,CAAC;;AAGJ,IAAA,OAAO,CAAC,GAAG,YAAY,EAAE,GAAG,aAAa,CAAC;AAC5C;AAEM,SAAU,4BAA4B,CAC1C,QAAuB,EACvB,iBAA0B,EAC1B,qCAA8C,KAAK,EAAA;IAEnD,OAAO,QAAQ,CAAC,MAAM,CAIpB,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,KAAI;AACtB,QAAA,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;AAC3B,YAAA,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC;;AAE9C,QAAA,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC;QACxC,IAAI,MAAM,KAAK,QAAQ,IAAI,KAAK,KAAK,CAAC,EAAE;AACtC,YAAA,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC;;AAE3D,QAAA,MAAM,IAAI,GAAG,mBAAmB,CAAC,MAAM,CAAC;AAExC,QAAA,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;QACrD,IACE,CAAC,GAAG,CAAC,wBAAwB;YAC7B,WAAW;AACX,YAAA,WAAW,CAAC,IAAI,KAAK,IAAI,EACzB;AACA,YAAA,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE;;AAGH,QAAA,MAAM,KAAK,GAAG,4BAA4B,CACxC,OAAO,EACP,iBAAiB,EACjB,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CACzB;AAED,QAAA,IAAI,GAAG,CAAC,wBAAwB,EAAE;AAChC,YAAA,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YACzD,IAAI,CAAC,WAAW,EAAE;AAChB,gBAAA,MAAM,IAAI,KAAK,CACb,mFAAmF,CACpF;;YAEH,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAEhC,OAAO;AACL,gBAAA,wBAAwB,EAAE,KAAK;gBAC/B,OAAO,EAAE,GAAG,CAAC,OAAO;aACrB;;QAEH,IAAI,UAAU,GAAG,IAAI;QACrB,IACE,UAAU,KAAK,UAAU;aACxB,UAAU,KAAK,QAAQ,IAAI,CAAC,kCAAkC,CAAC,EAChE;;YAEA,UAAU,GAAG,MAAM;;AAErB,QAAA,MAAM,OAAO,GAAY;AACvB,YAAA,IAAI,EAAE,UAAU;YAChB,KAAK;SACN;QACD,OAAO;AACL,YAAA,wBAAwB,EACtB,MAAM,KAAK,QAAQ,IAAI,CAAC,kCAAkC;AAC5D,YAAA,OAAO,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC;SAC3C;AACH,KAAC,EACD,EAAE,OAAO,EAAE,EAAE,EAAE,wBAAwB,EAAE,KAAK,EAAE,CACjD,CAAC,OAAO;AACX;AAEgB,SAAA,2CAA2C,CACzD,QAAyC,EACzC,KAGC,EAAA;AAED,IAAA,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;AAC5D,QAAA,OAAO,IAAI;;AAEb,IAAA,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,EAAE;AAC9C,IAAA,MAAM,CAAC,SAAS,CAAC,GAAG,QAAQ,CAAC,UAE5B;AACD,IAAA,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,cAAc,EAAE,GAAG,SAAS,IAAI,EAAE;AACxE,IAAA,IAAI,OAAmC;;IAEvC,MAAM,cAAc,GAAa,EAAE;IACnC,IACE,gBAAgB,IAAI,IAAI;AACxB,QAAA,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC;AACrC,QAAA,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,CAAC,CAAC,EAChD;;QAEA,MAAM,SAAS,GAAa,EAAE;AAC9B,QAAA,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,KAAK,EAAE;YACzC,IAAI,SAAS,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE;gBAC9C,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;gBACpC;;YAEF,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;;AAEjC,QAAA,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;;SACvB,IAAI,gBAAgB,IAAI,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE;QACpE,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAI;AACzC,YAAA,IAAI,MAAM,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,KAAK,IAAI,EAAE;gBACvD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;;AAC5B,iBAAA,IAAI,MAAM,IAAI,CAAC,EAAE;gBACtB,OAAO;AACL,oBAAA,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb;;AACI,iBAAA,IAAI,gBAAgB,IAAI,CAAC,EAAE;gBAChC,OAAO;AACL,oBAAA,IAAI,EAAE,gBAAgB;oBACtB,cAAc,EAAE,CAAC,CAAC,cAAc;iBACjC;;AACI,iBAAA,IAAI,qBAAqB,IAAI,CAAC,EAAE;gBACrC,OAAO;AACL,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,mBAAmB,EAAE,CAAC,CAAC,mBAAmB;iBAC3C;;AAEH,YAAA,OAAO,CAAC;AACV,SAAC,CAAC;;SACG;;QAEL,OAAO,GAAG,EAAE;;IAGd,IAAI,IAAI,GAAG,EAAE;AACb,IAAA,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,EAAE;QAC1C,IAAI,GAAG,OAAO;;AACT,SAAA,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;AACjC,QAAA,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,CAAC,CAEhC;AACb,QAAA,IAAI,GAAG,KAAK,EAAE,IAAI,IAAI,EAAE;;IAG1B,MAAM,cAAc,GAAoB,EAAE;IAC1C,IAAI,aAAa,EAAE;AACjB,QAAA,cAAc,CAAC,IAAI,CACjB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM;AAC5B,YAAA,GAAG,EAAE;YACL,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC;;;AAG7B,YAAA,IAAI,EAAE,iBAA0B;YAChC,EAAE,EAAE,IAAI,IAAI,EAAE,IAAI,OAAO,EAAE,CAAC,EAAE,KAAK,QAAQ,GAAG,EAAE,CAAC,EAAE,GAAGA,EAAM,EAAE;SAC/D,CAAC,CAAC,CACJ;;IAGH,MAAM,iBAAiB,GAAmD,EAAE;AAC5E,IAAA,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,iBAAiB,CAAC,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;;AAGvD,IAAA,IAAI,SAAS,EAAE,iBAAiB,EAAE;AAChC,QAAA,iBAAiB,CAAC,iBAAiB,GAAG,SAAS,CAAC,iBAAiB;;IAGnE,MAAM,YAAY,GAChB,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,KAAK,MAAM;QAC/C,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,KAAK,YAAY;QACrD,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,KAAK,QAAQ;IAEnD,OAAO,IAAI,mBAAmB,CAAC;QAC7B,IAAI;QACJ,OAAO,EAAE,IAAI,cAAc,CAAC;AAC1B,YAAA,OAAO,EAAE,OAAO;AAChB,YAAA,IAAI,EAAE,CAAC,gBAAgB,GAAG,SAAS,GAAG,gBAAgB,CAAC,IAAI;AAC3D,YAAA,gBAAgB,EAAE,cAAc;;;YAGhC,iBAAiB;YACjB,cAAc,EAAE,YAAY,GAAG,KAAK,CAAC,aAAa,GAAG,SAAS;SAC/D,CAAC;QACF,cAAc;AACf,KAAA,CAAC;AACJ;;;;"}
+{"version":3,"file":"common.mjs","sources":["../../../../../src/llm/google/utils/common.ts"],"sourcesContent":["import {\n  POSSIBLE_ROLES,\n  type Part,\n  type Content,\n  type TextPart,\n  type FileDataPart,\n  type InlineDataPart,\n  type FunctionCallPart,\n  type GenerateContentCandidate,\n  type EnhancedGenerateContentResponse,\n  type FunctionDeclaration as GenerativeAIFunctionDeclaration,\n  type FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,\n} from '@google/generative-ai';\nimport {\n  AIMessageChunk,\n  BaseMessage,\n  ChatMessage,\n  ToolMessage,\n  ToolMessageChunk,\n  MessageContent,\n  MessageContentComplex,\n  UsageMetadata,\n  isAIMessage,\n  isBaseMessage,\n  isToolMessage,\n  StandardContentBlockConverter,\n  parseBase64DataUrl,\n  convertToProviderContentBlock,\n  isDataContentBlock,\n} from '@langchain/core/messages';\nimport { ChatGenerationChunk } from '@langchain/core/outputs';\nimport type { ChatGeneration } from '@langchain/core/outputs';\nimport { isLangChainTool } from '@langchain/core/utils/function_calling';\nimport { isOpenAITool } from '@langchain/core/language_models/base';\nimport { ToolCallChunk } from '@langchain/core/messages/tool';\nimport { v4 as uuidv4 } from 'uuid';\nimport {\n  jsonSchemaToGeminiParameters,\n  schemaToGenerativeAIParameters,\n} from './zod_to_genai_parameters';\nimport { GoogleGenerativeAIToolType } from '../types';\n\nexport function getMessageAuthor(message: BaseMessage): string {\n  const type = message._getType();\n  if (ChatMessage.isInstance(message)) {\n    return message.role;\n  }\n  if (type === 'tool') {\n    return type;\n  }\n  return message.name ?? type;\n}\n\n/**\n * Maps a message type to a Google Generative AI chat author.\n * @param message The message to map.\n * @param model The model to use for mapping.\n * @returns The message type mapped to a Google Generative AI chat author.\n */\nexport function convertAuthorToRole(\n  author: string\n): (typeof POSSIBLE_ROLES)[number] {\n  switch (author) {\n  /**\n     *  Note: Gemini currently is not supporting system messages\n     *  we will convert them to human messages and merge with following\n     * */\n  case 'supervisor':\n  case 'ai':\n  case 'model': // getMessageAuthor returns message.name. code ex.: return message.name ?? type;\n    return 'model';\n  case 'system':\n    return 'system';\n  case 'human':\n    return 'user';\n  case 'tool':\n  case 'function':\n    return 'function';\n  default:\n    throw new Error(`Unknown / unsupported author: ${author}`);\n  }\n}\n\nfunction messageContentMedia(content: MessageContentComplex): Part {\n  if ('mimeType' in content && 'data' in content) {\n    return {\n      inlineData: {\n        mimeType: content.mimeType,\n        data: content.data,\n      },\n    };\n  }\n  if ('mimeType' in content && 'fileUri' in content) {\n    return {\n      fileData: {\n        mimeType: content.mimeType,\n        fileUri: content.fileUri,\n      },\n    };\n  }\n\n  throw new Error('Invalid media content');\n}\n\nfunction inferToolNameFromPreviousMessages(\n  message: ToolMessage | ToolMessageChunk,\n  previousMessages: BaseMessage[]\n): string | undefined {\n  return previousMessages\n    .map((msg) => {\n      if (isAIMessage(msg)) {\n        return msg.tool_calls ?? [];\n      }\n      return [];\n    })\n    .flat()\n    .find((toolCall) => {\n      return toolCall.id === message.tool_call_id;\n    })?.name;\n}\n\nfunction _getStandardContentBlockConverter(\n  isMultimodalModel: boolean\n): StandardContentBlockConverter<{\n  text: TextPart;\n  image: FileDataPart | InlineDataPart;\n  audio: FileDataPart | InlineDataPart;\n  file: FileDataPart | InlineDataPart | TextPart;\n}> {\n  const standardContentBlockConverter: StandardContentBlockConverter<{\n    text: TextPart;\n    image: FileDataPart | InlineDataPart;\n    audio: FileDataPart | InlineDataPart;\n    file: FileDataPart | InlineDataPart | TextPart;\n  }> = {\n    providerName: 'Google Gemini',\n\n    fromStandardTextBlock(block) {\n      return {\n        text: block.text,\n      };\n    },\n\n    fromStandardImageBlock(block): FileDataPart | InlineDataPart {\n      if (!isMultimodalModel) {\n        throw new Error('This model does not support images');\n      }\n      if (block.source_type === 'url') {\n        const data = parseBase64DataUrl({ dataUrl: block.url });\n        if (data) {\n          return {\n            inlineData: {\n              mimeType: data.mime_type,\n              data: data.data,\n            },\n          };\n        } else {\n          return {\n            fileData: {\n              mimeType: block.mime_type ?? '',\n              fileUri: block.url,\n            },\n          };\n        }\n      }\n\n      if (block.source_type === 'base64') {\n        return {\n          inlineData: {\n            mimeType: block.mime_type ?? '',\n            data: block.data,\n          },\n        };\n      }\n\n      throw new Error(`Unsupported source type: ${block.source_type}`);\n    },\n\n    fromStandardAudioBlock(block): FileDataPart | InlineDataPart {\n      if (!isMultimodalModel) {\n        throw new Error('This model does not support audio');\n      }\n      if (block.source_type === 'url') {\n        const data = parseBase64DataUrl({ dataUrl: block.url });\n        if (data) {\n          return {\n            inlineData: {\n              mimeType: data.mime_type,\n              data: data.data,\n            },\n          };\n        } else {\n          return {\n            fileData: {\n              mimeType: block.mime_type ?? '',\n              fileUri: block.url,\n            },\n          };\n        }\n      }\n\n      if (block.source_type === 'base64') {\n        return {\n          inlineData: {\n            mimeType: block.mime_type ?? '',\n            data: block.data,\n          },\n        };\n      }\n\n      throw new Error(`Unsupported source type: ${block.source_type}`);\n    },\n\n    fromStandardFileBlock(block): FileDataPart | InlineDataPart | TextPart {\n      if (!isMultimodalModel) {\n        throw new Error('This model does not support files');\n      }\n      if (block.source_type === 'text') {\n        return {\n          text: block.text,\n        };\n      }\n      if (block.source_type === 'url') {\n        const data = parseBase64DataUrl({ dataUrl: block.url });\n        if (data) {\n          return {\n            inlineData: {\n              mimeType: data.mime_type,\n              data: data.data,\n            },\n          };\n        } else {\n          return {\n            fileData: {\n              mimeType: block.mime_type ?? '',\n              fileUri: block.url,\n            },\n          };\n        }\n      }\n\n      if (block.source_type === 'base64') {\n        return {\n          inlineData: {\n            mimeType: block.mime_type ?? '',\n            data: block.data,\n          },\n        };\n      }\n      throw new Error(`Unsupported source type: ${block.source_type}`);\n    },\n  };\n  return standardContentBlockConverter;\n}\n\nfunction _convertLangChainContentToPart(\n  content: MessageContentComplex,\n  isMultimodalModel: boolean\n): Part | undefined {\n  if (isDataContentBlock(content)) {\n    return convertToProviderContentBlock(\n      content,\n      _getStandardContentBlockConverter(isMultimodalModel)\n    );\n  }\n\n  if (content.type === 'text') {\n    return { text: content.text };\n  } else if (content.type === 'executableCode') {\n    return { executableCode: content.executableCode };\n  } else if (content.type === 'codeExecutionResult') {\n    return { codeExecutionResult: content.codeExecutionResult };\n  } else if (content.type === 'image_url') {\n    if (!isMultimodalModel) {\n      throw new Error('This model does not support images');\n    }\n    let source: string;\n    if (typeof content.image_url === 'string') {\n      source = content.image_url;\n    } else if (\n      typeof content.image_url === 'object' &&\n      'url' in content.image_url\n    ) {\n      source = content.image_url.url;\n    } else {\n      throw new Error('Please provide image as base64 encoded data URL');\n    }\n    const [dm, data] = source.split(',');\n    if (!dm.startsWith('data:')) {\n      throw new Error('Please provide image as base64 encoded data URL');\n    }\n\n    const [mimeType, encoding] = dm.replace(/^data:/, '').split(';');\n    if (encoding !== 'base64') {\n      throw new Error('Please provide image as base64 encoded data URL');\n    }\n\n    return {\n      inlineData: {\n        data,\n        mimeType,\n      },\n    };\n  } else if (\n    content.type === 'document' ||\n    content.type === 'audio' ||\n    content.type === 'video'\n  ) {\n    if (!isMultimodalModel) {\n      throw new Error(`This model does not support ${content.type}s`);\n    }\n    return {\n      inlineData: {\n        data: content.data,\n        mimeType: content.mimeType,\n      },\n    };\n  } else if (content.type === 'media') {\n    return messageContentMedia(content);\n  } else if (content.type === 'tool_use') {\n    return {\n      functionCall: {\n        name: content.name,\n        args: content.input,\n      },\n    };\n  } else if (\n    content.type?.includes('/') === true &&\n    // Ensure it's a single slash.\n    content.type.split('/').length === 2 &&\n    'data' in content &&\n    typeof content.data === 'string'\n  ) {\n    return {\n      inlineData: {\n        mimeType: content.type,\n        data: content.data,\n      },\n    };\n  } else if ('functionCall' in content) {\n    // No action needed here — function calls will be added later from message.tool_calls\n    return undefined;\n  } else {\n    if ('type' in content) {\n      throw new Error(`Unknown content type ${content.type}`);\n    } else {\n      throw new Error(`Unknown content ${JSON.stringify(content)}`);\n    }\n  }\n}\n\nexport function convertMessageContentToParts(\n  message: BaseMessage,\n  isMultimodalModel: boolean,\n  previousMessages: BaseMessage[]\n): Part[] {\n  if (isToolMessage(message)) {\n    const messageName =\n      message.name ??\n      inferToolNameFromPreviousMessages(message, previousMessages);\n    if (messageName === undefined) {\n      throw new Error(\n        `Google requires a tool name for each tool call response, and we could not infer a called tool name for ToolMessage \"${message.id}\" from your passed messages. Please populate a \"name\" field on that ToolMessage explicitly.`\n      );\n    }\n\n    const result = Array.isArray(message.content)\n      ? (message.content\n        .map((c) => _convertLangChainContentToPart(c, isMultimodalModel))\n        .filter((p) => p !== undefined) as Part[])\n      : message.content;\n\n    if (message.status === 'error') {\n      return [\n        {\n          functionResponse: {\n            name: messageName,\n            // The API expects an object with an `error` field if the function call fails.\n            // `error` must be a valid object (not a string or array), so we wrap `message.content` here\n            response: { error: { details: result } },\n          },\n        },\n      ];\n    }\n\n    return [\n      {\n        functionResponse: {\n          name: messageName,\n          // again, can't have a string or array value for `response`, so we wrap it as an object here\n          response: { result },\n        },\n      },\n    ];\n  }\n\n  let functionCalls: FunctionCallPart[] = [];\n  const messageParts: Part[] = [];\n\n  if (typeof message.content === 'string' && message.content) {\n    messageParts.push({ text: message.content });\n  }\n\n  if (Array.isArray(message.content)) {\n    messageParts.push(\n      ...(message.content\n        .map((c) => _convertLangChainContentToPart(c, isMultimodalModel))\n        .filter((p) => p !== undefined) as Part[])\n    );\n  }\n\n  if (isAIMessage(message) && message.tool_calls?.length != null) {\n    functionCalls = message.tool_calls.map((tc) => {\n      return {\n        functionCall: {\n          name: tc.name,\n          args: tc.args,\n        },\n      };\n    });\n  }\n\n  return [...messageParts, ...functionCalls];\n}\n\nexport function convertBaseMessagesToContent(\n  messages: BaseMessage[],\n  isMultimodalModel: boolean,\n  convertSystemMessageToHumanContent: boolean = false\n): Content[] | undefined {\n  return messages.reduce<{\n    content: Content[] | undefined;\n    mergeWithPreviousContent: boolean;\n  }>(\n    (acc, message, index) => {\n      if (!isBaseMessage(message)) {\n        throw new Error('Unsupported message input');\n      }\n      const author = getMessageAuthor(message);\n      if (author === 'system' && index !== 0) {\n        throw new Error('System message should be the first one');\n      }\n      const role = convertAuthorToRole(author);\n\n      const prevContent = acc.content?.[acc.content.length];\n      if (\n        !acc.mergeWithPreviousContent &&\n        prevContent &&\n        prevContent.role === role\n      ) {\n        throw new Error(\n          'Google Generative AI requires alternate messages between authors'\n        );\n      }\n\n      const parts = convertMessageContentToParts(\n        message,\n        isMultimodalModel,\n        messages.slice(0, index)\n      );\n\n      if (acc.mergeWithPreviousContent) {\n        const prevContent = acc.content?.[acc.content.length - 1];\n        if (!prevContent) {\n          throw new Error(\n            'There was a problem parsing your system message. Please try a prompt without one.'\n          );\n        }\n        prevContent.parts.push(...parts);\n\n        return {\n          mergeWithPreviousContent: false,\n          content: acc.content,\n        };\n      }\n      let actualRole = role;\n      if (\n        actualRole === 'function' ||\n        (actualRole === 'system' && !convertSystemMessageToHumanContent)\n      ) {\n        // GenerativeAI API will throw an error if the role is not \"user\" or \"model.\"\n        actualRole = 'user';\n      }\n      const content: Content = {\n        role: actualRole,\n        parts,\n      };\n      return {\n        mergeWithPreviousContent:\n          author === 'system' && !convertSystemMessageToHumanContent,\n        content: [...(acc.content ?? []), content],\n      };\n    },\n    { content: [], mergeWithPreviousContent: false }\n  ).content;\n}\n\nexport function convertResponseContentToChatGenerationChunk(\n  response: EnhancedGenerateContentResponse,\n  extra: {\n    usageMetadata?: UsageMetadata | undefined;\n    index: number;\n  }\n): ChatGenerationChunk | null {\n  if (!response.candidates || response.candidates.length === 0) {\n    return null;\n  }\n  const functionCalls = response.functionCalls();\n  const [candidate] = response.candidates as [\n    Partial<GenerateContentCandidate> | undefined,\n  ];\n  const { content: candidateContent, ...generationInfo } = candidate ?? {};\n  let content: MessageContent | undefined;\n  // Checks if some parts do not have text. If false, it means that the content is a string.\n  const reasoningParts: string[] = [];\n  if (\n    candidateContent != null &&\n    Array.isArray(candidateContent.parts) &&\n    candidateContent.parts.every((p) => 'text' in p)\n  ) {\n    // content = candidateContent.parts.map((p) => p.text).join('');\n    const textParts: string[] = [];\n    for (const part of candidateContent.parts) {\n      if ('thought' in part && part.thought === true) {\n        reasoningParts.push(part.text ?? '');\n        continue;\n      }\n      textParts.push(part.text ?? '');\n    }\n    content = textParts.join('');\n  } else if (candidateContent && Array.isArray(candidateContent.parts)) {\n    content = candidateContent.parts.map((p) => {\n      if ('text' in p && 'thought' in p && p.thought === true) {\n        reasoningParts.push(p.text ?? '');\n      } else if ('text' in p) {\n        return {\n          type: 'text',\n          text: p.text,\n        };\n      } else if ('executableCode' in p) {\n        return {\n          type: 'executableCode',\n          executableCode: p.executableCode,\n        };\n      } else if ('codeExecutionResult' in p) {\n        return {\n          type: 'codeExecutionResult',\n          codeExecutionResult: p.codeExecutionResult,\n        };\n      }\n      return p;\n    });\n  } else {\n    // no content returned - likely due to abnormal stop reason, e.g. malformed function call\n    content = [];\n  }\n\n  let text = '';\n  if (typeof content === 'string' && content) {\n    text = content;\n  } else if (Array.isArray(content)) {\n    const block = content.find((b) => 'text' in b) as\n      | { text: string }\n      | undefined;\n    text = block?.text ?? '';\n  }\n\n  const toolCallChunks: ToolCallChunk[] = [];\n  if (functionCalls) {\n    toolCallChunks.push(\n      ...functionCalls.map((fc) => ({\n        ...fc,\n        args: JSON.stringify(fc.args),\n        // Un-commenting this causes LangChain to incorrectly merge tool calls together\n        // index: extra.index,\n        type: 'tool_call_chunk' as const,\n        id: 'id' in fc && typeof fc.id === 'string' ? fc.id : uuidv4(),\n      }))\n    );\n  }\n\n  const additional_kwargs: ChatGeneration['message']['additional_kwargs'] = {};\n  if (reasoningParts.length > 0) {\n    additional_kwargs.reasoning = reasoningParts.join('');\n  }\n\n  if (candidate?.groundingMetadata) {\n    additional_kwargs.groundingMetadata = candidate.groundingMetadata;\n  }\n\n  const isFinalChunk =\n    response.candidates[0]?.finishReason === 'STOP' ||\n    response.candidates[0]?.finishReason === 'MAX_TOKENS' ||\n    response.candidates[0]?.finishReason === 'SAFETY';\n\n  return new ChatGenerationChunk({\n    text,\n    message: new AIMessageChunk({\n      content: content,\n      name: !candidateContent ? undefined : candidateContent.role,\n      tool_call_chunks: toolCallChunks,\n      // Each chunk can have unique \"generationInfo\", and merging strategy is unclear,\n      // so leave blank for now.\n      additional_kwargs,\n      usage_metadata: isFinalChunk ? extra.usageMetadata : undefined,\n    }),\n    generationInfo,\n  });\n}\n\nexport function convertToGenerativeAITools(\n  tools: GoogleGenerativeAIToolType[]\n): GoogleGenerativeAIFunctionDeclarationsTool[] {\n  if (\n    tools.every(\n      (tool) =>\n        'functionDeclarations' in tool &&\n        Array.isArray(tool.functionDeclarations)\n    )\n  ) {\n    return tools as GoogleGenerativeAIFunctionDeclarationsTool[];\n  }\n  return [\n    {\n      functionDeclarations: tools.map(\n        (tool): GenerativeAIFunctionDeclaration => {\n          if (isLangChainTool(tool)) {\n            const jsonSchema = schemaToGenerativeAIParameters(tool.schema);\n            if (\n              jsonSchema.type === 'object' &&\n              'properties' in jsonSchema &&\n              Object.keys(jsonSchema.properties).length === 0\n            ) {\n              return {\n                name: tool.name,\n                description: tool.description,\n              };\n            }\n            return {\n              name: tool.name,\n              description: tool.description,\n              parameters: jsonSchema,\n            };\n          }\n          if (isOpenAITool(tool)) {\n            return {\n              name: tool.function.name,\n              description:\n                tool.function.description ?? 'A function available to call.',\n              parameters: jsonSchemaToGeminiParameters(\n                tool.function.parameters\n              ),\n            };\n          }\n          return tool as unknown as GenerativeAIFunctionDeclaration;\n        }\n      ),\n    },\n  ];\n}\n"],"names":["uuidv4"],"mappings":";;;;;;;;AA0CM,SAAU,gBAAgB,CAAC,OAAoB,EAAA;AACnD,IAAA,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE;AAC/B,IAAA,IAAI,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QACnC,OAAO,OAAO,CAAC,IAAI;;AAErB,IAAA,IAAI,IAAI,KAAK,MAAM,EAAE;AACnB,QAAA,OAAO,IAAI;;AAEb,IAAA,OAAO,OAAO,CAAC,IAAI,IAAI,IAAI;AAC7B;AAEA;;;;;AAKG;AACG,SAAU,mBAAmB,CACjC,MAAc,EAAA;IAEd,QAAQ,MAAM;AACd;;;AAGO;AACP,QAAA,KAAK,YAAY;AACjB,QAAA,KAAK,IAAI;QACT,KAAK,OAAO;AACV,YAAA,OAAO,OAAO;AAChB,QAAA,KAAK,QAAQ;AACX,YAAA,OAAO,QAAQ;AACjB,QAAA,KAAK,OAAO;AACV,YAAA,OAAO,MAAM;AACf,QAAA,KAAK,MAAM;AACX,QAAA,KAAK,UAAU;AACb,YAAA,OAAO,UAAU;AACnB,QAAA;AACE,YAAA,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,CAAA,CAAE,CAAC;;AAE9D;AAEA,SAAS,mBAAmB,CAAC,OAA8B,EAAA;IACzD,IAAI,UAAU,IAAI,OAAO,IAAI,MAAM,IAAI,OAAO,EAAE;QAC9C,OAAO;AACL,YAAA,UAAU,EAAE;gBACV,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;AACnB,aAAA;SACF;;IAEH,IAAI,UAAU,IAAI,OAAO,IAAI,SAAS,IAAI,OAAO,EAAE;QACjD,OAAO;AACL,YAAA,QAAQ,EAAE;gBACR,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,OAAO,EAAE,OAAO,CAAC,OAAO;AACzB,aAAA;SACF;;AAGH,IAAA,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC;AAC1C;AAEA,SAAS,iCAAiC,CACxC,OAAuC,EACvC,gBAA+B,EAAA;AAE/B,IAAA,OAAO;AACJ,SAAA,GAAG,CAAC,CAAC,GAAG,KAAI;AACX,QAAA,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE;AACpB,YAAA,OAAO,GAAG,CAAC,UAAU,IAAI,EAAE;;AAE7B,QAAA,OAAO,EAAE;AACX,KAAC;AACA,SAAA,IAAI;AACJ,SAAA,IAAI,CAAC,CAAC,QAAQ,KAAI;AACjB,QAAA,OAAO,QAAQ,CAAC,EAAE,KAAK,OAAO,CAAC,YAAY;KAC5C,CAAC,EAAE,IAAI;AACZ;AAEA,SAAS,iCAAiC,CACxC,iBAA0B,EAAA;AAO1B,IAAA,MAAM,6BAA6B,GAK9B;AACH,QAAA,YAAY,EAAE,eAAe;AAE7B,QAAA,qBAAqB,CAAC,KAAK,EAAA;YACzB,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;aACjB;SACF;AAED,QAAA,sBAAsB,CAAC,KAAK,EAAA;YAC1B,IAAI,CAAC,iBAAiB,EAAE;AACtB,gBAAA,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC;;AAEvD,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,EAAE;AAC/B,gBAAA,MAAM,IAAI,GAAG,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC;gBACvD,IAAI,IAAI,EAAE;oBACR,OAAO;AACL,wBAAA,UAAU,EAAE;4BACV,QAAQ,EAAE,IAAI,CAAC,SAAS;4BACxB,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,yBAAA;qBACF;;qBACI;oBACL,OAAO;AACL,wBAAA,QAAQ,EAAE;AACR,4BAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;4BAC/B,OAAO,EAAE,KAAK,CAAC,GAAG;AACnB,yBAAA;qBACF;;;AAIL,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE;gBAClC,OAAO;AACL,oBAAA,UAAU,EAAE;AACV,wBAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;wBAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;AACjB,qBAAA;iBACF;;YAGH,MAAM,IAAI,KAAK,CAAC,CAAA,yBAAA,EAA4B,KAAK,CAAC,WAAW,CAAE,CAAA,CAAC;SACjE;AAED,QAAA,sBAAsB,CAAC,KAAK,EAAA;YAC1B,IAAI,CAAC,iBAAiB,EAAE;AACtB,gBAAA,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC;;AAEtD,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,EAAE;AAC/B,gBAAA,MAAM,IAAI,GAAG,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC;gBACvD,IAAI,IAAI,EAAE;oBACR,OAAO;AACL,wBAAA,UAAU,EAAE;4BACV,QAAQ,EAAE,IAAI,CAAC,SAAS;4BACxB,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,yBAAA;qBACF;;qBACI;oBACL,OAAO;AACL,wBAAA,QAAQ,EAAE;AACR,4BAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;4BAC/B,OAAO,EAAE,KAAK,CAAC,GAAG;AACnB,yBAAA;qBACF;;;AAIL,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE;gBAClC,OAAO;AACL,oBAAA,UAAU,EAAE;AACV,wBAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;wBAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;AACjB,qBAAA;iBACF;;YAGH,MAAM,IAAI,KAAK,CAAC,CAAA,yBAAA,EAA4B,KAAK,CAAC,WAAW,CAAE,CAAA,CAAC;SACjE;AAED,QAAA,qBAAqB,CAAC,KAAK,EAAA;YACzB,IAAI,CAAC,iBAAiB,EAAE;AACtB,gBAAA,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC;;AAEtD,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,MAAM,EAAE;gBAChC,OAAO;oBACL,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB;;AAEH,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,EAAE;AAC/B,gBAAA,MAAM,IAAI,GAAG,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC;gBACvD,IAAI,IAAI,EAAE;oBACR,OAAO;AACL,wBAAA,UAAU,EAAE;4BACV,QAAQ,EAAE,IAAI,CAAC,SAAS;4BACxB,IAAI,EAAE,IAAI,CAAC,IAAI;AAChB,yBAAA;qBACF;;qBACI;oBACL,OAAO;AACL,wBAAA,QAAQ,EAAE;AACR,4BAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;4BAC/B,OAAO,EAAE,KAAK,CAAC,GAAG;AACnB,yBAAA;qBACF;;;AAIL,YAAA,IAAI,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE;gBAClC,OAAO;AACL,oBAAA,UAAU,EAAE;AACV,wBAAA,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;wBAC/B,IAAI,EAAE,KAAK,CAAC,IAAI;AACjB,qBAAA;iBACF;;YAEH,MAAM,IAAI,KAAK,CAAC,CAAA,yBAAA,EAA4B,KAAK,CAAC,WAAW,CAAE,CAAA,CAAC;SACjE;KACF;AACD,IAAA,OAAO,6BAA6B;AACtC;AAEA,SAAS,8BAA8B,CACrC,OAA8B,EAC9B,iBAA0B,EAAA;AAE1B,IAAA,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE;QAC/B,OAAO,6BAA6B,CAClC,OAAO,EACP,iCAAiC,CAAC,iBAAiB,CAAC,CACrD;;AAGH,IAAA,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE;AAC3B,QAAA,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;;AACxB,SAAA,IAAI,OAAO,CAAC,IAAI,KAAK,gBAAgB,EAAE;AAC5C,QAAA,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE;;AAC5C,SAAA,IAAI,OAAO,CAAC,IAAI,KAAK,qBAAqB,EAAE;AACjD,QAAA,OAAO,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE;;AACtD,SAAA,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE;QACvC,IAAI,CAAC,iBAAiB,EAAE;AACtB,YAAA,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC;;AAEvD,QAAA,IAAI,MAAc;AAClB,QAAA,IAAI,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,EAAE;AACzC,YAAA,MAAM,GAAG,OAAO,CAAC,SAAS;;AACrB,aAAA,IACL,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ;AACrC,YAAA,KAAK,IAAI,OAAO,CAAC,SAAS,EAC1B;AACA,YAAA,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG;;aACzB;AACL,YAAA,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC;;AAEpE,QAAA,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;QACpC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;AAC3B,YAAA,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC;;AAGpE,QAAA,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;AAChE,QAAA,IAAI,QAAQ,KAAK,QAAQ,EAAE;AACzB,YAAA,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC;;QAGpE,OAAO;AACL,YAAA,UAAU,EAAE;gBACV,IAAI;gBACJ,QAAQ;AACT,aAAA;SACF;;AACI,SAAA,IACL,OAAO,CAAC,IAAI,KAAK,UAAU;QAC3B,OAAO,CAAC,IAAI,KAAK,OAAO;AACxB,QAAA,OAAO,CAAC,IAAI,KAAK,OAAO,EACxB;QACA,IAAI,CAAC,iBAAiB,EAAE;YACtB,MAAM,IAAI,KAAK,CAAC,CAAA,4BAAA,EAA+B,OAAO,CAAC,IAAI,CAAG,CAAA,CAAA,CAAC;;QAEjE,OAAO;AACL,YAAA,UAAU,EAAE;gBACV,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;AAC3B,aAAA;SACF;;AACI,SAAA,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE;AACnC,QAAA,OAAO,mBAAmB,CAAC,OAAO,CAAC;;AAC9B,SAAA,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE;QACtC,OAAO;AACL,YAAA,YAAY,EAAE;gBACZ,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,IAAI,EAAE,OAAO,CAAC,KAAK;AACpB,aAAA;SACF;;SACI,IACL,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI;;QAEpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC;AACpC,QAAA,MAAM,IAAI,OAAO;AACjB,QAAA,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,EAChC;QACA,OAAO;AACL,YAAA,UAAU,EAAE;gBACV,QAAQ,EAAE,OAAO,CAAC,IAAI;gBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;AACnB,aAAA;SACF;;AACI,SAAA,IAAI,cAAc,IAAI,OAAO,EAAE;;AAEpC,QAAA,OAAO,SAAS;;SACX;AACL,QAAA,IAAI,MAAM,IAAI,OAAO,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,CAAA,qBAAA,EAAwB,OAAO,CAAC,IAAI,CAAE,CAAA,CAAC;;aAClD;AACL,YAAA,MAAM,IAAI,KAAK,CAAC,CAAA,gBAAA,EAAmB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAE,CAAA,CAAC;;;AAGnE;SAEgB,4BAA4B,CAC1C,OAAoB,EACpB,iBAA0B,EAC1B,gBAA+B,EAAA;AAE/B,IAAA,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE;AAC1B,QAAA,MAAM,WAAW,GACf,OAAO,CAAC,IAAI;AACZ,YAAA,iCAAiC,CAAC,OAAO,EAAE,gBAAgB,CAAC;AAC9D,QAAA,IAAI,WAAW,KAAK,SAAS,EAAE;YAC7B,MAAM,IAAI,KAAK,CACb,CAAA,oHAAA,EAAuH,OAAO,CAAC,EAAE,CAA6F,2FAAA,CAAA,CAC/N;;QAGH,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO;cACvC,OAAO,CAAC;AACR,iBAAA,GAAG,CAAC,CAAC,CAAC,KAAK,8BAA8B,CAAC,CAAC,EAAE,iBAAiB,CAAC;iBAC/D,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,SAAS;AAChC,cAAE,OAAO,CAAC,OAAO;AAEnB,QAAA,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE;YAC9B,OAAO;AACL,gBAAA;AACE,oBAAA,gBAAgB,EAAE;AAChB,wBAAA,IAAI,EAAE,WAAW;;;wBAGjB,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;AACzC,qBAAA;AACF,iBAAA;aACF;;QAGH,OAAO;AACL,YAAA;AACE,gBAAA,gBAAgB,EAAE;AAChB,oBAAA,IAAI,EAAE,WAAW;;oBAEjB,QAAQ,EAAE,EAAE,MAAM,EAAE;AACrB,iBAAA;AACF,aAAA;SACF;;IAGH,IAAI,aAAa,GAAuB,EAAE;IAC1C,MAAM,YAAY,GAAW,EAAE;IAE/B,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE;QAC1D,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;;IAG9C,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;AAClC,QAAA,YAAY,CAAC,IAAI,CACf,GAAI,OAAO,CAAC;AACT,aAAA,GAAG,CAAC,CAAC,CAAC,KAAK,8BAA8B,CAAC,CAAC,EAAE,iBAAiB,CAAC;aAC/D,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,SAAS,CAAY,CAC7C;;AAGH,IAAA,IAAI,WAAW,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,UAAU,EAAE,MAAM,IAAI,IAAI,EAAE;QAC9D,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,KAAI;YAC5C,OAAO;AACL,gBAAA,YAAY,EAAE;oBACZ,IAAI,EAAE,EAAE,CAAC,IAAI;oBACb,IAAI,EAAE,EAAE,CAAC,IAAI;AACd,iBAAA;aACF;AACH,SAAC,CAAC;;AAGJ,IAAA,OAAO,CAAC,GAAG,YAAY,EAAE,GAAG,aAAa,CAAC;AAC5C;AAEM,SAAU,4BAA4B,CAC1C,QAAuB,EACvB,iBAA0B,EAC1B,qCAA8C,KAAK,EAAA;IAEnD,OAAO,QAAQ,CAAC,MAAM,CAIpB,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,KAAI;AACtB,QAAA,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE;AAC3B,YAAA,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC;;AAE9C,QAAA,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC;QACxC,IAAI,MAAM,KAAK,QAAQ,IAAI,KAAK,KAAK,CAAC,EAAE;AACtC,YAAA,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC;;AAE3D,QAAA,MAAM,IAAI,GAAG,mBAAmB,CAAC,MAAM,CAAC;AAExC,QAAA,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;QACrD,IACE,CAAC,GAAG,CAAC,wBAAwB;YAC7B,WAAW;AACX,YAAA,WAAW,CAAC,IAAI,KAAK,IAAI,EACzB;AACA,YAAA,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE;;AAGH,QAAA,MAAM,KAAK,GAAG,4BAA4B,CACxC,OAAO,EACP,iBAAiB,EACjB,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CACzB;AAED,QAAA,IAAI,GAAG,CAAC,wBAAwB,EAAE;AAChC,YAAA,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YACzD,IAAI,CAAC,WAAW,EAAE;AAChB,gBAAA,MAAM,IAAI,KAAK,CACb,mFAAmF,CACpF;;YAEH,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAEhC,OAAO;AACL,gBAAA,wBAAwB,EAAE,KAAK;gBAC/B,OAAO,EAAE,GAAG,CAAC,OAAO;aACrB;;QAEH,IAAI,UAAU,GAAG,IAAI;QACrB,IACE,UAAU,KAAK,UAAU;aACxB,UAAU,KAAK,QAAQ,IAAI,CAAC,kCAAkC,CAAC,EAChE;;YAEA,UAAU,GAAG,MAAM;;AAErB,QAAA,MAAM,OAAO,GAAY;AACvB,YAAA,IAAI,EAAE,UAAU;YAChB,KAAK;SACN;QACD,OAAO;AACL,YAAA,wBAAwB,EACtB,MAAM,KAAK,QAAQ,IAAI,CAAC,kCAAkC;AAC5D,YAAA,OAAO,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC;SAC3C;AACH,KAAC,EACD,EAAE,OAAO,EAAE,EAAE,EAAE,wBAAwB,EAAE,KAAK,EAAE,CACjD,CAAC,OAAO;AACX;AAEgB,SAAA,2CAA2C,CACzD,QAAyC,EACzC,KAGC,EAAA;AAED,IAAA,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;AAC5D,QAAA,OAAO,IAAI;;AAEb,IAAA,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,EAAE;AAC9C,IAAA,MAAM,CAAC,SAAS,CAAC,GAAG,QAAQ,CAAC,UAE5B;AACD,IAAA,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,cAAc,EAAE,GAAG,SAAS,IAAI,EAAE;AACxE,IAAA,IAAI,OAAmC;;IAEvC,MAAM,cAAc,GAAa,EAAE;IACnC,IACE,gBAAgB,IAAI,IAAI;AACxB,QAAA,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC;AACrC,QAAA,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,CAAC,CAAC,EAChD;;QAEA,MAAM,SAAS,GAAa,EAAE;AAC9B,QAAA,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,KAAK,EAAE;YACzC,IAAI,SAAS,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE;gBAC9C,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;gBACpC;;YAEF,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;;AAEjC,QAAA,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;;SACvB,IAAI,gBAAgB,IAAI,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE;QACpE,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAI;AACzC,YAAA,IAAI,MAAM,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,KAAK,IAAI,EAAE;gBACvD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;;AAC5B,iBAAA,IAAI,MAAM,IAAI,CAAC,EAAE;gBACtB,OAAO;AACL,oBAAA,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb;;AACI,iBAAA,IAAI,gBAAgB,IAAI,CAAC,EAAE;gBAChC,OAAO;AACL,oBAAA,IAAI,EAAE,gBAAgB;oBACtB,cAAc,EAAE,CAAC,CAAC,cAAc;iBACjC;;AACI,iBAAA,IAAI,qBAAqB,IAAI,CAAC,EAAE;gBACrC,OAAO;AACL,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,mBAAmB,EAAE,CAAC,CAAC,mBAAmB;iBAC3C;;AAEH,YAAA,OAAO,CAAC;AACV,SAAC,CAAC;;SACG;;QAEL,OAAO,GAAG,EAAE;;IAGd,IAAI,IAAI,GAAG,EAAE;AACb,IAAA,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,EAAE;QAC1C,IAAI,GAAG,OAAO;;AACT,SAAA,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;AACjC,QAAA,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,CAAC,CAEhC;AACb,QAAA,IAAI,GAAG,KAAK,EAAE,IAAI,IAAI,EAAE;;IAG1B,MAAM,cAAc,GAAoB,EAAE;IAC1C,IAAI,aAAa,EAAE;AACjB,QAAA,cAAc,CAAC,IAAI,CACjB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM;AAC5B,YAAA,GAAG,EAAE;YACL,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC;;;AAG7B,YAAA,IAAI,EAAE,iBAA0B;YAChC,EAAE,EAAE,IAAI,IAAI,EAAE,IAAI,OAAO,EAAE,CAAC,EAAE,KAAK,QAAQ,GAAG,EAAE,CAAC,EAAE,GAAGA,EAAM,EAAE;SAC/D,CAAC,CAAC,CACJ;;IAGH,MAAM,iBAAiB,GAAmD,EAAE;AAC5E,IAAA,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7B,iBAAiB,CAAC,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;;AAGvD,IAAA,IAAI,SAAS,EAAE,iBAAiB,EAAE;AAChC,QAAA,iBAAiB,CAAC,iBAAiB,GAAG,SAAS,CAAC,iBAAiB;;IAGnE,MAAM,YAAY,GAChB,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,KAAK,MAAM;QAC/C,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,KAAK,YAAY;QACrD,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,KAAK,QAAQ;IAEnD,OAAO,IAAI,mBAAmB,CAAC;QAC7B,IAAI;QACJ,OAAO,EAAE,IAAI,cAAc,CAAC;AAC1B,YAAA,OAAO,EAAE,OAAO;AAChB,YAAA,IAAI,EAAE,CAAC,gBAAgB,GAAG,SAAS,GAAG,gBAAgB,CAAC,IAAI;AAC3D,YAAA,gBAAgB,EAAE,cAAc;;;YAGhC,iBAAiB;YACjB,cAAc,EAAE,YAAY,GAAG,KAAK,CAAC,aAAa,GAAG,SAAS;SAC/D,CAAC;QACF,cAAc;AACf,KAAA,CAAC;AACJ;;;;"}

package/dist/esm/main.mjs

@@ -5,7 +5,7 @@ export { HandlerRegistry, LLMStreamHandler, ModelEndHandler, TestChatStreamHandl
 export { convertMessagesToContent, findLastIndex, formatAnthropicArtifactContent, formatAnthropicMessage, formatArtifactPayload, getConverseOverrideMessage, modifyDeltaProperties } from './messages/core.mjs';
 export { getMessageId } from './messages/ids.mjs';
 export { calculateTotalTokens, checkValidNumber, createPruneMessages, getMessagesWithinTokenLimit } from './messages/prune.mjs';
-export { formatAgentMessages, formatContentStrings, formatFromLangChain, formatLangChainMessages, formatMessage, formatVisionMessage, shiftIndexTokenCountMap } from './messages/format.mjs';
+export { formatAgentMessages, formatContentStrings, formatFromLangChain, formatLangChainMessages, formatMediaMessage, formatMessage, shiftIndexTokenCountMap } from './messages/format.mjs';
 export { Graph, StandardGraph } from './graphs/Graph.mjs';
 export { createCodeExecutionTool, getCodeBaseURL, imageExtRegex } from './tools/CodeExecutor.mjs';
 export { handleServerToolResult, handleToolCallChunks, handleToolCalls, toolResultTypes } from './tools/handlers.mjs';

package/dist/esm/messages/format.mjs

@@ -3,12 +3,12 @@ import { Providers, ContentTypes } from '../common/enum.mjs';
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 /**
- * Formats a message to OpenAI Vision API payload format.
+ * Formats a message with media content (images, documents, videos, audios) to API payload format.
  *
- * @param {VisionMessageParams} params - The parameters for formatting.
- * @returns {Object} - The formatted message.
+ * @param params - The parameters for formatting.
+ * @returns - The formatted message.
  */
-const formatVisionMessage = ({ message, image_urls, endpoint, }) => {
+const formatMediaMessage = ({ message, endpoint, mediaParts, }) => {
     // Create a new object to avoid mutating the input
     const result = {
         ...message,
@@ -16,24 +16,24 @@ const formatVisionMessage = ({ message, image_urls, endpoint, }) => {
     };
     if (endpoint === Providers.ANTHROPIC) {
         result.content = [
-            ...image_urls,
+            ...mediaParts,
             { type: ContentTypes.TEXT, text: message.content },
         ];
         return result;
     }
     result.content = [
         { type: ContentTypes.TEXT, text: message.content },
-        ...image_urls,
+        ...mediaParts,
     ];
     return result;
 };
 /**
  * Formats a message to OpenAI payload format based on the provided options.
  *
- * @param {FormatMessageParams} params - The parameters for formatting.
- * @returns {FormattedMessage | HumanMessage | AIMessage | SystemMessage} - The formatted message.
+ * @param params - The parameters for formatting.
+ * @returns - The formatted message.
  */
-const formatMessage = ({ message, userName, assistantName, endpoint, langChain = false, }) => {
+const formatMessage = ({ message, userName, endpoint, assistantName, langChain = false, }) => {
     // eslint-disable-next-line prefer-const
     let { role: _role, _name, sender, text, content: _content, lc_id } = message;
     if (lc_id && lc_id[2] && !langChain) {
@@ -53,19 +53,7 @@ const formatMessage = ({ message, userName, assistantName, endpoint, langChain =
         role,
         content,
     };
-    const { image_urls } = message;
-    if (Array.isArray(image_urls) && image_urls.length > 0 && role === 'user') {
-        return formatVisionMessage({
-            message: {
-                ...formattedMessage,
-                content: typeof formattedMessage.content === 'string'
-                    ? formattedMessage.content
-                    : '',
-            },
-            image_urls,
-            endpoint,
-        });
-    }
+    // Set name fields first
     if (_name != null && _name) {
         formattedMessage.name = _name;
     }
@@ -85,6 +73,36 @@ const formatMessage = ({ message, userName, assistantName, endpoint, langChain =
             formattedMessage.name = formattedMessage.name.substring(0, 64);
         }
     }
+    const { image_urls, documents, videos, audios } = message;
+    const mediaParts = [];
+    if (Array.isArray(documents) && documents.length > 0) {
+        mediaParts.push(...documents);
+    }
+    if (Array.isArray(videos) && videos.length > 0) {
+        mediaParts.push(...videos);
+    }
+    if (Array.isArray(audios) && audios.length > 0) {
+        mediaParts.push(...audios);
+    }
+    if (Array.isArray(image_urls) && image_urls.length > 0) {
+        mediaParts.push(...image_urls);
+    }
+    if (mediaParts.length > 0 && role === 'user') {
+        const mediaMessage = formatMediaMessage({
+            message: {
+                ...formattedMessage,
+                content: typeof formattedMessage.content === 'string'
+                    ? formattedMessage.content
+                    : '',
+            },
+            mediaParts,
+            endpoint,
+        });
+        if (!langChain) {
+            return mediaMessage;
+        }
+        return new HumanMessage(mediaMessage);
+    }
     if (!langChain) {
         return formattedMessage;
     }
@@ -101,9 +119,9 @@ const formatMessage = ({ message, userName, assistantName, endpoint, langChain =
 /**
  * Formats an array of messages for LangChain.
  *
- * @param {Array<MessageInput>} messages - The array of messages to format.
- * @param {Omit<FormatMessageParams, 'message' | 'langChain'>} formatOptions - The options for formatting each message.
- * @returns {Array<HumanMessage | AIMessage | SystemMessage>} - The array of formatted LangChain messages.
+ * @param messages - The array of messages to format.
+ * @param formatOptions - The options for formatting each message.
+ * @returns - The array of formatted LangChain messages.
  */
 const formatLangChainMessages = (messages, formatOptions) => {
     return messages.map((msg) => {
@@ -118,8 +136,8 @@ const formatLangChainMessages = (messages, formatOptions) => {
 /**
  * Formats a LangChain message object by merging properties from `lc_kwargs` or `kwargs` and `additional_kwargs`.
  *
- * @param {LangChainMessage} message - The message object to format.
- * @returns {Record<string, any>} The formatted LangChain message.
+ * @param message - The message object to format.
+ * @returns - The formatted LangChain message.
  */
 const formatFromLangChain = (message) => {
     const kwargs = message.lc_kwargs ?? message.kwargs ?? {};
@@ -232,10 +250,10 @@ function formatAssistantMessage(message) {
 /**
  * Formats an array of messages for LangChain, handling tool calls and creating ToolMessage instances.
  *
- * @param {TPayload} payload - The array of messages to format.
- * @param {Record<number, number>} [indexTokenCountMap] - Optional map of message indices to token counts.
- * @param {Set<string>} [tools] - Optional set of tool names that are allowed in the request.
- * @returns {Object} - Object containing formatted messages and updated indexTokenCountMap if provided.
+ * @param payload - The array of messages to format.
+ * @param indexTokenCountMap - Optional map of message indices to token counts.
+ * @param tools - Optional set of tool names that are allowed in the request.
+ * @returns - Object containing formatted messages and updated indexTokenCountMap if provided.
  */
 const formatAgentMessages = (payload, indexTokenCountMap, tools) => {
     const messages = [];
@@ -381,8 +399,8 @@ const formatAgentMessages = (payload, indexTokenCountMap, tools) => {
 };
 /**
  * Formats an array of messages for LangChain, making sure all content fields are strings
- * @param {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} payload - The array of messages to format.
- * @returns {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} - The array of formatted LangChain messages, including ToolMessages for tool calls.
+ * @param payload - The array of messages to format.
+ * @returns - The array of formatted LangChain messages, including ToolMessages for tool calls.
  */
 const formatContentStrings = (payload) => {
     // Create a copy of the payload to avoid modifying the original
@@ -425,5 +443,5 @@ function shiftIndexTokenCountMap(indexTokenCountMap, instructionsTokenCount) {
     return shiftedMap;
 }
 
-export { formatAgentMessages, formatContentStrings, formatFromLangChain, formatLangChainMessages, formatMessage, formatVisionMessage, shiftIndexTokenCountMap };
+export { formatAgentMessages, formatContentStrings, formatFromLangChain, formatLangChainMessages, formatMediaMessage, formatMessage, shiftIndexTokenCountMap };
 //# sourceMappingURL=format.mjs.map

package/dist/esm/messages/format.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"format.mjs","sources":["../../../src/messages/format.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\nimport {\n  AIMessage,\n  ToolMessage,\n  BaseMessage,\n  HumanMessage,\n  SystemMessage,\n  getBufferString,\n} from '@langchain/core/messages';\nimport type { MessageContentImageUrl } from '@langchain/core/messages';\nimport type { ToolCall } from '@langchain/core/messages/tool';\nimport type {\n  MessageContentComplex,\n  ToolCallPart,\n  TPayload,\n  TMessage,\n} from '@/types';\nimport { Providers, ContentTypes } from '@/common';\n\ninterface VisionMessageParams {\n  message: {\n    role: string;\n    content: string;\n    name?: string;\n    [key: string]: any;\n  };\n  image_urls: MessageContentImageUrl[];\n  endpoint?: Providers;\n}\n\n/**\n * Formats a message to OpenAI Vision API payload format.\n *\n * @param {VisionMessageParams} params - The parameters for formatting.\n * @returns {Object} - The formatted message.\n */\nexport const formatVisionMessage = ({\n  message,\n  image_urls,\n  endpoint,\n}: VisionMessageParams): {\n  role: string;\n  content: MessageContentComplex[];\n  name?: string;\n  [key: string]: any;\n} => {\n  // Create a new object to avoid mutating the input\n  const result: {\n    role: string;\n    content: MessageContentComplex[];\n    name?: string;\n    [key: string]: any;\n  } = {\n    ...message,\n    content: [] as MessageContentComplex[],\n  };\n\n  if (endpoint === Providers.ANTHROPIC) {\n    result.content = [\n      ...image_urls,\n      { type: ContentTypes.TEXT, text: message.content },\n    ] as MessageContentComplex[];\n    return result;\n  }\n\n  result.content = [\n    { type: ContentTypes.TEXT, text: message.content },\n    ...image_urls,\n  ] as MessageContentComplex[];\n\n  return result;\n};\n\ninterface MessageInput {\n  role?: string;\n  _name?: string;\n  sender?: string;\n  text?: string;\n  content?: string | MessageContentComplex[];\n  image_urls?: MessageContentImageUrl[];\n  lc_id?: string[];\n  [key: string]: any;\n}\n\ninterface FormatMessageParams {\n  message: MessageInput;\n  userName?: string;\n  assistantName?: string;\n  endpoint?: Providers;\n  langChain?: boolean;\n}\n\ninterface FormattedMessage {\n  role: string;\n  content: string | MessageContentComplex[];\n  name?: string;\n  [key: string]: any;\n}\n\n/**\n * Formats a message to OpenAI payload format based on the provided options.\n *\n * @param {FormatMessageParams} params - The parameters for formatting.\n * @returns {FormattedMessage | HumanMessage | AIMessage | SystemMessage} - The formatted message.\n */\nexport const formatMessage = ({\n  message,\n  userName,\n  assistantName,\n  endpoint,\n  langChain = false,\n}: FormatMessageParams):\n  | FormattedMessage\n  | HumanMessage\n  | AIMessage\n  | SystemMessage => {\n  // eslint-disable-next-line prefer-const\n  let { role: _role, _name, sender, text, content: _content, lc_id } = message;\n  if (lc_id && lc_id[2] && !langChain) {\n    const roleMapping: Record<string, string> = {\n      SystemMessage: 'system',\n      HumanMessage: 'user',\n      AIMessage: 'assistant',\n    };\n    _role = roleMapping[lc_id[2]] || _role;\n  }\n  const role =\n    _role ??\n    (sender != null && sender && sender.toLowerCase() === 'user'\n      ? 'user'\n      : 'assistant');\n  const content = _content ?? text ?? '';\n  const formattedMessage: FormattedMessage = {\n    role,\n    content,\n  };\n\n  const { image_urls } = message;\n  if (Array.isArray(image_urls) && image_urls.length > 0 && role === 'user') {\n    return formatVisionMessage({\n      message: {\n        ...formattedMessage,\n        content:\n          typeof formattedMessage.content === 'string'\n            ? formattedMessage.content\n            : '',\n      },\n      image_urls,\n      endpoint,\n    });\n  }\n\n  if (_name != null && _name) {\n    formattedMessage.name = _name;\n  }\n\n  if (userName != null && userName && formattedMessage.role === 'user') {\n    formattedMessage.name = userName;\n  }\n\n  if (\n    assistantName != null &&\n    assistantName &&\n    formattedMessage.role === 'assistant'\n  ) {\n    formattedMessage.name = assistantName;\n  }\n\n  if (formattedMessage.name != null && formattedMessage.name) {\n    // Conform to API regex: ^[a-zA-Z0-9_-]{1,64}$\n    // https://community.openai.com/t/the-format-of-the-name-field-in-the-documentation-is-incorrect/175684/2\n    formattedMessage.name = formattedMessage.name.replace(\n      /[^a-zA-Z0-9_-]/g,\n      '_'\n    );\n\n    if (formattedMessage.name.length > 64) {\n      formattedMessage.name = formattedMessage.name.substring(0, 64);\n    }\n  }\n\n  if (!langChain) {\n    return formattedMessage;\n  }\n\n  if (role === 'user') {\n    return new HumanMessage(formattedMessage);\n  } else if (role === 'assistant') {\n    return new AIMessage(formattedMessage);\n  } else {\n    return new SystemMessage(formattedMessage);\n  }\n};\n\n/**\n * Formats an array of messages for LangChain.\n *\n * @param {Array<MessageInput>} messages - The array of messages to format.\n * @param {Omit<FormatMessageParams, 'message' | 'langChain'>} formatOptions - The options for formatting each message.\n * @returns {Array<HumanMessage | AIMessage | SystemMessage>} - The array of formatted LangChain messages.\n */\nexport const formatLangChainMessages = (\n  messages: Array<MessageInput>,\n  formatOptions: Omit<FormatMessageParams, 'message' | 'langChain'>\n): Array<HumanMessage | AIMessage | SystemMessage> => {\n  return messages.map((msg) => {\n    const formatted = formatMessage({\n      ...formatOptions,\n      message: msg,\n      langChain: true,\n    });\n    return formatted as HumanMessage | AIMessage | SystemMessage;\n  });\n};\n\ninterface LangChainMessage {\n  lc_kwargs?: {\n    additional_kwargs?: Record<string, any>;\n    [key: string]: any;\n  };\n  kwargs?: {\n    additional_kwargs?: Record<string, any>;\n    [key: string]: any;\n  };\n  [key: string]: any;\n}\n\n/**\n * Formats a LangChain message object by merging properties from `lc_kwargs` or `kwargs` and `additional_kwargs`.\n *\n * @param {LangChainMessage} message - The message object to format.\n * @returns {Record<string, any>} The formatted LangChain message.\n */\nexport const formatFromLangChain = (\n  message: LangChainMessage\n): Record<string, any> => {\n  const kwargs = message.lc_kwargs ?? message.kwargs ?? {};\n  const { additional_kwargs = {}, ...message_kwargs } = kwargs;\n  return {\n    ...message_kwargs,\n    ...additional_kwargs,\n  };\n};\n\n/**\n * Helper function to format an assistant message\n * @param message The message to format\n * @returns Array of formatted messages\n */\nfunction formatAssistantMessage(\n  message: Partial<TMessage>\n): Array<AIMessage | ToolMessage> {\n  const formattedMessages: Array<AIMessage | ToolMessage> = [];\n  let currentContent: MessageContentComplex[] = [];\n  let lastAIMessage: AIMessage | null = null;\n  let hasReasoning = false;\n\n  if (Array.isArray(message.content)) {\n    for (const part of message.content) {\n      if (part.type === ContentTypes.TEXT && part.tool_call_ids) {\n        /*\n        If there's pending content, it needs to be aggregated as a single string to prepare for tool calls.\n        For Anthropic models, the \"tool_calls\" field on a message is only respected if content is a string.\n        */\n        if (currentContent.length > 0) {\n          let content = currentContent.reduce((acc, curr) => {\n            if (curr.type === ContentTypes.TEXT) {\n              return `${acc}${curr[ContentTypes.TEXT] || ''}\\n`;\n            }\n            return acc;\n          }, '');\n          content =\n            `${content}\\n${part[ContentTypes.TEXT] ?? part.text ?? ''}`.trim();\n          lastAIMessage = new AIMessage({ content });\n          formattedMessages.push(lastAIMessage);\n          currentContent = [];\n          continue;\n        }\n        // Create a new AIMessage with this text and prepare for tool calls\n        lastAIMessage = new AIMessage({\n          content: part.text || '',\n        });\n        formattedMessages.push(lastAIMessage);\n      } else if (part.type === ContentTypes.TOOL_CALL) {\n        if (!lastAIMessage) {\n          // \"Heal\" the payload by creating an AIMessage to precede the tool call\n          lastAIMessage = new AIMessage({ content: '' });\n          formattedMessages.push(lastAIMessage);\n        }\n\n        // Note: `tool_calls` list is defined when constructed by `AIMessage` class, and outputs should be excluded from it\n        const {\n          output,\n          args: _args,\n          ..._tool_call\n        } = part.tool_call as ToolCallPart;\n        const tool_call: ToolCallPart = _tool_call;\n        // TODO: investigate; args as dictionary may need to be providers-or-tool-specific\n        let args: any = _args;\n        try {\n          if (typeof _args === 'string') {\n            args = JSON.parse(_args);\n          }\n        } catch {\n          if (typeof _args === 'string') {\n            args = { input: _args };\n          }\n        }\n\n        tool_call.args = args;\n        if (!lastAIMessage.tool_calls) {\n          lastAIMessage.tool_calls = [];\n        }\n        lastAIMessage.tool_calls.push(tool_call as ToolCall);\n\n        formattedMessages.push(\n          new ToolMessage({\n            tool_call_id: tool_call.id ?? '',\n            name: tool_call.name,\n            content: output || '',\n          })\n        );\n      } else if (part.type === ContentTypes.THINK) {\n        hasReasoning = true;\n        continue;\n      } else if (\n        part.type === ContentTypes.ERROR ||\n        part.type === ContentTypes.AGENT_UPDATE\n      ) {\n        continue;\n      } else {\n        currentContent.push(part);\n      }\n    }\n  }\n\n  if (hasReasoning && currentContent.length > 0) {\n    const content = currentContent\n      .reduce((acc, curr) => {\n        if (curr.type === ContentTypes.TEXT) {\n          return `${acc}${curr[ContentTypes.TEXT] || ''}\\n`;\n        }\n        return acc;\n      }, '')\n      .trim();\n\n    if (content) {\n      formattedMessages.push(new AIMessage({ content }));\n    }\n  } else if (currentContent.length > 0) {\n    formattedMessages.push(new AIMessage({ content: currentContent }));\n  }\n\n  return formattedMessages;\n}\n\n/**\n * Formats an array of messages for LangChain, handling tool calls and creating ToolMessage instances.\n *\n * @param {TPayload} payload - The array of messages to format.\n * @param {Record<number, number>} [indexTokenCountMap] - Optional map of message indices to token counts.\n * @param {Set<string>} [tools] - Optional set of tool names that are allowed in the request.\n * @returns {Object} - Object containing formatted messages and updated indexTokenCountMap if provided.\n */\nexport const formatAgentMessages = (\n  payload: TPayload,\n  indexTokenCountMap?: Record<number, number>,\n  tools?: Set<string>\n): {\n  messages: Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>;\n  indexTokenCountMap?: Record<number, number>;\n} => {\n  const messages: Array<\n    HumanMessage | AIMessage | SystemMessage | ToolMessage\n  > = [];\n  // If indexTokenCountMap is provided, create a new map to track the updated indices\n  const updatedIndexTokenCountMap: Record<number, number> = {};\n  // Keep track of the mapping from original payload indices to result indices\n  const indexMapping: Record<number, number[]> = {};\n\n  // Process messages with tool conversion if tools set is provided\n  for (let i = 0; i < payload.length; i++) {\n    const message = payload[i];\n    // Q: Store the current length of messages to track where this payload message starts in the result?\n    // const startIndex = messages.length;\n    if (typeof message.content === 'string') {\n      message.content = [\n        { type: ContentTypes.TEXT, [ContentTypes.TEXT]: message.content },\n      ];\n    }\n    if (message.role !== 'assistant') {\n      messages.push(\n        formatMessage({\n          message: message as MessageInput,\n          langChain: true,\n        }) as HumanMessage | AIMessage | SystemMessage\n      );\n\n      // Update the index mapping for this message\n      indexMapping[i] = [messages.length - 1];\n      continue;\n    }\n\n    // For assistant messages, track the starting index before processing\n    const startMessageIndex = messages.length;\n\n    // If tools set is provided, we need to check if we need to convert tool messages to a string\n    if (tools) {\n      // First, check if this message contains tool calls\n      let hasToolCalls = false;\n      let hasInvalidTool = false;\n      const toolNames: string[] = [];\n\n      const content = message.content;\n      if (content && Array.isArray(content)) {\n        for (const part of content) {\n          if (part.type === ContentTypes.TOOL_CALL) {\n            hasToolCalls = true;\n            if (tools.size === 0) {\n              hasInvalidTool = true;\n              break;\n            }\n            const toolName = part.tool_call.name;\n            toolNames.push(toolName);\n            if (!tools.has(toolName)) {\n              hasInvalidTool = true;\n            }\n          }\n        }\n      }\n\n      // If this message has tool calls and at least one is invalid, we need to convert it\n      if (hasToolCalls && hasInvalidTool) {\n        // We need to collect all related messages (this message and any subsequent tool messages)\n        const toolSequence: BaseMessage[] = [];\n        let sequenceEndIndex = i;\n\n        // Process the current assistant message to get the AIMessage with tool calls\n        const formattedMessages = formatAssistantMessage(message);\n        toolSequence.push(...formattedMessages);\n\n        // Look ahead for any subsequent assistant messages that might be part of this tool sequence\n        let j = i + 1;\n        while (j < payload.length && payload[j].role === 'assistant') {\n          // Check if this is a continuation of the tool sequence\n          let isToolResponse = false;\n          const content = payload[j].content;\n          if (content && Array.isArray(content)) {\n            for (const part of content) {\n              if (part.type === ContentTypes.TOOL_CALL) {\n                isToolResponse = true;\n                break;\n              }\n            }\n          }\n\n          if (isToolResponse) {\n            // This is part of the tool sequence, add it\n            const nextMessages = formatAssistantMessage(payload[j]);\n            toolSequence.push(...nextMessages);\n            sequenceEndIndex = j;\n            j++;\n          } else {\n            // This is not part of the tool sequence, stop looking\n            break;\n          }\n        }\n\n        // Convert the sequence to a string\n        const bufferString = getBufferString(toolSequence);\n        messages.push(new AIMessage({ content: bufferString }));\n\n        // Skip the messages we've already processed\n        i = sequenceEndIndex;\n\n        // Update the index mapping for this sequence\n        const resultIndices = [messages.length - 1];\n        for (let k = i; k >= i && k <= sequenceEndIndex; k++) {\n          indexMapping[k] = resultIndices;\n        }\n\n        continue;\n      }\n    }\n\n    // Process the assistant message using the helper function\n    const formattedMessages = formatAssistantMessage(message);\n    messages.push(...formattedMessages);\n\n    // Update the index mapping for this assistant message\n    // Store all indices that were created from this original message\n    const endMessageIndex = messages.length;\n    const resultIndices = [];\n    for (let j = startMessageIndex; j < endMessageIndex; j++) {\n      resultIndices.push(j);\n    }\n    indexMapping[i] = resultIndices;\n  }\n\n  // Update the token count map if it was provided\n  if (indexTokenCountMap) {\n    for (\n      let originalIndex = 0;\n      originalIndex < payload.length;\n      originalIndex++\n    ) {\n      const resultIndices = indexMapping[originalIndex] || [];\n      const tokenCount = indexTokenCountMap[originalIndex];\n\n      if (tokenCount !== undefined) {\n        if (resultIndices.length === 1) {\n          // Simple 1:1 mapping\n          updatedIndexTokenCountMap[resultIndices[0]] = tokenCount;\n        } else if (resultIndices.length > 1) {\n          // If one message was split into multiple, distribute the token count\n          // This is a simplification - in reality, you might want a more sophisticated distribution\n          const countPerMessage = Math.floor(tokenCount / resultIndices.length);\n          resultIndices.forEach((resultIndex, idx) => {\n            if (idx === resultIndices.length - 1) {\n              // Give any remainder to the last message\n              updatedIndexTokenCountMap[resultIndex] =\n                tokenCount - countPerMessage * (resultIndices.length - 1);\n            } else {\n              updatedIndexTokenCountMap[resultIndex] = countPerMessage;\n            }\n          });\n        }\n      }\n    }\n  }\n\n  return {\n    messages,\n    indexTokenCountMap: indexTokenCountMap\n      ? updatedIndexTokenCountMap\n      : undefined,\n  };\n};\n\n/**\n * Formats an array of messages for LangChain, making sure all content fields are strings\n * @param {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} payload - The array of messages to format.\n * @returns {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} - The array of formatted LangChain messages, including ToolMessages for tool calls.\n */\nexport const formatContentStrings = (\n  payload: Array<BaseMessage>\n): Array<BaseMessage> => {\n  // Create a copy of the payload to avoid modifying the original\n  const result = [...payload];\n\n  for (const message of result) {\n    if (typeof message.content === 'string') {\n      continue;\n    }\n\n    if (!Array.isArray(message.content)) {\n      continue;\n    }\n\n    // Reduce text types to a single string, ignore all other types\n    const content = message.content.reduce((acc, curr) => {\n      if (curr.type === ContentTypes.TEXT) {\n        return `${acc}${curr[ContentTypes.TEXT] || ''}\\n`;\n      }\n      return acc;\n    }, '');\n\n    message.content = content.trim();\n  }\n\n  return result;\n};\n\n/**\n * Adds a value at key 0 for system messages and shifts all key indices by one in an indexTokenCountMap.\n * This is useful when adding a system message at the beginning of a conversation.\n *\n * @param indexTokenCountMap - The original map of message indices to token counts\n * @param instructionsTokenCount - The token count for the system message to add at index 0\n * @returns A new map with the system message at index 0 and all other indices shifted by 1\n */\nexport function shiftIndexTokenCountMap(\n  indexTokenCountMap: Record<number, number>,\n  instructionsTokenCount: number\n): Record<number, number> {\n  // Create a new map to avoid modifying the original\n  const shiftedMap: Record<number, number> = {};\n  shiftedMap[0] = instructionsTokenCount;\n\n  // Shift all existing indices by 1\n  for (const [indexStr, tokenCount] of Object.entries(indexTokenCountMap)) {\n    const index = Number(indexStr);\n    shiftedMap[index + 1] = tokenCount;\n  }\n\n  return shiftedMap;\n}\n"],"names":[],"mappings":";;;AAAA;AA8BA;;;;;AAKG;AACI,MAAM,mBAAmB,GAAG,CAAC,EAClC,OAAO,EACP,UAAU,EACV,QAAQ,GACY,KAKlB;;AAEF,IAAA,MAAM,MAAM,GAKR;AACF,QAAA,GAAG,OAAO;AACV,QAAA,OAAO,EAAE,EAA6B;KACvC;AAED,IAAA,IAAI,QAAQ,KAAK,SAAS,CAAC,SAAS,EAAE;QACpC,MAAM,CAAC,OAAO,GAAG;AACf,YAAA,GAAG,UAAU;YACb,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE;SACxB;AAC5B,QAAA,OAAO,MAAM;;IAGf,MAAM,CAAC,OAAO,GAAG;QACf,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE;AAClD,QAAA,GAAG,UAAU;KACa;AAE5B,IAAA,OAAO,MAAM;AACf;AA4BA;;;;;AAKG;AACU,MAAA,aAAa,GAAG,CAAC,EAC5B,OAAO,EACP,QAAQ,EACR,aAAa,EACb,QAAQ,EACR,SAAS,GAAG,KAAK,GACG,KAIF;;AAElB,IAAA,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,OAAO;IAC5E,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE;AACnC,QAAA,MAAM,WAAW,GAA2B;AAC1C,YAAA,aAAa,EAAE,QAAQ;AACvB,YAAA,YAAY,EAAE,MAAM;AACpB,YAAA,SAAS,EAAE,WAAW;SACvB;QACD,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK;;IAExC,MAAM,IAAI,GACR,KAAK;SACJ,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK;AACpD,cAAE;cACA,WAAW,CAAC;AAClB,IAAA,MAAM,OAAO,GAAG,QAAQ,IAAI,IAAI,IAAI,EAAE;AACtC,IAAA,MAAM,gBAAgB,GAAqB;QACzC,IAAI;QACJ,OAAO;KACR;AAED,IAAA,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO;AAC9B,IAAA,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,KAAK,MAAM,EAAE;AACzE,QAAA,OAAO,mBAAmB,CAAC;AACzB,YAAA,OAAO,EAAE;AACP,gBAAA,GAAG,gBAAgB;AACnB,gBAAA,OAAO,EACL,OAAO,gBAAgB,CAAC,OAAO,KAAK;sBAChC,gBAAgB,CAAC;AACnB,sBAAE,EAAE;AACT,aAAA;YACD,UAAU;YACV,QAAQ;AACT,SAAA,CAAC;;AAGJ,IAAA,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,EAAE;AAC1B,QAAA,gBAAgB,CAAC,IAAI,GAAG,KAAK;;AAG/B,IAAA,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,IAAI,gBAAgB,CAAC,IAAI,KAAK,MAAM,EAAE;AACpE,QAAA,gBAAgB,CAAC,IAAI,GAAG,QAAQ;;IAGlC,IACE,aAAa,IAAI,IAAI;QACrB,aAAa;AACb,QAAA,gBAAgB,CAAC,IAAI,KAAK,WAAW,EACrC;AACA,QAAA,gBAAgB,CAAC,IAAI,GAAG,aAAa;;IAGvC,IAAI,gBAAgB,CAAC,IAAI,IAAI,IAAI,IAAI,gBAAgB,CAAC,IAAI,EAAE;;;AAG1D,QAAA,gBAAgB,CAAC,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CACnD,iBAAiB,EACjB,GAAG,CACJ;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE;AACrC,YAAA,gBAAgB,CAAC,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC;;;IAIlE,IAAI,CAAC,SAAS,EAAE;AACd,QAAA,OAAO,gBAAgB;;AAGzB,IAAA,IAAI,IAAI,KAAK,MAAM,EAAE;AACnB,QAAA,OAAO,IAAI,YAAY,CAAC,gBAAgB,CAAC;;AACpC,SAAA,IAAI,IAAI,KAAK,WAAW,EAAE;AAC/B,QAAA,OAAO,IAAI,SAAS,CAAC,gBAAgB,CAAC;;SACjC;AACL,QAAA,OAAO,IAAI,aAAa,CAAC,gBAAgB,CAAC;;AAE9C;AAEA;;;;;;AAMG;MACU,uBAAuB,GAAG,CACrC,QAA6B,EAC7B,aAAiE,KACd;AACnD,IAAA,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,KAAI;QAC1B,MAAM,SAAS,GAAG,aAAa,CAAC;AAC9B,YAAA,GAAG,aAAa;AAChB,YAAA,OAAO,EAAE,GAAG;AACZ,YAAA,SAAS,EAAE,IAAI;AAChB,SAAA,CAAC;AACF,QAAA,OAAO,SAAqD;AAC9D,KAAC,CAAC;AACJ;AAcA;;;;;AAKG;AACU,MAAA,mBAAmB,GAAG,CACjC,OAAyB,KACF;IACvB,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,IAAI,EAAE;IACxD,MAAM,EAAE,iBAAiB,GAAG,EAAE,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM;IAC5D,OAAO;AACL,QAAA,GAAG,cAAc;AACjB,QAAA,GAAG,iBAAiB;KACrB;AACH;AAEA;;;;AAIG;AACH,SAAS,sBAAsB,CAC7B,OAA0B,EAAA;IAE1B,MAAM,iBAAiB,GAAmC,EAAE;IAC5D,IAAI,cAAc,GAA4B,EAAE;IAChD,IAAI,aAAa,GAAqB,IAAI;IAC1C,IAAI,YAAY,GAAG,KAAK;IAExB,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;AAClC,QAAA,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE;AAClC,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE;AACzD;;;AAGE;AACF,gBAAA,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC7B,IAAI,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,KAAI;wBAChD,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE;AACnC,4BAAA,OAAO,CAAG,EAAA,GAAG,CAAG,EAAA,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,EAAA,CAAI;;AAEnD,wBAAA,OAAO,GAAG;qBACX,EAAE,EAAE,CAAC;oBACN,OAAO;AACL,wBAAA,CAAA,EAAG,OAAO,CAAK,EAAA,EAAA,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE;oBACpE,aAAa,GAAG,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;AAC1C,oBAAA,iBAAiB,CAAC,IAAI,CAAC,aAAa,CAAC;oBACrC,cAAc,GAAG,EAAE;oBACnB;;;gBAGF,aAAa,GAAG,IAAI,SAAS,CAAC;AAC5B,oBAAA,OAAO,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;AACzB,iBAAA,CAAC;AACF,gBAAA,iBAAiB,CAAC,IAAI,CAAC,aAAa,CAAC;;iBAChC,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,EAAE;gBAC/C,IAAI,CAAC,aAAa,EAAE;;oBAElB,aAAa,GAAG,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;AAC9C,oBAAA,iBAAiB,CAAC,IAAI,CAAC,aAAa,CAAC;;;AAIvC,gBAAA,MAAM,EACJ,MAAM,EACN,IAAI,EAAE,KAAK,EACX,GAAG,UAAU,EACd,GAAG,IAAI,CAAC,SAAyB;gBAClC,MAAM,SAAS,GAAiB,UAAU;;gBAE1C,IAAI,IAAI,GAAQ,KAAK;AACrB,gBAAA,IAAI;AACF,oBAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AAC7B,wBAAA,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;;;AAE1B,gBAAA,MAAM;AACN,oBAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AAC7B,wBAAA,IAAI,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE;;;AAI3B,gBAAA,SAAS,CAAC,IAAI,GAAG,IAAI;AACrB,gBAAA,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE;AAC7B,oBAAA,aAAa,CAAC,UAAU,GAAG,EAAE;;AAE/B,gBAAA,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,SAAqB,CAAC;AAEpD,gBAAA,iBAAiB,CAAC,IAAI,CACpB,IAAI,WAAW,CAAC;AACd,oBAAA,YAAY,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE;oBAChC,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,OAAO,EAAE,MAAM,IAAI,EAAE;AACtB,iBAAA,CAAC,CACH;;iBACI,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,KAAK,EAAE;gBAC3C,YAAY,GAAG,IAAI;gBACnB;;AACK,iBAAA,IACL,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,KAAK;AAChC,gBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,YAAY,EACvC;gBACA;;iBACK;AACL,gBAAA,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;;;;IAK/B,IAAI,YAAY,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7C,MAAM,OAAO,GAAG;AACb,aAAA,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,KAAI;YACpB,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE;AACnC,gBAAA,OAAO,CAAG,EAAA,GAAG,CAAG,EAAA,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,EAAA,CAAI;;AAEnD,YAAA,OAAO,GAAG;SACX,EAAE,EAAE;AACJ,aAAA,IAAI,EAAE;QAET,IAAI,OAAO,EAAE;YACX,iBAAiB,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;;;AAE/C,SAAA,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;AACpC,QAAA,iBAAiB,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;;AAGpE,IAAA,OAAO,iBAAiB;AAC1B;AAEA;;;;;;;AAOG;AACU,MAAA,mBAAmB,GAAG,CACjC,OAAiB,EACjB,kBAA2C,EAC3C,KAAmB,KAIjB;IACF,MAAM,QAAQ,GAEV,EAAE;;IAEN,MAAM,yBAAyB,GAA2B,EAAE;;IAE5D,MAAM,YAAY,GAA6B,EAAE;;AAGjD,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACvC,QAAA,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;;;AAG1B,QAAA,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE;YACvC,OAAO,CAAC,OAAO,GAAG;AAChB,gBAAA,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE;aAClE;;AAEH,QAAA,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE;AAChC,YAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;AACZ,gBAAA,OAAO,EAAE,OAAuB;AAChC,gBAAA,SAAS,EAAE,IAAI;AAChB,aAAA,CAA6C,CAC/C;;YAGD,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACvC;;;AAIF,QAAA,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM;;QAGzC,IAAI,KAAK,EAAE;;YAET,IAAI,YAAY,GAAG,KAAK;YACxB,IAAI,cAAc,GAAG,KAAK;AAG1B,YAAA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO;YAC/B,IAAI,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;AACrC,gBAAA,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE;oBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,EAAE;wBACxC,YAAY,GAAG,IAAI;AACnB,wBAAA,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE;4BACpB,cAAc,GAAG,IAAI;4BACrB;;AAEF,wBAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI;wBAEpC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;4BACxB,cAAc,GAAG,IAAI;;;;;;AAO7B,YAAA,IAAI,YAAY,IAAI,cAAc,EAAE;;gBAElC,MAAM,YAAY,GAAkB,EAAE;gBACtC,IAAI,gBAAgB,GAAG,CAAC;;AAGxB,gBAAA,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,OAAO,CAAC;AACzD,gBAAA,YAAY,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC;;AAGvC,gBAAA,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AACb,gBAAA,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE;;oBAE5D,IAAI,cAAc,GAAG,KAAK;oBAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;oBAClC,IAAI,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;AACrC,wBAAA,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE;4BAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,EAAE;gCACxC,cAAc,GAAG,IAAI;gCACrB;;;;oBAKN,IAAI,cAAc,EAAE;;wBAElB,MAAM,YAAY,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACvD,wBAAA,YAAY,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;wBAClC,gBAAgB,GAAG,CAAC;AACpB,wBAAA,CAAC,EAAE;;yBACE;;wBAEL;;;;AAKJ,gBAAA,MAAM,YAAY,GAAG,eAAe,CAAC,YAAY,CAAC;AAClD,gBAAA,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;;gBAGvD,CAAC,GAAG,gBAAgB;;gBAGpB,MAAM,aAAa,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;AAC3C,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,gBAAgB,EAAE,CAAC,EAAE,EAAE;AACpD,oBAAA,YAAY,CAAC,CAAC,CAAC,GAAG,aAAa;;gBAGjC;;;;AAKJ,QAAA,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,OAAO,CAAC;AACzD,QAAA,QAAQ,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC;;;AAInC,QAAA,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM;QACvC,MAAM,aAAa,GAAG,EAAE;AACxB,QAAA,KAAK,IAAI,CAAC,GAAG,iBAAiB,EAAE,CAAC,GAAG,eAAe,EAAE,CAAC,EAAE,EAAE;AACxD,YAAA,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;;AAEvB,QAAA,YAAY,CAAC,CAAC,CAAC,GAAG,aAAa;;;IAIjC,IAAI,kBAAkB,EAAE;AACtB,QAAA,KACE,IAAI,aAAa,GAAG,CAAC,EACrB,aAAa,GAAG,OAAO,CAAC,MAAM,EAC9B,aAAa,EAAE,EACf;YACA,MAAM,aAAa,GAAG,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE;AACvD,YAAA,MAAM,UAAU,GAAG,kBAAkB,CAAC,aAAa,CAAC;AAEpD,YAAA,IAAI,UAAU,KAAK,SAAS,EAAE;AAC5B,gBAAA,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE;;oBAE9B,yBAAyB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU;;AACnD,qBAAA,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;;;AAGnC,oBAAA,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC;oBACrE,aAAa,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,GAAG,KAAI;wBACzC,IAAI,GAAG,KAAK,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;;4BAEpC,yBAAyB,CAAC,WAAW,CAAC;gCACpC,UAAU,GAAG,eAAe,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;;6BACtD;AACL,4BAAA,yBAAyB,CAAC,WAAW,CAAC,GAAG,eAAe;;AAE5D,qBAAC,CAAC;;;;;IAMV,OAAO;QACL,QAAQ;AACR,QAAA,kBAAkB,EAAE;AAClB,cAAE;AACF,cAAE,SAAS;KACd;AACH;AAEA;;;;AAIG;AACU,MAAA,oBAAoB,GAAG,CAClC,OAA2B,KACL;;AAEtB,IAAA,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC;AAE3B,IAAA,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE;AAC5B,QAAA,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE;YACvC;;QAGF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;YACnC;;;AAIF,QAAA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,KAAI;YACnD,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE;AACnC,gBAAA,OAAO,CAAG,EAAA,GAAG,CAAG,EAAA,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,EAAA,CAAI;;AAEnD,YAAA,OAAO,GAAG;SACX,EAAE,EAAE,CAAC;AAEN,QAAA,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE;;AAGlC,IAAA,OAAO,MAAM;AACf;AAEA;;;;;;;AAOG;AACa,SAAA,uBAAuB,CACrC,kBAA0C,EAC1C,sBAA8B,EAAA;;IAG9B,MAAM,UAAU,GAA2B,EAAE;AAC7C,IAAA,UAAU,CAAC,CAAC,CAAC,GAAG,sBAAsB;;AAGtC,IAAA,KAAK,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAAE;AACvE,QAAA,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC;AAC9B,QAAA,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,UAAU;;AAGpC,IAAA,OAAO,UAAU;AACnB;;;;"}
+{"version":3,"file":"format.mjs","sources":["../../../src/messages/format.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\nimport {\n  AIMessage,\n  ToolMessage,\n  BaseMessage,\n  HumanMessage,\n  SystemMessage,\n  getBufferString,\n} from '@langchain/core/messages';\nimport type { MessageContentImageUrl } from '@langchain/core/messages';\nimport type { ToolCall } from '@langchain/core/messages/tool';\nimport type {\n  MessageContentComplex,\n  ToolCallPart,\n  TPayload,\n  TMessage,\n} from '@/types';\nimport { Providers, ContentTypes } from '@/common';\n\ninterface MediaMessageParams {\n  message: {\n    role: string;\n    content: string;\n    name?: string;\n    [key: string]: any;\n  };\n  mediaParts: MessageContentComplex[];\n  endpoint?: Providers;\n}\n\n/**\n * Formats a message with media content (images, documents, videos, audios) to API payload format.\n *\n * @param params - The parameters for formatting.\n * @returns - The formatted message.\n */\nexport const formatMediaMessage = ({\n  message,\n  endpoint,\n  mediaParts,\n}: MediaMessageParams): {\n  role: string;\n  content: MessageContentComplex[];\n  name?: string;\n  [key: string]: any;\n} => {\n  // Create a new object to avoid mutating the input\n  const result: {\n    role: string;\n    content: MessageContentComplex[];\n    name?: string;\n    [key: string]: any;\n  } = {\n    ...message,\n    content: [] as MessageContentComplex[],\n  };\n\n  if (endpoint === Providers.ANTHROPIC) {\n    result.content = [\n      ...mediaParts,\n      { type: ContentTypes.TEXT, text: message.content },\n    ] as MessageContentComplex[];\n    return result;\n  }\n\n  result.content = [\n    { type: ContentTypes.TEXT, text: message.content },\n    ...mediaParts,\n  ] as MessageContentComplex[];\n\n  return result;\n};\n\ninterface MessageInput {\n  role?: string;\n  _name?: string;\n  sender?: string;\n  text?: string;\n  content?: string | MessageContentComplex[];\n  image_urls?: MessageContentImageUrl[];\n  documents?: MessageContentComplex[];\n  videos?: MessageContentComplex[];\n  audios?: MessageContentComplex[];\n  lc_id?: string[];\n  [key: string]: any;\n}\n\ninterface FormatMessageParams {\n  message: MessageInput;\n  userName?: string;\n  assistantName?: string;\n  endpoint?: Providers;\n  langChain?: boolean;\n}\n\ninterface FormattedMessage {\n  role: string;\n  content: string | MessageContentComplex[];\n  name?: string;\n  [key: string]: any;\n}\n\n/**\n * Formats a message to OpenAI payload format based on the provided options.\n *\n * @param params - The parameters for formatting.\n * @returns - The formatted message.\n */\nexport const formatMessage = ({\n  message,\n  userName,\n  endpoint,\n  assistantName,\n  langChain = false,\n}: FormatMessageParams):\n  | FormattedMessage\n  | HumanMessage\n  | AIMessage\n  | SystemMessage => {\n  // eslint-disable-next-line prefer-const\n  let { role: _role, _name, sender, text, content: _content, lc_id } = message;\n  if (lc_id && lc_id[2] && !langChain) {\n    const roleMapping: Record<string, string> = {\n      SystemMessage: 'system',\n      HumanMessage: 'user',\n      AIMessage: 'assistant',\n    };\n    _role = roleMapping[lc_id[2]] || _role;\n  }\n  const role =\n    _role ??\n    (sender != null && sender && sender.toLowerCase() === 'user'\n      ? 'user'\n      : 'assistant');\n  const content = _content ?? text ?? '';\n  const formattedMessage: FormattedMessage = {\n    role,\n    content,\n  };\n\n  // Set name fields first\n  if (_name != null && _name) {\n    formattedMessage.name = _name;\n  }\n\n  if (userName != null && userName && formattedMessage.role === 'user') {\n    formattedMessage.name = userName;\n  }\n\n  if (\n    assistantName != null &&\n    assistantName &&\n    formattedMessage.role === 'assistant'\n  ) {\n    formattedMessage.name = assistantName;\n  }\n\n  if (formattedMessage.name != null && formattedMessage.name) {\n    // Conform to API regex: ^[a-zA-Z0-9_-]{1,64}$\n    // https://community.openai.com/t/the-format-of-the-name-field-in-the-documentation-is-incorrect/175684/2\n    formattedMessage.name = formattedMessage.name.replace(\n      /[^a-zA-Z0-9_-]/g,\n      '_'\n    );\n\n    if (formattedMessage.name.length > 64) {\n      formattedMessage.name = formattedMessage.name.substring(0, 64);\n    }\n  }\n\n  const { image_urls, documents, videos, audios } = message;\n  const mediaParts: MessageContentComplex[] = [];\n\n  if (Array.isArray(documents) && documents.length > 0) {\n    mediaParts.push(...documents);\n  }\n\n  if (Array.isArray(videos) && videos.length > 0) {\n    mediaParts.push(...videos);\n  }\n\n  if (Array.isArray(audios) && audios.length > 0) {\n    mediaParts.push(...audios);\n  }\n\n  if (Array.isArray(image_urls) && image_urls.length > 0) {\n    mediaParts.push(...image_urls);\n  }\n\n  if (mediaParts.length > 0 && role === 'user') {\n    const mediaMessage = formatMediaMessage({\n      message: {\n        ...formattedMessage,\n        content:\n          typeof formattedMessage.content === 'string'\n            ? formattedMessage.content\n            : '',\n      },\n      mediaParts,\n      endpoint,\n    });\n\n    if (!langChain) {\n      return mediaMessage;\n    }\n\n    return new HumanMessage(mediaMessage);\n  }\n\n  if (!langChain) {\n    return formattedMessage;\n  }\n\n  if (role === 'user') {\n    return new HumanMessage(formattedMessage);\n  } else if (role === 'assistant') {\n    return new AIMessage(formattedMessage);\n  } else {\n    return new SystemMessage(formattedMessage);\n  }\n};\n\n/**\n * Formats an array of messages for LangChain.\n *\n * @param messages - The array of messages to format.\n * @param formatOptions - The options for formatting each message.\n * @returns - The array of formatted LangChain messages.\n */\nexport const formatLangChainMessages = (\n  messages: Array<MessageInput>,\n  formatOptions: Omit<FormatMessageParams, 'message' | 'langChain'>\n): Array<HumanMessage | AIMessage | SystemMessage> => {\n  return messages.map((msg) => {\n    const formatted = formatMessage({\n      ...formatOptions,\n      message: msg,\n      langChain: true,\n    });\n    return formatted as HumanMessage | AIMessage | SystemMessage;\n  });\n};\n\ninterface LangChainMessage {\n  lc_kwargs?: {\n    additional_kwargs?: Record<string, any>;\n    [key: string]: any;\n  };\n  kwargs?: {\n    additional_kwargs?: Record<string, any>;\n    [key: string]: any;\n  };\n  [key: string]: any;\n}\n\n/**\n * Formats a LangChain message object by merging properties from `lc_kwargs` or `kwargs` and `additional_kwargs`.\n *\n * @param message - The message object to format.\n * @returns - The formatted LangChain message.\n */\nexport const formatFromLangChain = (\n  message: LangChainMessage\n): Record<string, any> => {\n  const kwargs = message.lc_kwargs ?? message.kwargs ?? {};\n  const { additional_kwargs = {}, ...message_kwargs } = kwargs;\n  return {\n    ...message_kwargs,\n    ...additional_kwargs,\n  };\n};\n\n/**\n * Helper function to format an assistant message\n * @param message The message to format\n * @returns Array of formatted messages\n */\nfunction formatAssistantMessage(\n  message: Partial<TMessage>\n): Array<AIMessage | ToolMessage> {\n  const formattedMessages: Array<AIMessage | ToolMessage> = [];\n  let currentContent: MessageContentComplex[] = [];\n  let lastAIMessage: AIMessage | null = null;\n  let hasReasoning = false;\n\n  if (Array.isArray(message.content)) {\n    for (const part of message.content) {\n      if (part.type === ContentTypes.TEXT && part.tool_call_ids) {\n        /*\n        If there's pending content, it needs to be aggregated as a single string to prepare for tool calls.\n        For Anthropic models, the \"tool_calls\" field on a message is only respected if content is a string.\n        */\n        if (currentContent.length > 0) {\n          let content = currentContent.reduce((acc, curr) => {\n            if (curr.type === ContentTypes.TEXT) {\n              return `${acc}${curr[ContentTypes.TEXT] || ''}\\n`;\n            }\n            return acc;\n          }, '');\n          content =\n            `${content}\\n${part[ContentTypes.TEXT] ?? part.text ?? ''}`.trim();\n          lastAIMessage = new AIMessage({ content });\n          formattedMessages.push(lastAIMessage);\n          currentContent = [];\n          continue;\n        }\n        // Create a new AIMessage with this text and prepare for tool calls\n        lastAIMessage = new AIMessage({\n          content: part.text || '',\n        });\n        formattedMessages.push(lastAIMessage);\n      } else if (part.type === ContentTypes.TOOL_CALL) {\n        if (!lastAIMessage) {\n          // \"Heal\" the payload by creating an AIMessage to precede the tool call\n          lastAIMessage = new AIMessage({ content: '' });\n          formattedMessages.push(lastAIMessage);\n        }\n\n        // Note: `tool_calls` list is defined when constructed by `AIMessage` class, and outputs should be excluded from it\n        const {\n          output,\n          args: _args,\n          ..._tool_call\n        } = part.tool_call as ToolCallPart;\n        const tool_call: ToolCallPart = _tool_call;\n        // TODO: investigate; args as dictionary may need to be providers-or-tool-specific\n        let args: any = _args;\n        try {\n          if (typeof _args === 'string') {\n            args = JSON.parse(_args);\n          }\n        } catch {\n          if (typeof _args === 'string') {\n            args = { input: _args };\n          }\n        }\n\n        tool_call.args = args;\n        if (!lastAIMessage.tool_calls) {\n          lastAIMessage.tool_calls = [];\n        }\n        lastAIMessage.tool_calls.push(tool_call as ToolCall);\n\n        formattedMessages.push(\n          new ToolMessage({\n            tool_call_id: tool_call.id ?? '',\n            name: tool_call.name,\n            content: output || '',\n          })\n        );\n      } else if (part.type === ContentTypes.THINK) {\n        hasReasoning = true;\n        continue;\n      } else if (\n        part.type === ContentTypes.ERROR ||\n        part.type === ContentTypes.AGENT_UPDATE\n      ) {\n        continue;\n      } else {\n        currentContent.push(part);\n      }\n    }\n  }\n\n  if (hasReasoning && currentContent.length > 0) {\n    const content = currentContent\n      .reduce((acc, curr) => {\n        if (curr.type === ContentTypes.TEXT) {\n          return `${acc}${curr[ContentTypes.TEXT] || ''}\\n`;\n        }\n        return acc;\n      }, '')\n      .trim();\n\n    if (content) {\n      formattedMessages.push(new AIMessage({ content }));\n    }\n  } else if (currentContent.length > 0) {\n    formattedMessages.push(new AIMessage({ content: currentContent }));\n  }\n\n  return formattedMessages;\n}\n\n/**\n * Formats an array of messages for LangChain, handling tool calls and creating ToolMessage instances.\n *\n * @param payload - The array of messages to format.\n * @param indexTokenCountMap - Optional map of message indices to token counts.\n * @param tools - Optional set of tool names that are allowed in the request.\n * @returns - Object containing formatted messages and updated indexTokenCountMap if provided.\n */\nexport const formatAgentMessages = (\n  payload: TPayload,\n  indexTokenCountMap?: Record<number, number>,\n  tools?: Set<string>\n): {\n  messages: Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>;\n  indexTokenCountMap?: Record<number, number>;\n} => {\n  const messages: Array<\n    HumanMessage | AIMessage | SystemMessage | ToolMessage\n  > = [];\n  // If indexTokenCountMap is provided, create a new map to track the updated indices\n  const updatedIndexTokenCountMap: Record<number, number> = {};\n  // Keep track of the mapping from original payload indices to result indices\n  const indexMapping: Record<number, number[]> = {};\n\n  // Process messages with tool conversion if tools set is provided\n  for (let i = 0; i < payload.length; i++) {\n    const message = payload[i];\n    // Q: Store the current length of messages to track where this payload message starts in the result?\n    // const startIndex = messages.length;\n    if (typeof message.content === 'string') {\n      message.content = [\n        { type: ContentTypes.TEXT, [ContentTypes.TEXT]: message.content },\n      ];\n    }\n    if (message.role !== 'assistant') {\n      messages.push(\n        formatMessage({\n          message: message as MessageInput,\n          langChain: true,\n        }) as HumanMessage | AIMessage | SystemMessage\n      );\n\n      // Update the index mapping for this message\n      indexMapping[i] = [messages.length - 1];\n      continue;\n    }\n\n    // For assistant messages, track the starting index before processing\n    const startMessageIndex = messages.length;\n\n    // If tools set is provided, we need to check if we need to convert tool messages to a string\n    if (tools) {\n      // First, check if this message contains tool calls\n      let hasToolCalls = false;\n      let hasInvalidTool = false;\n      const toolNames: string[] = [];\n\n      const content = message.content;\n      if (content && Array.isArray(content)) {\n        for (const part of content) {\n          if (part.type === ContentTypes.TOOL_CALL) {\n            hasToolCalls = true;\n            if (tools.size === 0) {\n              hasInvalidTool = true;\n              break;\n            }\n            const toolName = part.tool_call.name;\n            toolNames.push(toolName);\n            if (!tools.has(toolName)) {\n              hasInvalidTool = true;\n            }\n          }\n        }\n      }\n\n      // If this message has tool calls and at least one is invalid, we need to convert it\n      if (hasToolCalls && hasInvalidTool) {\n        // We need to collect all related messages (this message and any subsequent tool messages)\n        const toolSequence: BaseMessage[] = [];\n        let sequenceEndIndex = i;\n\n        // Process the current assistant message to get the AIMessage with tool calls\n        const formattedMessages = formatAssistantMessage(message);\n        toolSequence.push(...formattedMessages);\n\n        // Look ahead for any subsequent assistant messages that might be part of this tool sequence\n        let j = i + 1;\n        while (j < payload.length && payload[j].role === 'assistant') {\n          // Check if this is a continuation of the tool sequence\n          let isToolResponse = false;\n          const content = payload[j].content;\n          if (content && Array.isArray(content)) {\n            for (const part of content) {\n              if (part.type === ContentTypes.TOOL_CALL) {\n                isToolResponse = true;\n                break;\n              }\n            }\n          }\n\n          if (isToolResponse) {\n            // This is part of the tool sequence, add it\n            const nextMessages = formatAssistantMessage(payload[j]);\n            toolSequence.push(...nextMessages);\n            sequenceEndIndex = j;\n            j++;\n          } else {\n            // This is not part of the tool sequence, stop looking\n            break;\n          }\n        }\n\n        // Convert the sequence to a string\n        const bufferString = getBufferString(toolSequence);\n        messages.push(new AIMessage({ content: bufferString }));\n\n        // Skip the messages we've already processed\n        i = sequenceEndIndex;\n\n        // Update the index mapping for this sequence\n        const resultIndices = [messages.length - 1];\n        for (let k = i; k >= i && k <= sequenceEndIndex; k++) {\n          indexMapping[k] = resultIndices;\n        }\n\n        continue;\n      }\n    }\n\n    // Process the assistant message using the helper function\n    const formattedMessages = formatAssistantMessage(message);\n    messages.push(...formattedMessages);\n\n    // Update the index mapping for this assistant message\n    // Store all indices that were created from this original message\n    const endMessageIndex = messages.length;\n    const resultIndices = [];\n    for (let j = startMessageIndex; j < endMessageIndex; j++) {\n      resultIndices.push(j);\n    }\n    indexMapping[i] = resultIndices;\n  }\n\n  // Update the token count map if it was provided\n  if (indexTokenCountMap) {\n    for (\n      let originalIndex = 0;\n      originalIndex < payload.length;\n      originalIndex++\n    ) {\n      const resultIndices = indexMapping[originalIndex] || [];\n      const tokenCount = indexTokenCountMap[originalIndex];\n\n      if (tokenCount !== undefined) {\n        if (resultIndices.length === 1) {\n          // Simple 1:1 mapping\n          updatedIndexTokenCountMap[resultIndices[0]] = tokenCount;\n        } else if (resultIndices.length > 1) {\n          // If one message was split into multiple, distribute the token count\n          // This is a simplification - in reality, you might want a more sophisticated distribution\n          const countPerMessage = Math.floor(tokenCount / resultIndices.length);\n          resultIndices.forEach((resultIndex, idx) => {\n            if (idx === resultIndices.length - 1) {\n              // Give any remainder to the last message\n              updatedIndexTokenCountMap[resultIndex] =\n                tokenCount - countPerMessage * (resultIndices.length - 1);\n            } else {\n              updatedIndexTokenCountMap[resultIndex] = countPerMessage;\n            }\n          });\n        }\n      }\n    }\n  }\n\n  return {\n    messages,\n    indexTokenCountMap: indexTokenCountMap\n      ? updatedIndexTokenCountMap\n      : undefined,\n  };\n};\n\n/**\n * Formats an array of messages for LangChain, making sure all content fields are strings\n * @param payload - The array of messages to format.\n * @returns - The array of formatted LangChain messages, including ToolMessages for tool calls.\n */\nexport const formatContentStrings = (\n  payload: Array<BaseMessage>\n): Array<BaseMessage> => {\n  // Create a copy of the payload to avoid modifying the original\n  const result = [...payload];\n\n  for (const message of result) {\n    if (typeof message.content === 'string') {\n      continue;\n    }\n\n    if (!Array.isArray(message.content)) {\n      continue;\n    }\n\n    // Reduce text types to a single string, ignore all other types\n    const content = message.content.reduce((acc, curr) => {\n      if (curr.type === ContentTypes.TEXT) {\n        return `${acc}${curr[ContentTypes.TEXT] || ''}\\n`;\n      }\n      return acc;\n    }, '');\n\n    message.content = content.trim();\n  }\n\n  return result;\n};\n\n/**\n * Adds a value at key 0 for system messages and shifts all key indices by one in an indexTokenCountMap.\n * This is useful when adding a system message at the beginning of a conversation.\n *\n * @param indexTokenCountMap - The original map of message indices to token counts\n * @param instructionsTokenCount - The token count for the system message to add at index 0\n * @returns A new map with the system message at index 0 and all other indices shifted by 1\n */\nexport function shiftIndexTokenCountMap(\n  indexTokenCountMap: Record<number, number>,\n  instructionsTokenCount: number\n): Record<number, number> {\n  // Create a new map to avoid modifying the original\n  const shiftedMap: Record<number, number> = {};\n  shiftedMap[0] = instructionsTokenCount;\n\n  // Shift all existing indices by 1\n  for (const [indexStr, tokenCount] of Object.entries(indexTokenCountMap)) {\n    const index = Number(indexStr);\n    shiftedMap[index + 1] = tokenCount;\n  }\n\n  return shiftedMap;\n}\n"],"names":[],"mappings":";;;AAAA;AA8BA;;;;;AAKG;AACI,MAAM,kBAAkB,GAAG,CAAC,EACjC,OAAO,EACP,QAAQ,EACR,UAAU,GACS,KAKjB;;AAEF,IAAA,MAAM,MAAM,GAKR;AACF,QAAA,GAAG,OAAO;AACV,QAAA,OAAO,EAAE,EAA6B;KACvC;AAED,IAAA,IAAI,QAAQ,KAAK,SAAS,CAAC,SAAS,EAAE;QACpC,MAAM,CAAC,OAAO,GAAG;AACf,YAAA,GAAG,UAAU;YACb,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE;SACxB;AAC5B,QAAA,OAAO,MAAM;;IAGf,MAAM,CAAC,OAAO,GAAG;QACf,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE;AAClD,QAAA,GAAG,UAAU;KACa;AAE5B,IAAA,OAAO,MAAM;AACf;AA+BA;;;;;AAKG;AACU,MAAA,aAAa,GAAG,CAAC,EAC5B,OAAO,EACP,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,SAAS,GAAG,KAAK,GACG,KAIF;;AAElB,IAAA,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,OAAO;IAC5E,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE;AACnC,QAAA,MAAM,WAAW,GAA2B;AAC1C,YAAA,aAAa,EAAE,QAAQ;AACvB,YAAA,YAAY,EAAE,MAAM;AACpB,YAAA,SAAS,EAAE,WAAW;SACvB;QACD,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK;;IAExC,MAAM,IAAI,GACR,KAAK;SACJ,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK;AACpD,cAAE;cACA,WAAW,CAAC;AAClB,IAAA,MAAM,OAAO,GAAG,QAAQ,IAAI,IAAI,IAAI,EAAE;AACtC,IAAA,MAAM,gBAAgB,GAAqB;QACzC,IAAI;QACJ,OAAO;KACR;;AAGD,IAAA,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,EAAE;AAC1B,QAAA,gBAAgB,CAAC,IAAI,GAAG,KAAK;;AAG/B,IAAA,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,IAAI,gBAAgB,CAAC,IAAI,KAAK,MAAM,EAAE;AACpE,QAAA,gBAAgB,CAAC,IAAI,GAAG,QAAQ;;IAGlC,IACE,aAAa,IAAI,IAAI;QACrB,aAAa;AACb,QAAA,gBAAgB,CAAC,IAAI,KAAK,WAAW,EACrC;AACA,QAAA,gBAAgB,CAAC,IAAI,GAAG,aAAa;;IAGvC,IAAI,gBAAgB,CAAC,IAAI,IAAI,IAAI,IAAI,gBAAgB,CAAC,IAAI,EAAE;;;AAG1D,QAAA,gBAAgB,CAAC,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CACnD,iBAAiB,EACjB,GAAG,CACJ;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE;AACrC,YAAA,gBAAgB,CAAC,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC;;;IAIlE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO;IACzD,MAAM,UAAU,GAA4B,EAAE;AAE9C,IAAA,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;AACpD,QAAA,UAAU,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC;;AAG/B,IAAA,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;AAC9C,QAAA,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;;AAG5B,IAAA,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;AAC9C,QAAA,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;;AAG5B,IAAA,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;AACtD,QAAA,UAAU,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;;IAGhC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,KAAK,MAAM,EAAE;QAC5C,MAAM,YAAY,GAAG,kBAAkB,CAAC;AACtC,YAAA,OAAO,EAAE;AACP,gBAAA,GAAG,gBAAgB;AACnB,gBAAA,OAAO,EACL,OAAO,gBAAgB,CAAC,OAAO,KAAK;sBAChC,gBAAgB,CAAC;AACnB,sBAAE,EAAE;AACT,aAAA;YACD,UAAU;YACV,QAAQ;AACT,SAAA,CAAC;QAEF,IAAI,CAAC,SAAS,EAAE;AACd,YAAA,OAAO,YAAY;;AAGrB,QAAA,OAAO,IAAI,YAAY,CAAC,YAAY,CAAC;;IAGvC,IAAI,CAAC,SAAS,EAAE;AACd,QAAA,OAAO,gBAAgB;;AAGzB,IAAA,IAAI,IAAI,KAAK,MAAM,EAAE;AACnB,QAAA,OAAO,IAAI,YAAY,CAAC,gBAAgB,CAAC;;AACpC,SAAA,IAAI,IAAI,KAAK,WAAW,EAAE;AAC/B,QAAA,OAAO,IAAI,SAAS,CAAC,gBAAgB,CAAC;;SACjC;AACL,QAAA,OAAO,IAAI,aAAa,CAAC,gBAAgB,CAAC;;AAE9C;AAEA;;;;;;AAMG;MACU,uBAAuB,GAAG,CACrC,QAA6B,EAC7B,aAAiE,KACd;AACnD,IAAA,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,KAAI;QAC1B,MAAM,SAAS,GAAG,aAAa,CAAC;AAC9B,YAAA,GAAG,aAAa;AAChB,YAAA,OAAO,EAAE,GAAG;AACZ,YAAA,SAAS,EAAE,IAAI;AAChB,SAAA,CAAC;AACF,QAAA,OAAO,SAAqD;AAC9D,KAAC,CAAC;AACJ;AAcA;;;;;AAKG;AACU,MAAA,mBAAmB,GAAG,CACjC,OAAyB,KACF;IACvB,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,IAAI,EAAE;IACxD,MAAM,EAAE,iBAAiB,GAAG,EAAE,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM;IAC5D,OAAO;AACL,QAAA,GAAG,cAAc;AACjB,QAAA,GAAG,iBAAiB;KACrB;AACH;AAEA;;;;AAIG;AACH,SAAS,sBAAsB,CAC7B,OAA0B,EAAA;IAE1B,MAAM,iBAAiB,GAAmC,EAAE;IAC5D,IAAI,cAAc,GAA4B,EAAE;IAChD,IAAI,aAAa,GAAqB,IAAI;IAC1C,IAAI,YAAY,GAAG,KAAK;IAExB,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;AAClC,QAAA,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE;AAClC,YAAA,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE;AACzD;;;AAGE;AACF,gBAAA,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC7B,IAAI,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,KAAI;wBAChD,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE;AACnC,4BAAA,OAAO,CAAG,EAAA,GAAG,CAAG,EAAA,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,EAAA,CAAI;;AAEnD,wBAAA,OAAO,GAAG;qBACX,EAAE,EAAE,CAAC;oBACN,OAAO;AACL,wBAAA,CAAA,EAAG,OAAO,CAAK,EAAA,EAAA,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE;oBACpE,aAAa,GAAG,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;AAC1C,oBAAA,iBAAiB,CAAC,IAAI,CAAC,aAAa,CAAC;oBACrC,cAAc,GAAG,EAAE;oBACnB;;;gBAGF,aAAa,GAAG,IAAI,SAAS,CAAC;AAC5B,oBAAA,OAAO,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;AACzB,iBAAA,CAAC;AACF,gBAAA,iBAAiB,CAAC,IAAI,CAAC,aAAa,CAAC;;iBAChC,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,EAAE;gBAC/C,IAAI,CAAC,aAAa,EAAE;;oBAElB,aAAa,GAAG,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;AAC9C,oBAAA,iBAAiB,CAAC,IAAI,CAAC,aAAa,CAAC;;;AAIvC,gBAAA,MAAM,EACJ,MAAM,EACN,IAAI,EAAE,KAAK,EACX,GAAG,UAAU,EACd,GAAG,IAAI,CAAC,SAAyB;gBAClC,MAAM,SAAS,GAAiB,UAAU;;gBAE1C,IAAI,IAAI,GAAQ,KAAK;AACrB,gBAAA,IAAI;AACF,oBAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AAC7B,wBAAA,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;;;AAE1B,gBAAA,MAAM;AACN,oBAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AAC7B,wBAAA,IAAI,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE;;;AAI3B,gBAAA,SAAS,CAAC,IAAI,GAAG,IAAI;AACrB,gBAAA,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE;AAC7B,oBAAA,aAAa,CAAC,UAAU,GAAG,EAAE;;AAE/B,gBAAA,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,SAAqB,CAAC;AAEpD,gBAAA,iBAAiB,CAAC,IAAI,CACpB,IAAI,WAAW,CAAC;AACd,oBAAA,YAAY,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE;oBAChC,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,OAAO,EAAE,MAAM,IAAI,EAAE;AACtB,iBAAA,CAAC,CACH;;iBACI,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,KAAK,EAAE;gBAC3C,YAAY,GAAG,IAAI;gBACnB;;AACK,iBAAA,IACL,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,KAAK;AAChC,gBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,YAAY,EACvC;gBACA;;iBACK;AACL,gBAAA,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;;;;IAK/B,IAAI,YAAY,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;QAC7C,MAAM,OAAO,GAAG;AACb,aAAA,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,KAAI;YACpB,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE;AACnC,gBAAA,OAAO,CAAG,EAAA,GAAG,CAAG,EAAA,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,EAAA,CAAI;;AAEnD,YAAA,OAAO,GAAG;SACX,EAAE,EAAE;AACJ,aAAA,IAAI,EAAE;QAET,IAAI,OAAO,EAAE;YACX,iBAAiB,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;;;AAE/C,SAAA,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;AACpC,QAAA,iBAAiB,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;;AAGpE,IAAA,OAAO,iBAAiB;AAC1B;AAEA;;;;;;;AAOG;AACU,MAAA,mBAAmB,GAAG,CACjC,OAAiB,EACjB,kBAA2C,EAC3C,KAAmB,KAIjB;IACF,MAAM,QAAQ,GAEV,EAAE;;IAEN,MAAM,yBAAyB,GAA2B,EAAE;;IAE5D,MAAM,YAAY,GAA6B,EAAE;;AAGjD,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACvC,QAAA,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;;;AAG1B,QAAA,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE;YACvC,OAAO,CAAC,OAAO,GAAG;AAChB,gBAAA,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE;aAClE;;AAEH,QAAA,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE;AAChC,YAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;AACZ,gBAAA,OAAO,EAAE,OAAuB;AAChC,gBAAA,SAAS,EAAE,IAAI;AAChB,aAAA,CAA6C,CAC/C;;YAGD,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACvC;;;AAIF,QAAA,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM;;QAGzC,IAAI,KAAK,EAAE;;YAET,IAAI,YAAY,GAAG,KAAK;YACxB,IAAI,cAAc,GAAG,KAAK;AAG1B,YAAA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO;YAC/B,IAAI,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;AACrC,gBAAA,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE;oBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,EAAE;wBACxC,YAAY,GAAG,IAAI;AACnB,wBAAA,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE;4BACpB,cAAc,GAAG,IAAI;4BACrB;;AAEF,wBAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI;wBAEpC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;4BACxB,cAAc,GAAG,IAAI;;;;;;AAO7B,YAAA,IAAI,YAAY,IAAI,cAAc,EAAE;;gBAElC,MAAM,YAAY,GAAkB,EAAE;gBACtC,IAAI,gBAAgB,GAAG,CAAC;;AAGxB,gBAAA,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,OAAO,CAAC;AACzD,gBAAA,YAAY,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC;;AAGvC,gBAAA,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AACb,gBAAA,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE;;oBAE5D,IAAI,cAAc,GAAG,KAAK;oBAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;oBAClC,IAAI,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;AACrC,wBAAA,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE;4BAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,EAAE;gCACxC,cAAc,GAAG,IAAI;gCACrB;;;;oBAKN,IAAI,cAAc,EAAE;;wBAElB,MAAM,YAAY,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACvD,wBAAA,YAAY,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;wBAClC,gBAAgB,GAAG,CAAC;AACpB,wBAAA,CAAC,EAAE;;yBACE;;wBAEL;;;;AAKJ,gBAAA,MAAM,YAAY,GAAG,eAAe,CAAC,YAAY,CAAC;AAClD,gBAAA,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;;gBAGvD,CAAC,GAAG,gBAAgB;;gBAGpB,MAAM,aAAa,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;AAC3C,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,gBAAgB,EAAE,CAAC,EAAE,EAAE;AACpD,oBAAA,YAAY,CAAC,CAAC,CAAC,GAAG,aAAa;;gBAGjC;;;;AAKJ,QAAA,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,OAAO,CAAC;AACzD,QAAA,QAAQ,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC;;;AAInC,QAAA,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM;QACvC,MAAM,aAAa,GAAG,EAAE;AACxB,QAAA,KAAK,IAAI,CAAC,GAAG,iBAAiB,EAAE,CAAC,GAAG,eAAe,EAAE,CAAC,EAAE,EAAE;AACxD,YAAA,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;;AAEvB,QAAA,YAAY,CAAC,CAAC,CAAC,GAAG,aAAa;;;IAIjC,IAAI,kBAAkB,EAAE;AACtB,QAAA,KACE,IAAI,aAAa,GAAG,CAAC,EACrB,aAAa,GAAG,OAAO,CAAC,MAAM,EAC9B,aAAa,EAAE,EACf;YACA,MAAM,aAAa,GAAG,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE;AACvD,YAAA,MAAM,UAAU,GAAG,kBAAkB,CAAC,aAAa,CAAC;AAEpD,YAAA,IAAI,UAAU,KAAK,SAAS,EAAE;AAC5B,gBAAA,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE;;oBAE9B,yBAAyB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU;;AACnD,qBAAA,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;;;AAGnC,oBAAA,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC;oBACrE,aAAa,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,GAAG,KAAI;wBACzC,IAAI,GAAG,KAAK,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;;4BAEpC,yBAAyB,CAAC,WAAW,CAAC;gCACpC,UAAU,GAAG,eAAe,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;;6BACtD;AACL,4BAAA,yBAAyB,CAAC,WAAW,CAAC,GAAG,eAAe;;AAE5D,qBAAC,CAAC;;;;;IAMV,OAAO;QACL,QAAQ;AACR,QAAA,kBAAkB,EAAE;AAClB,cAAE;AACF,cAAE,SAAS;KACd;AACH;AAEA;;;;AAIG;AACU,MAAA,oBAAoB,GAAG,CAClC,OAA2B,KACL;;AAEtB,IAAA,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC;AAE3B,IAAA,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE;AAC5B,QAAA,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE;YACvC;;QAGF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;YACnC;;;AAIF,QAAA,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,KAAI;YACnD,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE;AACnC,gBAAA,OAAO,CAAG,EAAA,GAAG,CAAG,EAAA,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA,EAAA,CAAI;;AAEnD,YAAA,OAAO,GAAG;SACX,EAAE,EAAE,CAAC;AAEN,QAAA,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE;;AAGlC,IAAA,OAAO,MAAM;AACf;AAEA;;;;;;;AAOG;AACa,SAAA,uBAAuB,CACrC,kBAA0C,EAC1C,sBAA8B,EAAA;;IAG9B,MAAM,UAAU,GAA2B,EAAE;AAC7C,IAAA,UAAU,CAAC,CAAC,CAAC,GAAG,sBAAsB;;AAGtC,IAAA,KAAK,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAAE;AACvE,QAAA,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC;AAC9B,QAAA,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,UAAU;;AAGpC,IAAA,OAAO,UAAU;AACnB;;;;"}

package/dist/types/messages/format.d.ts

@@ -2,23 +2,23 @@ import { AIMessage, ToolMessage, BaseMessage, HumanMessage, SystemMessage } from
 import type { MessageContentImageUrl } from '@langchain/core/messages';
 import type { MessageContentComplex, TPayload } from '@/types';
 import { Providers } from '@/common';
-interface VisionMessageParams {
+interface MediaMessageParams {
     message: {
         role: string;
         content: string;
         name?: string;
         [key: string]: any;
     };
-    image_urls: MessageContentImageUrl[];
+    mediaParts: MessageContentComplex[];
     endpoint?: Providers;
 }
 /**
- * Formats a message to OpenAI Vision API payload format.
+ * Formats a message with media content (images, documents, videos, audios) to API payload format.
  *
- * @param {VisionMessageParams} params - The parameters for formatting.
- * @returns {Object} - The formatted message.
+ * @param params - The parameters for formatting.
+ * @returns - The formatted message.
  */
-export declare const formatVisionMessage: ({ message, image_urls, endpoint, }: VisionMessageParams) => {
+export declare const formatMediaMessage: ({ message, endpoint, mediaParts, }: MediaMessageParams) => {
     role: string;
     content: MessageContentComplex[];
     name?: string;
@@ -31,6 +31,9 @@ interface MessageInput {
     text?: string;
     content?: string | MessageContentComplex[];
     image_urls?: MessageContentImageUrl[];
+    documents?: MessageContentComplex[];
+    videos?: MessageContentComplex[];
+    audios?: MessageContentComplex[];
     lc_id?: string[];
     [key: string]: any;
 }
@@ -50,16 +53,16 @@ interface FormattedMessage {
 /**
  * Formats a message to OpenAI payload format based on the provided options.
  *
- * @param {FormatMessageParams} params - The parameters for formatting.
- * @returns {FormattedMessage | HumanMessage | AIMessage | SystemMessage} - The formatted message.
+ * @param params - The parameters for formatting.
+ * @returns - The formatted message.
  */
-export declare const formatMessage: ({ message, userName, assistantName, endpoint, langChain, }: FormatMessageParams) => FormattedMessage | HumanMessage | AIMessage | SystemMessage;
+export declare const formatMessage: ({ message, userName, endpoint, assistantName, langChain, }: FormatMessageParams) => FormattedMessage | HumanMessage | AIMessage | SystemMessage;
 /**
  * Formats an array of messages for LangChain.
  *
- * @param {Array<MessageInput>} messages - The array of messages to format.
- * @param {Omit<FormatMessageParams, 'message' | 'langChain'>} formatOptions - The options for formatting each message.
- * @returns {Array<HumanMessage | AIMessage | SystemMessage>} - The array of formatted LangChain messages.
+ * @param messages - The array of messages to format.
+ * @param formatOptions - The options for formatting each message.
+ * @returns - The array of formatted LangChain messages.
  */
 export declare const formatLangChainMessages: (messages: Array<MessageInput>, formatOptions: Omit<FormatMessageParams, "message" | "langChain">) => Array<HumanMessage | AIMessage | SystemMessage>;
 interface LangChainMessage {
@@ -76,17 +79,17 @@ interface LangChainMessage {
 /**
  * Formats a LangChain message object by merging properties from `lc_kwargs` or `kwargs` and `additional_kwargs`.
  *
- * @param {LangChainMessage} message - The message object to format.
- * @returns {Record<string, any>} The formatted LangChain message.
+ * @param message - The message object to format.
+ * @returns - The formatted LangChain message.
  */
 export declare const formatFromLangChain: (message: LangChainMessage) => Record<string, any>;
 /**
  * Formats an array of messages for LangChain, handling tool calls and creating ToolMessage instances.
  *
- * @param {TPayload} payload - The array of messages to format.
- * @param {Record<number, number>} [indexTokenCountMap] - Optional map of message indices to token counts.
- * @param {Set<string>} [tools] - Optional set of tool names that are allowed in the request.
- * @returns {Object} - Object containing formatted messages and updated indexTokenCountMap if provided.
+ * @param payload - The array of messages to format.
+ * @param indexTokenCountMap - Optional map of message indices to token counts.
+ * @param tools - Optional set of tool names that are allowed in the request.
+ * @returns - Object containing formatted messages and updated indexTokenCountMap if provided.
  */
 export declare const formatAgentMessages: (payload: TPayload, indexTokenCountMap?: Record<number, number>, tools?: Set<string>) => {
     messages: Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>;
@@ -94,8 +97,8 @@ export declare const formatAgentMessages: (payload: TPayload, indexTokenCountMap
 };
 /**
  * Formats an array of messages for LangChain, making sure all content fields are strings
- * @param {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} payload - The array of messages to format.
- * @returns {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} - The array of formatted LangChain messages, including ToolMessages for tool calls.
+ * @param payload - The array of messages to format.
+ * @returns - The array of formatted LangChain messages, including ToolMessages for tool calls.
  */
 export declare const formatContentStrings: (payload: Array<BaseMessage>) => Array<BaseMessage>;
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "2.4.84",
+  "version": "2.4.85",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/llm/google/utils/common.ts

@@ -301,6 +301,20 @@ function _convertLangChainContentToPart(
         mimeType,
       },
     };
+  } else if (
+    content.type === 'document' ||
+    content.type === 'audio' ||
+    content.type === 'video'
+  ) {
+    if (!isMultimodalModel) {
+      throw new Error(`This model does not support ${content.type}s`);
+    }
+    return {
+      inlineData: {
+        data: content.data,
+        mimeType: content.mimeType,
+      },
+    };
   } else if (content.type === 'media') {
     return messageContentMedia(content);
   } else if (content.type === 'tool_use') {