@assistant-ui/react-langchain 0.0.21 → 0.0.23

package/dist/convertMessages.d.ts

@@ -1,6 +1,6 @@
 import { LangChainBaseMessage, UIMessage } from "./types.js";
-import { AppendMessage, MessageTiming } from "@assistant-ui/core";
 import { useExternalMessageConverter } from "@assistant-ui/core/react";
+import { AppendMessage, MessageTiming } from "@assistant-ui/core";
 //#region src/convertMessages.d.ts
 type LangChainMessageConverterMetadata = useExternalMessageConverter.Metadata & {
   uiMessagesByParent?: Map<string, UIMessage[]>;
@@ -12,10 +12,12 @@ declare const getMessageContent: (msg: AppendMessage) => string | ({
   type: "text";
   text: string;
   image_url?: never;
-  url?: never;
+  id?: never;
   mime_type?: never;
+  filename?: never;
   metadata?: never;
   source_type?: never;
+  url?: never;
   data?: never;
 } | {
   type: "image_url";
@@ -23,32 +25,62 @@ declare const getMessageContent: (msg: AppendMessage) => string | ({
     url: string;
   };
   text?: never;
-  url?: never;
+  id?: never;
   mime_type?: never;
+  filename?: never;
   metadata?: never;
   source_type?: never;
+  url?: never;
+  data?: never;
+} | {
+  type: "file";
+  id: string;
+  mime_type: string;
+  filename: string;
+  metadata: {
+    filename: string;
+  };
+  source_type: "id";
+  text?: never;
+  image_url?: never;
+  url?: never;
   data?: never;
 } | {
   type: "file";
   url: string;
   mime_type: string;
+  filename: string;
   metadata: {
     filename: string;
   };
   source_type: "url";
   text?: never;
   image_url?: never;
+  id?: never;
   data?: never;
 } | {
   type: "file";
   data: string;
   mime_type: string;
+  filename: string;
   metadata: {
     filename: string;
   };
   source_type: "base64";
   text?: never;
   image_url?: never;
+  id?: never;
+  url?: never;
+} | {
+  type: "audio";
+  data: string;
+  mime_type: string;
+  source_type: "base64";
+  text?: never;
+  image_url?: never;
+  id?: never;
+  filename?: never;
+  metadata?: never;
   url?: never;
 })[];
 //#endregion

package/dist/convertMessages.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"convertMessages.d.ts","names":[],"sources":["../src/convertMessages.ts"],"mappings":";;;;KAgBK,oCACH,4BAA4B;EAC1B,qBAAqB,YAAY;EACjC,gBAAgB,eAAe;;cAStB,iBAAkB,SAAS;cAyE3B,8BACX,SAAS,sBACT,WAAU,sCACT,4BAA4B;cAyFlB,oBAAqB,KAAK"}
+{"version":3,"file":"convertMessages.d.ts","names":[],"sources":["../src/convertMessages.ts"],"mappings":";;;;KAgBK,oCACH,4BAA4B;EAC1B,qBAAqB,YAAY;EACjC,gBAAgB,eAAe;;cAStB,iBAAkB,SAAS;cAiH3B,8BACX,SAAS,sBACT,WAAU,sCACT,4BAA4B;cA0GlB,oBAAqB,KAAK"}

package/dist/convertMessages.js

@@ -35,8 +35,19 @@ const contentToParts = (content) => {
 				type: "file",
 				filename: part.metadata?.filename ?? "file",
 				data: part.source_type === "url" ? part.url : part.source_type === "id" ? part.id : part.data,
-				mimeType: part.mime_type ?? "application/octet-stream"
+				mimeType: part.mime_type ?? "application/octet-stream",
+				...(part.source_type === "url" || part.source_type === "id") && { sourceType: part.source_type }
 			};
+			case "audio": {
+				const mimeType = part.mime_type ?? "application/octet-stream";
+				const subtype = mimeType.startsWith("audio/") ? mimeType.slice(6) : void 0;
+				return {
+					type: "file",
+					filename: subtype ? `audio.${subtype}` : "audio",
+					data: part.data,
+					mimeType
+				};
+			}
 			case "thinking": return {
 				type: "reasoning",
 				text: part.thinking
@@ -51,6 +62,22 @@ const contentToParts = (content) => {
 		}
 	}).filter((p) => p !== null);
 };
+const hasVisibleText = (text) => typeof text === "string" && text.trim() !== "";
+/**
+* Audio output arrives outside the content array: providers leave `content`
+* empty and put the spoken text in `additional_kwargs.audio.transcript`. The
+* audio bytes stay behind because no provider reports their media type, and a
+* streamed response carries raw PCM rather than a playable file.
+*/
+const withAudioTranscript = (parts, additionalKwargs) => {
+	const transcript = (additionalKwargs?.audio)?.transcript;
+	if (typeof transcript !== "string" || !hasVisibleText(transcript)) return parts;
+	if (parts.some((part) => part.type === "text" && hasVisibleText(part.text))) return parts;
+	return [...parts.filter((part) => part.type !== "text"), {
+		type: "text",
+		text: transcript
+	}];
+};
 const getCustomMetadata = (additionalKwargs) => additionalKwargs?.metadata ?? {};
 const getStringContent = (content) => {
 	if (typeof content === "string") return content;
@@ -88,7 +115,7 @@ const convertLangChainBaseMessage = (message, metadata = {}) => {
 				role: "assistant",
 				id: message.id,
 				content: [
-					...contentToParts(message.content),
+					...withAudioTranscript(contentToParts(message.content), message.additional_kwargs),
 					...toolCallParts,
 					...uiDataParts
 				],
@@ -117,15 +144,28 @@ const convertLangChainBaseMessage = (message, metadata = {}) => {
 		};
 	}
 };
+/**
+* Audio media types that reach a provider's audio input through the LangChain
+* `audio` block. langchain-core derives OpenAI's `input_audio.format` by
+* splitting `mime_type` on `/`, and that format is a wav-or-mp3 enum, so
+* `audio/mpeg` passes the converter and is rejected at the provider.
+*/
+const audioBlockMimeTypes = /* @__PURE__ */ new Map([
+	["audio/mp3", "audio/mp3"],
+	["audio/mpeg", "audio/mp3"],
+	["audio/wav", "audio/wav"],
+	["audio/wave", "audio/wav"],
+	["audio/x-wav", "audio/wav"]
+]);
 const getMessageContent = (msg) => {
 	const allContent = [...msg.content, ...msg.attachments?.flatMap((a) => a.content) ?? []];
-	const hasNonText = allContent.some((part) => part.type === "file" || part.type === "image");
+	const hasNonText = allContent.some((part) => part.type === "file" || part.type === "image" || part.type === "audio");
 	const hasText = allContent.some((part) => part.type === "text");
 	if (hasNonText && !hasText) allContent.unshift({
 		type: "text",
 		text: " "
 	});
-	const content = allContent.map((part) => {
+	const content = allContent.flatMap((part) => {
 		const type = part.type;
 		switch (type) {
 			case "text": return {
@@ -138,22 +178,46 @@ const getMessageContent = (msg) => {
 			};
 			case "file": {
 				const metadata = { filename: part.filename ?? "file" };
-				if (httpUrlPattern.test(part.data)) return {
+				if (part.sourceType === "id") return {
+					type: "file",
+					id: part.data,
+					mime_type: part.mimeType,
+					filename: metadata.filename,
+					metadata,
+					source_type: "id"
+				};
+				if (part.sourceType === "url" || httpUrlPattern.test(part.data)) return {
 					type: "file",
 					url: part.data,
 					mime_type: part.mimeType,
+					filename: metadata.filename,
 					metadata,
 					source_type: "url"
 				};
 				const parsed = parseDataUrl(part.data);
+				const audioMimeType = audioBlockMimeTypes.get((parsed?.mimeType ?? part.mimeType).toLowerCase());
+				if (audioMimeType) return {
+					type: "audio",
+					data: parsed?.data ?? part.data,
+					mime_type: audioMimeType,
+					source_type: "base64"
+				};
 				return {
 					type: "file",
 					data: parsed?.data ?? part.data,
 					mime_type: parsed?.mimeType ?? part.mimeType,
+					filename: metadata.filename,
 					metadata,
 					source_type: "base64"
 				};
 			}
+			case "audio": return {
+				type: "audio",
+				data: parseDataUrl(part.audio.data)?.data ?? part.audio.data,
+				mime_type: `audio/${part.audio.format}`,
+				source_type: "base64"
+			};
+			case "data": return [];
 			case "tool-call": throw new Error("Tool call appends are not supported.");
 			default: throw new Error(`Unsupported append message part type: ${type}`);
 		}

package/dist/convertMessages.js.map

@@ -1 +1 @@
-{"version":3,"file":"convertMessages.js","names":["parts","_exhaustiveCheck"],"sources":["../src/convertMessages.ts"],"sourcesContent":["\"use client\";\n\nimport type { useExternalMessageConverter } from \"@assistant-ui/core/react\";\nimport type {\n  AppendMessage,\n  DataMessagePart,\n  MessageTiming,\n} from \"@assistant-ui/core\";\nimport { httpUrlPattern, parseDataUrl } from \"@assistant-ui/core/internal\";\nimport type { ReadonlyJSONObject } from \"assistant-stream/utils\";\nimport type {\n  LangChainBaseMessage,\n  LangChainContentBlock,\n  UIMessage,\n} from \"./types\";\n\ntype LangChainMessageConverterMetadata =\n  useExternalMessageConverter.Metadata & {\n    uiMessagesByParent?: Map<string, UIMessage[]>;\n    messageTiming?: Record<string, MessageTiming>;\n  };\n\nconst uiMessageToDataPart = (ui: UIMessage): DataMessagePart => ({\n  type: \"data\",\n  name: ui.name,\n  data: ui.props,\n});\n\nexport const getMessageType = (message: LangChainBaseMessage): string => {\n  if (typeof message._getType === \"function\") return message._getType();\n  if (\"type\" in message)\n    return (message as Record<string, unknown>).type as string;\n  throw new Error(\"Cannot determine message type\");\n};\n\nconst contentToParts = (content: unknown) => {\n  if (typeof content === \"string\")\n    return [{ type: \"text\" as const, text: content }];\n\n  const parts = content as readonly LangChainContentBlock[];\n  return parts\n    .map((part) => {\n      const type = part.type;\n      switch (type) {\n        case \"text\":\n        case \"text_delta\":\n          return { type: \"text\" as const, text: part.text };\n        case \"image_url\": {\n          const image =\n            typeof part.image_url === \"string\"\n              ? part.image_url\n              : part.image_url?.url;\n          if (!image) return null;\n          return { type: \"image\" as const, image };\n        }\n        case \"file\":\n          return {\n            type: \"file\" as const,\n            filename: part.metadata?.filename ?? \"file\",\n            data:\n              part.source_type === \"url\"\n                ? part.url\n                : part.source_type === \"id\"\n                  ? part.id\n                  : part.data,\n            mimeType: part.mime_type ?? \"application/octet-stream\",\n          };\n        case \"thinking\":\n          return { type: \"reasoning\" as const, text: part.thinking };\n        case \"reasoning\":\n          return {\n            type: \"reasoning\" as const,\n            text:\n              part.summary?.map((s) => s?.text ?? \"\").join(\"\\n\\n\\n\") ??\n              part.reasoning ??\n              \"\",\n          };\n        case \"tool_use\":\n        case \"input_json_delta\":\n          return null;\n        default:\n          return null;\n      }\n    })\n    .filter((p) => p !== null);\n};\n\nconst getCustomMetadata = (\n  additionalKwargs: Record<string, unknown> | undefined,\n): Record<string, unknown> =>\n  (additionalKwargs?.metadata as Record<string, unknown>) ?? {};\n\nconst getStringContent = (content: unknown): string => {\n  if (typeof content === \"string\") return content;\n  const parts = content as readonly LangChainContentBlock[];\n  return parts\n    .filter((c): c is { type: \"text\"; text: string } => c.type === \"text\")\n    .map((c) => c.text)\n    .join(\"\");\n};\n\nexport const convertLangChainBaseMessage = (\n  message: LangChainBaseMessage,\n  metadata: LangChainMessageConverterMetadata = {},\n): useExternalMessageConverter.Message => {\n  const type = getMessageType(message);\n\n  switch (type) {\n    case \"system\":\n      return {\n        role: \"system\",\n        id: message.id,\n        content: [{ type: \"text\", text: getStringContent(message.content) }],\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n        },\n      };\n\n    case \"human\":\n      return {\n        role: \"user\",\n        id: message.id,\n        content: contentToParts(message.content),\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n        },\n      };\n\n    case \"ai\": {\n      const toolCallParts =\n        message.tool_calls?.map((tc) => ({\n          type: \"tool-call\" as const,\n          toolCallId: tc.id,\n          toolName: tc.name,\n          args: tc.args as ReadonlyJSONObject,\n          argsText: JSON.stringify(tc.args),\n        })) ?? [];\n\n      const assistantStatus =\n        typeof message.status === \"object\" ? message.status : undefined;\n\n      const uiDataParts =\n        (message.id\n          ? metadata.uiMessagesByParent\n              ?.get(message.id)\n              ?.map(uiMessageToDataPart)\n          : undefined) ?? [];\n\n      const timing = metadata.messageTiming?.[message.id ?? \"\"];\n\n      return {\n        role: \"assistant\",\n        id: message.id,\n        content: [\n          ...contentToParts(message.content),\n          ...toolCallParts,\n          ...uiDataParts,\n        ],\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n          ...(timing && { timing }),\n        },\n        ...(assistantStatus && { status: assistantStatus }),\n      };\n    }\n\n    case \"tool\":\n      return {\n        role: \"tool\",\n        toolName: message.name ?? \"\",\n        toolCallId: message.tool_call_id ?? \"\",\n        result: message.content,\n        artifact: message.artifact,\n        isError: message.status === \"error\",\n      };\n\n    default:\n      return {\n        role: \"system\",\n        id: message.id,\n        content: [\n          {\n            type: \"text\",\n            text:\n              typeof message.content === \"string\"\n                ? message.content\n                : JSON.stringify(message.content),\n          },\n        ],\n      };\n  }\n};\n\nexport const getMessageContent = (msg: AppendMessage) => {\n  const allContent = [\n    ...msg.content,\n    ...(msg.attachments?.flatMap((a) => a.content) ?? []),\n  ];\n\n  const hasNonText = allContent.some(\n    (part) => part.type === \"file\" || part.type === \"image\",\n  );\n  const hasText = allContent.some((part) => part.type === \"text\");\n  if (hasNonText && !hasText) {\n    allContent.unshift({ type: \"text\", text: \" \" });\n  }\n\n  const content = allContent.map((part) => {\n    const type = part.type;\n    switch (type) {\n      case \"text\":\n        return { type: \"text\" as const, text: part.text };\n      case \"image\":\n        return { type: \"image_url\" as const, image_url: { url: part.image } };\n      case \"file\": {\n        const metadata = { filename: part.filename ?? \"file\" };\n        if (httpUrlPattern.test(part.data)) {\n          return {\n            type: \"file\" as const,\n            url: part.data,\n            mime_type: part.mimeType,\n            metadata,\n            source_type: \"url\" as const,\n          };\n        }\n        const parsed = parseDataUrl(part.data);\n        return {\n          type: \"file\" as const,\n          data: parsed?.data ?? part.data,\n          mime_type: parsed?.mimeType ?? part.mimeType,\n          metadata,\n          source_type: \"base64\" as const,\n        };\n      }\n      case \"tool-call\":\n        throw new Error(\"Tool call appends are not supported.\");\n      default: {\n        const _exhaustiveCheck:\n          | \"reasoning\"\n          | \"source\"\n          | \"audio\"\n          | \"data\"\n          | \"generative-ui\" = type;\n        throw new Error(\n          `Unsupported append message part type: ${_exhaustiveCheck}`,\n        );\n      }\n    }\n  });\n\n  if (content.length === 1 && content[0]?.type === \"text\") {\n    return content[0].text ?? \"\";\n  }\n  return content;\n};\n"],"mappings":";;;AAsBA,MAAM,uBAAuB,QAAoC;CAC/D,MAAM;CACN,MAAM,GAAG;CACT,MAAM,GAAG;AACX;AAEA,MAAa,kBAAkB,YAA0C;CACvE,IAAI,OAAO,QAAQ,aAAa,YAAY,OAAO,QAAQ,SAAS;CACpE,IAAI,UAAU,SACZ,OAAQ,QAAoC;CAC9C,MAAM,IAAI,MAAM,+BAA+B;AACjD;AAEA,MAAM,kBAAkB,YAAqB;CAC3C,IAAI,OAAO,YAAY,UACrB,OAAO,CAAC;EAAE,MAAM;EAAiB,MAAM;CAAQ,CAAC;CAGlD,OAAOA,QACJ,KAAK,SAAS;EAEb,QADa,KAAK,MAClB;GACE,KAAK;GACL,KAAK,cACH,OAAO;IAAE,MAAM;IAAiB,MAAM,KAAK;GAAK;GAClD,KAAK,aAAa;IAChB,MAAM,QACJ,OAAO,KAAK,cAAc,WACtB,KAAK,YACL,KAAK,WAAW;IACtB,IAAI,CAAC,OAAO,OAAO;IACnB,OAAO;KAAE,MAAM;KAAkB;IAAM;GACzC;GACA,KAAK,QACH,OAAO;IACL,MAAM;IACN,UAAU,KAAK,UAAU,YAAY;IACrC,MACE,KAAK,gBAAgB,QACjB,KAAK,MACL,KAAK,gBAAgB,OACnB,KAAK,KACL,KAAK;IACb,UAAU,KAAK,aAAa;GAC9B;GACF,KAAK,YACH,OAAO;IAAE,MAAM;IAAsB,MAAM,KAAK;GAAS;GAC3D,KAAK,aACH,OAAO;IACL,MAAM;IACN,MACE,KAAK,SAAS,KAAK,MAAM,GAAG,QAAQ,EAAE,CAAC,CAAC,KAAK,QAAQ,KACrD,KAAK,aACL;GACJ;GACF,KAAK;GACL,KAAK,oBACH,OAAO;GACT,SACE,OAAO;EACX;CACF,CAAC,CAAC,CACD,QAAQ,MAAM,MAAM,IAAI;AAC7B;AAEA,MAAM,qBACJ,qBAEC,kBAAkB,YAAwC,CAAC;AAE9D,MAAM,oBAAoB,YAA6B;CACrD,IAAI,OAAO,YAAY,UAAU,OAAO;CAExC,OAAOA,QACJ,QAAQ,MAA2C,EAAE,SAAS,MAAM,CAAC,CACrE,KAAK,MAAM,EAAE,IAAI,CAAC,CAClB,KAAK,EAAE;AACZ;AAEA,MAAa,+BACX,SACA,WAA8C,CAAC,MACP;CAGxC,QAFa,eAAe,OAEjB,GAAX;EACE,KAAK,UACH,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,CAAC;IAAE,MAAM;IAAQ,MAAM,iBAAiB,QAAQ,OAAO;GAAE,CAAC;GACnE,UAAU,EACR,QAAQ,kBAAkB,QAAQ,iBAAiB,EACrD;EACF;EAEF,KAAK,SACH,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,eAAe,QAAQ,OAAO;GACvC,UAAU,EACR,QAAQ,kBAAkB,QAAQ,iBAAiB,EACrD;EACF;EAEF,KAAK,MAAM;GACT,MAAM,gBACJ,QAAQ,YAAY,KAAK,QAAQ;IAC/B,MAAM;IACN,YAAY,GAAG;IACf,UAAU,GAAG;IACb,MAAM,GAAG;IACT,UAAU,KAAK,UAAU,GAAG,IAAI;GAClC,EAAE,KAAK,CAAC;GAEV,MAAM,kBACJ,OAAO,QAAQ,WAAW,WAAW,QAAQ,SAAS,KAAA;GAExD,MAAM,eACH,QAAQ,KACL,SAAS,oBACL,IAAI,QAAQ,EAAE,CAAC,EACf,IAAI,mBAAmB,IAC3B,KAAA,MAAc,CAAC;GAErB,MAAM,SAAS,SAAS,gBAAgB,QAAQ,MAAM;GAEtD,OAAO;IACL,MAAM;IACN,IAAI,QAAQ;IACZ,SAAS;KACP,GAAG,eAAe,QAAQ,OAAO;KACjC,GAAG;KACH,GAAG;IACL;IACA,UAAU;KACR,QAAQ,kBAAkB,QAAQ,iBAAiB;KACnD,GAAI,UAAU,EAAE,OAAO;IACzB;IACA,GAAI,mBAAmB,EAAE,QAAQ,gBAAgB;GACnD;EACF;EAEA,KAAK,QACH,OAAO;GACL,MAAM;GACN,UAAU,QAAQ,QAAQ;GAC1B,YAAY,QAAQ,gBAAgB;GACpC,QAAQ,QAAQ;GAChB,UAAU,QAAQ;GAClB,SAAS,QAAQ,WAAW;EAC9B;EAEF,SACE,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,CACP;IACE,MAAM;IACN,MACE,OAAO,QAAQ,YAAY,WACvB,QAAQ,UACR,KAAK,UAAU,QAAQ,OAAO;GACtC,CACF;EACF;CACJ;AACF;AAEA,MAAa,qBAAqB,QAAuB;CACvD,MAAM,aAAa,CACjB,GAAG,IAAI,SACP,GAAI,IAAI,aAAa,SAAS,MAAM,EAAE,OAAO,KAAK,CAAC,CACrD;CAEA,MAAM,aAAa,WAAW,MAC3B,SAAS,KAAK,SAAS,UAAU,KAAK,SAAS,OAClD;CACA,MAAM,UAAU,WAAW,MAAM,SAAS,KAAK,SAAS,MAAM;CAC9D,IAAI,cAAc,CAAC,SACjB,WAAW,QAAQ;EAAE,MAAM;EAAQ,MAAM;CAAI,CAAC;CAGhD,MAAM,UAAU,WAAW,KAAK,SAAS;EACvC,MAAM,OAAO,KAAK;EAClB,QAAQ,MAAR;GACE,KAAK,QACH,OAAO;IAAE,MAAM;IAAiB,MAAM,KAAK;GAAK;GAClD,KAAK,SACH,OAAO;IAAE,MAAM;IAAsB,WAAW,EAAE,KAAK,KAAK,MAAM;GAAE;GACtE,KAAK,QAAQ;IACX,MAAM,WAAW,EAAE,UAAU,KAAK,YAAY,OAAO;IACrD,IAAI,eAAe,KAAK,KAAK,IAAI,GAC/B,OAAO;KACL,MAAM;KACN,KAAK,KAAK;KACV,WAAW,KAAK;KAChB;KACA,aAAa;IACf;IAEF,MAAM,SAAS,aAAa,KAAK,IAAI;IACrC,OAAO;KACL,MAAM;KACN,MAAM,QAAQ,QAAQ,KAAK;KAC3B,WAAW,QAAQ,YAAY,KAAK;KACpC;KACA,aAAa;IACf;GACF;GACA,KAAK,aACH,MAAM,IAAI,MAAM,sCAAsC;GACxD,SAOE,MAAM,IAAI,MACR,yCAAyCC,MAC3C;EAEJ;CACF,CAAC;CAED,IAAI,QAAQ,WAAW,KAAK,QAAQ,EAAE,EAAE,SAAS,QAC/C,OAAO,QAAQ,EAAE,CAAC,QAAQ;CAE5B,OAAO;AACT"}
+{"version":3,"file":"convertMessages.js","names":["parts","_exhaustiveCheck"],"sources":["../src/convertMessages.ts"],"sourcesContent":["\"use client\";\n\nimport type { useExternalMessageConverter } from \"@assistant-ui/core/react\";\nimport type {\n  AppendMessage,\n  DataMessagePart,\n  MessageTiming,\n} from \"@assistant-ui/core\";\nimport { httpUrlPattern, parseDataUrl } from \"@assistant-ui/core/internal\";\nimport type { ReadonlyJSONObject } from \"assistant-stream/utils\";\nimport type {\n  LangChainBaseMessage,\n  LangChainContentBlock,\n  UIMessage,\n} from \"./types\";\n\ntype LangChainMessageConverterMetadata =\n  useExternalMessageConverter.Metadata & {\n    uiMessagesByParent?: Map<string, UIMessage[]>;\n    messageTiming?: Record<string, MessageTiming>;\n  };\n\nconst uiMessageToDataPart = (ui: UIMessage): DataMessagePart => ({\n  type: \"data\",\n  name: ui.name,\n  data: ui.props,\n});\n\nexport const getMessageType = (message: LangChainBaseMessage): string => {\n  if (typeof message._getType === \"function\") return message._getType();\n  if (\"type\" in message)\n    return (message as Record<string, unknown>).type as string;\n  throw new Error(\"Cannot determine message type\");\n};\n\nconst contentToParts = (content: unknown) => {\n  if (typeof content === \"string\")\n    return [{ type: \"text\" as const, text: content }];\n\n  const parts = content as readonly LangChainContentBlock[];\n  return parts\n    .map((part) => {\n      const type = part.type;\n      switch (type) {\n        case \"text\":\n        case \"text_delta\":\n          return { type: \"text\" as const, text: part.text };\n        case \"image_url\": {\n          const image =\n            typeof part.image_url === \"string\"\n              ? part.image_url\n              : part.image_url?.url;\n          if (!image) return null;\n          return { type: \"image\" as const, image };\n        }\n        case \"file\":\n          return {\n            type: \"file\" as const,\n            filename: part.metadata?.filename ?? \"file\",\n            data:\n              part.source_type === \"url\"\n                ? part.url\n                : part.source_type === \"id\"\n                  ? part.id\n                  : part.data,\n            mimeType: part.mime_type ?? \"application/octet-stream\",\n            ...((part.source_type === \"url\" || part.source_type === \"id\") && {\n              sourceType: part.source_type,\n            }),\n          };\n        case \"audio\": {\n          const mimeType = part.mime_type ?? \"application/octet-stream\";\n          const subtype = mimeType.startsWith(\"audio/\")\n            ? mimeType.slice(\"audio/\".length)\n            : undefined;\n          return {\n            type: \"file\" as const,\n            filename: subtype ? `audio.${subtype}` : \"audio\",\n            data: part.data,\n            mimeType,\n          };\n        }\n        case \"thinking\":\n          return { type: \"reasoning\" as const, text: part.thinking };\n        case \"reasoning\":\n          return {\n            type: \"reasoning\" as const,\n            text:\n              part.summary?.map((s) => s?.text ?? \"\").join(\"\\n\\n\\n\") ??\n              part.reasoning ??\n              \"\",\n          };\n        case \"tool_use\":\n        case \"input_json_delta\":\n          return null;\n        default:\n          return null;\n      }\n    })\n    .filter((p) => p !== null);\n};\n\nconst hasVisibleText = (text: unknown): boolean =>\n  typeof text === \"string\" && text.trim() !== \"\";\n\n/**\n * Audio output arrives outside the content array: providers leave `content`\n * empty and put the spoken text in `additional_kwargs.audio.transcript`. The\n * audio bytes stay behind because no provider reports their media type, and a\n * streamed response carries raw PCM rather than a playable file.\n */\nconst withAudioTranscript = (\n  parts: ReturnType<typeof contentToParts>,\n  additionalKwargs: Record<string, unknown> | undefined,\n): ReturnType<typeof contentToParts> => {\n  const audio = additionalKwargs?.audio as { transcript?: unknown } | undefined;\n  const transcript = audio?.transcript;\n  if (typeof transcript !== \"string\" || !hasVisibleText(transcript))\n    return parts;\n  if (parts.some((part) => part.type === \"text\" && hasVisibleText(part.text)))\n    return parts;\n  return [\n    ...parts.filter((part) => part.type !== \"text\"),\n    { type: \"text\" as const, text: transcript },\n  ];\n};\n\nconst getCustomMetadata = (\n  additionalKwargs: Record<string, unknown> | undefined,\n): Record<string, unknown> =>\n  (additionalKwargs?.metadata as Record<string, unknown>) ?? {};\n\nconst getStringContent = (content: unknown): string => {\n  if (typeof content === \"string\") return content;\n  const parts = content as readonly LangChainContentBlock[];\n  return parts\n    .filter((c): c is { type: \"text\"; text: string } => c.type === \"text\")\n    .map((c) => c.text)\n    .join(\"\");\n};\n\nexport const convertLangChainBaseMessage = (\n  message: LangChainBaseMessage,\n  metadata: LangChainMessageConverterMetadata = {},\n): useExternalMessageConverter.Message => {\n  const type = getMessageType(message);\n\n  switch (type) {\n    case \"system\":\n      return {\n        role: \"system\",\n        id: message.id,\n        content: [{ type: \"text\", text: getStringContent(message.content) }],\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n        },\n      };\n\n    case \"human\":\n      return {\n        role: \"user\",\n        id: message.id,\n        content: contentToParts(message.content),\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n        },\n      };\n\n    case \"ai\": {\n      const toolCallParts =\n        message.tool_calls?.map((tc) => ({\n          type: \"tool-call\" as const,\n          toolCallId: tc.id,\n          toolName: tc.name,\n          args: tc.args as ReadonlyJSONObject,\n          argsText: JSON.stringify(tc.args),\n        })) ?? [];\n\n      const assistantStatus =\n        typeof message.status === \"object\" ? message.status : undefined;\n\n      const uiDataParts =\n        (message.id\n          ? metadata.uiMessagesByParent\n              ?.get(message.id)\n              ?.map(uiMessageToDataPart)\n          : undefined) ?? [];\n\n      const timing = metadata.messageTiming?.[message.id ?? \"\"];\n\n      return {\n        role: \"assistant\",\n        id: message.id,\n        content: [\n          ...withAudioTranscript(\n            contentToParts(message.content),\n            message.additional_kwargs,\n          ),\n          ...toolCallParts,\n          ...uiDataParts,\n        ],\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n          ...(timing && { timing }),\n        },\n        ...(assistantStatus && { status: assistantStatus }),\n      };\n    }\n\n    case \"tool\":\n      return {\n        role: \"tool\",\n        toolName: message.name ?? \"\",\n        toolCallId: message.tool_call_id ?? \"\",\n        result: message.content,\n        artifact: message.artifact,\n        isError: message.status === \"error\",\n      };\n\n    default:\n      return {\n        role: \"system\",\n        id: message.id,\n        content: [\n          {\n            type: \"text\",\n            text:\n              typeof message.content === \"string\"\n                ? message.content\n                : JSON.stringify(message.content),\n          },\n        ],\n      };\n  }\n};\n\n/**\n * Audio media types that reach a provider's audio input through the LangChain\n * `audio` block. langchain-core derives OpenAI's `input_audio.format` by\n * splitting `mime_type` on `/`, and that format is a wav-or-mp3 enum, so\n * `audio/mpeg` passes the converter and is rejected at the provider.\n */\nconst audioBlockMimeTypes = new Map<string, \"audio/mp3\" | \"audio/wav\">([\n  [\"audio/mp3\", \"audio/mp3\"],\n  [\"audio/mpeg\", \"audio/mp3\"],\n  [\"audio/wav\", \"audio/wav\"],\n  [\"audio/wave\", \"audio/wav\"],\n  [\"audio/x-wav\", \"audio/wav\"],\n]);\n\nexport const getMessageContent = (msg: AppendMessage) => {\n  const allContent = [\n    ...msg.content,\n    ...(msg.attachments?.flatMap((a) => a.content) ?? []),\n  ];\n\n  const hasNonText = allContent.some(\n    (part) =>\n      part.type === \"file\" || part.type === \"image\" || part.type === \"audio\",\n  );\n  const hasText = allContent.some((part) => part.type === \"text\");\n  if (hasNonText && !hasText) {\n    allContent.unshift({ type: \"text\", text: \" \" });\n  }\n\n  const content = allContent.flatMap((part) => {\n    const type = part.type;\n    switch (type) {\n      case \"text\":\n        return { type: \"text\" as const, text: part.text };\n      case \"image\":\n        return { type: \"image_url\" as const, image_url: { url: part.image } };\n      case \"file\": {\n        const metadata = { filename: part.filename ?? \"file\" };\n        if (part.sourceType === \"id\") {\n          return {\n            type: \"file\" as const,\n            id: part.data,\n            mime_type: part.mimeType,\n            filename: metadata.filename,\n            metadata,\n            source_type: \"id\" as const,\n          };\n        }\n        if (part.sourceType === \"url\" || httpUrlPattern.test(part.data)) {\n          return {\n            type: \"file\" as const,\n            url: part.data,\n            mime_type: part.mimeType,\n            filename: metadata.filename,\n            metadata,\n            source_type: \"url\" as const,\n          };\n        }\n        const parsed = parseDataUrl(part.data);\n        const audioMimeType = audioBlockMimeTypes.get(\n          (parsed?.mimeType ?? part.mimeType).toLowerCase(),\n        );\n        if (audioMimeType) {\n          return {\n            type: \"audio\" as const,\n            data: parsed?.data ?? part.data,\n            mime_type: audioMimeType,\n            source_type: \"base64\" as const,\n          };\n        }\n        return {\n          type: \"file\" as const,\n          data: parsed?.data ?? part.data,\n          mime_type: parsed?.mimeType ?? part.mimeType,\n          filename: metadata.filename,\n          metadata,\n          source_type: \"base64\" as const,\n        };\n      }\n      case \"audio\": {\n        const parsed = parseDataUrl(part.audio.data);\n        return {\n          type: \"audio\" as const,\n          data: parsed?.data ?? part.audio.data,\n          mime_type: `audio/${part.audio.format}`,\n          source_type: \"base64\" as const,\n        };\n      }\n      case \"data\":\n        return [];\n      case \"tool-call\":\n        throw new Error(\"Tool call appends are not supported.\");\n      default: {\n        const _exhaustiveCheck: \"reasoning\" | \"source\" | \"generative-ui\" = type;\n        throw new Error(\n          `Unsupported append message part type: ${_exhaustiveCheck}`,\n        );\n      }\n    }\n  });\n\n  if (content.length === 1 && content[0]?.type === \"text\") {\n    return content[0].text ?? \"\";\n  }\n  return content;\n};\n"],"mappings":";;;AAsBA,MAAM,uBAAuB,QAAoC;CAC/D,MAAM;CACN,MAAM,GAAG;CACT,MAAM,GAAG;AACX;AAEA,MAAa,kBAAkB,YAA0C;CACvE,IAAI,OAAO,QAAQ,aAAa,YAAY,OAAO,QAAQ,SAAS;CACpE,IAAI,UAAU,SACZ,OAAQ,QAAoC;CAC9C,MAAM,IAAI,MAAM,+BAA+B;AACjD;AAEA,MAAM,kBAAkB,YAAqB;CAC3C,IAAI,OAAO,YAAY,UACrB,OAAO,CAAC;EAAE,MAAM;EAAiB,MAAM;CAAQ,CAAC;CAGlD,OAAOA,QACJ,KAAK,SAAS;EAEb,QADa,KAAK,MAClB;GACE,KAAK;GACL,KAAK,cACH,OAAO;IAAE,MAAM;IAAiB,MAAM,KAAK;GAAK;GAClD,KAAK,aAAa;IAChB,MAAM,QACJ,OAAO,KAAK,cAAc,WACtB,KAAK,YACL,KAAK,WAAW;IACtB,IAAI,CAAC,OAAO,OAAO;IACnB,OAAO;KAAE,MAAM;KAAkB;IAAM;GACzC;GACA,KAAK,QACH,OAAO;IACL,MAAM;IACN,UAAU,KAAK,UAAU,YAAY;IACrC,MACE,KAAK,gBAAgB,QACjB,KAAK,MACL,KAAK,gBAAgB,OACnB,KAAK,KACL,KAAK;IACb,UAAU,KAAK,aAAa;IAC5B,IAAK,KAAK,gBAAgB,SAAS,KAAK,gBAAgB,SAAS,EAC/D,YAAY,KAAK,YACnB;GACF;GACF,KAAK,SAAS;IACZ,MAAM,WAAW,KAAK,aAAa;IACnC,MAAM,UAAU,SAAS,WAAW,QAAQ,IACxC,SAAS,MAAM,CAAe,IAC9B,KAAA;IACJ,OAAO;KACL,MAAM;KACN,UAAU,UAAU,SAAS,YAAY;KACzC,MAAM,KAAK;KACX;IACF;GACF;GACA,KAAK,YACH,OAAO;IAAE,MAAM;IAAsB,MAAM,KAAK;GAAS;GAC3D,KAAK,aACH,OAAO;IACL,MAAM;IACN,MACE,KAAK,SAAS,KAAK,MAAM,GAAG,QAAQ,EAAE,CAAC,CAAC,KAAK,QAAQ,KACrD,KAAK,aACL;GACJ;GACF,KAAK;GACL,KAAK,oBACH,OAAO;GACT,SACE,OAAO;EACX;CACF,CAAC,CAAC,CACD,QAAQ,MAAM,MAAM,IAAI;AAC7B;AAEA,MAAM,kBAAkB,SACtB,OAAO,SAAS,YAAY,KAAK,KAAK,MAAM;;;;;;;AAQ9C,MAAM,uBACJ,OACA,qBACsC;CAEtC,MAAM,cADQ,kBAAkB,MAAA,EACN;CAC1B,IAAI,OAAO,eAAe,YAAY,CAAC,eAAe,UAAU,GAC9D,OAAO;CACT,IAAI,MAAM,MAAM,SAAS,KAAK,SAAS,UAAU,eAAe,KAAK,IAAI,CAAC,GACxE,OAAO;CACT,OAAO,CACL,GAAG,MAAM,QAAQ,SAAS,KAAK,SAAS,MAAM,GAC9C;EAAE,MAAM;EAAiB,MAAM;CAAW,CAC5C;AACF;AAEA,MAAM,qBACJ,qBAEC,kBAAkB,YAAwC,CAAC;AAE9D,MAAM,oBAAoB,YAA6B;CACrD,IAAI,OAAO,YAAY,UAAU,OAAO;CAExC,OAAOA,QACJ,QAAQ,MAA2C,EAAE,SAAS,MAAM,CAAC,CACrE,KAAK,MAAM,EAAE,IAAI,CAAC,CAClB,KAAK,EAAE;AACZ;AAEA,MAAa,+BACX,SACA,WAA8C,CAAC,MACP;CAGxC,QAFa,eAAe,OAEjB,GAAX;EACE,KAAK,UACH,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,CAAC;IAAE,MAAM;IAAQ,MAAM,iBAAiB,QAAQ,OAAO;GAAE,CAAC;GACnE,UAAU,EACR,QAAQ,kBAAkB,QAAQ,iBAAiB,EACrD;EACF;EAEF,KAAK,SACH,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,eAAe,QAAQ,OAAO;GACvC,UAAU,EACR,QAAQ,kBAAkB,QAAQ,iBAAiB,EACrD;EACF;EAEF,KAAK,MAAM;GACT,MAAM,gBACJ,QAAQ,YAAY,KAAK,QAAQ;IAC/B,MAAM;IACN,YAAY,GAAG;IACf,UAAU,GAAG;IACb,MAAM,GAAG;IACT,UAAU,KAAK,UAAU,GAAG,IAAI;GAClC,EAAE,KAAK,CAAC;GAEV,MAAM,kBACJ,OAAO,QAAQ,WAAW,WAAW,QAAQ,SAAS,KAAA;GAExD,MAAM,eACH,QAAQ,KACL,SAAS,oBACL,IAAI,QAAQ,EAAE,CAAC,EACf,IAAI,mBAAmB,IAC3B,KAAA,MAAc,CAAC;GAErB,MAAM,SAAS,SAAS,gBAAgB,QAAQ,MAAM;GAEtD,OAAO;IACL,MAAM;IACN,IAAI,QAAQ;IACZ,SAAS;KACP,GAAG,oBACD,eAAe,QAAQ,OAAO,GAC9B,QAAQ,iBACV;KACA,GAAG;KACH,GAAG;IACL;IACA,UAAU;KACR,QAAQ,kBAAkB,QAAQ,iBAAiB;KACnD,GAAI,UAAU,EAAE,OAAO;IACzB;IACA,GAAI,mBAAmB,EAAE,QAAQ,gBAAgB;GACnD;EACF;EAEA,KAAK,QACH,OAAO;GACL,MAAM;GACN,UAAU,QAAQ,QAAQ;GAC1B,YAAY,QAAQ,gBAAgB;GACpC,QAAQ,QAAQ;GAChB,UAAU,QAAQ;GAClB,SAAS,QAAQ,WAAW;EAC9B;EAEF,SACE,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,CACP;IACE,MAAM;IACN,MACE,OAAO,QAAQ,YAAY,WACvB,QAAQ,UACR,KAAK,UAAU,QAAQ,OAAO;GACtC,CACF;EACF;CACJ;AACF;;;;;;;AAQA,MAAM,sCAAsB,IAAI,IAAuC;CACrE,CAAC,aAAa,WAAW;CACzB,CAAC,cAAc,WAAW;CAC1B,CAAC,aAAa,WAAW;CACzB,CAAC,cAAc,WAAW;CAC1B,CAAC,eAAe,WAAW;AAC7B,CAAC;AAED,MAAa,qBAAqB,QAAuB;CACvD,MAAM,aAAa,CACjB,GAAG,IAAI,SACP,GAAI,IAAI,aAAa,SAAS,MAAM,EAAE,OAAO,KAAK,CAAC,CACrD;CAEA,MAAM,aAAa,WAAW,MAC3B,SACC,KAAK,SAAS,UAAU,KAAK,SAAS,WAAW,KAAK,SAAS,OACnE;CACA,MAAM,UAAU,WAAW,MAAM,SAAS,KAAK,SAAS,MAAM;CAC9D,IAAI,cAAc,CAAC,SACjB,WAAW,QAAQ;EAAE,MAAM;EAAQ,MAAM;CAAI,CAAC;CAGhD,MAAM,UAAU,WAAW,SAAS,SAAS;EAC3C,MAAM,OAAO,KAAK;EAClB,QAAQ,MAAR;GACE,KAAK,QACH,OAAO;IAAE,MAAM;IAAiB,MAAM,KAAK;GAAK;GAClD,KAAK,SACH,OAAO;IAAE,MAAM;IAAsB,WAAW,EAAE,KAAK,KAAK,MAAM;GAAE;GACtE,KAAK,QAAQ;IACX,MAAM,WAAW,EAAE,UAAU,KAAK,YAAY,OAAO;IACrD,IAAI,KAAK,eAAe,MACtB,OAAO;KACL,MAAM;KACN,IAAI,KAAK;KACT,WAAW,KAAK;KAChB,UAAU,SAAS;KACnB;KACA,aAAa;IACf;IAEF,IAAI,KAAK,eAAe,SAAS,eAAe,KAAK,KAAK,IAAI,GAC5D,OAAO;KACL,MAAM;KACN,KAAK,KAAK;KACV,WAAW,KAAK;KAChB,UAAU,SAAS;KACnB;KACA,aAAa;IACf;IAEF,MAAM,SAAS,aAAa,KAAK,IAAI;IACrC,MAAM,gBAAgB,oBAAoB,KACvC,QAAQ,YAAY,KAAK,SAAA,CAAU,YAAY,CAClD;IACA,IAAI,eACF,OAAO;KACL,MAAM;KACN,MAAM,QAAQ,QAAQ,KAAK;KAC3B,WAAW;KACX,aAAa;IACf;IAEF,OAAO;KACL,MAAM;KACN,MAAM,QAAQ,QAAQ,KAAK;KAC3B,WAAW,QAAQ,YAAY,KAAK;KACpC,UAAU,SAAS;KACnB;KACA,aAAa;IACf;GACF;GACA,KAAK,SAEH,OAAO;IACL,MAAM;IACN,MAHa,aAAa,KAAK,MAAM,IAG1B,CAAC,EAAE,QAAQ,KAAK,MAAM;IACjC,WAAW,SAAS,KAAK,MAAM;IAC/B,aAAa;GACf;GAEF,KAAK,QACH,OAAO,CAAC;GACV,KAAK,aACH,MAAM,IAAI,MAAM,sCAAsC;GACxD,SAEE,MAAM,IAAI,MACR,yCAAyCC,MAC3C;EAEJ;CACF,CAAC;CAED,IAAI,QAAQ,WAAW,KAAK,QAAQ,EAAE,EAAE,SAAS,QAC/C,OAAO,QAAQ,EAAE,CAAC,QAAQ;CAE5B,OAAO;AACT"}

package/dist/runtimeExtras.d.ts

@@ -1,6 +1,6 @@
 import { LangChainRuntimeExtras } from "./types.js";
 //#region src/runtimeExtras.d.ts
-declare const langChainExtras: import("@assistant-ui/core/internal").RuntimeExtras<LangChainRuntimeExtras>;
+declare const langChainExtras: import("@assistant-ui/core/react").RuntimeExtras<LangChainRuntimeExtras>;
 //#endregion
 export { langChainExtras };
 //# sourceMappingURL=runtimeExtras.d.ts.map

package/dist/runtimeExtras.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runtimeExtras.d.ts","names":[],"sources":["../src/runtimeExtras.ts"],"mappings":";;cAGa,uDAAe,cAAA"}
+{"version":3,"file":"runtimeExtras.d.ts","names":[],"sources":["../src/runtimeExtras.ts"],"mappings":";;cAGa,oDAAe,cAAA"}

package/dist/runtimeExtras.js

@@ -1,4 +1,4 @@
-import { createRuntimeExtras } from "@assistant-ui/core/internal";
+import { createRuntimeExtras } from "@assistant-ui/core/react";
 //#region src/runtimeExtras.ts
 const langChainExtras = createRuntimeExtras("useStreamRuntime");
 //#endregion

package/dist/runtimeExtras.js.map

@@ -1 +1 @@
-{"version":3,"file":"runtimeExtras.js","names":[],"sources":["../src/runtimeExtras.ts"],"sourcesContent":["import { createRuntimeExtras } from \"@assistant-ui/core/internal\";\nimport type { LangChainRuntimeExtras } from \"./types\";\n\nexport const langChainExtras =\n  createRuntimeExtras<LangChainRuntimeExtras>(\"useStreamRuntime\");\n"],"mappings":";;AAGA,MAAa,kBACX,oBAA4C,kBAAkB"}
+{"version":3,"file":"runtimeExtras.js","names":[],"sources":["../src/runtimeExtras.ts"],"sourcesContent":["import { createRuntimeExtras } from \"@assistant-ui/core/react\";\nimport type { LangChainRuntimeExtras } from \"./types\";\n\nexport const langChainExtras =\n  createRuntimeExtras<LangChainRuntimeExtras>(\"useStreamRuntime\");\n"],"mappings":";;AAGA,MAAa,kBACX,oBAA4C,kBAAkB"}

package/dist/streamingTiming.d.ts

@@ -1,6 +1,6 @@
 import { LangChainBaseMessage } from "./types.js";
-import { MessageTiming } from "@assistant-ui/core";
 import { StreamingTimingAccessors } from "@assistant-ui/core/react";
+import { MessageTiming } from "@assistant-ui/core";
 //#region src/streamingTiming.d.ts
 declare const langChainStreamingTimingAccessors: StreamingTimingAccessors<LangChainBaseMessage>;
 /**

package/dist/types.d.ts

@@ -48,6 +48,11 @@ type LangChainContentBlock = {
   metadata?: {
     filename?: string;
   };
+} | {
+  type: "audio";
+  data: string;
+  mime_type: string;
+  source_type: "base64";
 } | {
   type: "tool_use" | "input_json_delta";
 };

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";;;;;KAoBY;EACN;EAAc;;EACd;EAAoB;;EACpB;EAAmB;IAAsB;;;EACzC;EAAkB;;EAElB;EACA,UAAU;IAAQ;IAAsB;;EACxC;;EAGA;EACA;EACA;EACA;EACA;IAAa;;;EAGb;EACA;EACA;EACA;EACA;IAAa;;;EAGb;EACA;EACA;EACA;EACA;IAAa;;;EAEb;;KAEM;EACV;EACA;EACA,MAAM;;;;;;;;;KAUI,UACV,+BACA,eAAe,0BAA0B;EAEzC;EACA;EACA,MAAM;EACN,OAAO;EACP;IACE;IACA;IACA;IACA;IACA;IACA;KACC;;;;;;;KAQO;EACV;EACA;;;;;;;;;KAUU;EACV;EACA;EACA;EACA;EACA,oBAAoB;;EAEpB,sBAAsB;;EAEtB;;EAEA,SAAS;;EAET;;KAGU;EACV;IAAa;;EACb;IAAuB;IAAa;;EACpC,oBAAoB;EACpB,WAAW,oBAAoB;EAC/B,WAAW,oBAAoB;EAC/B,QAAQ;EACR;EACA,SACE,QAAQ,4CACR,UAAU,4BACP;EACL,UACE,mBACA,UAAU,4BACP;EACL,aACE,eAAe,yBACf,UAAU,4BACP;EACL,QAAQ;EACR;;KAGU,+BAA+B;;;;;;;;;EASzC,qBAAqB;EACrB,QAAQ;EACR;IAEM,cAAc;IACd,SAAS;IACT,YAAY;IACZ,QAAQ;IACR,WAAW;;;;;;;;EASjB;;;;;EAKA;;;;;EAKA,6BAA6B;;EAE7B,gBAAgB;IAAU;;;EAE1B,WAAW,qBAAqB;;;;;;;EAOhC;;KAOU,0BAA0B,gCAA+B,KACjE,UAAU,mBACR,IAAI"}
+{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";;;;;KAoBY;EACN;EAAc;;EACd;EAAoB;;EACpB;EAAmB;IAAsB;;;EACzC;EAAkB;;EAElB;EACA,UAAU;IAAQ;IAAsB;;EACxC;;EAGA;EACA;EACA;EACA;EACA;IAAa;;;EAGb;EACA;EACA;EACA;EACA;IAAa;;;EAGb;EACA;EACA;EACA;EACA;IAAa;;;EAGb;EACA;EACA;EACA;;EAEA;;KAEM;EACV;EACA;EACA,MAAM;;;;;;;;;KAUI,UACV,+BACA,eAAe,0BAA0B;EAEzC;EACA;EACA,MAAM;EACN,OAAO;EACP;IACE;IACA;IACA;IACA;IACA;IACA;KACC;;;;;;;KAQO;EACV;EACA;;;;;;;;;KAUU;EACV;EACA;EACA;EACA;EACA,oBAAoB;;EAEpB,sBAAsB;;EAEtB;;EAEA,SAAS;;EAET;;KAGU;EACV;IAAa;;EACb;IAAuB;IAAa;;EACpC,oBAAoB;EACpB,WAAW,oBAAoB;EAC/B,WAAW,oBAAoB;EAC/B,QAAQ;EACR;EACA,SACE,QAAQ,4CACR,UAAU,4BACP;EACL,UACE,mBACA,UAAU,4BACP;EACL,aACE,eAAe,yBACf,UAAU,4BACP;EACL,QAAQ;EACR;;KAGU,+BAA+B;;;;;;;;;EASzC,qBAAqB;EACrB,QAAQ;EACR;IAEM,cAAc;IACd,SAAS;IACT,YAAY;IACZ,QAAQ;IACR,WAAW;;;;;;;;EASjB;;;;;EAKA;;;;;EAKA,6BAA6B;;EAE7B,gBAAgB;IAAU;;;EAE1B,WAAW,qBAAqB;;;;;;;EAOhC;;KAOU,0BAA0B,gCAA+B,KACjE,UAAU,mBACR,IAAI"}

package/dist/useStreamRuntime.js

@@ -5,9 +5,9 @@ import { foldUIUpdates, mergeUIMessages } from "./uiMessages.js";
 import { resolveForkCheckpoint } from "./resolveForkCheckpoint.js";
 import { useLangChainStreamingTiming } from "./streamingTiming.js";
 import { useAuiState } from "@assistant-ui/store";
+import { useCloudThreadListAdapter, useExternalMessageConverter, useExternalStoreRuntime, useRemoteThreadListRuntime } from "@assistant-ui/core/react";
 import { useEffect, useMemo, useRef, useState } from "react";
 import { generateId, getExternalStoreMessages, pickExternalStoreSharedOptions } from "@assistant-ui/core";
-import { useCloudThreadListAdapter, useExternalMessageConverter, useExternalStoreRuntime, useRemoteThreadListRuntime } from "@assistant-ui/core/react";
 import { STREAM_CONTROLLER, useChannel, useStream } from "@langchain/react";
 //#region src/useStreamRuntime.ts
 const UI_CUSTOM_CHANNELS = ["custom"];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@assistant-ui/react-langchain",
-  "version": "0.0.21",
+  "version": "0.0.23",
   "description": "LangChain useStream adapter for assistant-ui",
   "keywords": [
     "langchain",
@@ -29,10 +29,10 @@
   ],
   "sideEffects": false,
   "dependencies": {
-    "@assistant-ui/core": "^0.3.0",
-    "@assistant-ui/store": "^0.3.0",
+    "@assistant-ui/core": "^0.3.4",
+    "@assistant-ui/store": "^0.3.3",
     "assistant-cloud": "*",
-    "assistant-stream": "^0.3.29"
+    "assistant-stream": "^0.3.32"
   },
   "peerDependencies": {
     "@langchain/react": "^1.0.2",

package/src/convertMessages.test.ts

@@ -87,6 +87,7 @@ describe("convertLangChainBaseMessage file content parts", () => {
         filename: "file.pdf",
         data: "https://r2.example/u/abc/file.pdf",
         mimeType: "application/pdf",
+        sourceType: "url",
       },
     ]);
   });
@@ -103,6 +104,7 @@ describe("convertLangChainBaseMessage file content parts", () => {
         filename: "file",
         data: "file-abc123",
         mimeType: "application/octet-stream",
+        sourceType: "id",
       },
     ]);
   });
@@ -128,6 +130,7 @@ describe("getMessageContent file blocks", () => {
         type: "file",
         data: "ZmFrZQ==",
         mime_type: "application/pdf",
+        filename: "a.pdf",
         metadata: { filename: "a.pdf" },
         source_type: "base64",
       },
@@ -150,6 +153,7 @@ describe("getMessageContent file blocks", () => {
         type: "file",
         url: "https://r2.example/u/abc/file.pdf",
         mime_type: "application/pdf",
+        filename: "file.pdf",
         metadata: { filename: "file.pdf" },
         source_type: "url",
       },
@@ -173,6 +177,7 @@ describe("getMessageContent file blocks", () => {
         type: "file",
         data: "ZmFrZQ==",
         mime_type: "application/pdf",
+        filename: "a.pdf",
         metadata: { filename: "a.pdf" },
         source_type: "base64",
       },
@@ -195,11 +200,509 @@ describe("getMessageContent file blocks", () => {
         type: "file",
         data: "blob:https://app.example/123",
         mime_type: "application/pdf",
+        filename: "a.pdf",
+        metadata: { filename: "a.pdf" },
+        source_type: "base64",
+      },
+    ]);
+  });
+
+  it("emits an id source block with the value in the id key for sourceType id", () => {
+    const content = getMessageContent(
+      appendMessage({
+        type: "file",
+        data: "flx::storage:file_object:abc",
+        mimeType: "application/pdf",
+        filename: "invoice.pdf",
+        sourceType: "id",
+      }),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "file",
+        id: "flx::storage:file_object:abc",
+        mime_type: "application/pdf",
+        filename: "invoice.pdf",
+        metadata: { filename: "invoice.pdf" },
+        source_type: "id",
+      },
+    ]);
+    expect(content[1]).not.toHaveProperty("data");
+  });
+
+  it("lets sourceType url override sniffing for non-http data", () => {
+    const content = getMessageContent(
+      appendMessage({
+        type: "file",
+        data: "s3://bucket/key.pdf",
+        mimeType: "application/pdf",
+        filename: "key.pdf",
+        sourceType: "url",
+      }),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "file",
+        url: "s3://bucket/key.pdf",
+        mime_type: "application/pdf",
+        filename: "key.pdf",
+        metadata: { filename: "key.pdf" },
+        source_type: "url",
+      },
+    ]);
+  });
+
+  it("emits an id source block for attachment content parts", () => {
+    const content = getMessageContent({
+      content: [{ type: "text", text: "see attached" }],
+      attachments: [
+        {
+          content: [
+            {
+              type: "file",
+              data: "file-abc123",
+              mimeType: "application/pdf",
+              filename: "a.pdf",
+              sourceType: "id",
+            },
+          ],
+        },
+      ],
+    } as unknown as AppendMessage);
+
+    expect(content).toEqual([
+      { type: "text", text: "see attached" },
+      {
+        type: "file",
+        id: "file-abc123",
+        mime_type: "application/pdf",
+        filename: "a.pdf",
+        metadata: { filename: "a.pdf" },
+        source_type: "id",
+      },
+    ]);
+  });
+
+  it("round-trips an id source block through both converters", () => {
+    const converted = convertLangChainBaseMessage(
+      humanMessage([
+        {
+          type: "file",
+          id: "file-abc123",
+          mime_type: "application/pdf",
+          source_type: "id",
+          metadata: { filename: "a.pdf" },
+        },
+      ]),
+      {},
+    );
+
+    const content = getMessageContent(converted as unknown as AppendMessage);
+
+    expect(content).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "file",
+        id: "file-abc123",
+        mime_type: "application/pdf",
+        filename: "a.pdf",
         metadata: { filename: "a.pdf" },
+        source_type: "id",
+      },
+    ]);
+  });
+
+  it("emits an audio block for a base64 file part with an audio mime type", () => {
+    const content = getMessageContent(
+      appendMessage({
+        type: "file",
+        data: "c291bmQ=",
+        mimeType: "audio/mp3",
+        filename: "memo.mp3",
+      }),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "audio",
+        data: "c291bmQ=",
+        mime_type: "audio/mp3",
         source_type: "base64",
       },
     ]);
   });
+
+  it("normalizes audio/mpeg and audio/x-wav to the accepted spellings", () => {
+    expect(
+      getMessageContent(
+        appendMessage({
+          type: "file",
+          data: "c291bmQ=",
+          mimeType: "audio/mpeg",
+        }),
+      )[1],
+    ).toMatchObject({ type: "audio", mime_type: "audio/mp3" });
+
+    expect(
+      getMessageContent(
+        appendMessage({
+          type: "file",
+          data: "c291bmQ=",
+          mimeType: "audio/x-wav",
+        }),
+      )[1],
+    ).toMatchObject({ type: "audio", mime_type: "audio/wav" });
+  });
+
+  it("strips the data URL envelope from an audio file part", () => {
+    const content = getMessageContent(
+      appendMessage({
+        type: "file",
+        data: "data:audio/mpeg;base64,c291bmQ=",
+        mimeType: "audio/mp3",
+      }),
+    );
+
+    expect(content[1]).toEqual({
+      type: "audio",
+      data: "c291bmQ=",
+      mime_type: "audio/mp3",
+      source_type: "base64",
+    });
+  });
+
+  it("keeps url and id audio references as file blocks", () => {
+    expect(
+      getMessageContent(
+        appendMessage({
+          type: "file",
+          data: "https://cdn.example.com/memo.mp3",
+          mimeType: "audio/mp3",
+          filename: "memo.mp3",
+        }),
+      )[1],
+    ).toMatchObject({ type: "file", source_type: "url" });
+
+    expect(
+      getMessageContent(
+        appendMessage({
+          type: "file",
+          data: "file-abc123",
+          mimeType: "audio/mp3",
+          filename: "memo.mp3",
+          sourceType: "id",
+        }),
+      )[1],
+    ).toMatchObject({ type: "file", source_type: "id" });
+  });
+
+  it("does not treat inherited object keys as audio media types", () => {
+    for (const mimeType of ["__proto__", "constructor"]) {
+      expect(
+        getMessageContent(
+          appendMessage({
+            type: "file",
+            data: "ZmFrZQ==",
+            mimeType,
+            filename: "a.bin",
+          }),
+        )[1],
+      ).toMatchObject({ type: "file", mime_type: mimeType });
+    }
+  });
+
+  it("detects audio from the data URL envelope when the declared type is generic", () => {
+    expect(
+      getMessageContent(
+        appendMessage({
+          type: "file",
+          data: "data:audio/mpeg;base64,c291bmQ=",
+          mimeType: "application/octet-stream",
+        }),
+      )[1],
+    ).toEqual({
+      type: "audio",
+      data: "c291bmQ=",
+      mime_type: "audio/mp3",
+      source_type: "base64",
+    });
+  });
+
+  it("leaves non-audio file parts as file blocks", () => {
+    expect(
+      getMessageContent(
+        appendMessage({
+          type: "file",
+          data: "ZmFrZQ==",
+          mimeType: "application/pdf",
+          filename: "a.pdf",
+        }),
+      )[1],
+    ).toMatchObject({ type: "file", mime_type: "application/pdf" });
+  });
+});
+
+describe("getMessageContent audio and data parts", () => {
+  const appendMessage = (...parts: Record<string, unknown>[]) =>
+    ({ content: parts }) as unknown as AppendMessage;
+
+  it("emits a base64 audio block with the format MIME type for audio parts", () => {
+    const content = getMessageContent(
+      appendMessage({
+        type: "audio",
+        audio: { data: "c291bmQ=", format: "mp3" },
+      }),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "audio",
+        data: "c291bmQ=",
+        mime_type: "audio/mp3",
+        source_type: "base64",
+      },
+    ]);
+  });
+
+  it("strips a data URL envelope from audio data", () => {
+    const content = getMessageContent(
+      appendMessage({
+        type: "audio",
+        audio: { data: "data:audio/wav;base64,d2F2", format: "wav" },
+      }),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "audio",
+        data: "d2F2",
+        mime_type: "audio/wav",
+        source_type: "base64",
+      },
+    ]);
+  });
+
+  it("keeps the format-derived MIME when a data URL carries a divergent one", () => {
+    const content = getMessageContent(
+      appendMessage({
+        type: "audio",
+        audio: { data: "data:audio/mpeg;base64,c291bmQ=", format: "mp3" },
+      }),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "audio",
+        data: "c291bmQ=",
+        mime_type: "audio/mp3",
+        source_type: "base64",
+      },
+    ]);
+  });
+
+  it("does not prepend a second placeholder when text accompanies audio", () => {
+    const content = getMessageContent(
+      appendMessage(
+        { type: "text", text: "listen" },
+        { type: "audio", audio: { data: "d2F2", format: "wav" } },
+      ),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: "listen" },
+      {
+        type: "audio",
+        data: "d2F2",
+        mime_type: "audio/wav",
+        source_type: "base64",
+      },
+    ]);
+  });
+
+  it("drops data parts while keeping the rest of the message", () => {
+    const content = getMessageContent(
+      appendMessage(
+        { type: "text", text: "hi" },
+        { type: "data", name: "chart", data: { values: [1, 2] } },
+      ),
+    );
+
+    expect(content).toBe("hi");
+  });
+
+  it("returns empty content for a data-only message", () => {
+    const content = getMessageContent(
+      appendMessage({ type: "data", name: "chart", data: { values: [1, 2] } }),
+    );
+
+    expect(content).toEqual([]);
+  });
+
+  it("still throws on assistant-only part types", () => {
+    expect(() =>
+      getMessageContent(appendMessage({ type: "reasoning", text: "hmm" })),
+    ).toThrow("Unsupported append message part type: reasoning");
+  });
+});
+
+describe("contentToParts audio blocks", () => {
+  const inboundAudioPart = (block: Record<string, unknown>) => {
+    const result = convertLangChainBaseMessage(
+      {
+        _getType: () => "human",
+        id: "h1",
+        content: [{ type: "audio", data: "c291bmQ=", ...block }],
+      },
+      {},
+    );
+    return contentOf(result)[0];
+  };
+
+  it("converts an inbound base64 audio block back to an audio part", () => {
+    const result = convertLangChainBaseMessage(
+      {
+        _getType: () => "human",
+        id: "h1",
+        content: [
+          {
+            type: "audio",
+            data: "c291bmQ=",
+            mime_type: "audio/mp3",
+            source_type: "base64",
+          },
+        ],
+      },
+      {},
+    );
+
+    expect(result).toMatchObject({
+      role: "user",
+      content: [
+        {
+          type: "file",
+          filename: "audio.mp3",
+          data: "c291bmQ=",
+          mimeType: "audio/mp3",
+        },
+      ],
+    });
+  });
+
+  it("keeps an inbound audio block whose mime type has no wire format", () => {
+    const result = convertLangChainBaseMessage(
+      {
+        _getType: () => "human",
+        id: "h1",
+        content: [
+          {
+            type: "audio",
+            data: "b2dn",
+            mime_type: "audio/ogg",
+            source_type: "base64",
+          },
+        ],
+      },
+      {},
+    );
+
+    expect(result).toMatchObject({
+      role: "user",
+      content: [
+        {
+          type: "file",
+          filename: "audio.ogg",
+          data: "b2dn",
+          mimeType: "audio/ogg",
+        },
+      ],
+    });
+  });
+
+  it("keeps an audio block on an assistant message", () => {
+    const result = convertLangChainBaseMessage(
+      {
+        _getType: () => "ai",
+        id: "ai-1",
+        content: [
+          {
+            type: "audio",
+            data: "c291bmQ=",
+            mime_type: "audio/mp3",
+            source_type: "base64",
+          },
+          { type: "text", text: "done" },
+        ],
+      },
+      {},
+    );
+
+    expect(result).toMatchObject({
+      role: "assistant",
+      content: [
+        {
+          type: "file",
+          filename: "audio.mp3",
+          data: "c291bmQ=",
+          mimeType: "audio/mp3",
+        },
+        { type: "text", text: "done" },
+      ],
+    });
+  });
+  it("round-trips an audio file part through both converters", () => {
+    const outbound = getMessageContent({
+      content: [
+        {
+          type: "file",
+          data: "data:audio/mpeg;base64,c291bmQ=",
+          mimeType: "audio/mpeg",
+          filename: "memo.mp3",
+        },
+      ],
+    } as unknown as AppendMessage);
+
+    const inbound = convertLangChainBaseMessage(
+      {
+        _getType: () => "human",
+        id: "h1",
+        content: outbound as never,
+      },
+      {},
+    );
+
+    expect(contentOf(inbound)).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "file",
+        filename: "audio.mp3",
+        data: "c291bmQ=",
+        mimeType: "audio/mp3",
+      },
+    ]);
+
+    expect(
+      getMessageContent({
+        content: contentOf(inbound),
+      } as unknown as AppendMessage),
+    ).toEqual(outbound);
+  });
+  it("names an inbound audio attachment from its media subtype", () => {
+    expect(inboundAudioPart({ mime_type: "audio/wav" })).toMatchObject({
+      filename: "audio.wav",
+    });
+    expect(inboundAudioPart({})).toMatchObject({
+      filename: "audio",
+      mimeType: "application/octet-stream",
+    });
+  });
 });
 
 describe("convertLangChainBaseMessage reasoning content parts", () => {
@@ -346,3 +849,102 @@ describe("convertLangChainBaseMessage image content parts", () => {
     expect(contentOf(result)).toEqual([]);
   });
 });
+
+describe("convertLangChainBaseMessage audio transcripts", () => {
+  const audioMessage = (
+    content: unknown,
+    audio: unknown,
+  ): LangChainBaseMessage => ({
+    _getType: () => "ai",
+    id: "msg-3",
+    content,
+    additional_kwargs: { audio },
+  });
+
+  it("surfaces the transcript when the provider leaves content empty", () => {
+    const result = convertLangChainBaseMessage(
+      audioMessage("", {
+        id: "audio_1",
+        data: "UklGRg==",
+        expires_at: 1,
+        transcript: "the secret number is four seven two",
+      }),
+      {},
+    );
+
+    expect(contentOf(result)).toEqual([
+      { type: "text", text: "the secret number is four seven two" },
+    ]);
+  });
+
+  it("treats a whitespace-only placeholder as no text", () => {
+    const result = convertLangChainBaseMessage(
+      audioMessage([{ type: "text", text: "   " }], {
+        transcript: "spoken words",
+      }),
+      {},
+    );
+
+    expect(contentOf(result)).toEqual([{ type: "text", text: "spoken words" }]);
+  });
+
+  it("does not throw on a non-spec text block whose text is missing or not a string", () => {
+    for (const block of [{ type: "text" }, { type: "text", text: 42 }]) {
+      const result = convertLangChainBaseMessage(
+        audioMessage([block], { transcript: "spoken words" }),
+        {},
+      );
+
+      expect(contentOf(result)).toEqual([
+        { type: "text", text: "spoken words" },
+      ]);
+    }
+  });
+
+  it("keeps non-text parts when it substitutes the transcript", () => {
+    const result = convertLangChainBaseMessage(
+      audioMessage(
+        [
+          { type: "text", text: "" },
+          {
+            type: "image_url",
+            image_url: { url: "https://example.com/a.png" },
+          },
+        ],
+        { transcript: "spoken words" },
+      ),
+      {},
+    );
+
+    expect(contentOf(result)).toEqual([
+      { type: "image", image: "https://example.com/a.png" },
+      { type: "text", text: "spoken words" },
+    ]);
+  });
+
+  it("leaves existing text alone so the transcript is not duplicated", () => {
+    const result = convertLangChainBaseMessage(
+      audioMessage([{ type: "text", text: "written answer" }], {
+        transcript: "written answer",
+      }),
+      {},
+    );
+
+    expect(contentOf(result)).toEqual([
+      { type: "text", text: "written answer" },
+    ]);
+  });
+
+  it("ignores an absent, blank, or non-string transcript", () => {
+    for (const audio of [
+      undefined,
+      {},
+      { transcript: "" },
+      { transcript: "   " },
+      { transcript: 42 },
+    ]) {
+      const result = convertLangChainBaseMessage(audioMessage("", audio), {});
+      expect(contentOf(result)).toEqual([{ type: "text", text: "" }]);
+    }
+  });
+});

package/src/convertMessages.ts

@@ -64,7 +64,22 @@ const contentToParts = (content: unknown) => {
                   ? part.id
                   : part.data,
             mimeType: part.mime_type ?? "application/octet-stream",
+            ...((part.source_type === "url" || part.source_type === "id") && {
+              sourceType: part.source_type,
+            }),
           };
+        case "audio": {
+          const mimeType = part.mime_type ?? "application/octet-stream";
+          const subtype = mimeType.startsWith("audio/")
+            ? mimeType.slice("audio/".length)
+            : undefined;
+          return {
+            type: "file" as const,
+            filename: subtype ? `audio.${subtype}` : "audio",
+            data: part.data,
+            mimeType,
+          };
+        }
         case "thinking":
           return { type: "reasoning" as const, text: part.thinking };
         case "reasoning":
@@ -85,6 +100,31 @@ const contentToParts = (content: unknown) => {
     .filter((p) => p !== null);
 };
 
+const hasVisibleText = (text: unknown): boolean =>
+  typeof text === "string" && text.trim() !== "";
+
+/**
+ * Audio output arrives outside the content array: providers leave `content`
+ * empty and put the spoken text in `additional_kwargs.audio.transcript`. The
+ * audio bytes stay behind because no provider reports their media type, and a
+ * streamed response carries raw PCM rather than a playable file.
+ */
+const withAudioTranscript = (
+  parts: ReturnType<typeof contentToParts>,
+  additionalKwargs: Record<string, unknown> | undefined,
+): ReturnType<typeof contentToParts> => {
+  const audio = additionalKwargs?.audio as { transcript?: unknown } | undefined;
+  const transcript = audio?.transcript;
+  if (typeof transcript !== "string" || !hasVisibleText(transcript))
+    return parts;
+  if (parts.some((part) => part.type === "text" && hasVisibleText(part.text)))
+    return parts;
+  return [
+    ...parts.filter((part) => part.type !== "text"),
+    { type: "text" as const, text: transcript },
+  ];
+};
+
 const getCustomMetadata = (
   additionalKwargs: Record<string, unknown> | undefined,
 ): Record<string, unknown> =>
@@ -152,7 +192,10 @@ export const convertLangChainBaseMessage = (
         role: "assistant",
         id: message.id,
         content: [
-          ...contentToParts(message.content),
+          ...withAudioTranscript(
+            contentToParts(message.content),
+            message.additional_kwargs,
+          ),
           ...toolCallParts,
           ...uiDataParts,
         ],
@@ -191,6 +234,20 @@ export const convertLangChainBaseMessage = (
   }
 };
 
+/**
+ * Audio media types that reach a provider's audio input through the LangChain
+ * `audio` block. langchain-core derives OpenAI's `input_audio.format` by
+ * splitting `mime_type` on `/`, and that format is a wav-or-mp3 enum, so
+ * `audio/mpeg` passes the converter and is rejected at the provider.
+ */
+const audioBlockMimeTypes = new Map<string, "audio/mp3" | "audio/wav">([
+  ["audio/mp3", "audio/mp3"],
+  ["audio/mpeg", "audio/mp3"],
+  ["audio/wav", "audio/wav"],
+  ["audio/wave", "audio/wav"],
+  ["audio/x-wav", "audio/wav"],
+]);
+
 export const getMessageContent = (msg: AppendMessage) => {
   const allContent = [
     ...msg.content,
@@ -198,14 +255,15 @@ export const getMessageContent = (msg: AppendMessage) => {
   ];
 
   const hasNonText = allContent.some(
-    (part) => part.type === "file" || part.type === "image",
+    (part) =>
+      part.type === "file" || part.type === "image" || part.type === "audio",
   );
   const hasText = allContent.some((part) => part.type === "text");
   if (hasNonText && !hasText) {
     allContent.unshift({ type: "text", text: " " });
   }
 
-  const content = allContent.map((part) => {
+  const content = allContent.flatMap((part) => {
     const type = part.type;
     switch (type) {
       case "text":
@@ -214,33 +272,62 @@ export const getMessageContent = (msg: AppendMessage) => {
         return { type: "image_url" as const, image_url: { url: part.image } };
       case "file": {
         const metadata = { filename: part.filename ?? "file" };
-        if (httpUrlPattern.test(part.data)) {
+        if (part.sourceType === "id") {
+          return {
+            type: "file" as const,
+            id: part.data,
+            mime_type: part.mimeType,
+            filename: metadata.filename,
+            metadata,
+            source_type: "id" as const,
+          };
+        }
+        if (part.sourceType === "url" || httpUrlPattern.test(part.data)) {
           return {
             type: "file" as const,
             url: part.data,
             mime_type: part.mimeType,
+            filename: metadata.filename,
             metadata,
             source_type: "url" as const,
           };
         }
         const parsed = parseDataUrl(part.data);
+        const audioMimeType = audioBlockMimeTypes.get(
+          (parsed?.mimeType ?? part.mimeType).toLowerCase(),
+        );
+        if (audioMimeType) {
+          return {
+            type: "audio" as const,
+            data: parsed?.data ?? part.data,
+            mime_type: audioMimeType,
+            source_type: "base64" as const,
+          };
+        }
         return {
           type: "file" as const,
           data: parsed?.data ?? part.data,
           mime_type: parsed?.mimeType ?? part.mimeType,
+          filename: metadata.filename,
           metadata,
           source_type: "base64" as const,
         };
       }
+      case "audio": {
+        const parsed = parseDataUrl(part.audio.data);
+        return {
+          type: "audio" as const,
+          data: parsed?.data ?? part.audio.data,
+          mime_type: `audio/${part.audio.format}`,
+          source_type: "base64" as const,
+        };
+      }
+      case "data":
+        return [];
       case "tool-call":
         throw new Error("Tool call appends are not supported.");
       default: {
-        const _exhaustiveCheck:
-          | "reasoning"
-          | "source"
-          | "audio"
-          | "data"
-          | "generative-ui" = type;
+        const _exhaustiveCheck: "reasoning" | "source" | "generative-ui" = type;
         throw new Error(
           `Unsupported append message part type: ${_exhaustiveCheck}`,
         );

package/src/runtimeExtras.ts

@@ -1,4 +1,4 @@
-import { createRuntimeExtras } from "@assistant-ui/core/internal";
+import { createRuntimeExtras } from "@assistant-ui/core/react";
 import type { LangChainRuntimeExtras } from "./types";
 
 export const langChainExtras =

package/src/types.ts

@@ -49,6 +49,12 @@ export type LangChainContentBlock =
       source_type: "id";
       metadata?: { filename?: string };
     }
+  | {
+      type: "audio";
+      data: string;
+      mime_type: string;
+      source_type: "base64";
+    }
   | { type: "tool_use" | "input_json_delta" };
 
 export type LangChainToolCall = {