@assistant-ui/react-langchain 0.0.21 → 0.0.22

package/dist/convertMessages.d.ts

@@ -1,6 +1,6 @@
 import { LangChainBaseMessage, UIMessage } from "./types.js";
-import { AppendMessage, MessageTiming } from "@assistant-ui/core";
 import { useExternalMessageConverter } from "@assistant-ui/core/react";
+import { AppendMessage, MessageTiming } from "@assistant-ui/core";
 //#region src/convertMessages.d.ts
 type LangChainMessageConverterMetadata = useExternalMessageConverter.Metadata & {
   uiMessagesByParent?: Map<string, UIMessage[]>;
@@ -50,6 +50,15 @@ declare const getMessageContent: (msg: AppendMessage) => string | ({
   text?: never;
   image_url?: never;
   url?: never;
+} | {
+  type: "audio";
+  data: string;
+  mime_type: string;
+  source_type: "base64";
+  text?: never;
+  image_url?: never;
+  url?: never;
+  metadata?: never;
 })[];
 //#endregion
 export { convertLangChainBaseMessage, getMessageContent, getMessageType };

package/dist/convertMessages.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"convertMessages.d.ts","names":[],"sources":["../src/convertMessages.ts"],"mappings":";;;;KAgBK,oCACH,4BAA4B;EAC1B,qBAAqB,YAAY;EACjC,gBAAgB,eAAe;;cAStB,iBAAkB,SAAS;cAyE3B,8BACX,SAAS,sBACT,WAAU,sCACT,4BAA4B;cAyFlB,oBAAqB,KAAK"}
+{"version":3,"file":"convertMessages.d.ts","names":[],"sources":["../src/convertMessages.ts"],"mappings":";;;;KAgBK,oCACH,4BAA4B;EAC1B,qBAAqB,YAAY;EACjC,gBAAgB,eAAe;;cAStB,iBAAkB,SAAS;cAuF3B,8BACX,SAAS,sBACT,WAAU,sCACT,4BAA4B;cAyFlB,oBAAqB,KAAK"}

package/dist/convertMessages.js

@@ -11,7 +11,7 @@ const getMessageType = (message) => {
 	if ("type" in message) return message.type;
 	throw new Error("Cannot determine message type");
 };
-const contentToParts = (content) => {
+const contentToParts = (content, role) => {
 	if (typeof content === "string") return [{
 		type: "text",
 		text: content
@@ -37,6 +37,18 @@ const contentToParts = (content) => {
 				data: part.source_type === "url" ? part.url : part.source_type === "id" ? part.id : part.data,
 				mimeType: part.mime_type ?? "application/octet-stream"
 			};
+			case "audio": {
+				if (role !== "user") return null;
+				const format = part.mime_type === "audio/wav" ? "wav" : part.mime_type === "audio/mp3" ? "mp3" : null;
+				if (!format) return null;
+				return {
+					type: "audio",
+					audio: {
+						data: part.data,
+						format
+					}
+				};
+			}
 			case "thinking": return {
 				type: "reasoning",
 				text: part.thinking
@@ -70,7 +82,7 @@ const convertLangChainBaseMessage = (message, metadata = {}) => {
 		case "human": return {
 			role: "user",
 			id: message.id,
-			content: contentToParts(message.content),
+			content: contentToParts(message.content, "user"),
 			metadata: { custom: getCustomMetadata(message.additional_kwargs) }
 		};
 		case "ai": {
@@ -88,7 +100,7 @@ const convertLangChainBaseMessage = (message, metadata = {}) => {
 				role: "assistant",
 				id: message.id,
 				content: [
-					...contentToParts(message.content),
+					...contentToParts(message.content, "assistant"),
 					...toolCallParts,
 					...uiDataParts
 				],
@@ -119,13 +131,13 @@ const convertLangChainBaseMessage = (message, metadata = {}) => {
 };
 const getMessageContent = (msg) => {
 	const allContent = [...msg.content, ...msg.attachments?.flatMap((a) => a.content) ?? []];
-	const hasNonText = allContent.some((part) => part.type === "file" || part.type === "image");
+	const hasNonText = allContent.some((part) => part.type === "file" || part.type === "image" || part.type === "audio");
 	const hasText = allContent.some((part) => part.type === "text");
 	if (hasNonText && !hasText) allContent.unshift({
 		type: "text",
 		text: " "
 	});
-	const content = allContent.map((part) => {
+	const content = allContent.flatMap((part) => {
 		const type = part.type;
 		switch (type) {
 			case "text": return {
@@ -154,6 +166,13 @@ const getMessageContent = (msg) => {
 					source_type: "base64"
 				};
 			}
+			case "audio": return {
+				type: "audio",
+				data: parseDataUrl(part.audio.data)?.data ?? part.audio.data,
+				mime_type: `audio/${part.audio.format}`,
+				source_type: "base64"
+			};
+			case "data": return [];
 			case "tool-call": throw new Error("Tool call appends are not supported.");
 			default: throw new Error(`Unsupported append message part type: ${type}`);
 		}

package/dist/convertMessages.js.map

@@ -1 +1 @@
-{"version":3,"file":"convertMessages.js","names":["parts","_exhaustiveCheck"],"sources":["../src/convertMessages.ts"],"sourcesContent":["\"use client\";\n\nimport type { useExternalMessageConverter } from \"@assistant-ui/core/react\";\nimport type {\n  AppendMessage,\n  DataMessagePart,\n  MessageTiming,\n} from \"@assistant-ui/core\";\nimport { httpUrlPattern, parseDataUrl } from \"@assistant-ui/core/internal\";\nimport type { ReadonlyJSONObject } from \"assistant-stream/utils\";\nimport type {\n  LangChainBaseMessage,\n  LangChainContentBlock,\n  UIMessage,\n} from \"./types\";\n\ntype LangChainMessageConverterMetadata =\n  useExternalMessageConverter.Metadata & {\n    uiMessagesByParent?: Map<string, UIMessage[]>;\n    messageTiming?: Record<string, MessageTiming>;\n  };\n\nconst uiMessageToDataPart = (ui: UIMessage): DataMessagePart => ({\n  type: \"data\",\n  name: ui.name,\n  data: ui.props,\n});\n\nexport const getMessageType = (message: LangChainBaseMessage): string => {\n  if (typeof message._getType === \"function\") return message._getType();\n  if (\"type\" in message)\n    return (message as Record<string, unknown>).type as string;\n  throw new Error(\"Cannot determine message type\");\n};\n\nconst contentToParts = (content: unknown) => {\n  if (typeof content === \"string\")\n    return [{ type: \"text\" as const, text: content }];\n\n  const parts = content as readonly LangChainContentBlock[];\n  return parts\n    .map((part) => {\n      const type = part.type;\n      switch (type) {\n        case \"text\":\n        case \"text_delta\":\n          return { type: \"text\" as const, text: part.text };\n        case \"image_url\": {\n          const image =\n            typeof part.image_url === \"string\"\n              ? part.image_url\n              : part.image_url?.url;\n          if (!image) return null;\n          return { type: \"image\" as const, image };\n        }\n        case \"file\":\n          return {\n            type: \"file\" as const,\n            filename: part.metadata?.filename ?? \"file\",\n            data:\n              part.source_type === \"url\"\n                ? part.url\n                : part.source_type === \"id\"\n                  ? part.id\n                  : part.data,\n            mimeType: part.mime_type ?? \"application/octet-stream\",\n          };\n        case \"thinking\":\n          return { type: \"reasoning\" as const, text: part.thinking };\n        case \"reasoning\":\n          return {\n            type: \"reasoning\" as const,\n            text:\n              part.summary?.map((s) => s?.text ?? \"\").join(\"\\n\\n\\n\") ??\n              part.reasoning ??\n              \"\",\n          };\n        case \"tool_use\":\n        case \"input_json_delta\":\n          return null;\n        default:\n          return null;\n      }\n    })\n    .filter((p) => p !== null);\n};\n\nconst getCustomMetadata = (\n  additionalKwargs: Record<string, unknown> | undefined,\n): Record<string, unknown> =>\n  (additionalKwargs?.metadata as Record<string, unknown>) ?? {};\n\nconst getStringContent = (content: unknown): string => {\n  if (typeof content === \"string\") return content;\n  const parts = content as readonly LangChainContentBlock[];\n  return parts\n    .filter((c): c is { type: \"text\"; text: string } => c.type === \"text\")\n    .map((c) => c.text)\n    .join(\"\");\n};\n\nexport const convertLangChainBaseMessage = (\n  message: LangChainBaseMessage,\n  metadata: LangChainMessageConverterMetadata = {},\n): useExternalMessageConverter.Message => {\n  const type = getMessageType(message);\n\n  switch (type) {\n    case \"system\":\n      return {\n        role: \"system\",\n        id: message.id,\n        content: [{ type: \"text\", text: getStringContent(message.content) }],\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n        },\n      };\n\n    case \"human\":\n      return {\n        role: \"user\",\n        id: message.id,\n        content: contentToParts(message.content),\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n        },\n      };\n\n    case \"ai\": {\n      const toolCallParts =\n        message.tool_calls?.map((tc) => ({\n          type: \"tool-call\" as const,\n          toolCallId: tc.id,\n          toolName: tc.name,\n          args: tc.args as ReadonlyJSONObject,\n          argsText: JSON.stringify(tc.args),\n        })) ?? [];\n\n      const assistantStatus =\n        typeof message.status === \"object\" ? message.status : undefined;\n\n      const uiDataParts =\n        (message.id\n          ? metadata.uiMessagesByParent\n              ?.get(message.id)\n              ?.map(uiMessageToDataPart)\n          : undefined) ?? [];\n\n      const timing = metadata.messageTiming?.[message.id ?? \"\"];\n\n      return {\n        role: \"assistant\",\n        id: message.id,\n        content: [\n          ...contentToParts(message.content),\n          ...toolCallParts,\n          ...uiDataParts,\n        ],\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n          ...(timing && { timing }),\n        },\n        ...(assistantStatus && { status: assistantStatus }),\n      };\n    }\n\n    case \"tool\":\n      return {\n        role: \"tool\",\n        toolName: message.name ?? \"\",\n        toolCallId: message.tool_call_id ?? \"\",\n        result: message.content,\n        artifact: message.artifact,\n        isError: message.status === \"error\",\n      };\n\n    default:\n      return {\n        role: \"system\",\n        id: message.id,\n        content: [\n          {\n            type: \"text\",\n            text:\n              typeof message.content === \"string\"\n                ? message.content\n                : JSON.stringify(message.content),\n          },\n        ],\n      };\n  }\n};\n\nexport const getMessageContent = (msg: AppendMessage) => {\n  const allContent = [\n    ...msg.content,\n    ...(msg.attachments?.flatMap((a) => a.content) ?? []),\n  ];\n\n  const hasNonText = allContent.some(\n    (part) => part.type === \"file\" || part.type === \"image\",\n  );\n  const hasText = allContent.some((part) => part.type === \"text\");\n  if (hasNonText && !hasText) {\n    allContent.unshift({ type: \"text\", text: \" \" });\n  }\n\n  const content = allContent.map((part) => {\n    const type = part.type;\n    switch (type) {\n      case \"text\":\n        return { type: \"text\" as const, text: part.text };\n      case \"image\":\n        return { type: \"image_url\" as const, image_url: { url: part.image } };\n      case \"file\": {\n        const metadata = { filename: part.filename ?? \"file\" };\n        if (httpUrlPattern.test(part.data)) {\n          return {\n            type: \"file\" as const,\n            url: part.data,\n            mime_type: part.mimeType,\n            metadata,\n            source_type: \"url\" as const,\n          };\n        }\n        const parsed = parseDataUrl(part.data);\n        return {\n          type: \"file\" as const,\n          data: parsed?.data ?? part.data,\n          mime_type: parsed?.mimeType ?? part.mimeType,\n          metadata,\n          source_type: \"base64\" as const,\n        };\n      }\n      case \"tool-call\":\n        throw new Error(\"Tool call appends are not supported.\");\n      default: {\n        const _exhaustiveCheck:\n          | \"reasoning\"\n          | \"source\"\n          | \"audio\"\n          | \"data\"\n          | \"generative-ui\" = type;\n        throw new Error(\n          `Unsupported append message part type: ${_exhaustiveCheck}`,\n        );\n      }\n    }\n  });\n\n  if (content.length === 1 && content[0]?.type === \"text\") {\n    return content[0].text ?? \"\";\n  }\n  return content;\n};\n"],"mappings":";;;AAsBA,MAAM,uBAAuB,QAAoC;CAC/D,MAAM;CACN,MAAM,GAAG;CACT,MAAM,GAAG;AACX;AAEA,MAAa,kBAAkB,YAA0C;CACvE,IAAI,OAAO,QAAQ,aAAa,YAAY,OAAO,QAAQ,SAAS;CACpE,IAAI,UAAU,SACZ,OAAQ,QAAoC;CAC9C,MAAM,IAAI,MAAM,+BAA+B;AACjD;AAEA,MAAM,kBAAkB,YAAqB;CAC3C,IAAI,OAAO,YAAY,UACrB,OAAO,CAAC;EAAE,MAAM;EAAiB,MAAM;CAAQ,CAAC;CAGlD,OAAOA,QACJ,KAAK,SAAS;EAEb,QADa,KAAK,MAClB;GACE,KAAK;GACL,KAAK,cACH,OAAO;IAAE,MAAM;IAAiB,MAAM,KAAK;GAAK;GAClD,KAAK,aAAa;IAChB,MAAM,QACJ,OAAO,KAAK,cAAc,WACtB,KAAK,YACL,KAAK,WAAW;IACtB,IAAI,CAAC,OAAO,OAAO;IACnB,OAAO;KAAE,MAAM;KAAkB;IAAM;GACzC;GACA,KAAK,QACH,OAAO;IACL,MAAM;IACN,UAAU,KAAK,UAAU,YAAY;IACrC,MACE,KAAK,gBAAgB,QACjB,KAAK,MACL,KAAK,gBAAgB,OACnB,KAAK,KACL,KAAK;IACb,UAAU,KAAK,aAAa;GAC9B;GACF,KAAK,YACH,OAAO;IAAE,MAAM;IAAsB,MAAM,KAAK;GAAS;GAC3D,KAAK,aACH,OAAO;IACL,MAAM;IACN,MACE,KAAK,SAAS,KAAK,MAAM,GAAG,QAAQ,EAAE,CAAC,CAAC,KAAK,QAAQ,KACrD,KAAK,aACL;GACJ;GACF,KAAK;GACL,KAAK,oBACH,OAAO;GACT,SACE,OAAO;EACX;CACF,CAAC,CAAC,CACD,QAAQ,MAAM,MAAM,IAAI;AAC7B;AAEA,MAAM,qBACJ,qBAEC,kBAAkB,YAAwC,CAAC;AAE9D,MAAM,oBAAoB,YAA6B;CACrD,IAAI,OAAO,YAAY,UAAU,OAAO;CAExC,OAAOA,QACJ,QAAQ,MAA2C,EAAE,SAAS,MAAM,CAAC,CACrE,KAAK,MAAM,EAAE,IAAI,CAAC,CAClB,KAAK,EAAE;AACZ;AAEA,MAAa,+BACX,SACA,WAA8C,CAAC,MACP;CAGxC,QAFa,eAAe,OAEjB,GAAX;EACE,KAAK,UACH,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,CAAC;IAAE,MAAM;IAAQ,MAAM,iBAAiB,QAAQ,OAAO;GAAE,CAAC;GACnE,UAAU,EACR,QAAQ,kBAAkB,QAAQ,iBAAiB,EACrD;EACF;EAEF,KAAK,SACH,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,eAAe,QAAQ,OAAO;GACvC,UAAU,EACR,QAAQ,kBAAkB,QAAQ,iBAAiB,EACrD;EACF;EAEF,KAAK,MAAM;GACT,MAAM,gBACJ,QAAQ,YAAY,KAAK,QAAQ;IAC/B,MAAM;IACN,YAAY,GAAG;IACf,UAAU,GAAG;IACb,MAAM,GAAG;IACT,UAAU,KAAK,UAAU,GAAG,IAAI;GAClC,EAAE,KAAK,CAAC;GAEV,MAAM,kBACJ,OAAO,QAAQ,WAAW,WAAW,QAAQ,SAAS,KAAA;GAExD,MAAM,eACH,QAAQ,KACL,SAAS,oBACL,IAAI,QAAQ,EAAE,CAAC,EACf,IAAI,mBAAmB,IAC3B,KAAA,MAAc,CAAC;GAErB,MAAM,SAAS,SAAS,gBAAgB,QAAQ,MAAM;GAEtD,OAAO;IACL,MAAM;IACN,IAAI,QAAQ;IACZ,SAAS;KACP,GAAG,eAAe,QAAQ,OAAO;KACjC,GAAG;KACH,GAAG;IACL;IACA,UAAU;KACR,QAAQ,kBAAkB,QAAQ,iBAAiB;KACnD,GAAI,UAAU,EAAE,OAAO;IACzB;IACA,GAAI,mBAAmB,EAAE,QAAQ,gBAAgB;GACnD;EACF;EAEA,KAAK,QACH,OAAO;GACL,MAAM;GACN,UAAU,QAAQ,QAAQ;GAC1B,YAAY,QAAQ,gBAAgB;GACpC,QAAQ,QAAQ;GAChB,UAAU,QAAQ;GAClB,SAAS,QAAQ,WAAW;EAC9B;EAEF,SACE,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,CACP;IACE,MAAM;IACN,MACE,OAAO,QAAQ,YAAY,WACvB,QAAQ,UACR,KAAK,UAAU,QAAQ,OAAO;GACtC,CACF;EACF;CACJ;AACF;AAEA,MAAa,qBAAqB,QAAuB;CACvD,MAAM,aAAa,CACjB,GAAG,IAAI,SACP,GAAI,IAAI,aAAa,SAAS,MAAM,EAAE,OAAO,KAAK,CAAC,CACrD;CAEA,MAAM,aAAa,WAAW,MAC3B,SAAS,KAAK,SAAS,UAAU,KAAK,SAAS,OAClD;CACA,MAAM,UAAU,WAAW,MAAM,SAAS,KAAK,SAAS,MAAM;CAC9D,IAAI,cAAc,CAAC,SACjB,WAAW,QAAQ;EAAE,MAAM;EAAQ,MAAM;CAAI,CAAC;CAGhD,MAAM,UAAU,WAAW,KAAK,SAAS;EACvC,MAAM,OAAO,KAAK;EAClB,QAAQ,MAAR;GACE,KAAK,QACH,OAAO;IAAE,MAAM;IAAiB,MAAM,KAAK;GAAK;GAClD,KAAK,SACH,OAAO;IAAE,MAAM;IAAsB,WAAW,EAAE,KAAK,KAAK,MAAM;GAAE;GACtE,KAAK,QAAQ;IACX,MAAM,WAAW,EAAE,UAAU,KAAK,YAAY,OAAO;IACrD,IAAI,eAAe,KAAK,KAAK,IAAI,GAC/B,OAAO;KACL,MAAM;KACN,KAAK,KAAK;KACV,WAAW,KAAK;KAChB;KACA,aAAa;IACf;IAEF,MAAM,SAAS,aAAa,KAAK,IAAI;IACrC,OAAO;KACL,MAAM;KACN,MAAM,QAAQ,QAAQ,KAAK;KAC3B,WAAW,QAAQ,YAAY,KAAK;KACpC;KACA,aAAa;IACf;GACF;GACA,KAAK,aACH,MAAM,IAAI,MAAM,sCAAsC;GACxD,SAOE,MAAM,IAAI,MACR,yCAAyCC,MAC3C;EAEJ;CACF,CAAC;CAED,IAAI,QAAQ,WAAW,KAAK,QAAQ,EAAE,EAAE,SAAS,QAC/C,OAAO,QAAQ,EAAE,CAAC,QAAQ;CAE5B,OAAO;AACT"}
+{"version":3,"file":"convertMessages.js","names":["parts","_exhaustiveCheck"],"sources":["../src/convertMessages.ts"],"sourcesContent":["\"use client\";\n\nimport type { useExternalMessageConverter } from \"@assistant-ui/core/react\";\nimport type {\n  AppendMessage,\n  DataMessagePart,\n  MessageTiming,\n} from \"@assistant-ui/core\";\nimport { httpUrlPattern, parseDataUrl } from \"@assistant-ui/core/internal\";\nimport type { ReadonlyJSONObject } from \"assistant-stream/utils\";\nimport type {\n  LangChainBaseMessage,\n  LangChainContentBlock,\n  UIMessage,\n} from \"./types\";\n\ntype LangChainMessageConverterMetadata =\n  useExternalMessageConverter.Metadata & {\n    uiMessagesByParent?: Map<string, UIMessage[]>;\n    messageTiming?: Record<string, MessageTiming>;\n  };\n\nconst uiMessageToDataPart = (ui: UIMessage): DataMessagePart => ({\n  type: \"data\",\n  name: ui.name,\n  data: ui.props,\n});\n\nexport const getMessageType = (message: LangChainBaseMessage): string => {\n  if (typeof message._getType === \"function\") return message._getType();\n  if (\"type\" in message)\n    return (message as Record<string, unknown>).type as string;\n  throw new Error(\"Cannot determine message type\");\n};\n\nconst contentToParts = (content: unknown, role: \"user\" | \"assistant\") => {\n  if (typeof content === \"string\")\n    return [{ type: \"text\" as const, text: content }];\n\n  const parts = content as readonly LangChainContentBlock[];\n  return parts\n    .map((part) => {\n      const type = part.type;\n      switch (type) {\n        case \"text\":\n        case \"text_delta\":\n          return { type: \"text\" as const, text: part.text };\n        case \"image_url\": {\n          const image =\n            typeof part.image_url === \"string\"\n              ? part.image_url\n              : part.image_url?.url;\n          if (!image) return null;\n          return { type: \"image\" as const, image };\n        }\n        case \"file\":\n          return {\n            type: \"file\" as const,\n            filename: part.metadata?.filename ?? \"file\",\n            data:\n              part.source_type === \"url\"\n                ? part.url\n                : part.source_type === \"id\"\n                  ? part.id\n                  : part.data,\n            mimeType: part.mime_type ?? \"application/octet-stream\",\n          };\n        case \"audio\": {\n          if (role !== \"user\") return null;\n          const format =\n            part.mime_type === \"audio/wav\"\n              ? (\"wav\" as const)\n              : part.mime_type === \"audio/mp3\"\n                ? (\"mp3\" as const)\n                : null;\n          if (!format) return null;\n          return {\n            type: \"audio\" as const,\n            audio: { data: part.data, format },\n          };\n        }\n        case \"thinking\":\n          return { type: \"reasoning\" as const, text: part.thinking };\n        case \"reasoning\":\n          return {\n            type: \"reasoning\" as const,\n            text:\n              part.summary?.map((s) => s?.text ?? \"\").join(\"\\n\\n\\n\") ??\n              part.reasoning ??\n              \"\",\n          };\n        case \"tool_use\":\n        case \"input_json_delta\":\n          return null;\n        default:\n          return null;\n      }\n    })\n    .filter((p) => p !== null);\n};\n\nconst getCustomMetadata = (\n  additionalKwargs: Record<string, unknown> | undefined,\n): Record<string, unknown> =>\n  (additionalKwargs?.metadata as Record<string, unknown>) ?? {};\n\nconst getStringContent = (content: unknown): string => {\n  if (typeof content === \"string\") return content;\n  const parts = content as readonly LangChainContentBlock[];\n  return parts\n    .filter((c): c is { type: \"text\"; text: string } => c.type === \"text\")\n    .map((c) => c.text)\n    .join(\"\");\n};\n\nexport const convertLangChainBaseMessage = (\n  message: LangChainBaseMessage,\n  metadata: LangChainMessageConverterMetadata = {},\n): useExternalMessageConverter.Message => {\n  const type = getMessageType(message);\n\n  switch (type) {\n    case \"system\":\n      return {\n        role: \"system\",\n        id: message.id,\n        content: [{ type: \"text\", text: getStringContent(message.content) }],\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n        },\n      };\n\n    case \"human\":\n      return {\n        role: \"user\",\n        id: message.id,\n        content: contentToParts(message.content, \"user\"),\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n        },\n      };\n\n    case \"ai\": {\n      const toolCallParts =\n        message.tool_calls?.map((tc) => ({\n          type: \"tool-call\" as const,\n          toolCallId: tc.id,\n          toolName: tc.name,\n          args: tc.args as ReadonlyJSONObject,\n          argsText: JSON.stringify(tc.args),\n        })) ?? [];\n\n      const assistantStatus =\n        typeof message.status === \"object\" ? message.status : undefined;\n\n      const uiDataParts =\n        (message.id\n          ? metadata.uiMessagesByParent\n              ?.get(message.id)\n              ?.map(uiMessageToDataPart)\n          : undefined) ?? [];\n\n      const timing = metadata.messageTiming?.[message.id ?? \"\"];\n\n      return {\n        role: \"assistant\",\n        id: message.id,\n        content: [\n          ...contentToParts(message.content, \"assistant\"),\n          ...toolCallParts,\n          ...uiDataParts,\n        ],\n        metadata: {\n          custom: getCustomMetadata(message.additional_kwargs),\n          ...(timing && { timing }),\n        },\n        ...(assistantStatus && { status: assistantStatus }),\n      };\n    }\n\n    case \"tool\":\n      return {\n        role: \"tool\",\n        toolName: message.name ?? \"\",\n        toolCallId: message.tool_call_id ?? \"\",\n        result: message.content,\n        artifact: message.artifact,\n        isError: message.status === \"error\",\n      };\n\n    default:\n      return {\n        role: \"system\",\n        id: message.id,\n        content: [\n          {\n            type: \"text\",\n            text:\n              typeof message.content === \"string\"\n                ? message.content\n                : JSON.stringify(message.content),\n          },\n        ],\n      };\n  }\n};\n\nexport const getMessageContent = (msg: AppendMessage) => {\n  const allContent = [\n    ...msg.content,\n    ...(msg.attachments?.flatMap((a) => a.content) ?? []),\n  ];\n\n  const hasNonText = allContent.some(\n    (part) =>\n      part.type === \"file\" || part.type === \"image\" || part.type === \"audio\",\n  );\n  const hasText = allContent.some((part) => part.type === \"text\");\n  if (hasNonText && !hasText) {\n    allContent.unshift({ type: \"text\", text: \" \" });\n  }\n\n  const content = allContent.flatMap((part) => {\n    const type = part.type;\n    switch (type) {\n      case \"text\":\n        return { type: \"text\" as const, text: part.text };\n      case \"image\":\n        return { type: \"image_url\" as const, image_url: { url: part.image } };\n      case \"file\": {\n        const metadata = { filename: part.filename ?? \"file\" };\n        if (httpUrlPattern.test(part.data)) {\n          return {\n            type: \"file\" as const,\n            url: part.data,\n            mime_type: part.mimeType,\n            metadata,\n            source_type: \"url\" as const,\n          };\n        }\n        const parsed = parseDataUrl(part.data);\n        return {\n          type: \"file\" as const,\n          data: parsed?.data ?? part.data,\n          mime_type: parsed?.mimeType ?? part.mimeType,\n          metadata,\n          source_type: \"base64\" as const,\n        };\n      }\n      case \"audio\": {\n        const parsed = parseDataUrl(part.audio.data);\n        return {\n          type: \"audio\" as const,\n          data: parsed?.data ?? part.audio.data,\n          mime_type: `audio/${part.audio.format}`,\n          source_type: \"base64\" as const,\n        };\n      }\n      case \"data\":\n        return [];\n      case \"tool-call\":\n        throw new Error(\"Tool call appends are not supported.\");\n      default: {\n        const _exhaustiveCheck: \"reasoning\" | \"source\" | \"generative-ui\" = type;\n        throw new Error(\n          `Unsupported append message part type: ${_exhaustiveCheck}`,\n        );\n      }\n    }\n  });\n\n  if (content.length === 1 && content[0]?.type === \"text\") {\n    return content[0].text ?? \"\";\n  }\n  return content;\n};\n"],"mappings":";;;AAsBA,MAAM,uBAAuB,QAAoC;CAC/D,MAAM;CACN,MAAM,GAAG;CACT,MAAM,GAAG;AACX;AAEA,MAAa,kBAAkB,YAA0C;CACvE,IAAI,OAAO,QAAQ,aAAa,YAAY,OAAO,QAAQ,SAAS;CACpE,IAAI,UAAU,SACZ,OAAQ,QAAoC;CAC9C,MAAM,IAAI,MAAM,+BAA+B;AACjD;AAEA,MAAM,kBAAkB,SAAkB,SAA+B;CACvE,IAAI,OAAO,YAAY,UACrB,OAAO,CAAC;EAAE,MAAM;EAAiB,MAAM;CAAQ,CAAC;CAGlD,OAAOA,QACJ,KAAK,SAAS;EAEb,QADa,KAAK,MAClB;GACE,KAAK;GACL,KAAK,cACH,OAAO;IAAE,MAAM;IAAiB,MAAM,KAAK;GAAK;GAClD,KAAK,aAAa;IAChB,MAAM,QACJ,OAAO,KAAK,cAAc,WACtB,KAAK,YACL,KAAK,WAAW;IACtB,IAAI,CAAC,OAAO,OAAO;IACnB,OAAO;KAAE,MAAM;KAAkB;IAAM;GACzC;GACA,KAAK,QACH,OAAO;IACL,MAAM;IACN,UAAU,KAAK,UAAU,YAAY;IACrC,MACE,KAAK,gBAAgB,QACjB,KAAK,MACL,KAAK,gBAAgB,OACnB,KAAK,KACL,KAAK;IACb,UAAU,KAAK,aAAa;GAC9B;GACF,KAAK,SAAS;IACZ,IAAI,SAAS,QAAQ,OAAO;IAC5B,MAAM,SACJ,KAAK,cAAc,cACd,QACD,KAAK,cAAc,cAChB,QACD;IACR,IAAI,CAAC,QAAQ,OAAO;IACpB,OAAO;KACL,MAAM;KACN,OAAO;MAAE,MAAM,KAAK;MAAM;KAAO;IACnC;GACF;GACA,KAAK,YACH,OAAO;IAAE,MAAM;IAAsB,MAAM,KAAK;GAAS;GAC3D,KAAK,aACH,OAAO;IACL,MAAM;IACN,MACE,KAAK,SAAS,KAAK,MAAM,GAAG,QAAQ,EAAE,CAAC,CAAC,KAAK,QAAQ,KACrD,KAAK,aACL;GACJ;GACF,KAAK;GACL,KAAK,oBACH,OAAO;GACT,SACE,OAAO;EACX;CACF,CAAC,CAAC,CACD,QAAQ,MAAM,MAAM,IAAI;AAC7B;AAEA,MAAM,qBACJ,qBAEC,kBAAkB,YAAwC,CAAC;AAE9D,MAAM,oBAAoB,YAA6B;CACrD,IAAI,OAAO,YAAY,UAAU,OAAO;CAExC,OAAOA,QACJ,QAAQ,MAA2C,EAAE,SAAS,MAAM,CAAC,CACrE,KAAK,MAAM,EAAE,IAAI,CAAC,CAClB,KAAK,EAAE;AACZ;AAEA,MAAa,+BACX,SACA,WAA8C,CAAC,MACP;CAGxC,QAFa,eAAe,OAEjB,GAAX;EACE,KAAK,UACH,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,CAAC;IAAE,MAAM;IAAQ,MAAM,iBAAiB,QAAQ,OAAO;GAAE,CAAC;GACnE,UAAU,EACR,QAAQ,kBAAkB,QAAQ,iBAAiB,EACrD;EACF;EAEF,KAAK,SACH,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,eAAe,QAAQ,SAAS,MAAM;GAC/C,UAAU,EACR,QAAQ,kBAAkB,QAAQ,iBAAiB,EACrD;EACF;EAEF,KAAK,MAAM;GACT,MAAM,gBACJ,QAAQ,YAAY,KAAK,QAAQ;IAC/B,MAAM;IACN,YAAY,GAAG;IACf,UAAU,GAAG;IACb,MAAM,GAAG;IACT,UAAU,KAAK,UAAU,GAAG,IAAI;GAClC,EAAE,KAAK,CAAC;GAEV,MAAM,kBACJ,OAAO,QAAQ,WAAW,WAAW,QAAQ,SAAS,KAAA;GAExD,MAAM,eACH,QAAQ,KACL,SAAS,oBACL,IAAI,QAAQ,EAAE,CAAC,EACf,IAAI,mBAAmB,IAC3B,KAAA,MAAc,CAAC;GAErB,MAAM,SAAS,SAAS,gBAAgB,QAAQ,MAAM;GAEtD,OAAO;IACL,MAAM;IACN,IAAI,QAAQ;IACZ,SAAS;KACP,GAAG,eAAe,QAAQ,SAAS,WAAW;KAC9C,GAAG;KACH,GAAG;IACL;IACA,UAAU;KACR,QAAQ,kBAAkB,QAAQ,iBAAiB;KACnD,GAAI,UAAU,EAAE,OAAO;IACzB;IACA,GAAI,mBAAmB,EAAE,QAAQ,gBAAgB;GACnD;EACF;EAEA,KAAK,QACH,OAAO;GACL,MAAM;GACN,UAAU,QAAQ,QAAQ;GAC1B,YAAY,QAAQ,gBAAgB;GACpC,QAAQ,QAAQ;GAChB,UAAU,QAAQ;GAClB,SAAS,QAAQ,WAAW;EAC9B;EAEF,SACE,OAAO;GACL,MAAM;GACN,IAAI,QAAQ;GACZ,SAAS,CACP;IACE,MAAM;IACN,MACE,OAAO,QAAQ,YAAY,WACvB,QAAQ,UACR,KAAK,UAAU,QAAQ,OAAO;GACtC,CACF;EACF;CACJ;AACF;AAEA,MAAa,qBAAqB,QAAuB;CACvD,MAAM,aAAa,CACjB,GAAG,IAAI,SACP,GAAI,IAAI,aAAa,SAAS,MAAM,EAAE,OAAO,KAAK,CAAC,CACrD;CAEA,MAAM,aAAa,WAAW,MAC3B,SACC,KAAK,SAAS,UAAU,KAAK,SAAS,WAAW,KAAK,SAAS,OACnE;CACA,MAAM,UAAU,WAAW,MAAM,SAAS,KAAK,SAAS,MAAM;CAC9D,IAAI,cAAc,CAAC,SACjB,WAAW,QAAQ;EAAE,MAAM;EAAQ,MAAM;CAAI,CAAC;CAGhD,MAAM,UAAU,WAAW,SAAS,SAAS;EAC3C,MAAM,OAAO,KAAK;EAClB,QAAQ,MAAR;GACE,KAAK,QACH,OAAO;IAAE,MAAM;IAAiB,MAAM,KAAK;GAAK;GAClD,KAAK,SACH,OAAO;IAAE,MAAM;IAAsB,WAAW,EAAE,KAAK,KAAK,MAAM;GAAE;GACtE,KAAK,QAAQ;IACX,MAAM,WAAW,EAAE,UAAU,KAAK,YAAY,OAAO;IACrD,IAAI,eAAe,KAAK,KAAK,IAAI,GAC/B,OAAO;KACL,MAAM;KACN,KAAK,KAAK;KACV,WAAW,KAAK;KAChB;KACA,aAAa;IACf;IAEF,MAAM,SAAS,aAAa,KAAK,IAAI;IACrC,OAAO;KACL,MAAM;KACN,MAAM,QAAQ,QAAQ,KAAK;KAC3B,WAAW,QAAQ,YAAY,KAAK;KACpC;KACA,aAAa;IACf;GACF;GACA,KAAK,SAEH,OAAO;IACL,MAAM;IACN,MAHa,aAAa,KAAK,MAAM,IAG1B,CAAC,EAAE,QAAQ,KAAK,MAAM;IACjC,WAAW,SAAS,KAAK,MAAM;IAC/B,aAAa;GACf;GAEF,KAAK,QACH,OAAO,CAAC;GACV,KAAK,aACH,MAAM,IAAI,MAAM,sCAAsC;GACxD,SAEE,MAAM,IAAI,MACR,yCAAyCC,MAC3C;EAEJ;CACF,CAAC;CAED,IAAI,QAAQ,WAAW,KAAK,QAAQ,EAAE,EAAE,SAAS,QAC/C,OAAO,QAAQ,EAAE,CAAC,QAAQ;CAE5B,OAAO;AACT"}

package/dist/runtimeExtras.d.ts

@@ -1,6 +1,6 @@
 import { LangChainRuntimeExtras } from "./types.js";
 //#region src/runtimeExtras.d.ts
-declare const langChainExtras: import("@assistant-ui/core/internal").RuntimeExtras<LangChainRuntimeExtras>;
+declare const langChainExtras: import("@assistant-ui/core/react").RuntimeExtras<LangChainRuntimeExtras>;
 //#endregion
 export { langChainExtras };
 //# sourceMappingURL=runtimeExtras.d.ts.map

package/dist/runtimeExtras.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runtimeExtras.d.ts","names":[],"sources":["../src/runtimeExtras.ts"],"mappings":";;cAGa,uDAAe,cAAA"}
+{"version":3,"file":"runtimeExtras.d.ts","names":[],"sources":["../src/runtimeExtras.ts"],"mappings":";;cAGa,oDAAe,cAAA"}

package/dist/runtimeExtras.js

@@ -1,4 +1,4 @@
-import { createRuntimeExtras } from "@assistant-ui/core/internal";
+import { createRuntimeExtras } from "@assistant-ui/core/react";
 //#region src/runtimeExtras.ts
 const langChainExtras = createRuntimeExtras("useStreamRuntime");
 //#endregion

package/dist/runtimeExtras.js.map

@@ -1 +1 @@
-{"version":3,"file":"runtimeExtras.js","names":[],"sources":["../src/runtimeExtras.ts"],"sourcesContent":["import { createRuntimeExtras } from \"@assistant-ui/core/internal\";\nimport type { LangChainRuntimeExtras } from \"./types\";\n\nexport const langChainExtras =\n  createRuntimeExtras<LangChainRuntimeExtras>(\"useStreamRuntime\");\n"],"mappings":";;AAGA,MAAa,kBACX,oBAA4C,kBAAkB"}
+{"version":3,"file":"runtimeExtras.js","names":[],"sources":["../src/runtimeExtras.ts"],"sourcesContent":["import { createRuntimeExtras } from \"@assistant-ui/core/react\";\nimport type { LangChainRuntimeExtras } from \"./types\";\n\nexport const langChainExtras =\n  createRuntimeExtras<LangChainRuntimeExtras>(\"useStreamRuntime\");\n"],"mappings":";;AAGA,MAAa,kBACX,oBAA4C,kBAAkB"}

package/dist/streamingTiming.d.ts

@@ -1,6 +1,6 @@
 import { LangChainBaseMessage } from "./types.js";
-import { MessageTiming } from "@assistant-ui/core";
 import { StreamingTimingAccessors } from "@assistant-ui/core/react";
+import { MessageTiming } from "@assistant-ui/core";
 //#region src/streamingTiming.d.ts
 declare const langChainStreamingTimingAccessors: StreamingTimingAccessors<LangChainBaseMessage>;
 /**

package/dist/types.d.ts

@@ -48,6 +48,11 @@ type LangChainContentBlock = {
   metadata?: {
     filename?: string;
   };
+} | {
+  type: "audio";
+  data: string;
+  mime_type: string;
+  source_type: "base64";
 } | {
   type: "tool_use" | "input_json_delta";
 };

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";;;;;KAoBY;EACN;EAAc;;EACd;EAAoB;;EACpB;EAAmB;IAAsB;;;EACzC;EAAkB;;EAElB;EACA,UAAU;IAAQ;IAAsB;;EACxC;;EAGA;EACA;EACA;EACA;EACA;IAAa;;;EAGb;EACA;EACA;EACA;EACA;IAAa;;;EAGb;EACA;EACA;EACA;EACA;IAAa;;;EAEb;;KAEM;EACV;EACA;EACA,MAAM;;;;;;;;;KAUI,UACV,+BACA,eAAe,0BAA0B;EAEzC;EACA;EACA,MAAM;EACN,OAAO;EACP;IACE;IACA;IACA;IACA;IACA;IACA;KACC;;;;;;;KAQO;EACV;EACA;;;;;;;;;KAUU;EACV;EACA;EACA;EACA;EACA,oBAAoB;;EAEpB,sBAAsB;;EAEtB;;EAEA,SAAS;;EAET;;KAGU;EACV;IAAa;;EACb;IAAuB;IAAa;;EACpC,oBAAoB;EACpB,WAAW,oBAAoB;EAC/B,WAAW,oBAAoB;EAC/B,QAAQ;EACR;EACA,SACE,QAAQ,4CACR,UAAU,4BACP;EACL,UACE,mBACA,UAAU,4BACP;EACL,aACE,eAAe,yBACf,UAAU,4BACP;EACL,QAAQ;EACR;;KAGU,+BAA+B;;;;;;;;;EASzC,qBAAqB;EACrB,QAAQ;EACR;IAEM,cAAc;IACd,SAAS;IACT,YAAY;IACZ,QAAQ;IACR,WAAW;;;;;;;;EASjB;;;;;EAKA;;;;;EAKA,6BAA6B;;EAE7B,gBAAgB;IAAU;;;EAE1B,WAAW,qBAAqB;;;;;;;EAOhC;;KAOU,0BAA0B,gCAA+B,KACjE,UAAU,mBACR,IAAI"}
+{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";;;;;KAoBY;EACN;EAAc;;EACd;EAAoB;;EACpB;EAAmB;IAAsB;;;EACzC;EAAkB;;EAElB;EACA,UAAU;IAAQ;IAAsB;;EACxC;;EAGA;EACA;EACA;EACA;EACA;IAAa;;;EAGb;EACA;EACA;EACA;EACA;IAAa;;;EAGb;EACA;EACA;EACA;EACA;IAAa;;;EAGb;EACA;EACA;EACA;;EAEA;;KAEM;EACV;EACA;EACA,MAAM;;;;;;;;;KAUI,UACV,+BACA,eAAe,0BAA0B;EAEzC;EACA;EACA,MAAM;EACN,OAAO;EACP;IACE;IACA;IACA;IACA;IACA;IACA;KACC;;;;;;;KAQO;EACV;EACA;;;;;;;;;KAUU;EACV;EACA;EACA;EACA;EACA,oBAAoB;;EAEpB,sBAAsB;;EAEtB;;EAEA,SAAS;;EAET;;KAGU;EACV;IAAa;;EACb;IAAuB;IAAa;;EACpC,oBAAoB;EACpB,WAAW,oBAAoB;EAC/B,WAAW,oBAAoB;EAC/B,QAAQ;EACR;EACA,SACE,QAAQ,4CACR,UAAU,4BACP;EACL,UACE,mBACA,UAAU,4BACP;EACL,aACE,eAAe,yBACf,UAAU,4BACP;EACL,QAAQ;EACR;;KAGU,+BAA+B;;;;;;;;;EASzC,qBAAqB;EACrB,QAAQ;EACR;IAEM,cAAc;IACd,SAAS;IACT,YAAY;IACZ,QAAQ;IACR,WAAW;;;;;;;;EASjB;;;;;EAKA;;;;;EAKA,6BAA6B;;EAE7B,gBAAgB;IAAU;;;EAE1B,WAAW,qBAAqB;;;;;;;EAOhC;;KAOU,0BAA0B,gCAA+B,KACjE,UAAU,mBACR,IAAI"}

package/dist/useStreamRuntime.js

@@ -5,9 +5,9 @@ import { foldUIUpdates, mergeUIMessages } from "./uiMessages.js";
 import { resolveForkCheckpoint } from "./resolveForkCheckpoint.js";
 import { useLangChainStreamingTiming } from "./streamingTiming.js";
 import { useAuiState } from "@assistant-ui/store";
+import { useCloudThreadListAdapter, useExternalMessageConverter, useExternalStoreRuntime, useRemoteThreadListRuntime } from "@assistant-ui/core/react";
 import { useEffect, useMemo, useRef, useState } from "react";
 import { generateId, getExternalStoreMessages, pickExternalStoreSharedOptions } from "@assistant-ui/core";
-import { useCloudThreadListAdapter, useExternalMessageConverter, useExternalStoreRuntime, useRemoteThreadListRuntime } from "@assistant-ui/core/react";
 import { STREAM_CONTROLLER, useChannel, useStream } from "@langchain/react";
 //#region src/useStreamRuntime.ts
 const UI_CUSTOM_CHANNELS = ["custom"];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@assistant-ui/react-langchain",
-  "version": "0.0.21",
+  "version": "0.0.22",
   "description": "LangChain useStream adapter for assistant-ui",
   "keywords": [
     "langchain",
@@ -29,10 +29,10 @@
   ],
   "sideEffects": false,
   "dependencies": {
-    "@assistant-ui/core": "^0.3.0",
-    "@assistant-ui/store": "^0.3.0",
+    "@assistant-ui/core": "^0.3.2",
+    "@assistant-ui/store": "^0.3.2",
     "assistant-cloud": "*",
-    "assistant-stream": "^0.3.29"
+    "assistant-stream": "^0.3.31"
   },
   "peerDependencies": {
     "@langchain/react": "^1.0.2",

package/src/convertMessages.test.ts

@@ -202,6 +202,181 @@ describe("getMessageContent file blocks", () => {
   });
 });
 
+describe("getMessageContent audio and data parts", () => {
+  const appendMessage = (...parts: Record<string, unknown>[]) =>
+    ({ content: parts }) as unknown as AppendMessage;
+
+  it("emits a base64 audio block with the format MIME type for audio parts", () => {
+    const content = getMessageContent(
+      appendMessage({
+        type: "audio",
+        audio: { data: "c291bmQ=", format: "mp3" },
+      }),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "audio",
+        data: "c291bmQ=",
+        mime_type: "audio/mp3",
+        source_type: "base64",
+      },
+    ]);
+  });
+
+  it("strips a data URL envelope from audio data", () => {
+    const content = getMessageContent(
+      appendMessage({
+        type: "audio",
+        audio: { data: "data:audio/wav;base64,d2F2", format: "wav" },
+      }),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "audio",
+        data: "d2F2",
+        mime_type: "audio/wav",
+        source_type: "base64",
+      },
+    ]);
+  });
+
+  it("keeps the format-derived MIME when a data URL carries a divergent one", () => {
+    const content = getMessageContent(
+      appendMessage({
+        type: "audio",
+        audio: { data: "data:audio/mpeg;base64,c291bmQ=", format: "mp3" },
+      }),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: " " },
+      {
+        type: "audio",
+        data: "c291bmQ=",
+        mime_type: "audio/mp3",
+        source_type: "base64",
+      },
+    ]);
+  });
+
+  it("does not prepend a second placeholder when text accompanies audio", () => {
+    const content = getMessageContent(
+      appendMessage(
+        { type: "text", text: "listen" },
+        { type: "audio", audio: { data: "d2F2", format: "wav" } },
+      ),
+    );
+
+    expect(content).toEqual([
+      { type: "text", text: "listen" },
+      {
+        type: "audio",
+        data: "d2F2",
+        mime_type: "audio/wav",
+        source_type: "base64",
+      },
+    ]);
+  });
+
+  it("drops data parts while keeping the rest of the message", () => {
+    const content = getMessageContent(
+      appendMessage(
+        { type: "text", text: "hi" },
+        { type: "data", name: "chart", data: { values: [1, 2] } },
+      ),
+    );
+
+    expect(content).toBe("hi");
+  });
+
+  it("returns empty content for a data-only message", () => {
+    const content = getMessageContent(
+      appendMessage({ type: "data", name: "chart", data: { values: [1, 2] } }),
+    );
+
+    expect(content).toEqual([]);
+  });
+
+  it("still throws on assistant-only part types", () => {
+    expect(() =>
+      getMessageContent(appendMessage({ type: "reasoning", text: "hmm" })),
+    ).toThrow("Unsupported append message part type: reasoning");
+  });
+});
+
+describe("contentToParts audio blocks", () => {
+  it("converts an inbound base64 audio block back to an audio part", () => {
+    const result = convertLangChainBaseMessage(
+      {
+        _getType: () => "human",
+        id: "h1",
+        content: [
+          {
+            type: "audio",
+            data: "c291bmQ=",
+            mime_type: "audio/mp3",
+            source_type: "base64",
+          },
+        ],
+      },
+      {},
+    );
+
+    expect(result).toMatchObject({
+      role: "user",
+      content: [{ type: "audio", audio: { data: "c291bmQ=", format: "mp3" } }],
+    });
+  });
+
+  it("drops an inbound audio block with an unrepresentable mime type", () => {
+    const result = convertLangChainBaseMessage(
+      {
+        _getType: () => "human",
+        id: "h1",
+        content: [
+          {
+            type: "audio",
+            data: "b2dn",
+            mime_type: "audio/ogg",
+            source_type: "base64",
+          },
+        ],
+      },
+      {},
+    );
+
+    expect(result).toMatchObject({ role: "user", content: [] });
+  });
+
+  it("drops an audio block on an assistant message", () => {
+    const result = convertLangChainBaseMessage(
+      {
+        _getType: () => "ai",
+        id: "ai-1",
+        content: [
+          {
+            type: "audio",
+            data: "c291bmQ=",
+            mime_type: "audio/mp3",
+            source_type: "base64",
+          },
+          { type: "text", text: "done" },
+        ],
+      },
+      {},
+    );
+
+    expect(result).toMatchObject({
+      role: "assistant",
+      content: [{ type: "text", text: "done" }],
+    });
+  });
+});
+
 describe("convertLangChainBaseMessage reasoning content parts", () => {
   it("joins summary parts into a single reasoning part", () => {
     const result = convertLangChainBaseMessage(

package/src/convertMessages.ts

@@ -33,7 +33,7 @@ export const getMessageType = (message: LangChainBaseMessage): string => {
   throw new Error("Cannot determine message type");
 };
 
-const contentToParts = (content: unknown) => {
+const contentToParts = (content: unknown, role: "user" | "assistant") => {
   if (typeof content === "string")
     return [{ type: "text" as const, text: content }];
 
@@ -65,6 +65,20 @@ const contentToParts = (content: unknown) => {
                   : part.data,
             mimeType: part.mime_type ?? "application/octet-stream",
           };
+        case "audio": {
+          if (role !== "user") return null;
+          const format =
+            part.mime_type === "audio/wav"
+              ? ("wav" as const)
+              : part.mime_type === "audio/mp3"
+                ? ("mp3" as const)
+                : null;
+          if (!format) return null;
+          return {
+            type: "audio" as const,
+            audio: { data: part.data, format },
+          };
+        }
         case "thinking":
           return { type: "reasoning" as const, text: part.thinking };
         case "reasoning":
@@ -120,7 +134,7 @@ export const convertLangChainBaseMessage = (
       return {
         role: "user",
         id: message.id,
-        content: contentToParts(message.content),
+        content: contentToParts(message.content, "user"),
         metadata: {
           custom: getCustomMetadata(message.additional_kwargs),
         },
@@ -152,7 +166,7 @@ export const convertLangChainBaseMessage = (
         role: "assistant",
         id: message.id,
         content: [
-          ...contentToParts(message.content),
+          ...contentToParts(message.content, "assistant"),
           ...toolCallParts,
           ...uiDataParts,
         ],
@@ -198,14 +212,15 @@ export const getMessageContent = (msg: AppendMessage) => {
   ];
 
   const hasNonText = allContent.some(
-    (part) => part.type === "file" || part.type === "image",
+    (part) =>
+      part.type === "file" || part.type === "image" || part.type === "audio",
   );
   const hasText = allContent.some((part) => part.type === "text");
   if (hasNonText && !hasText) {
     allContent.unshift({ type: "text", text: " " });
   }
 
-  const content = allContent.map((part) => {
+  const content = allContent.flatMap((part) => {
     const type = part.type;
     switch (type) {
       case "text":
@@ -232,15 +247,21 @@ export const getMessageContent = (msg: AppendMessage) => {
           source_type: "base64" as const,
         };
       }
+      case "audio": {
+        const parsed = parseDataUrl(part.audio.data);
+        return {
+          type: "audio" as const,
+          data: parsed?.data ?? part.audio.data,
+          mime_type: `audio/${part.audio.format}`,
+          source_type: "base64" as const,
+        };
+      }
+      case "data":
+        return [];
       case "tool-call":
         throw new Error("Tool call appends are not supported.");
       default: {
-        const _exhaustiveCheck:
-          | "reasoning"
-          | "source"
-          | "audio"
-          | "data"
-          | "generative-ui" = type;
+        const _exhaustiveCheck: "reasoning" | "source" | "generative-ui" = type;
         throw new Error(
           `Unsupported append message part type: ${_exhaustiveCheck}`,
         );

package/src/runtimeExtras.ts

@@ -1,4 +1,4 @@
-import { createRuntimeExtras } from "@assistant-ui/core/internal";
+import { createRuntimeExtras } from "@assistant-ui/core/react";
 import type { LangChainRuntimeExtras } from "./types";
 
 export const langChainExtras =

package/src/types.ts

@@ -49,6 +49,12 @@ export type LangChainContentBlock =
       source_type: "id";
       metadata?: { filename?: string };
     }
+  | {
+      type: "audio";
+      data: string;
+      mime_type: string;
+      source_type: "base64";
+    }
   | { type: "tool_use" | "input_json_delta" };
 
 export type LangChainToolCall = {