@lobehub/lobehub 2.0.0-next.114 → 2.0.0-next.116

package/packages/model-runtime/src/core/streams/protocol.ts

@@ -77,6 +77,10 @@ export interface StreamProtocolChunk {
     | 'reasoning_signature'
     // flagged reasoning signature
     | 'flagged_reasoning_signature'
+    // multimodal content part in reasoning
+    | 'reasoning_part'
+    // multimodal content part in content
+    | 'content_part'
     // Search or Grounding
     | 'grounding'
     // stop signal
@@ -91,6 +95,21 @@ export interface StreamProtocolChunk {
     | 'data';
 }
 
+/**
+ * Stream content part chunk data for multimodal support
+ */
+export interface StreamPartChunkData {
+  content: string;
+  // whether this part is in reasoning or regular content
+  inReasoning: boolean;
+  // image MIME type
+  mimeType?: string;
+  // text content or base64 image data
+  partType: 'text' | 'image';
+  // Optional signature for reasoning verification (Google Gemini feature)
+  thoughtSignature?: string;
+}
+
 export interface StreamToolCallChunkData {
   function?: {
     arguments?: string;

package/packages/types/src/message/common/base.ts

@@ -26,14 +26,40 @@ export interface ChatCitationItem {
   url: string;
 }
 
+/**
+ * Message content part types for multimodal content support
+ */
+export interface MessageContentPartText {
+  text: string;
+  thoughtSignature?: string;
+  type: 'text';
+}
+
+export interface MessageContentPartImage {
+  image: string;
+  thoughtSignature?: string;
+  type: 'image';
+}
+
+export type MessageContentPart = MessageContentPartText | MessageContentPartImage;
+
 export interface ModelReasoning {
+  /**
+   * Reasoning content, can be plain string or serialized JSON array of MessageContentPart[]
+   */
   content?: string;
   duration?: number;
+  /**
+   * Flag indicating if content is multimodal (serialized MessageContentPart[])
+   */
+  isMultimodal?: boolean;
   signature?: string;
+  tempDisplayContent?: MessageContentPart[];
 }
 
 export const ModelReasoningSchema = z.object({
   content: z.string().optional(),
   duration: z.number().optional(),
+  isMultimodal: z.boolean().optional(),
   signature: z.string().optional(),
 });

package/packages/types/src/message/common/metadata.ts

@@ -78,6 +78,7 @@ export const ModelPerformanceSchema = z.object({
 export const MessageMetadataSchema = ModelUsageSchema.merge(ModelPerformanceSchema).extend({
   collapsed: z.boolean().optional(),
   inspectExpanded: z.boolean().optional(),
+  isMultimodal: z.boolean().optional(),
 });
 
 export interface ModelUsage extends ModelTokensUsage {
@@ -123,4 +124,10 @@ export interface MessageMetadata extends ModelUsage, ModelPerformance {
   compare?: boolean;
   usage?: ModelUsage;
   performance?: ModelPerformance;
+  /**
+   * Flag indicating if message content is multimodal (serialized MessageContentPart[])
+   */
+  isMultimodal?: boolean;
+  // message content is multimodal, display content in the streaming, won't save to db
+  tempDisplayContent?: string;
 }

package/packages/utils/src/index.ts

@@ -5,6 +5,7 @@ export * from './format';
 export * from './imageToBase64';
 export * from './keyboard';
 export * from './merge';
+export * from './multimodalContent';
 export * from './number';
 export * from './object';
 export * from './pricing';

package/packages/utils/src/multimodalContent.ts

@@ -0,0 +1,25 @@
+import { MessageContentPart } from '@lobechat/types';
+
+/**
+ * Serialize message content parts to JSON string for storage
+ */
+export function serializePartsForStorage(parts: MessageContentPart[]): string {
+  return JSON.stringify(parts);
+}
+
+/**
+ * Deserialize content string to message content parts
+ * Returns null if content is not valid JSON array of parts
+ */
+export function deserializeParts(content: string): MessageContentPart[] | null {
+  try {
+    const parsed = JSON.parse(content);
+    // Validate it's an array with valid part structure
+    if (Array.isArray(parsed) && parsed.length > 0 && parsed[0]?.type) {
+      return parsed as MessageContentPart[];
+    }
+  } catch {
+    // Not JSON, treat as plain text
+  }
+  return null;
+}

package/src/components/Thinking/index.tsx

@@ -4,7 +4,7 @@ import { createStyles } from 'antd-style';
 import { AnimatePresence, motion } from 'framer-motion';
 import { AtomIcon } from 'lucide-react';
 import { rgba } from 'polished';
-import { CSSProperties, RefObject, memo, useEffect, useRef, useState } from 'react';
+import { CSSProperties, ReactNode, RefObject, memo, useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { Flexbox } from 'react-layout-kit';
 
@@ -76,7 +76,7 @@ const useStyles = createStyles(({ css, token }) => ({
 
 interface ThinkingProps {
   citations?: ChatCitationItem[];
-  content?: string;
+  content?: string | ReactNode;
   duration?: number;
   style?: CSSProperties;
   thinking?: boolean;
@@ -158,7 +158,7 @@ const Thinking = memo<ThinkingProps>((props) => {
           </Flexbox>
         )}
         <Flexbox gap={4} horizontal>
-          {showDetail && content && (
+          {showDetail && content && typeof content === 'string' && (
             <div
               onClick={(event) => {
                 event.stopPropagation();

package/src/features/ChatList/Messages/Assistant/DisplayContent.tsx

@@ -0,0 +1,44 @@
+import { deserializeParts } from '@lobechat/utils';
+import { Markdown, MarkdownProps } from '@lobehub/ui';
+import { memo } from 'react';
+
+import BubblesLoading from '@/components/BubblesLoading';
+import { LOADING_FLAT } from '@/const/message';
+import { RichContentRenderer } from '@/features/ChatList/components/RichContentRenderer';
+import { normalizeThinkTags, processWithArtifact } from '@/features/ChatList/utils/markdown';
+
+const MessageContent = memo<{
+  addIdOnDOM?: boolean;
+  content: string;
+  hasImages?: boolean;
+  isMultimodal?: boolean;
+  isToolCallGenerating?: boolean;
+  markdownProps?: Omit<MarkdownProps, 'className' | 'style' | 'children'>;
+  tempDisplayContent?: string;
+}>(
+  ({
+    markdownProps,
+    content,
+    isToolCallGenerating,
+    hasImages,
+    isMultimodal,
+    tempDisplayContent,
+  }) => {
+    const message = normalizeThinkTags(processWithArtifact(content));
+    if (isToolCallGenerating) return;
+
+    if ((!content && !hasImages) || content === LOADING_FLAT) return <BubblesLoading />;
+
+    const contentParts = isMultimodal ? deserializeParts(tempDisplayContent || content) : null;
+
+    return contentParts ? (
+      <RichContentRenderer parts={contentParts} />
+    ) : (
+      <Markdown {...markdownProps} variant={'chat'}>
+        {message}
+      </Markdown>
+    );
+  },
+);
+
+export default MessageContent;

package/src/features/ChatList/Messages/Assistant/MessageBody.tsx

@@ -0,0 +1,96 @@
+import { LOADING_FLAT } from '@lobechat/const';
+import { UIChatMessage } from '@lobechat/types';
+import { MarkdownProps } from '@lobehub/ui';
+import { ReactNode, memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { useChatStore } from '@/store/chat';
+import { aiChatSelectors, messageStateSelectors } from '@/store/chat/selectors';
+
+import { DefaultMessage } from '../Default';
+import ImageFileListViewer from '../User/ImageFileListViewer';
+import { CollapsedMessage } from './CollapsedMessage';
+import MessageContent from './DisplayContent';
+import FileChunks from './FileChunks';
+import IntentUnderstanding from './IntentUnderstanding';
+import Reasoning from './Reasoning';
+import SearchGrounding from './SearchGrounding';
+
+export const AssistantMessageBody = memo<
+  UIChatMessage & {
+    editableContent: ReactNode;
+    markdownProps?: Omit<MarkdownProps, 'className' | 'style' | 'children'>;
+  }
+>(
+  ({
+    id,
+    tools,
+    content,
+    chunksList,
+    search,
+    imageList,
+    metadata,
+    editableContent,
+    markdownProps,
+    ...props
+  }) => {
+    const [editing, generating, isCollapsed] = useChatStore((s) => [
+      messageStateSelectors.isMessageEditing(id)(s),
+      messageStateSelectors.isMessageGenerating(id)(s),
+      messageStateSelectors.isMessageCollapsed(id)(s),
+    ]);
+
+    const isToolCallGenerating = generating && (content === LOADING_FLAT || !content) && !!tools;
+
+    const isReasoning = useChatStore(aiChatSelectors.isMessageInReasoning(id));
+
+    const isIntentUnderstanding = useChatStore(aiChatSelectors.isIntentUnderstanding(id));
+
+    const showSearch = !!search && !!search.citations?.length;
+    const showImageItems = !!imageList && imageList.length > 0;
+
+    // remove \n to avoid empty content
+    // refs: https://github.com/lobehub/lobe-chat/pull/6153
+    const showReasoning =
+      (!!props.reasoning && props.reasoning.content?.trim() !== '') ||
+      (!props.reasoning && isReasoning);
+
+    const showFileChunks = !!chunksList && chunksList.length > 0;
+
+    if (editing)
+      return (
+        <DefaultMessage
+          content={content}
+          editableContent={editableContent}
+          id={id}
+          isToolCallGenerating={isToolCallGenerating}
+          {...props}
+        />
+      );
+
+    if (isCollapsed) return <CollapsedMessage content={content} id={id} />;
+
+    return (
+      <Flexbox gap={8} id={id}>
+        {showSearch && (
+          <SearchGrounding citations={search?.citations} searchQueries={search?.searchQueries} />
+        )}
+        {showFileChunks && <FileChunks data={chunksList} />}
+        {showReasoning && <Reasoning {...props.reasoning} id={id} />}
+        {isIntentUnderstanding ? (
+          <IntentUnderstanding />
+        ) : (
+          <MessageContent
+            content={content}
+            hasImages={showImageItems}
+            isMultimodal={metadata?.isMultimodal}
+            isToolCallGenerating={isToolCallGenerating}
+            markdownProps={markdownProps}
+            tempDisplayContent={metadata?.tempDisplayContent}
+          />
+        )}
+        {showImageItems && <ImageFileListViewer items={imageList} />}
+      </Flexbox>
+    );
+  },
+);

package/src/features/ChatList/Messages/Assistant/Reasoning/index.tsx

@@ -1,3 +1,5 @@
+import { MessageContentPart } from '@lobechat/types';
+import { deserializeParts } from '@lobechat/utils';
 import { memo } from 'react';
 
 import Thinking from '@/components/Thinking';
@@ -6,24 +8,35 @@ import { aiChatSelectors } from '@/store/chat/selectors';
 import { useUserStore } from '@/store/user';
 import { userGeneralSettingsSelectors } from '@/store/user/selectors';
 
+import { RichContentRenderer } from '../../../components/RichContentRenderer';
+
 interface ReasoningProps {
   content?: string;
   duration?: number;
   id: string;
+  isMultimodal?: boolean;
+  tempDisplayContent?: MessageContentPart[];
 }
 
-const Reasoning = memo<ReasoningProps>(({ content = '', duration, id }) => {
-  const isReasoning = useChatStore(aiChatSelectors.isMessageInReasoning(id));
-  const transitionMode = useUserStore(userGeneralSettingsSelectors.transitionMode);
-
-  return (
-    <Thinking
-      content={content}
-      duration={duration}
-      thinking={isReasoning}
-      thinkingAnimated={transitionMode === 'fadeIn' && isReasoning}
-    />
-  );
-});
+const Reasoning = memo<ReasoningProps>(
+  ({ content = '', duration, id, isMultimodal, tempDisplayContent }) => {
+    const isReasoning = useChatStore(aiChatSelectors.isMessageInReasoning(id));
+    const transitionMode = useUserStore(userGeneralSettingsSelectors.transitionMode);
+
+    const parts = tempDisplayContent || deserializeParts(content);
+
+    // If parts are provided, render multimodal content
+    const thinkingContent = isMultimodal && parts ? <RichContentRenderer parts={parts} /> : content;
+
+    return (
+      <Thinking
+        content={thinkingContent}
+        duration={duration}
+        thinking={isReasoning}
+        thinkingAnimated={transitionMode === 'fadeIn' && isReasoning}
+      />
+    );
+  },
+);
 
 export default Reasoning;

package/src/features/ChatList/Messages/Assistant/index.tsx

@@ -33,7 +33,7 @@ import { useDoubleClickEdit } from '../../hooks/useDoubleClickEdit';
 import { normalizeThinkTags, processWithArtifact } from '../../utils/markdown';
 import { AssistantActionsBar } from './Actions';
 import { AssistantMessageExtra } from './Extra';
-import { AssistantMessageContent } from './MessageContent';
+import { AssistantMessageBody } from './MessageBody';
 
 const rehypePlugins = markdownElements.map((element) => element.rehypePlugin).filter(Boolean);
 const remarkPlugins = markdownElements.map((element) => element.remarkPlugin).filter(Boolean);
@@ -75,7 +75,7 @@ export const useStyles = createStyles(
           justify-content: ${placement === 'left' ? 'flex-end' : 'flex-start'};
         `,
         editing &&
-        css`
+          css`
             pointer-events: none !important;
             opacity: 0 !important;
           `,
@@ -84,11 +84,9 @@ export const useStyles = createStyles(
         variant === 'docs' && rawContainerStylish,
         css`
           position: relative;
-
           width: 100%;
           max-width: 100vw;
           padding-block: 24px 12px;
-          padding-inline: 12px;
 
           @supports (content-visibility: auto) {
             contain-intrinsic-size: auto 100lvh;
@@ -305,9 +303,13 @@ const AssistantMessage = memo<AssistantMessageProps>(
 
     const renderMessage = useCallback(
       (editableContent: ReactNode) => (
-        <AssistantMessageContent {...item} editableContent={editableContent} />
+        <AssistantMessageBody
+          {...item}
+          editableContent={editableContent}
+          markdownProps={markdownProps}
+        />
       ),
-      [item],
+      [item, markdownProps],
     );
     const errorMessage = <ErrorMessageExtra data={item} />;
 

package/src/features/ChatList/Messages/Default.tsx

@@ -6,25 +6,22 @@ import { LOADING_FLAT } from '@/const/message';
 import { useChatStore } from '@/store/chat';
 import { messageStateSelectors } from '@/store/chat/selectors';
 
-export const MessageContentClassName = 'msg_content_flag'
+export const MessageContentClassName = 'msg_content_flag';
 
 export const DefaultMessage = memo<
   UIChatMessage & {
     addIdOnDOM?: boolean;
     editableContent: ReactNode;
+    hasImages?: boolean;
     isToolCallGenerating?: boolean;
   }
->(({ id, editableContent, content, isToolCallGenerating, addIdOnDOM = true }) => {
+>(({ id, editableContent, content, isToolCallGenerating, addIdOnDOM = true, hasImages }) => {
   const editing = useChatStore(messageStateSelectors.isMessageEditing(id));
 
   if (isToolCallGenerating) return;
 
-  if (!content) return <BubblesLoading />;
+  if (!content && !hasImages) return <BubblesLoading />;
   if (content === LOADING_FLAT && !editing) return <BubblesLoading />;
 
   return <div id={addIdOnDOM ? id : undefined}>{editableContent}</div>;
 });
-
-export const DefaultBelowMessage = memo<UIChatMessage>(() => {
-  return null;
-});

package/src/features/ChatList/components/RichContentRenderer.tsx

@@ -0,0 +1,35 @@
+import { Image, Markdown } from '@lobehub/ui';
+import { memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { MessageContentPart } from '@/types/index';
+
+interface RichContentRendererProps {
+  parts: MessageContentPart[];
+}
+
+export const RichContentRenderer = memo<RichContentRendererProps>(({ parts }) => {
+  return (
+    <Flexbox gap={8}>
+      {parts.map((part, index) => {
+        if (part.type === 'text') {
+          return (
+            <Markdown key={index} variant="chat">
+              {part.text}
+            </Markdown>
+          );
+        }
+
+        if (part.type === 'image') {
+          return (
+            <Image key={index} src={part.image} style={{ borderRadius: 8, maxWidth: '100%' }} />
+          );
+        }
+
+        return null;
+      })}
+    </Flexbox>
+  );
+});
+
+RichContentRenderer.displayName = 'RichContentRenderer';