@lobehub/lobehub 2.0.0-next.115 → 2.0.0-next.117

package/src/features/ChatList/Messages/Assistant/Reasoning/index.tsx

@@ -1,3 +1,5 @@
+import { MessageContentPart } from '@lobechat/types';
+import { deserializeParts } from '@lobechat/utils';
 import { memo } from 'react';
 
 import Thinking from '@/components/Thinking';
@@ -6,24 +8,35 @@ import { aiChatSelectors } from '@/store/chat/selectors';
 import { useUserStore } from '@/store/user';
 import { userGeneralSettingsSelectors } from '@/store/user/selectors';
 
+import { RichContentRenderer } from '../../../components/RichContentRenderer';
+
 interface ReasoningProps {
   content?: string;
   duration?: number;
   id: string;
+  isMultimodal?: boolean;
+  tempDisplayContent?: MessageContentPart[];
 }
 
-const Reasoning = memo<ReasoningProps>(({ content = '', duration, id }) => {
-  const isReasoning = useChatStore(aiChatSelectors.isMessageInReasoning(id));
-  const transitionMode = useUserStore(userGeneralSettingsSelectors.transitionMode);
-
-  return (
-    <Thinking
-      content={content}
-      duration={duration}
-      thinking={isReasoning}
-      thinkingAnimated={transitionMode === 'fadeIn' && isReasoning}
-    />
-  );
-});
+const Reasoning = memo<ReasoningProps>(
+  ({ content = '', duration, id, isMultimodal, tempDisplayContent }) => {
+    const isReasoning = useChatStore(aiChatSelectors.isMessageInReasoning(id));
+    const transitionMode = useUserStore(userGeneralSettingsSelectors.transitionMode);
+
+    const parts = tempDisplayContent || deserializeParts(content);
+
+    // If parts are provided, render multimodal content
+    const thinkingContent = isMultimodal && parts ? <RichContentRenderer parts={parts} /> : content;
+
+    return (
+      <Thinking
+        content={thinkingContent}
+        duration={duration}
+        thinking={isReasoning}
+        thinkingAnimated={transitionMode === 'fadeIn' && isReasoning}
+      />
+    );
+  },
+);
 
 export default Reasoning;

package/src/features/ChatList/Messages/Assistant/index.tsx

@@ -33,7 +33,7 @@ import { useDoubleClickEdit } from '../../hooks/useDoubleClickEdit';
 import { normalizeThinkTags, processWithArtifact } from '../../utils/markdown';
 import { AssistantActionsBar } from './Actions';
 import { AssistantMessageExtra } from './Extra';
-import { AssistantMessageContent } from './MessageContent';
+import { AssistantMessageBody } from './MessageBody';
 
 const rehypePlugins = markdownElements.map((element) => element.rehypePlugin).filter(Boolean);
 const remarkPlugins = markdownElements.map((element) => element.remarkPlugin).filter(Boolean);
@@ -75,7 +75,7 @@ export const useStyles = createStyles(
           justify-content: ${placement === 'left' ? 'flex-end' : 'flex-start'};
         `,
         editing &&
-        css`
+          css`
             pointer-events: none !important;
             opacity: 0 !important;
           `,
@@ -84,11 +84,9 @@ export const useStyles = createStyles(
         variant === 'docs' && rawContainerStylish,
         css`
           position: relative;
-
           width: 100%;
           max-width: 100vw;
           padding-block: 24px 12px;
-          padding-inline: 12px;
 
           @supports (content-visibility: auto) {
             contain-intrinsic-size: auto 100lvh;
@@ -305,9 +303,13 @@ const AssistantMessage = memo<AssistantMessageProps>(
 
     const renderMessage = useCallback(
       (editableContent: ReactNode) => (
-        <AssistantMessageContent {...item} editableContent={editableContent} />
+        <AssistantMessageBody
+          {...item}
+          editableContent={editableContent}
+          markdownProps={markdownProps}
+        />
       ),
-      [item],
+      [item, markdownProps],
     );
     const errorMessage = <ErrorMessageExtra data={item} />;
 

package/src/features/ChatList/Messages/Default.tsx

@@ -6,25 +6,22 @@ import { LOADING_FLAT } from '@/const/message';
 import { useChatStore } from '@/store/chat';
 import { messageStateSelectors } from '@/store/chat/selectors';
 
-export const MessageContentClassName = 'msg_content_flag'
+export const MessageContentClassName = 'msg_content_flag';
 
 export const DefaultMessage = memo<
   UIChatMessage & {
     addIdOnDOM?: boolean;
     editableContent: ReactNode;
+    hasImages?: boolean;
     isToolCallGenerating?: boolean;
   }
->(({ id, editableContent, content, isToolCallGenerating, addIdOnDOM = true }) => {
+>(({ id, editableContent, content, isToolCallGenerating, addIdOnDOM = true, hasImages }) => {
   const editing = useChatStore(messageStateSelectors.isMessageEditing(id));
 
   if (isToolCallGenerating) return;
 
-  if (!content) return <BubblesLoading />;
+  if (!content && !hasImages) return <BubblesLoading />;
   if (content === LOADING_FLAT && !editing) return <BubblesLoading />;
 
   return <div id={addIdOnDOM ? id : undefined}>{editableContent}</div>;
 });
-
-export const DefaultBelowMessage = memo<UIChatMessage>(() => {
-  return null;
-});

package/src/features/ChatList/components/RichContentRenderer.tsx

@@ -0,0 +1,35 @@
+import { Image, Markdown } from '@lobehub/ui';
+import { memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { MessageContentPart } from '@/types/index';
+
+interface RichContentRendererProps {
+  parts: MessageContentPart[];
+}
+
+export const RichContentRenderer = memo<RichContentRendererProps>(({ parts }) => {
+  return (
+    <Flexbox gap={8}>
+      {parts.map((part, index) => {
+        if (part.type === 'text') {
+          return (
+            <Markdown key={index} variant="chat">
+              {part.text}
+            </Markdown>
+          );
+        }
+
+        if (part.type === 'image') {
+          return (
+            <Image key={index} src={part.image} style={{ borderRadius: 8, maxWidth: '100%' }} />
+          );
+        }
+
+        return null;
+      })}
+    </Flexbox>
+  );
+});
+
+RichContentRenderer.displayName = 'RichContentRenderer';

package/src/store/agent/slices/chat/selectors/chatConfig.ts

@@ -1,4 +1,5 @@
-import { contextCachingModels, thinkingWithToolClaudeModels } from '@/const/models';
+import { isContextCachingModel, isThinkingWithToolClaudeModel } from '@lobechat/model-runtime';
+
 import { DEFAULT_AGENT_CHAT_CONFIG, DEFAULT_AGENT_SEARCH_FC_MODEL } from '@/const/settings';
 import { AgentStoreState } from '@/store/agent/initialState';
 import { LobeAgentChatConfig } from '@/types/agent';
@@ -24,12 +25,12 @@ const enableHistoryCount = (s: AgentStoreState) => {
   // 如果开启了上下文缓存，且当前模型类型匹配，则不开启历史记录
   const enableContextCaching = !chatConfig.disableContextCaching;
 
-  if (enableContextCaching && contextCachingModels.has(config.model)) return false;
+  if (enableContextCaching && isContextCachingModel(config.model)) return false;
 
   // 当开启搜索时，针对 claude 3.7 sonnet 模型不开启历史记录
   const enableSearch = isAgentEnableSearch(s);
 
-  if (enableSearch && thinkingWithToolClaudeModels.has(config.model)) return false;
+  if (enableSearch && isThinkingWithToolClaudeModel(config.model)) return false;
 
   return chatConfig.enableHistoryCount;
 };

package/src/store/chat/slices/aiChat/actions/streamingExecutor.ts

@@ -5,14 +5,17 @@ import { isDesktop } from '@lobechat/const';
 import {
   ChatImageItem,
   ChatToolPayload,
+  MessageContentPart,
   MessageToolCall,
   ModelUsage,
   TraceNameMap,
   UIChatMessage,
 } from '@lobechat/types';
+import { serializePartsForStorage } from '@lobechat/utils';
 import debug from 'debug';
 import { t } from 'i18next';
 import { throttle } from 'lodash-es';
+import pMap from 'p-map';
 import { StateCreator } from 'zustand/vanilla';
 
 import { createAgentToolsEngine } from '@/helpers/toolEngineering';
@@ -272,14 +275,21 @@ export const streamingExecutor: StateCreator<
     let finalUsage;
     let msgTraceId: string | undefined;
     let output = '';
-    let thinking = '';
+
+    let thinkingContent = '';
     let thinkingStartAt: number;
-    let duration: number | undefined;
+    let thinkingDuration: number | undefined;
     let reasoningOperationId: string | undefined;
     let finishType: string | undefined;
     // to upload image
     const uploadTasks: Map<string, Promise<{ id?: string; url?: string }>> = new Map();
 
+    // Multimodal content parts
+    let contentParts: MessageContentPart[] = [];
+    let reasoningParts: MessageContentPart[] = [];
+    const contentImageUploads: Map<number, Promise<string>> = new Map();
+    const reasoningImageUploads: Map<number, Promise<string>> = new Map();
+
     // Throttle tool_calls updates to prevent excessive re-renders (max once per 300ms)
     const throttledUpdateToolCalls = throttle(
       (toolCalls: MessageToolCall[]) => {
@@ -344,7 +354,9 @@ export const streamingExecutor: StateCreator<
         if (uploadTasks.size > 0) {
           try {
             // 等待所有上传任务完成
-            const uploadResults = await Promise.all(uploadTasks.values());
+            const uploadResults = await pMap(Array.from(uploadTasks.values()), (task) => task, {
+              concurrency: 5,
+            });
 
             // 使用上传后的 S3 URL 替换原始图像数据
             finalImages = uploadResults.filter((i) => !!i.url) as ChatImageItem[];
@@ -353,6 +365,14 @@ export const streamingExecutor: StateCreator<
           }
         }
 
+        // Wait for all multimodal image uploads to complete
+        // Note: Arrays are already updated in-place when uploads complete
+        // Use Promise.allSettled to continue even if some uploads fail
+        await Promise.allSettled([
+          ...Array.from(contentImageUploads.values()),
+          ...Array.from(reasoningImageUploads.values()),
+        ]);
+
         let parsedToolCalls = toolCalls;
         if (parsedToolCalls && parsedToolCalls.length > 0) {
           // Flush any pending throttled updates before finalizing
@@ -384,18 +404,58 @@ export const streamingExecutor: StateCreator<
           operationId,
         );
 
+        // Check if there are any image parts
+        const hasContentImages = contentParts.some((part) => part.type === 'image');
+        const hasReasoningImages = reasoningParts.some((part) => part.type === 'image');
+
+        // Determine final content
+        // If has images, serialize contentParts; otherwise use accumulated output text
+        const finalContent = hasContentImages ? serializePartsForStorage(contentParts) : output;
+
+        const finalDuration =
+          thinkingDuration && !isNaN(thinkingDuration) ? thinkingDuration : undefined;
+
+        // Determine final reasoning content
+        // Priority: reasoningParts (multimodal) > thinkingContent (from reasoning_part text) > reasoning (from old reasoning event)
+        let finalReasoning: any = undefined;
+        if (hasReasoningImages) {
+          // Has images, use multimodal format
+          finalReasoning = {
+            content: serializePartsForStorage(reasoningParts),
+            duration: finalDuration,
+            isMultimodal: true,
+          };
+        } else if (thinkingContent) {
+          // Has text from reasoning_part but no images
+          finalReasoning = {
+            content: thinkingContent,
+            duration: finalDuration,
+          };
+        } else if (reasoning?.content) {
+          // Fallback to old reasoning event content
+          finalReasoning = {
+            ...reasoning,
+            duration: finalDuration,
+          };
+        }
+
         // update the content after fetch result
         await optimisticUpdateMessageContent(
           messageId,
-          content,
+          finalContent,
           {
             tools,
-            reasoning: !!reasoning
-              ? { ...reasoning, duration: duration && !isNaN(duration) ? duration : undefined }
-              : undefined,
+            reasoning: finalReasoning,
             search: !!grounding?.citations ? grounding : undefined,
             imageList: finalImages.length > 0 ? finalImages : undefined,
-            metadata: { ...usage, ...speed, performance: speed, usage, finishType: type },
+            metadata: {
+              ...usage,
+              ...speed,
+              performance: speed,
+              usage,
+              finishType: type,
+              ...(hasContentImages && { isMultimodal: true }),
+            },
           },
           { operationId },
         );
@@ -457,8 +517,8 @@ export const streamingExecutor: StateCreator<
             output += chunk.text;
 
             // if there is no duration, it means the end of reasoning
-            if (!duration) {
-              duration = Date.now() - thinkingStartAt;
+            if (!thinkingDuration) {
+              thinkingDuration = Date.now() - thinkingStartAt;
 
               // Complete reasoning operation if it exists
               if (reasoningOperationId) {
@@ -480,7 +540,9 @@ export const streamingExecutor: StateCreator<
                 type: 'updateMessage',
                 value: {
                   content: output,
-                  reasoning: !!thinking ? { content: thinking, duration } : undefined,
+                  reasoning: !!thinkingContent
+                    ? { content: thinkingContent, duration: thinkingDuration }
+                    : undefined,
                 },
               },
               { operationId },
@@ -505,13 +567,178 @@ export const streamingExecutor: StateCreator<
               get().associateMessageWithOperation(messageId, reasoningOperationId);
             }
 
-            thinking += chunk.text;
+            thinkingContent += chunk.text;
 
             internal_dispatchMessage(
               {
                 id: messageId,
                 type: 'updateMessage',
-                value: { reasoning: { content: thinking } },
+                value: { reasoning: { content: thinkingContent } },
+              },
+              { operationId },
+            );
+            break;
+          }
+
+          case 'reasoning_part': {
+            // Start reasoning if not started
+            if (!thinkingStartAt) {
+              thinkingStartAt = Date.now();
+
+              const { operationId: reasoningOpId } = get().startOperation({
+                type: 'reasoning',
+                context: { sessionId, topicId, messageId },
+                parentOperationId: operationId,
+              });
+              reasoningOperationId = reasoningOpId;
+              get().associateMessageWithOperation(messageId, reasoningOperationId);
+            }
+
+            const { partType, content: partContent, mimeType } = chunk;
+
+            if (partType === 'text') {
+              const lastPart = reasoningParts.at(-1);
+
+              // If last part is also text, merge chunks together
+              if (lastPart?.type === 'text') {
+                reasoningParts = [
+                  ...reasoningParts.slice(0, -1),
+                  { type: 'text', text: lastPart.text + partContent },
+                ];
+              } else {
+                // Create new text part (first chunk, may contain thoughtSignature)
+                reasoningParts = [...reasoningParts, { type: 'text', text: partContent }];
+              }
+              thinkingContent += partContent;
+            } else if (partType === 'image') {
+              // Image part - create new array to avoid mutation
+              const tempImage = `data:${mimeType};base64,${partContent}`;
+              const partIndex = reasoningParts.length;
+              const newPart: MessageContentPart = { type: 'image', image: tempImage };
+              reasoningParts = [...reasoningParts, newPart];
+
+              // Start upload task and update array when done
+              const uploadTask = getFileStoreState()
+                .uploadBase64FileWithProgress(tempImage)
+                .then((file) => {
+                  const url = file?.url || tempImage;
+                  // Replace the part at index by creating a new array
+                  const updatedParts = [...reasoningParts];
+                  updatedParts[partIndex] = { type: 'image', image: url };
+                  reasoningParts = updatedParts;
+                  return url;
+                })
+                .catch((error) => {
+                  console.error('[reasoning_part] Image upload failed:', error);
+                  return tempImage;
+                });
+
+              reasoningImageUploads.set(partIndex, uploadTask);
+            }
+
+            // Real-time update with display format
+            // Check if there are any image parts to determine if it's multimodal
+            const hasReasoningImages = reasoningParts.some((part) => part.type === 'image');
+
+            internal_dispatchMessage(
+              {
+                id: messageId,
+                type: 'updateMessage',
+                value: {
+                  reasoning: hasReasoningImages
+                    ? { tempDisplayContent: reasoningParts, isMultimodal: true }
+                    : { content: thinkingContent },
+                },
+              },
+              { operationId },
+            );
+            break;
+          }
+
+          case 'content_part': {
+            const { partType, content: partContent, mimeType } = chunk;
+
+            // End reasoning when content starts
+            if (!thinkingDuration && reasoningOperationId) {
+              thinkingDuration = Date.now() - thinkingStartAt;
+              get().completeOperation(reasoningOperationId);
+              reasoningOperationId = undefined;
+            }
+
+            if (partType === 'text') {
+              const lastPart = contentParts.at(-1);
+
+              // If last part is also text, merge chunks together
+              if (lastPart?.type === 'text') {
+                contentParts = [
+                  ...contentParts.slice(0, -1),
+                  { type: 'text', text: lastPart.text + partContent },
+                ];
+              } else {
+                // Create new text part (first chunk, may contain thoughtSignature)
+                contentParts = [...contentParts, { type: 'text', text: partContent }];
+              }
+              output += partContent;
+            } else if (partType === 'image') {
+              // Image part - create new array to avoid mutation
+              const tempImage = `data:${mimeType};base64,${partContent}`;
+              const partIndex = contentParts.length;
+              const newPart: MessageContentPart = {
+                type: 'image',
+                image: tempImage,
+              };
+              contentParts = [...contentParts, newPart];
+
+              // Start upload task and update array when done
+              const uploadTask = getFileStoreState()
+                .uploadBase64FileWithProgress(tempImage)
+                .then((file) => {
+                  const url = file?.url || tempImage;
+                  // Replace the part at index by creating a new array
+                  const updatedParts = [...contentParts];
+                  updatedParts[partIndex] = {
+                    type: 'image',
+                    image: url,
+                  };
+                  contentParts = updatedParts;
+                  return url;
+                })
+                .catch((error) => {
+                  console.error('[content_part] Image upload failed:', error);
+                  return tempImage;
+                });
+
+              contentImageUploads.set(partIndex, uploadTask);
+            }
+
+            // Real-time update with display format
+            // Check if there are any image parts to determine if it's multimodal
+            const hasContentImages = contentParts.some((part) => part.type === 'image');
+
+            const hasReasoningImages = reasoningParts.some((part) => part.type === 'image');
+
+            internal_dispatchMessage(
+              {
+                id: messageId,
+                type: 'updateMessage',
+                value: {
+                  content: output,
+                  reasoning: hasReasoningImages
+                    ? {
+                        tempDisplayContent: reasoningParts,
+                        isMultimodal: true,
+                        duration: thinkingDuration,
+                      }
+                    : !!thinkingContent
+                      ? { content: thinkingContent, duration: thinkingDuration }
+                      : undefined,
+                  ...(hasContentImages && {
+                    metadata: {
+                      isMultimodal: true,
+                      tempDisplayContent: serializePartsForStorage(contentParts),
+                    },
+                  }),
+                },
               },
               { operationId },
             );
@@ -525,8 +752,8 @@ export const streamingExecutor: StateCreator<
             isFunctionCall = true;
 
             // Complete reasoning operation if it exists
-            if (!duration && reasoningOperationId) {
-              duration = Date.now() - thinkingStartAt;
+            if (!thinkingDuration && reasoningOperationId) {
+              thinkingDuration = Date.now() - thinkingStartAt;
               get().completeOperation(reasoningOperationId);
               reasoningOperationId = undefined;
             }
@@ -535,8 +762,8 @@ export const streamingExecutor: StateCreator<
 
           case 'stop': {
             // Complete reasoning operation when receiving stop signal
-            if (!duration && reasoningOperationId) {
-              duration = Date.now() - thinkingStartAt;
+            if (!thinkingDuration && reasoningOperationId) {
+              thinkingDuration = Date.now() - thinkingStartAt;
               get().completeOperation(reasoningOperationId);
               reasoningOperationId = undefined;
             }

package/packages/const/src/models.ts

@@ -1,93 +0,0 @@
-export const systemToUserModels = new Set([
-  'o1-preview',
-  'o1-preview-2024-09-12',
-  'o1-mini',
-  'o1-mini-2024-09-12',
-]);
-
-// TODO: temporary implementation, needs to be refactored into model card display configuration
-export const disableStreamModels = new Set([
-  'o1',
-  'o1-2024-12-17',
-  'o1-pro',
-  'o1-pro-2025-03-19',
-  /*
-  Official documentation shows no support, but actual testing shows Streaming is supported, temporarily commented out
-  'o3-pro',
-  'o3-pro-2025-06-10',
-  */
-  'computer-use-preview',
-  'computer-use-preview-2025-03-11',
-]);
-
-/**
- * models use Responses API only
- */
-export const responsesAPIModels = new Set([
-  'o1-pro',
-  'o1-pro-2025-03-19',
-  'o3-deep-research',
-  'o3-deep-research-2025-06-26',
-  'o3-pro',
-  'o3-pro-2025-06-10',
-  'o4-mini-deep-research',
-  'o4-mini-deep-research-2025-06-26',
-  'codex-mini-latest',
-  'computer-use-preview',
-  'computer-use-preview-2025-03-11',
-  'gpt-5-codex',
-  'gpt-5-pro',
-  'gpt-5-pro-2025-10-06',
-  'gpt-5.1-codex',
-  'gpt-5.1-codex-mini',
-]);
-
-/**
- * models support context caching
- */
-export const contextCachingModels = new Set([
-  'claude-opus-4-5-20251101',
-  'claude-haiku-4-5-20251001',
-  'claude-sonnet-4-5-latest',
-  'claude-sonnet-4-5-20250929',
-  'anthropic/claude-sonnet-4.5',
-  'claude-opus-4-latest',
-  'claude-opus-4-20250514',
-  'claude-sonnet-4-latest',
-  'claude-sonnet-4-20250514',
-  'claude-3-7-sonnet-latest',
-  'claude-3-7-sonnet-20250219',
-  'claude-3-5-sonnet-latest',
-  'claude-3-5-sonnet-20241022',
-  'claude-3-5-sonnet-20240620',
-  'claude-3-5-haiku-latest',
-  'claude-3-5-haiku-20241022',
-  // Bedrock model IDs
-  'us.anthropic.claude-sonnet-4-5-20250929-v1:0',
-  'anthropic.claude-sonnet-4-5-20250929-v1:0',
-  'us.anthropic.claude-haiku-4-5-20251001-v1:0',
-  'anthropic.claude-haiku-4-5-20251001-v1:0',
-  'global.anthropic.claude-opus-4-5-20251101-v1:0',
-  'anthropic.claude-opus-4-5-20251101-v1:0',
-]);
-
-export const thinkingWithToolClaudeModels = new Set([
-  'claude-opus-4-5-20251101',
-  'claude-opus-4-latest',
-  'claude-opus-4-20250514',
-  'claude-sonnet-4-latest',
-  'claude-sonnet-4-20250514',
-  'claude-sonnet-4-5-latest',
-  'claude-sonnet-4-5-20250929',
-  'claude-haiku-4-5-20251001',
-  'anthropic/claude-sonnet-4.5',
-  'claude-3-7-sonnet-latest',
-  'claude-3-7-sonnet-20250219',
-  // Bedrock model IDs
-  'us.anthropic.claude-sonnet-4-5-20250929-v1:0',
-  'anthropic.claude-sonnet-4-5-20250929-v1:0',
-  'us.anthropic.claude-haiku-4-5-20251001-v1:0',
-  'anthropic.claude-haiku-4-5-20251001-v1:0',
-  'global.anthropic.claude-opus-4-5-20251101-v1:0',
-  'anthropic.claude-opus-4-5-20251101-v1:0',
-]);