@opensumi/ide-ai-native 3.8.3-next-1742180589.0 → 3.8.3-next-1742193440.0

package/src/browser/chat/chat.view.tsx

@@ -281,7 +281,7 @@ export const AIChatView = () => {
           if (loading) {
             return;
           }
-          await handleSend(message.message, message.agentId, message.command);
+          await handleSend(message.message, message.images, message.agentId, message.command);
         } else {
           if (message.agentId) {
             setAgentId(message.agentId);
@@ -462,10 +462,16 @@ export const AIChatView = () => {
   );
 
   const renderUserMessage = React.useCallback(
-    async (renderModel: { message: string; agentId?: string; relationId: string; command?: string }) => {
+    async (renderModel: {
+      message: string;
+      images?: string[];
+      agentId?: string;
+      relationId: string;
+      command?: string;
+    }) => {
       const ChatUserRoleRender = chatRenderRegistry.chatUserRoleRender;
 
-      const { message, agentId, relationId, command } = renderModel;
+      const { message, images, agentId, relationId, command } = renderModel;
 
       const visibleAgentId = agentId === ChatProxyService.AGENT_ID ? '' : agentId;
 
@@ -474,12 +480,13 @@ export const AIChatView = () => {
           id: uuid(6),
           relationId,
           text: ChatUserRoleRender ? (
-            <ChatUserRoleRender content={message} agentId={visibleAgentId} command={command} />
+            <ChatUserRoleRender content={message} images={images} agentId={visibleAgentId} command={command} />
           ) : (
             <CodeBlockWrapperInput
               labelService={labelService}
               relationId={relationId}
               text={message}
+              images={images}
               agentId={visibleAgentId}
               command={command}
               workspaceService={workspaceService}
@@ -598,10 +605,10 @@ export const AIChatView = () => {
 
   const handleAgentReply = React.useCallback(
     async (value: IChatMessageStructure) => {
-      const { message, agentId, command, reportExtra } = value;
+      const { message, images, agentId, command, reportExtra } = value;
       const { actionType, actionSource } = reportExtra || {};
 
-      const request = aiChatService.createRequest(message, agentId!, command);
+      const request = aiChatService.createRequest(message, agentId!, images, command);
       if (!request) {
         return;
       }
@@ -627,6 +634,7 @@ export const AIChatView = () => {
 
       msgHistoryManager.addUserMessage({
         content: message,
+        images: images || [],
         agentId: agentId!,
         agentCommand: command!,
         relationId,
@@ -635,6 +643,7 @@ export const AIChatView = () => {
       await renderUserMessage({
         relationId,
         message,
+        images,
         command,
         agentId,
       });
@@ -668,7 +677,7 @@ export const AIChatView = () => {
   );
 
   const handleSend = React.useCallback(
-    async (message: string, agentId?: string, command?: string) => {
+    async (message: string, images?: string[], agentId?: string, command?: string) => {
       const reportExtra = {
         actionSource: ActionSourceEnum.Chat,
         actionType: ActionTypeEnum.Send,
@@ -707,7 +716,7 @@ export const AIChatView = () => {
           processedContent = processedContent.replace(match, `\`<attached_folder>${relativePath}\``);
         }
       }
-      return handleAgentReply({ message: processedContent, agentId, command, reportExtra });
+      return handleAgentReply({ message: processedContent, images, agentId, command, reportExtra });
     },
     [handleAgentReply],
   );
@@ -750,6 +759,7 @@ export const AIChatView = () => {
             message: msg.content,
             agentId: msg.agentId,
             command: msg.agentCommand,
+            images: msg.images,
           });
         } else if (msg.role === ChatMessageRole.Assistant && msg.requestId) {
           const request = aiChatService.sessionModel.getRequest(msg.requestId)!;

package/src/browser/components/ChatEditor.tsx

@@ -2,6 +2,7 @@ import capitalize from 'lodash/capitalize';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 import Highlight from 'react-highlight';
 
+import { Image } from '@opensumi/ide-components/lib/image';
 import {
   EDITOR_COMMANDS,
   FILE_COMMANDS,
@@ -336,6 +337,7 @@ export const CodeBlockWrapper = ({
 
 export const CodeBlockWrapperInput = ({
   text,
+  images,
   relationId,
   agentId,
   command,
@@ -344,6 +346,7 @@ export const CodeBlockWrapperInput = ({
   commandService,
 }: {
   text: string;
+  images?: string[];
   relationId: string;
   agentId?: string;
   command?: string;
@@ -370,6 +373,11 @@ export const CodeBlockWrapperInput = ({
 
   return (
     <div className={styles.ai_chat_code_wrapper}>
+      {images?.map((image) => (
+        <div className={styles.image_wrapper}>
+          <Image src={image} />
+        </div>
+      ))}
       <div className={styles.render_text}>
         {tag && (
           <div className={styles.tag_wrapper}>

package/src/browser/components/ChatInput.tsx

@@ -162,7 +162,7 @@ const AgentWidget = ({ agentId, command }) => (
 );
 
 export interface IChatInputProps {
-  onSend: (value: string, agentId?: string, command?: string) => void;
+  onSend: (value: string, images?: string[], agentId?: string, command?: string) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -341,7 +341,7 @@ export const ChatInput = React.forwardRef((props: IChatInputProps, ref) => {
     }
 
     const handleSendLogic = (newValue: string = value) => {
-      onSend(newValue, agentId, command);
+      onSend(newValue, [], agentId, command);
       setValue('');
       setTheme('');
       setAgentId('');

package/src/browser/components/ChatMentionInput.tsx

@@ -1,14 +1,18 @@
+import { DataContent } from 'ai';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
+import { Image } from '@opensumi/ide-components/lib/image';
 import { LabelService, RecentFilesManager, useInjectable } from '@opensumi/ide-core-browser';
-import { getIcon } from '@opensumi/ide-core-browser/lib/components';
-import { URI, localize } from '@opensumi/ide-core-common';
+import { Icon, getIcon } from '@opensumi/ide-core-browser/lib/components';
+import { ChatFeatureRegistryToken, URI, localize } from '@opensumi/ide-core-common';
 import { CommandService } from '@opensumi/ide-core-common/lib/command';
 import { WorkbenchEditorService } from '@opensumi/ide-editor';
 import { FileSearchServicePath, IFileSearchService } from '@opensumi/ide-file-search';
+import { IMessageService } from '@opensumi/ide-overlay';
 import { IWorkspaceService } from '@opensumi/ide-workspace';
 
 import { IChatInternalService } from '../../common';
+import { ChatFeatureRegistry } from '../chat/chat.feature.registry';
 import { ChatInternalService } from '../chat/chat.internal.service';
 import { OPEN_MCP_CONFIG_COMMAND } from '../mcp/config/mcp-config.commands';
 
@@ -17,7 +21,13 @@ import { MentionInput } from './mention-input/mention-input';
 import { FooterButtonPosition, FooterConfig, MentionItem, MentionType } from './mention-input/types';
 
 export interface IChatMentionInputProps {
-  onSend: (value: string, agentId?: string, command?: string, option?: { model: string; [key: string]: any }) => void;
+  onSend: (
+    value: string,
+    images?: string[],
+    agentId?: string,
+    command?: string,
+    option?: { model: string; [key: string]: any },
+  ) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -26,6 +36,7 @@ export interface IChatMentionInputProps {
   sendBtnClassName?: string;
   defaultHeight?: number;
   value?: string;
+  images?: Array<DataContent | URL>;
   autoFocus?: boolean;
   theme?: string | null;
   setTheme: (theme: string | null) => void;
@@ -41,6 +52,7 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
   const { onSend, disabled = false } = props;
 
   const [value, setValue] = useState(props.value || '');
+  const [images, setImages] = useState(props.images || []);
   const aiChatService = useInjectable<ChatInternalService>(IChatInternalService);
   const commandService = useInjectable<CommandService>(CommandService);
   const searchService = useInjectable<IFileSearchService>(FileSearchServicePath);
@@ -48,7 +60,8 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
   const workspaceService = useInjectable<IWorkspaceService>(IWorkspaceService);
   const editorService = useInjectable<WorkbenchEditorService>(WorkbenchEditorService);
   const labelService = useInjectable<LabelService>(LabelService);
-
+  const messageService = useInjectable<IMessageService>(IMessageService);
+  const chatFeatureRegistry = useInjectable<ChatFeatureRegistry>(ChatFeatureRegistryToken);
   const handleShowMCPConfig = React.useCallback(() => {
     commandService.executeCommand(OPEN_MCP_CONFIG_COMMAND.id);
   }, [commandService]);
@@ -239,6 +252,24 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
           onClick: handleShowMCPConfig,
           position: FooterButtonPosition.LEFT,
         },
+        {
+          id: 'upload-image',
+          icon: 'image',
+          title: localize('aiNative.chat.imageUpload'),
+          onClick: () => {
+            const input = document.createElement('input');
+            input.type = 'file';
+            input.accept = 'image/*';
+            input.onchange = (e) => {
+              const files = (e.target as HTMLInputElement).files;
+              if (files?.length) {
+                handleImageUpload(Array.from(files));
+              }
+            };
+            input.click();
+          },
+          position: FooterButtonPosition.LEFT,
+        },
       ],
       showModelSelector: true,
     }),
@@ -254,13 +285,54 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
       if (disabled) {
         return;
       }
-      onSend(content, undefined, undefined, option);
+      onSend(
+        content,
+        images.map((image) => image.toString()),
+        undefined,
+        undefined,
+        option,
+      );
+      setImages(props.images || []);
     },
-    [onSend, editorService, disabled],
+    [onSend, images, disabled],
+  );
+
+  const handleImageUpload = useCallback(
+    async (files: File[]) => {
+      const allowedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp', 'image/gif'];
+
+      // Validate file types
+      const invalidFiles = files.filter((file) => !allowedTypes.includes(file.type));
+      if (invalidFiles.length > 0) {
+        messageService.error('Only JPG, PNG, WebP and GIF images are supported');
+        return;
+      }
+
+      const imageUploadProvider = chatFeatureRegistry.getImageUploadProvider();
+      if (!imageUploadProvider) {
+        messageService.error('No image upload provider found');
+        return;
+      }
+
+      // Upload all files
+      const uploadedData = await Promise.all(files.map((file) => imageUploadProvider.imageUpload(file)));
+
+      const newImages = [...images, ...uploadedData];
+      setImages(newImages);
+    },
+    [images],
+  );
+
+  const handleDeleteImage = useCallback(
+    (index: number) => {
+      setImages(images.filter((_, i) => i !== index));
+    },
+    [images],
   );
 
   return (
     <div className={styles.chat_input_container}>
+      {images.length > 0 && <ImagePreviewer images={images} onDelete={handleDeleteImage} />}
       <MentionInput
         mentionItems={defaultMenuItems}
         onSend={handleSend}
@@ -270,7 +342,29 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
         workspaceService={workspaceService}
         placeholder={localize('aiNative.chat.input.placeholder.default')}
         footerConfig={defaultMentionInputFooterOptions}
+        onImageUpload={handleImageUpload}
       />
     </div>
   );
 };
+
+const ImagePreviewer = ({
+  images,
+  onDelete,
+}: {
+  images: Array<DataContent | URL>;
+  onDelete: (index: number) => void;
+}) => (
+  <div>
+    <div className={styles.thumbnail_container}>
+      {images.map((image, index) => (
+        <div key={index} className={styles.thumbnail}>
+          <Image src={image.toString()} />
+          <button onClick={() => onDelete(index)} className={styles.delete_button}>
+            <Icon iconClass='codicon codicon-close' />
+          </button>
+        </div>
+      ))}
+    </div>
+  </div>
+);

package/src/browser/components/components.module.less

@@ -601,3 +601,52 @@
   vertical-align: middle;
   font-size: 12px;
 }
+
+.thumbnail_container {
+  display: flex;
+  gap: 4px;
+  padding: 4px 12px;
+  .thumbnail {
+    width: 36px;
+    height: 36px;
+    border-radius: 4px;
+    position: relative;
+    box-shadow: 0 0 10px 0 rgba(0, 0, 0, 0.3);
+    padding: 2px;
+    display: inline-flex;
+    align-items: center;
+    img {
+      width: 100%;
+      height: 100%;
+      object-fit: cover;
+    }
+  }
+  .delete_button {
+    position: absolute;
+    top: -5px;
+    right: -5px;
+    font-size: 12px;
+    padding: 2px;
+    border: 0;
+    border-radius: 50%;
+    transition: transform 0.2s ease-in-out;
+    width: 16px;
+    height: 16px;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    background-color: var(--badge-background);
+    &:hover {
+      cursor: pointer;
+      transform: scale(1.2);
+    }
+    :global(.codicon) {
+      font-size: 12px;
+      color: var(--badge-foreground);
+    }
+  }
+}
+
+.image_wrapper {
+  display: inline-block;
+}

package/src/browser/components/mention-input/mention-input.tsx

@@ -18,6 +18,7 @@ export const MentionInput: React.FC<MentionInputProps> = ({
   loading = false,
   mentionKeyword = MENTION_KEYWORD,
   onSelectionChange,
+  onImageUpload,
   labelService,
   workspaceService,
   placeholder = 'Ask anything, @ to mention',
@@ -318,14 +319,74 @@ export const MentionInput: React.FC<MentionInputProps> = ({
         inlineSearchStartPos: null,
         loading: false,
       });
+    }
+
+    // 处理上下方向键导航历史记录
+    if ((e.key === 'ArrowUp' || e.key === 'ArrowDown') && !e.shiftKey && !e.ctrlKey && !e.metaKey && !e.altKey) {
+      // 只有在非提及面板激活状态下才处理历史导航
+      if (!mentionState.active && !mentionState.inlineSearchActive && editorRef.current && history.length > 0) {
+        const currentContent = editorRef.current.innerHTML;
+
+        // 检查是否应该触发历史导航
+        const shouldTriggerHistory =
+          // 当前内容为空
+          !currentContent ||
+          currentContent === '<br>' ||
+          // 或者当前内容与历史记录中的某一项匹配（正在浏览历史）
+          (isNavigatingHistory && historyIndex >= 0 && history[history.length - 1 - historyIndex] === currentContent);
+
+        if (shouldTriggerHistory) {
+          e.preventDefault();
+
+          // 如果是第一次按上下键，保存当前输入
+          if (!isNavigatingHistory) {
+            setCurrentInput(currentContent);
+            setIsNavigatingHistory(true);
+          }
+
+          // 计算新的历史索引
+          let newIndex = historyIndex;
+          if (e.key === 'ArrowUp') {
+            // 向上导航到较早的历史记录
+            newIndex = Math.min(history.length - 1, historyIndex + 1);
+          } else {
+            // 向下导航到较新的历史记录
+            newIndex = Math.max(-1, historyIndex - 1);
+          }
+
+          setHistoryIndex(newIndex);
+
+          // 更新编辑器内容
+          if (newIndex === -1) {
+            // 恢复到当前输入
+            editorRef.current.innerHTML = currentInput;
+          } else {
+            // 显示历史记录
+            editorRef.current.innerHTML = history[history.length - 1 - newIndex];
+          }
 
-      // 不要阻止默认行为，让 @ 正常输入到编辑器中
+          // 将光标移到末尾
+          const range = document.createRange();
+          range.selectNodeContents(editorRef.current);
+          range.collapse(false);
+          const selection = window.getSelection();
+          if (selection) {
+            selection.removeAllRanges();
+            selection.addRange(range);
+          }
+
+          return;
+        }
+      }
+    } else if (isNavigatingHistory && e.key !== 'ArrowUp' && e.key !== 'ArrowDown') {
+      // 如果用户在浏览历史记录后开始输入其他内容，退出历史导航模式
+      setIsNavigatingHistory(false);
+      setHistoryIndex(-1);
     }
 
     // 添加对 Enter 键的处理，只有在按下 Shift+Enter 时才允许换行
     if (e.key === 'Enter') {
       // 检查是否是输入法的回车键
-      // isComposing 属性表示是否正在进行输入法组合输入
       if (e.nativeEvent.isComposing) {
         return; // 如果是输入法组合输入过程中的回车，不做任何处理
       }
@@ -339,57 +400,6 @@ export const MentionInput: React.FC<MentionInputProps> = ({
       }
     }
 
-    // 处理上下方向键导航历史记录
-    if (e.key === 'ArrowUp' || e.key === 'ArrowDown') {
-      // 只有在非提及面板激活状态下才处理历史导航
-      if (!mentionState.active && !mentionState.inlineSearchActive && editorRef.current) {
-        e.preventDefault();
-
-        // 如果是第一次按上下键，保存当前输入
-        if (!isNavigatingHistory) {
-          setCurrentInput(editorRef.current.innerHTML);
-          setIsNavigatingHistory(true);
-        }
-
-        // 计算新的历史索引
-        let newIndex = historyIndex;
-        if (e.key === 'ArrowUp') {
-          // 向上导航到较早的历史记录
-          newIndex = Math.min(history.length - 1, historyIndex + 1);
-        } else {
-          // 向下导航到较新的历史记录
-          newIndex = Math.max(-1, historyIndex - 1);
-        }
-
-        setHistoryIndex(newIndex);
-
-        // 更新编辑器内容
-        if (newIndex === -1) {
-          // 恢复到当前输入
-          editorRef.current.innerHTML = currentInput;
-        } else {
-          // 显示历史记录
-          editorRef.current.innerHTML = history[history.length - 1 - newIndex];
-        }
-
-        // 将光标移到末尾
-        const range = document.createRange();
-        range.selectNodeContents(editorRef.current);
-        range.collapse(false);
-        const selection = window.getSelection();
-        if (selection) {
-          selection.removeAllRanges();
-          selection.addRange(range);
-        }
-
-        return;
-      }
-    } else if (isNavigatingHistory && e.key !== 'ArrowUp' && e.key !== 'ArrowDown') {
-      // 如果用户在浏览历史记录后开始输入，退出历史导航模式
-      setIsNavigatingHistory(false);
-      setHistoryIndex(-1);
-    }
-
     // 如果提及面板未激活，不处理其他键盘事件
     if (!mentionState.active) {
       return;
@@ -404,8 +414,6 @@ export const MentionInput: React.FC<MentionInputProps> = ({
       filteredItems = filteredItems.filter((item) => item.text.toLowerCase().includes(searchText));
     }
 
-    // 二级菜单过滤已经在 getCurrentItems 中处理
-
     if (filteredItems.length === 0) {
       return;
     }
@@ -431,14 +439,6 @@ export const MentionInput: React.FC<MentionInputProps> = ({
         e.preventDefault();
       }
     }
-
-    // 处理 Backspace 键，检查是否需要清空编辑器
-    if (e.key === 'Backspace' && editorRef.current) {
-      const content = editorRef.current.innerHTML;
-      if (content === '<br>' || content === '<br/>') {
-        editorRef.current.innerHTML = '';
-      }
-    }
   };
 
   // 添加对输入法事件的处理
@@ -447,6 +447,86 @@ export const MentionInput: React.FC<MentionInputProps> = ({
     // 这里可以添加额外的逻辑，如果需要的话
   };
 
+  // 添加粘贴事件处理
+  const handlePaste = async (e: React.ClipboardEvent<HTMLDivElement>) => {
+    const items = e.clipboardData.items;
+
+    // 先收集所有图片文件
+    const imageFiles: File[] = [];
+    // eslint-disable-next-line @typescript-eslint/prefer-for-of
+    for (let i = 0; i < items.length; i++) {
+      if (items[i].kind === 'file' && items[i].type.startsWith('image/')) {
+        const file = items[i].getAsFile();
+        if (file) {
+          imageFiles.push(file);
+        }
+      }
+    }
+
+    e.preventDefault();
+
+    // 处理所有收集到的图片
+    if (imageFiles.length > 0 && onImageUpload) {
+      await onImageUpload(imageFiles);
+      return;
+    }
+
+    const text = e.clipboardData.getData('text/plain');
+
+    // 处理文本，保留换行和缩进
+    const processedText = text
+      .replace(/\t/g, '    ')
+      .replace(/\n\s*\n/g, '\n\n')
+      .replace(/[ \t]+$/gm, '');
+
+    const selection = window.getSelection();
+    if (!selection || !selection.rangeCount) {
+      return;
+    }
+
+    const range = selection.getRangeAt(0);
+    range.deleteContents();
+
+    // 将处理后的文本按行分割
+    const lines = processedText.split('\n');
+    const fragment = document.createDocumentFragment();
+
+    lines.forEach((line, index) => {
+      // 处理行首空格，将每个空格转换为 &nbsp;
+      const processedLine = line.replace(/^[ ]+/g, (match) => {
+        const span = document.createElement('span');
+        span.innerHTML = '\u00A0'.repeat(match.length);
+        return span.innerHTML;
+      });
+
+      // 创建一个临时容器来保持 HTML 内容
+      const container = document.createElement('span');
+      container.innerHTML = processedLine;
+
+      // 将容器的内容添加到文档片段
+      while (container.firstChild) {
+        fragment.appendChild(container.firstChild);
+      }
+
+      // 如果不是最后一行，添加换行符
+      if (index < lines.length - 1) {
+        fragment.appendChild(document.createElement('br'));
+      }
+    });
+
+    // 插入处理后的内容
+    range.insertNode(fragment);
+
+    // 将光标移动到插入内容的末尾
+    range.setStartAfter(fragment);
+    range.setEndAfter(fragment);
+    selection.removeAllRanges();
+    selection.addRange(range);
+
+    // 触发 input 事件以更新状态
+    handleInput();
+  };
+
   // 初始化编辑器
   React.useEffect(() => {
     if (editorRef.current) {
@@ -867,7 +947,7 @@ export const MentionInput: React.FC<MentionInputProps> = ({
             title={button.title}
           >
             <EnhanceIcon
-              className={cls(getIcon(button.icon), styles[`${button.id}_logo`])}
+              className={cls(button.icon ? getIcon(button.icon) : button.iconClass, styles[`${button.id}_logo`])}
               tabIndex={0}
               role='button'
               ariaLabel={button.title}
@@ -901,6 +981,7 @@ export const MentionInput: React.FC<MentionInputProps> = ({
           contentEditable={true}
           onInput={handleInput}
           onKeyDown={handleKeyDown}
+          onPaste={handlePaste}
           onCompositionEnd={handleCompositionEnd}
         />
       </div>

package/src/browser/components/mention-input/types.ts

@@ -55,7 +55,8 @@ export enum MentionType {
 
 interface FooterButton {
   id: string;
-  icon: string;
+  icon?: string;
+  iconClass?: string;
   title: string;
   onClick?: () => void;
   position: FooterButtonPosition;
@@ -75,6 +76,7 @@ export interface MentionInputProps {
   placeholder?: string;
   loading?: boolean;
   onSelectionChange?: (value: string) => void;
+  onImageUpload?: (files: File[]) => Promise<void>;
   footerConfig?: FooterConfig; // 新增配置项
   mentionKeyword?: string;
   labelService?: LabelService;

package/src/browser/context/llm-context.service.ts

@@ -1,3 +1,5 @@
+import { DataContent } from 'ai';
+
 import { Autowired, Injectable } from '@opensumi/di';
 import { AppConfig } from '@opensumi/ide-core-browser/lib/react-providers/config-provider';
 import { WithEventBus } from '@opensumi/ide-core-common/lib/event-bus/event-decorator';