@opensumi/ide-ai-native 3.8.3-next-1741945847.0 → 3.8.3-next-1741949132.0

package/src/browser/components/ChatMentionInput.tsx

@@ -1,14 +1,18 @@
+import { DataContent } from 'ai';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
+import { Image } from '@opensumi/ide-components/lib/image';
 import { LabelService, RecentFilesManager, useInjectable } from '@opensumi/ide-core-browser';
-import { getIcon } from '@opensumi/ide-core-browser/lib/components';
-import { URI, localize } from '@opensumi/ide-core-common';
+import { Icon, getIcon } from '@opensumi/ide-core-browser/lib/components';
+import { ChatFeatureRegistryToken, URI, localize } from '@opensumi/ide-core-common';
 import { CommandService } from '@opensumi/ide-core-common/lib/command';
 import { WorkbenchEditorService } from '@opensumi/ide-editor';
 import { FileSearchServicePath, IFileSearchService } from '@opensumi/ide-file-search';
+import { IMessageService } from '@opensumi/ide-overlay';
 import { IWorkspaceService } from '@opensumi/ide-workspace';
 
 import { IChatInternalService } from '../../common';
+import { ChatFeatureRegistry } from '../chat/chat.feature.registry';
 import { ChatInternalService } from '../chat/chat.internal.service';
 import { OPEN_MCP_CONFIG_COMMAND } from '../mcp/config/mcp-config.commands';
 
@@ -17,7 +21,13 @@ import { MentionInput } from './mention-input/mention-input';
 import { FooterButtonPosition, FooterConfig, MentionItem, MentionType } from './mention-input/types';
 
 export interface IChatMentionInputProps {
-  onSend: (value: string, agentId?: string, command?: string, option?: { model: string; [key: string]: any }) => void;
+  onSend: (
+    value: string,
+    images?: string[],
+    agentId?: string,
+    command?: string,
+    option?: { model: string; [key: string]: any },
+  ) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -26,6 +36,7 @@ export interface IChatMentionInputProps {
   sendBtnClassName?: string;
   defaultHeight?: number;
   value?: string;
+  images?: Array<DataContent | URL>;
   autoFocus?: boolean;
   theme?: string | null;
   setTheme: (theme: string | null) => void;
@@ -41,6 +52,7 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
   const { onSend, disabled = false } = props;
 
   const [value, setValue] = useState(props.value || '');
+  const [images, setImages] = useState(props.images || []);
   const aiChatService = useInjectable<ChatInternalService>(IChatInternalService);
   const commandService = useInjectable<CommandService>(CommandService);
   const searchService = useInjectable<IFileSearchService>(FileSearchServicePath);
@@ -48,7 +60,8 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
   const workspaceService = useInjectable<IWorkspaceService>(IWorkspaceService);
   const editorService = useInjectable<WorkbenchEditorService>(WorkbenchEditorService);
   const labelService = useInjectable<LabelService>(LabelService);
-
+  const messageService = useInjectable<IMessageService>(IMessageService);
+  const chatFeatureRegistry = useInjectable<ChatFeatureRegistry>(ChatFeatureRegistryToken);
   const handleShowMCPConfig = React.useCallback(() => {
     commandService.executeCommand(OPEN_MCP_CONFIG_COMMAND.id);
   }, [commandService]);
@@ -239,6 +252,24 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
           onClick: handleShowMCPConfig,
           position: FooterButtonPosition.LEFT,
         },
+        {
+          id: 'upload-image',
+          iconClass: 'codicon codicon-file-media',
+          title: 'Upload Image',
+          onClick: () => {
+            const input = document.createElement('input');
+            input.type = 'file';
+            input.accept = 'image/*';
+            input.onchange = (e) => {
+              const file = (e.target as HTMLInputElement).files?.[0];
+              if (file) {
+                handleImageUpload(file);
+              }
+            };
+            input.click();
+          },
+          position: FooterButtonPosition.LEFT,
+        },
       ],
       showModelSelector: true,
     }),
@@ -254,13 +285,47 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
       if (disabled) {
         return;
       }
-      onSend(content, undefined, undefined, option);
+      onSend(
+        content,
+        images.map((image) => image.toString()),
+        undefined,
+        undefined,
+        option,
+      );
+      setImages(props.images || []);
+    },
+    [onSend, images, disabled],
+  );
+
+  const handleImageUpload = useCallback(
+    async (file: File) => {
+      const allowedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp', 'image/gif'];
+      if (!allowedTypes.includes(file.type)) {
+        messageService.error('Only JPG, PNG, WebP and GIF images are supported');
+        return;
+      }
+
+      const imageUploadProvider = chatFeatureRegistry.getImageUploadProvider();
+      if (!imageUploadProvider) {
+        messageService.error('No image upload provider found');
+        return;
+      }
+      const data = await imageUploadProvider.imageUpload(file);
+      setImages([...images, data]);
     },
-    [onSend, editorService, disabled],
+    [images],
+  );
+
+  const handleDeleteImage = useCallback(
+    (index: number) => {
+      setImages(images.filter((_, i) => i !== index));
+    },
+    [images],
   );
 
   return (
     <div className={styles.chat_input_container}>
+      {images.length > 0 && <ImagePreviewer images={images} onDelete={handleDeleteImage} />}
       <MentionInput
         mentionItems={defaultMenuItems}
         onSend={handleSend}
@@ -270,7 +335,29 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
         workspaceService={workspaceService}
         placeholder={localize('aiNative.chat.input.placeholder.default')}
         footerConfig={defaultMentionInputFooterOptions}
+        onImageUpload={handleImageUpload}
       />
     </div>
   );
 };
+
+const ImagePreviewer = ({
+  images,
+  onDelete,
+}: {
+  images: Array<DataContent | URL>;
+  onDelete: (index: number) => void;
+}) => (
+  <div>
+    <div className={styles.thumbnail_container}>
+      {images.map((image, index) => (
+        <div key={index} className={styles.thumbnail}>
+          <Image src={image.toString()} />
+          <button onClick={() => onDelete(index)} className={styles.delete_button}>
+            <Icon iconClass='codicon codicon-close' />
+          </button>
+        </div>
+      ))}
+    </div>
+  </div>
+);

package/src/browser/components/components.module.less

@@ -601,3 +601,52 @@
   vertical-align: middle;
   font-size: 12px;
 }
+
+.thumbnail_container {
+  display: flex;
+  gap: 4px;
+  padding: 4px 12px;
+  .thumbnail {
+    width: 36px;
+    height: 36px;
+    border-radius: 4px;
+    position: relative;
+    box-shadow: 0 0 10px 0 rgba(0, 0, 0, 0.3);
+    padding: 2px;
+    display: inline-flex;
+    align-items: center;
+    img {
+      width: 100%;
+      height: 100%;
+      object-fit: cover;
+    }
+  }
+  .delete_button {
+    position: absolute;
+    top: -5px;
+    right: -5px;
+    font-size: 12px;
+    padding: 2px;
+    border: 0;
+    border-radius: 50%;
+    transition: transform 0.2s ease-in-out;
+    width: 16px;
+    height: 16px;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    background-color: var(--badge-background);
+    &:hover {
+      cursor: pointer;
+      transform: scale(1.2);
+    }
+    :global(.codicon) {
+      font-size: 12px;
+      color: var(--badge-foreground);
+    }
+  }
+}
+
+.image_wrapper {
+  display: inline-block;
+}

package/src/browser/components/mention-input/mention-input.module.less

@@ -50,6 +50,7 @@
     display: flex;
     align-items: center;
     cursor: pointer;
+    text-wrap: nowrap;
     white-space: nowrap;
     box-sizing: border-box;
 

package/src/browser/components/mention-input/mention-input.tsx

@@ -18,6 +18,7 @@ export const MentionInput: React.FC<MentionInputProps> = ({
   loading = false,
   mentionKeyword = MENTION_KEYWORD,
   onSelectionChange,
+  onImageUpload,
   labelService,
   workspaceService,
   placeholder = 'Ask anything, @ to mention',
@@ -275,6 +276,21 @@ export const MentionInput: React.FC<MentionInputProps> = ({
     }
   };
 
+  // 处理图片粘贴事件
+  const handlePaste = (e: React.ClipboardEvent<HTMLDivElement>) => {
+    const items = e.clipboardData.items;
+    // eslint-disable-next-line @typescript-eslint/prefer-for-of
+    for (let i = 0; i < items.length; i++) {
+      if (items[i].kind === 'file' && items[i].type.startsWith('image/')) {
+        const file = items[i].getAsFile();
+        if (file && onImageUpload) {
+          e.preventDefault();
+          onImageUpload(file);
+        }
+      }
+    }
+  };
+
   // 处理键盘事件
   const handleKeyDown = (e: React.KeyboardEvent<HTMLDivElement>) => {
     // 如果按下ESC键且提及面板处于活动状态或内联搜索处于活动状态
@@ -318,74 +334,14 @@ export const MentionInput: React.FC<MentionInputProps> = ({
         inlineSearchStartPos: null,
         loading: false,
       });
-    }
-
-    // 处理上下方向键导航历史记录
-    if ((e.key === 'ArrowUp' || e.key === 'ArrowDown') && !e.shiftKey && !e.ctrlKey && !e.metaKey && !e.altKey) {
-      // 只有在非提及面板激活状态下才处理历史导航
-      if (!mentionState.active && !mentionState.inlineSearchActive && editorRef.current && history.length > 0) {
-        const currentContent = editorRef.current.innerHTML;
-
-        // 检查是否应该触发历史导航
-        const shouldTriggerHistory =
-          // 当前内容为空
-          !currentContent ||
-          currentContent === '<br>' ||
-          // 或者当前内容与历史记录中的某一项匹配（正在浏览历史）
-          (isNavigatingHistory && historyIndex >= 0 && history[history.length - 1 - historyIndex] === currentContent);
-
-        if (shouldTriggerHistory) {
-          e.preventDefault();
-
-          // 如果是第一次按上下键，保存当前输入
-          if (!isNavigatingHistory) {
-            setCurrentInput(currentContent);
-            setIsNavigatingHistory(true);
-          }
-
-          // 计算新的历史索引
-          let newIndex = historyIndex;
-          if (e.key === 'ArrowUp') {
-            // 向上导航到较早的历史记录
-            newIndex = Math.min(history.length - 1, historyIndex + 1);
-          } else {
-            // 向下导航到较新的历史记录
-            newIndex = Math.max(-1, historyIndex - 1);
-          }
-
-          setHistoryIndex(newIndex);
 
-          // 更新编辑器内容
-          if (newIndex === -1) {
-            // 恢复到当前输入
-            editorRef.current.innerHTML = currentInput;
-          } else {
-            // 显示历史记录
-            editorRef.current.innerHTML = history[history.length - 1 - newIndex];
-          }
-
-          // 将光标移到末尾
-          const range = document.createRange();
-          range.selectNodeContents(editorRef.current);
-          range.collapse(false);
-          const selection = window.getSelection();
-          if (selection) {
-            selection.removeAllRanges();
-            selection.addRange(range);
-          }
-
-          return;
-        }
-      }
-    } else if (isNavigatingHistory && e.key !== 'ArrowUp' && e.key !== 'ArrowDown') {
-      // 如果用户在浏览历史记录后开始输入其他内容，退出历史导航模式
-      setIsNavigatingHistory(false);
-      setHistoryIndex(-1);
+      // 不要阻止默认行为，让 @ 正常输入到编辑器中
     }
 
     // 添加对 Enter 键的处理，只有在按下 Shift+Enter 时才允许换行
     if (e.key === 'Enter') {
       // 检查是否是输入法的回车键
+      // isComposing 属性表示是否正在进行输入法组合输入
       if (e.nativeEvent.isComposing) {
         return; // 如果是输入法组合输入过程中的回车，不做任何处理
       }
@@ -399,6 +355,57 @@ export const MentionInput: React.FC<MentionInputProps> = ({
       }
     }
 
+    // 处理上下方向键导航历史记录
+    if (e.key === 'ArrowUp' || e.key === 'ArrowDown') {
+      // 只有在非提及面板激活状态下才处理历史导航
+      if (!mentionState.active && !mentionState.inlineSearchActive && editorRef.current) {
+        e.preventDefault();
+
+        // 如果是第一次按上下键，保存当前输入
+        if (!isNavigatingHistory) {
+          setCurrentInput(editorRef.current.innerHTML);
+          setIsNavigatingHistory(true);
+        }
+
+        // 计算新的历史索引
+        let newIndex = historyIndex;
+        if (e.key === 'ArrowUp') {
+          // 向上导航到较早的历史记录
+          newIndex = Math.min(history.length - 1, historyIndex + 1);
+        } else {
+          // 向下导航到较新的历史记录
+          newIndex = Math.max(-1, historyIndex - 1);
+        }
+
+        setHistoryIndex(newIndex);
+
+        // 更新编辑器内容
+        if (newIndex === -1) {
+          // 恢复到当前输入
+          editorRef.current.innerHTML = currentInput;
+        } else {
+          // 显示历史记录
+          editorRef.current.innerHTML = history[history.length - 1 - newIndex];
+        }
+
+        // 将光标移到末尾
+        const range = document.createRange();
+        range.selectNodeContents(editorRef.current);
+        range.collapse(false);
+        const selection = window.getSelection();
+        if (selection) {
+          selection.removeAllRanges();
+          selection.addRange(range);
+        }
+
+        return;
+      }
+    } else if (isNavigatingHistory && e.key !== 'ArrowUp' && e.key !== 'ArrowDown') {
+      // 如果用户在浏览历史记录后开始输入，退出历史导航模式
+      setIsNavigatingHistory(false);
+      setHistoryIndex(-1);
+    }
+
     // 如果提及面板未激活，不处理其他键盘事件
     if (!mentionState.active) {
       return;
@@ -413,6 +420,8 @@ export const MentionInput: React.FC<MentionInputProps> = ({
       filteredItems = filteredItems.filter((item) => item.text.toLowerCase().includes(searchText));
     }
 
+    // 二级菜单过滤已经在 getCurrentItems 中处理
+
     if (filteredItems.length === 0) {
       return;
     }
@@ -438,6 +447,14 @@ export const MentionInput: React.FC<MentionInputProps> = ({
         e.preventDefault();
       }
     }
+
+    // 处理 Backspace 键，检查是否需要清空编辑器
+    if (e.key === 'Backspace' && editorRef.current) {
+      const content = editorRef.current.innerHTML;
+      if (content === '<br>' || content === '<br/>') {
+        editorRef.current.innerHTML = '';
+      }
+    }
   };
 
   // 添加对输入法事件的处理
@@ -446,71 +463,6 @@ export const MentionInput: React.FC<MentionInputProps> = ({
     // 这里可以添加额外的逻辑，如果需要的话
   };
 
-  // 添加粘贴事件处理
-  const handlePaste = (e: React.ClipboardEvent<HTMLDivElement>) => {
-    // 阻止默认粘贴行为
-    e.preventDefault();
-
-    // 获取剪贴板中的纯文本内容
-    const text = e.clipboardData.getData('text/plain');
-
-    // 处理文本，保留换行和缩进
-    const processedText = text
-      // 将制表符转换为4个空格
-      .replace(/\t/g, '    ')
-      // 将连续的换行符转换为单个换行
-      .replace(/\n\s*\n/g, '\n\n')
-      // 移除行尾空格
-      .replace(/[ \t]+$/gm, '');
-
-    const selection = window.getSelection();
-    if (!selection || !selection.rangeCount) {
-      return;
-    }
-
-    const range = selection.getRangeAt(0);
-    range.deleteContents();
-
-    // 将处理后的文本按行分割
-    const lines = processedText.split('\n');
-    const fragment = document.createDocumentFragment();
-
-    lines.forEach((line, index) => {
-      // 处理行首空格，将每个空格转换为 &nbsp;
-      const processedLine = line.replace(/^[ ]+/g, (match) => {
-        const span = document.createElement('span');
-        span.innerHTML = '\u00A0'.repeat(match.length);
-        return span.innerHTML;
-      });
-
-      // 创建一个临时容器来保持 HTML 内容
-      const container = document.createElement('span');
-      container.innerHTML = processedLine;
-
-      // 将容器的内容添加到文档片段
-      while (container.firstChild) {
-        fragment.appendChild(container.firstChild);
-      }
-
-      // 如果不是最后一行，添加换行符
-      if (index < lines.length - 1) {
-        fragment.appendChild(document.createElement('br'));
-      }
-    });
-
-    // 插入处理后的内容
-    range.insertNode(fragment);
-
-    // 将光标移动到插入内容的末尾
-    range.setStartAfter(fragment);
-    range.setEndAfter(fragment);
-    selection.removeAllRanges();
-    selection.addRange(range);
-
-    // 触发 input 事件以更新状态
-    handleInput();
-  };
-
   // 初始化编辑器
   React.useEffect(() => {
     if (editorRef.current) {
@@ -931,7 +883,7 @@ export const MentionInput: React.FC<MentionInputProps> = ({
             title={button.title}
           >
             <EnhanceIcon
-              className={cls(getIcon(button.icon), styles[`${button.id}_logo`])}
+              className={cls(button.icon ? getIcon(button.icon) : button.iconClass, styles[`${button.id}_logo`])}
               tabIndex={0}
               role='button'
               ariaLabel={button.title}
@@ -965,8 +917,8 @@ export const MentionInput: React.FC<MentionInputProps> = ({
           contentEditable={true}
           onInput={handleInput}
           onKeyDown={handleKeyDown}
-          onCompositionEnd={handleCompositionEnd}
           onPaste={handlePaste}
+          onCompositionEnd={handleCompositionEnd}
         />
       </div>
       <div className={styles.footer}>

package/src/browser/components/mention-input/types.ts

@@ -55,7 +55,8 @@ export enum MentionType {
 
 interface FooterButton {
   id: string;
-  icon: string;
+  icon?: string;
+  iconClass?: string;
   title: string;
   onClick?: () => void;
   position: FooterButtonPosition;
@@ -75,6 +76,7 @@ export interface MentionInputProps {
   placeholder?: string;
   loading?: boolean;
   onSelectionChange?: (value: string) => void;
+  onImageUpload?: (file: File) => void;
   footerConfig?: FooterConfig; // 新增配置项
   mentionKeyword?: string;
   labelService?: LabelService;

package/src/browser/context/llm-context.service.ts

@@ -1,3 +1,5 @@
+import { DataContent } from 'ai';
+
 import { Autowired, Injectable } from '@opensumi/di';
 import { AppConfig } from '@opensumi/ide-core-browser/lib/react-providers/config-provider';
 import { WithEventBus } from '@opensumi/ide-core-common/lib/event-bus/event-decorator';

package/src/browser/mcp/config/components/mcp-config.view.tsx

@@ -26,21 +26,7 @@ export const MCPConfigView: React.FC = () => {
   const [editingServer, setEditingServer] = React.useState<MCPServerFormData | undefined>();
   const [loadingServer, setLoadingServer] = React.useState<string | undefined>();
   const loadServers = useCallback(async () => {
-    const userServers = preferenceService.get<MCPServerDescription[]>(AINativeSettingSectionsId.MCPServers, []);
-    const runningServers = await mcpServerProxyService.$getServers();
-    const builtinServer = runningServers.find((server) => server.name === BUILTIN_MCP_SERVER_NAME);
-    const allServers = userServers.map((server) => {
-      const runningServer = runningServers.find((s) => s.name === server.name);
-      return {
-        ...server,
-        name: server.name,
-        isStarted: !!runningServer,
-        tools: runningServer?.tools,
-      };
-    }) as MCPServer[];
-    if (builtinServer) {
-      allServers.unshift(builtinServer);
-    }
+    const allServers = await mcpServerProxyService.$getServers();
     setServers(allServers);
   }, [mcpServerProxyService]);
 

package/src/browser/model/msg-history-manager.ts

@@ -62,7 +62,7 @@ export class MsgHistoryManager extends Disposable {
   }
 
   public addUserMessage(
-    message: Required<Pick<IExcludeMessage, 'agentId' | 'agentCommand' | 'content' | 'relationId'>>,
+    message: Required<Pick<IExcludeMessage, 'agentId' | 'agentCommand' | 'content' | 'relationId' | 'images'>>,
   ): string {
     return this.doAddMessage({
       ...message,

package/src/browser/types.ts

@@ -1,3 +1,4 @@
+import { DataContent } from 'ai';
 import React from 'react';
 import { ZodSchema } from 'zod';
 
@@ -129,6 +130,7 @@ export interface IChatSlashCommandHandler {
 }
 
 export interface IChatFeatureRegistry {
+  registerImageUploadProvider(provider: IImageUploadProvider): void;
   registerWelcome(content: IChatWelcomeMessageContent | React.ReactNode, sampleQuestions?: ISampleQuestions[]): void;
   registerSlashCommand(command: IChatSlashCommandItem, handler: IChatSlashCommandHandler): void;
 }
@@ -140,13 +142,14 @@ export type ChatWelcomeRender = (props: {
 export type ChatAIRoleRender = (props: { content: string }) => React.ReactElement | React.JSX.Element;
 export type ChatUserRoleRender = (props: {
   content: string;
+  images?: string[];
   agentId?: string;
   command?: string;
 }) => React.ReactElement | React.JSX.Element;
 export type ChatThinkingRender = (props: { thinkingText?: string }) => React.ReactElement | React.JSX.Element;
 export type ChatThinkingResultRender = (props: { thinkingResult?: string }) => React.ReactElement | React.JSX.Element;
 export type ChatInputRender = (props: {
-  onSend: (value: string, agentId?: string, command?: string) => void;
+  onSend: (value: string, images?: string[], agentId?: string, command?: string) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -289,6 +292,10 @@ export interface IProblemFixProviderRegistry {
   registerHoverFixProvider(handler: IHoverFixHandler): void;
 }
 
+export interface IImageUploadProvider {
+  imageUpload(file: File): Promise<DataContent | URL>;
+}
+
 export const AINativeCoreContribution = Symbol('AINativeCoreContribution');
 
 export interface AINativeCoreContribution {

package/src/common/index.ts

@@ -52,6 +52,10 @@ export interface IChatMessageStructure {
    * 用于 chat 面板展示
    */
   message: string;
+  /**
+   * 图片
+   */
+  images?: string[];
   /**
    * 实际调用的 prompt
    */
@@ -134,14 +138,7 @@ export interface ISumiMCPServerBackend {
   initBuiltinMCPServer(enabled: boolean): void;
   initExternalMCPServers(servers: MCPServerDescription[]): void;
   getAllMCPTools(): Promise<MCPTool[]>;
-  getServers(): Promise<
-    Array<{
-      name: string;
-      isStarted: boolean;
-      type: string;
-      tools: MCPTool[];
-    }>
-  >;
+  getServers(): Promise<Array<{ name: string; isStarted: boolean; tools: MCPTool[] }>>;
   startServer(serverName: string): Promise<void>;
   stopServer(serverName: string): Promise<void>;
   addOrUpdateServer(description: MCPServerDescription): void;
@@ -207,6 +204,7 @@ export interface IChatAgentRequest {
   requestId: string;
   command?: string;
   message: string;
+  images?: string[];
   regenerate?: boolean;
 }
 
@@ -244,6 +242,7 @@ export type IChatFollowup = IChatReplyFollowup | IChatResponseCommandFollowup;
 
 export interface IChatRequestMessage {
   prompt: string;
+  images?: string[];
   agentId: string;
   command?: string;
 }

package/src/common/llm-context.ts

@@ -1,3 +1,5 @@
+import { DataContent } from 'ai';
+
 import { Event, URI } from '@opensumi/ide-core-common/lib/utils';
 
 export interface LLMContextService {