@opensumi/ide-ai-native 3.8.3-next-1741934038.0 → 3.8.3-next-1741942697.0

package/src/browser/chat/chat.internal.service.ts

@@ -62,8 +62,8 @@ export class ChatInternalService extends Disposable {
     this._onChangeRequestId.fire(id);
   }
 
-  createRequest(input: string, agentId: string, command?: string) {
-    return this.chatManagerService.createRequest(this.#sessionModel.sessionId, input, agentId, command);
+  createRequest(input: string, agentId: string, images?: string[], command?: string) {
+    return this.chatManagerService.createRequest(this.#sessionModel.sessionId, input, agentId, command, images);
   }
 
   sendRequest(request: ChatRequestModel, regenerate = false) {

package/src/browser/chat/chat.view.tsx

@@ -281,7 +281,7 @@ export const AIChatView = () => {
           if (loading) {
             return;
           }
-          await handleSend(message.message, message.agentId, message.command);
+          await handleSend(message.message, message.images, message.agentId, message.command);
         } else {
           if (message.agentId) {
             setAgentId(message.agentId);
@@ -462,10 +462,16 @@ export const AIChatView = () => {
   );
 
   const renderUserMessage = React.useCallback(
-    async (renderModel: { message: string; agentId?: string; relationId: string; command?: string }) => {
+    async (renderModel: {
+      message: string;
+      images?: string[];
+      agentId?: string;
+      relationId: string;
+      command?: string;
+    }) => {
       const ChatUserRoleRender = chatRenderRegistry.chatUserRoleRender;
 
-      const { message, agentId, relationId, command } = renderModel;
+      const { message, images, agentId, relationId, command } = renderModel;
 
       const visibleAgentId = agentId === ChatProxyService.AGENT_ID ? '' : agentId;
 
@@ -474,12 +480,13 @@ export const AIChatView = () => {
           id: uuid(6),
           relationId,
           text: ChatUserRoleRender ? (
-            <ChatUserRoleRender content={message} agentId={visibleAgentId} command={command} />
+            <ChatUserRoleRender content={message} images={images} agentId={visibleAgentId} command={command} />
           ) : (
             <CodeBlockWrapperInput
               labelService={labelService}
               relationId={relationId}
               text={message}
+              images={images}
               agentId={visibleAgentId}
               command={command}
               workspaceService={workspaceService}
@@ -598,10 +605,10 @@ export const AIChatView = () => {
 
   const handleAgentReply = React.useCallback(
     async (value: IChatMessageStructure) => {
-      const { message, agentId, command, reportExtra } = value;
+      const { message, images, agentId, command, reportExtra } = value;
       const { actionType, actionSource } = reportExtra || {};
 
-      const request = aiChatService.createRequest(message, agentId!, command);
+      const request = aiChatService.createRequest(message, agentId!, images, command);
       if (!request) {
         return;
       }
@@ -627,6 +634,7 @@ export const AIChatView = () => {
 
       msgHistoryManager.addUserMessage({
         content: message,
+        images: images || [],
         agentId: agentId!,
         agentCommand: command!,
         relationId,
@@ -635,6 +643,7 @@ export const AIChatView = () => {
       await renderUserMessage({
         relationId,
         message,
+        images,
         command,
         agentId,
       });
@@ -668,7 +677,7 @@ export const AIChatView = () => {
   );
 
   const handleSend = React.useCallback(
-    async (message: string, agentId?: string, command?: string) => {
+    async (message: string, images?: string[], agentId?: string, command?: string) => {
       const reportExtra = {
         actionSource: ActionSourceEnum.Chat,
         actionType: ActionTypeEnum.Send,
@@ -707,7 +716,7 @@ export const AIChatView = () => {
           processedContent = processedContent.replace(match, `\`<attached_folder>${relativePath}\``);
         }
       }
-      return handleAgentReply({ message: processedContent, agentId, command, reportExtra });
+      return handleAgentReply({ message: processedContent, images, agentId, command, reportExtra });
     },
     [handleAgentReply],
   );
@@ -750,6 +759,7 @@ export const AIChatView = () => {
             message: msg.content,
             agentId: msg.agentId,
             command: msg.agentCommand,
+            images: msg.images,
           });
         } else if (msg.role === ChatMessageRole.Assistant && msg.requestId) {
           const request = aiChatService.sessionModel.getRequest(msg.requestId)!;

package/src/browser/components/ChatEditor.tsx

@@ -2,6 +2,7 @@ import capitalize from 'lodash/capitalize';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 import Highlight from 'react-highlight';
 
+import { Image } from '@opensumi/ide-components/lib/image';
 import {
   EDITOR_COMMANDS,
   FILE_COMMANDS,
@@ -336,6 +337,7 @@ export const CodeBlockWrapper = ({
 
 export const CodeBlockWrapperInput = ({
   text,
+  images,
   relationId,
   agentId,
   command,
@@ -344,6 +346,7 @@ export const CodeBlockWrapperInput = ({
   commandService,
 }: {
   text: string;
+  images?: string[];
   relationId: string;
   agentId?: string;
   command?: string;
@@ -370,6 +373,11 @@ export const CodeBlockWrapperInput = ({
 
   return (
     <div className={styles.ai_chat_code_wrapper}>
+      {images?.map((image) => (
+        <div className={styles.image_wrapper}>
+          <Image src={image} />
+        </div>
+      ))}
       <div className={styles.render_text}>
         {tag && (
           <div className={styles.tag_wrapper}>

package/src/browser/components/ChatInput.tsx

@@ -162,7 +162,7 @@ const AgentWidget = ({ agentId, command }) => (
 );
 
 export interface IChatInputProps {
-  onSend: (value: string, agentId?: string, command?: string) => void;
+  onSend: (value: string, images?: string[], agentId?: string, command?: string) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -341,7 +341,7 @@ export const ChatInput = React.forwardRef((props: IChatInputProps, ref) => {
     }
 
     const handleSendLogic = (newValue: string = value) => {
-      onSend(newValue, agentId, command);
+      onSend(newValue, [], agentId, command);
       setValue('');
       setTheme('');
       setAgentId('');

package/src/browser/components/ChatMentionInput.tsx

@@ -1,23 +1,34 @@
+import { DataContent } from 'ai';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
+import { Image } from '@opensumi/ide-components/lib/image';
 import { LabelService, RecentFilesManager, useInjectable } from '@opensumi/ide-core-browser';
-import { getIcon } from '@opensumi/ide-core-browser/lib/components';
+import { Icon, getIcon } from '@opensumi/ide-core-browser/lib/components';
 import { URI, localize } from '@opensumi/ide-core-common';
 import { CommandService } from '@opensumi/ide-core-common/lib/command';
 import { WorkbenchEditorService } from '@opensumi/ide-editor';
 import { FileSearchServicePath, IFileSearchService } from '@opensumi/ide-file-search';
+import { IMessageService } from '@opensumi/ide-overlay';
 import { IWorkspaceService } from '@opensumi/ide-workspace';
 
 import { IChatInternalService } from '../../common';
 import { ChatInternalService } from '../chat/chat.internal.service';
+import { ImageUploadProviderRegistryToken } from '../contrib/image-upload/image-upload.feature.registry';
 import { OPEN_MCP_CONFIG_COMMAND } from '../mcp/config/mcp-config.commands';
+import { IImageUploadProviderRegistry } from '../types';
 
 import styles from './components.module.less';
 import { MentionInput } from './mention-input/mention-input';
 import { FooterButtonPosition, FooterConfig, MentionItem, MentionType } from './mention-input/types';
 
 export interface IChatMentionInputProps {
-  onSend: (value: string, agentId?: string, command?: string, option?: { model: string; [key: string]: any }) => void;
+  onSend: (
+    value: string,
+    images?: string[],
+    agentId?: string,
+    command?: string,
+    option?: { model: string; [key: string]: any },
+  ) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -26,6 +37,7 @@ export interface IChatMentionInputProps {
   sendBtnClassName?: string;
   defaultHeight?: number;
   value?: string;
+  images?: Array<DataContent | URL>;
   autoFocus?: boolean;
   theme?: string | null;
   setTheme: (theme: string | null) => void;
@@ -41,6 +53,7 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
   const { onSend, disabled = false } = props;
 
   const [value, setValue] = useState(props.value || '');
+  const [images, setImages] = useState(props.images || []);
   const aiChatService = useInjectable<ChatInternalService>(IChatInternalService);
   const commandService = useInjectable<CommandService>(CommandService);
   const searchService = useInjectable<IFileSearchService>(FileSearchServicePath);
@@ -48,6 +61,8 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
   const workspaceService = useInjectable<IWorkspaceService>(IWorkspaceService);
   const editorService = useInjectable<WorkbenchEditorService>(WorkbenchEditorService);
   const labelService = useInjectable<LabelService>(LabelService);
+  const messageService = useInjectable<IMessageService>(IMessageService);
+  const imageUploadProviderRegistry = useInjectable<IImageUploadProviderRegistry>(ImageUploadProviderRegistryToken);
 
   const handleShowMCPConfig = React.useCallback(() => {
     commandService.executeCommand(OPEN_MCP_CONFIG_COMMAND.id);
@@ -239,6 +254,24 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
           onClick: handleShowMCPConfig,
           position: FooterButtonPosition.LEFT,
         },
+        {
+          id: 'upload-image',
+          iconClass: 'codicon codicon-file-media',
+          title: 'Upload Image',
+          onClick: () => {
+            const input = document.createElement('input');
+            input.type = 'file';
+            input.accept = 'image/*';
+            input.onchange = (e) => {
+              const file = (e.target as HTMLInputElement).files?.[0];
+              if (file) {
+                handleImageUpload(file);
+              }
+            };
+            input.click();
+          },
+          position: FooterButtonPosition.LEFT,
+        },
       ],
       showModelSelector: true,
     }),
@@ -254,13 +287,46 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
       if (disabled) {
         return;
       }
-      onSend(content, undefined, undefined, option);
+      onSend(
+        content,
+        images.map((image) => image.toString()),
+        undefined,
+        undefined,
+        option,
+      );
     },
-    [onSend, editorService, disabled],
+    [onSend, images, disabled],
+  );
+
+  const handleImageUpload = useCallback(
+    async (file: File) => {
+      const allowedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp', 'image/gif'];
+      if (!allowedTypes.includes(file.type)) {
+        messageService.error('Only JPG, PNG, WebP and GIF images are supported');
+        return;
+      }
+
+      const imageUploadProvider = imageUploadProviderRegistry.getImageUploadProvider();
+      if (!imageUploadProvider) {
+        messageService.error('No image upload provider found');
+        return;
+      }
+      const data = await imageUploadProvider.imageUpload(file);
+      setImages([...images, data]);
+    },
+    [images],
+  );
+
+  const handleDeleteImage = useCallback(
+    (index: number) => {
+      setImages(images.filter((_, i) => i !== index));
+    },
+    [images],
   );
 
   return (
     <div className={styles.chat_input_container}>
+      {images.length > 0 && <ImagePreviewer images={images} onDelete={handleDeleteImage} />}
       <MentionInput
         mentionItems={defaultMenuItems}
         onSend={handleSend}
@@ -270,7 +336,29 @@ export const ChatMentionInput = (props: IChatMentionInputProps) => {
         workspaceService={workspaceService}
         placeholder={localize('aiNative.chat.input.placeholder.default')}
         footerConfig={defaultMentionInputFooterOptions}
+        onImageUpload={handleImageUpload}
       />
     </div>
   );
 };
+
+const ImagePreviewer = ({
+  images,
+  onDelete,
+}: {
+  images: Array<DataContent | URL>;
+  onDelete: (index: number) => void;
+}) => (
+  <div>
+    <div className={styles.thumbnail_container}>
+      {images.map((image, index) => (
+        <div key={index} className={styles.thumbnail}>
+          <Image src={image.toString()} />
+          <button onClick={() => onDelete(index)} className={styles.delete_button}>
+            <Icon iconClass='codicon codicon-close' />
+          </button>
+        </div>
+      ))}
+    </div>
+  </div>
+);

package/src/browser/components/components.module.less

@@ -601,3 +601,52 @@
   vertical-align: middle;
   font-size: 12px;
 }
+
+.thumbnail_container {
+  display: flex;
+  gap: 4px;
+  padding: 4px 12px;
+  .thumbnail {
+    width: 36px;
+    height: 36px;
+    border-radius: 4px;
+    position: relative;
+    box-shadow: 0 0 10px 0 rgba(0, 0, 0, 0.3);
+    padding: 2px;
+    display: inline-flex;
+    align-items: center;
+    img {
+      width: 100%;
+      height: 100%;
+      object-fit: cover;
+    }
+  }
+  .delete_button {
+    position: absolute;
+    top: -5px;
+    right: -5px;
+    font-size: 12px;
+    padding: 2px;
+    border: 0;
+    border-radius: 50%;
+    transition: transform 0.2s ease-in-out;
+    width: 16px;
+    height: 16px;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    background-color: var(--badge-background);
+    &:hover {
+      cursor: pointer;
+      transform: scale(1.2);
+    }
+    :global(.codicon) {
+      font-size: 12px;
+      color: var(--badge-foreground);
+    }
+  }
+}
+
+.image_wrapper {
+  display: inline-block;
+}

package/src/browser/components/mention-input/mention-input.module.less

@@ -50,6 +50,7 @@
     display: flex;
     align-items: center;
     cursor: pointer;
+    text-wrap: nowrap;
     white-space: nowrap;
     box-sizing: border-box;
 

package/src/browser/components/mention-input/mention-input.tsx

@@ -18,6 +18,7 @@ export const MentionInput: React.FC<MentionInputProps> = ({
   loading = false,
   mentionKeyword = MENTION_KEYWORD,
   onSelectionChange,
+  onImageUpload,
   labelService,
   workspaceService,
   placeholder = 'Ask anything, @ to mention',
@@ -275,6 +276,21 @@ export const MentionInput: React.FC<MentionInputProps> = ({
     }
   };
 
+  // 处理图片粘贴事件
+  const handlePaste = (e: React.ClipboardEvent<HTMLDivElement>) => {
+    const items = e.clipboardData.items;
+    // eslint-disable-next-line @typescript-eslint/prefer-for-of
+    for (let i = 0; i < items.length; i++) {
+      if (items[i].kind === 'file' && items[i].type.startsWith('image/')) {
+        const file = items[i].getAsFile();
+        if (file && onImageUpload) {
+          e.preventDefault();
+          onImageUpload(file);
+        }
+      }
+    }
+  };
+
   // 处理键盘事件
   const handleKeyDown = (e: React.KeyboardEvent<HTMLDivElement>) => {
     // 如果按下ESC键且提及面板处于活动状态或内联搜索处于活动状态
@@ -867,7 +883,7 @@ export const MentionInput: React.FC<MentionInputProps> = ({
             title={button.title}
           >
             <EnhanceIcon
-              className={cls(getIcon(button.icon), styles[`${button.id}_logo`])}
+              className={cls(button.icon ? getIcon(button.icon) : button.iconClass, styles[`${button.id}_logo`])}
               tabIndex={0}
               role='button'
               ariaLabel={button.title}
@@ -901,6 +917,7 @@ export const MentionInput: React.FC<MentionInputProps> = ({
           contentEditable={true}
           onInput={handleInput}
           onKeyDown={handleKeyDown}
+          onPaste={handlePaste}
           onCompositionEnd={handleCompositionEnd}
         />
       </div>

package/src/browser/components/mention-input/types.ts

@@ -55,7 +55,8 @@ export enum MentionType {
 
 interface FooterButton {
   id: string;
-  icon: string;
+  icon?: string;
+  iconClass?: string;
   title: string;
   onClick?: () => void;
   position: FooterButtonPosition;
@@ -75,6 +76,7 @@ export interface MentionInputProps {
   placeholder?: string;
   loading?: boolean;
   onSelectionChange?: (value: string) => void;
+  onImageUpload?: (file: File) => void;
   footerConfig?: FooterConfig; // 新增配置项
   mentionKeyword?: string;
   labelService?: LabelService;

package/src/browser/context/llm-context.service.ts

@@ -1,3 +1,5 @@
+import { DataContent } from 'ai';
+
 import { Autowired, Injectable } from '@opensumi/di';
 import { AppConfig } from '@opensumi/ide-core-browser/lib/react-providers/config-provider';
 import { WithEventBus } from '@opensumi/ide-core-common/lib/event-bus/event-decorator';

package/src/browser/contrib/image-upload/image-upload.feature.registry.ts

@@ -0,0 +1,18 @@
+import { Injectable } from '@opensumi/di';
+
+import { IImageUploadProvider, IImageUploadProviderRegistry } from '../../types';
+
+@Injectable()
+export class ImageUploadProviderRegistry implements IImageUploadProviderRegistry {
+  private imageUploadProvider: IImageUploadProvider | undefined;
+
+  registerImageUploadProvider(provider: IImageUploadProvider): void {
+    this.imageUploadProvider = provider;
+  }
+
+  getImageUploadProvider(): IImageUploadProvider | undefined {
+    return this.imageUploadProvider;
+  }
+}
+
+export const ImageUploadProviderRegistryToken = Symbol('ImageUploadProviderRegistry');

package/src/browser/index.ts

@@ -46,6 +46,10 @@ import { ChatRenderRegistry } from './chat/chat.render.registry';
 import { LlmContextContribution } from './context/llm-context.contribution';
 import { LLMContextServiceImpl } from './context/llm-context.service';
 import { AICodeActionContribution } from './contrib/code-action/code-action.contribution';
+import {
+  ImageUploadProviderRegistry,
+  ImageUploadProviderRegistryToken,
+} from './contrib/image-upload/image-upload.feature.registry';
 import { AIInlineCompletionsProvider } from './contrib/inline-completions/completeProvider';
 import { IntelligentCompletionsContribution } from './contrib/intelligent-completions/intelligent-completions.contribution';
 import { IntelligentCompletionsRegistry } from './contrib/intelligent-completions/intelligent-completions.feature.registry';
@@ -186,6 +190,10 @@ export class AINativeModule extends BrowserModule {
       token: TerminalRegistryToken,
       useClass: TerminalFeatureRegistry,
     },
+    {
+      token: ImageUploadProviderRegistryToken,
+      useClass: ImageUploadProviderRegistry,
+    },
     {
       token: LanguageParserService,
       useClass: LanguageParserService,

package/src/browser/model/msg-history-manager.ts

@@ -62,7 +62,7 @@ export class MsgHistoryManager extends Disposable {
   }
 
   public addUserMessage(
-    message: Required<Pick<IExcludeMessage, 'agentId' | 'agentCommand' | 'content' | 'relationId'>>,
+    message: Required<Pick<IExcludeMessage, 'agentId' | 'agentCommand' | 'content' | 'relationId' | 'images'>>,
   ): string {
     return this.doAddMessage({
       ...message,

package/src/browser/types.ts

@@ -1,3 +1,4 @@
+import { DataContent } from 'ai';
 import React from 'react';
 import { ZodSchema } from 'zod';
 
@@ -140,13 +141,14 @@ export type ChatWelcomeRender = (props: {
 export type ChatAIRoleRender = (props: { content: string }) => React.ReactElement | React.JSX.Element;
 export type ChatUserRoleRender = (props: {
   content: string;
+  images?: string[];
   agentId?: string;
   command?: string;
 }) => React.ReactElement | React.JSX.Element;
 export type ChatThinkingRender = (props: { thinkingText?: string }) => React.ReactElement | React.JSX.Element;
 export type ChatThinkingResultRender = (props: { thinkingResult?: string }) => React.ReactElement | React.JSX.Element;
 export type ChatInputRender = (props: {
-  onSend: (value: string, agentId?: string, command?: string) => void;
+  onSend: (value: string, images?: string[], agentId?: string, command?: string) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -289,6 +291,15 @@ export interface IProblemFixProviderRegistry {
   registerHoverFixProvider(handler: IHoverFixHandler): void;
 }
 
+export interface IImageUploadProvider {
+  imageUpload(file: File): Promise<DataContent | URL>;
+}
+
+export interface IImageUploadProviderRegistry {
+  registerImageUploadProvider(provider: IImageUploadProvider): void;
+  getImageUploadProvider(): IImageUploadProvider | undefined;
+}
+
 export const AINativeCoreContribution = Symbol('AINativeCoreContribution');
 
 export interface AINativeCoreContribution {
@@ -332,6 +343,11 @@ export interface AINativeCoreContribution {
    * @param provider
    */
   registerChatAgentPromptProvider?(): void;
+
+  /**
+   * 注册图片上传的能力
+   */
+  registerImageUploadProvider?(registry: IImageUploadProviderRegistry): void;
 }
 
 // MCP Server 的 贡献点

package/src/common/index.ts

@@ -52,6 +52,10 @@ export interface IChatMessageStructure {
    * 用于 chat 面板展示
    */
   message: string;
+  /**
+   * 图片
+   */
+  images?: string[];
   /**
    * 实际调用的 prompt
    */
@@ -200,6 +204,7 @@ export interface IChatAgentRequest {
   requestId: string;
   command?: string;
   message: string;
+  images?: string[];
   regenerate?: boolean;
 }
 
@@ -237,6 +242,7 @@ export type IChatFollowup = IChatReplyFollowup | IChatResponseCommandFollowup;
 
 export interface IChatRequestMessage {
   prompt: string;
+  images?: string[];
   agentId: string;
   command?: string;
 }

package/src/common/llm-context.ts

@@ -1,3 +1,5 @@
+import { DataContent } from 'ai';
+
 import { Event, URI } from '@opensumi/ide-core-common/lib/utils';
 
 export interface LLMContextService {

package/src/node/base-language-model.ts

@@ -1,4 +1,13 @@
-import { CoreMessage, ToolExecutionOptions, jsonSchema, streamText, tool } from 'ai';
+import {
+  CoreMessage,
+  CoreUserMessage,
+  ImagePart,
+  TextPart,
+  ToolExecutionOptions,
+  jsonSchema,
+  streamText,
+  tool,
+} from 'ai';
 
 import { Autowired, Injectable } from '@opensumi/di';
 import { IAIBackServiceOption } from '@opensumi/ide-core-common';
@@ -48,6 +57,7 @@ export abstract class BaseLanguageModel {
       options.trimTexts,
       options.system,
       options.maxTokens,
+      options.images,
       cancellationToken,
     );
   }
@@ -77,6 +87,7 @@ export abstract class BaseLanguageModel {
     trimTexts?: [string, string],
     systemPrompt?: string,
     maxTokens?: number,
+    images?: string[],
     cancellationToken?: CancellationToken,
   ): Promise<any> {
     try {
@@ -89,7 +100,18 @@ export abstract class BaseLanguageModel {
         });
       }
 
-      const messages: CoreMessage[] = [...history, { role: 'user', content: request }];
+      const messages: CoreMessage[] = [
+        ...history,
+        {
+          role: 'user',
+          content: images?.length
+            ? [
+                { type: 'text', text: request } as TextPart,
+                ...images.map((image) => ({ type: 'image', image: new URL(image) } as ImagePart)),
+              ]
+            : request,
+        },
+      ];
       const modelInfo = modelId ? this.getModelInfo(modelId) : undefined;
       const stream = streamText({
         model: this.getModelIdentifier(provider, modelId),
@@ -101,9 +123,9 @@ export abstract class BaseLanguageModel {
         maxTokens,
         temperature: modelInfo?.temperature || 0,
         topP: modelInfo?.topP || 0.8,
-        topK: modelInfo?.topK || 1,
         system: systemPrompt,
         providerOptions,
+        ...(!images?.length && { topK: modelInfo?.topK || 1 }),
       });
 
       // 状态跟踪变量