@opensumi/ide-ai-native 3.8.3-next-1741940731.0 → 3.8.3-next-1741942697.0

package/src/browser/types.ts

@@ -1,3 +1,4 @@
+import { DataContent } from 'ai';
 import React from 'react';
 import { ZodSchema } from 'zod';
 
@@ -140,13 +141,14 @@ export type ChatWelcomeRender = (props: {
 export type ChatAIRoleRender = (props: { content: string }) => React.ReactElement | React.JSX.Element;
 export type ChatUserRoleRender = (props: {
   content: string;
+  images?: string[];
   agentId?: string;
   command?: string;
 }) => React.ReactElement | React.JSX.Element;
 export type ChatThinkingRender = (props: { thinkingText?: string }) => React.ReactElement | React.JSX.Element;
 export type ChatThinkingResultRender = (props: { thinkingResult?: string }) => React.ReactElement | React.JSX.Element;
 export type ChatInputRender = (props: {
-  onSend: (value: string, agentId?: string, command?: string) => void;
+  onSend: (value: string, images?: string[], agentId?: string, command?: string) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -289,6 +291,15 @@ export interface IProblemFixProviderRegistry {
   registerHoverFixProvider(handler: IHoverFixHandler): void;
 }
 
+export interface IImageUploadProvider {
+  imageUpload(file: File): Promise<DataContent | URL>;
+}
+
+export interface IImageUploadProviderRegistry {
+  registerImageUploadProvider(provider: IImageUploadProvider): void;
+  getImageUploadProvider(): IImageUploadProvider | undefined;
+}
+
 export const AINativeCoreContribution = Symbol('AINativeCoreContribution');
 
 export interface AINativeCoreContribution {
@@ -332,6 +343,11 @@ export interface AINativeCoreContribution {
    * @param provider
    */
   registerChatAgentPromptProvider?(): void;
+
+  /**
+   * 注册图片上传的能力
+   */
+  registerImageUploadProvider?(registry: IImageUploadProviderRegistry): void;
 }
 
 // MCP Server 的 贡献点

package/src/common/index.ts

@@ -52,6 +52,10 @@ export interface IChatMessageStructure {
    * 用于 chat 面板展示
    */
   message: string;
+  /**
+   * 图片
+   */
+  images?: string[];
   /**
    * 实际调用的 prompt
    */
@@ -134,14 +138,7 @@ export interface ISumiMCPServerBackend {
   initBuiltinMCPServer(enabled: boolean): void;
   initExternalMCPServers(servers: MCPServerDescription[]): void;
   getAllMCPTools(): Promise<MCPTool[]>;
-  getServers(): Promise<
-    Array<{
-      name: string;
-      isStarted: boolean;
-      type: string;
-      tools: MCPTool[];
-    }>
-  >;
+  getServers(): Promise<Array<{ name: string; isStarted: boolean; tools: MCPTool[] }>>;
   startServer(serverName: string): Promise<void>;
   stopServer(serverName: string): Promise<void>;
   addOrUpdateServer(description: MCPServerDescription): void;
@@ -207,6 +204,7 @@ export interface IChatAgentRequest {
   requestId: string;
   command?: string;
   message: string;
+  images?: string[];
   regenerate?: boolean;
 }
 
@@ -244,6 +242,7 @@ export type IChatFollowup = IChatReplyFollowup | IChatResponseCommandFollowup;
 
 export interface IChatRequestMessage {
   prompt: string;
+  images?: string[];
   agentId: string;
   command?: string;
 }

package/src/common/llm-context.ts

@@ -1,3 +1,5 @@
+import { DataContent } from 'ai';
+
 import { Event, URI } from '@opensumi/ide-core-common/lib/utils';
 
 export interface LLMContextService {

package/src/node/base-language-model.ts

@@ -1,4 +1,13 @@
-import { CoreMessage, ToolExecutionOptions, jsonSchema, streamText, tool } from 'ai';
+import {
+  CoreMessage,
+  CoreUserMessage,
+  ImagePart,
+  TextPart,
+  ToolExecutionOptions,
+  jsonSchema,
+  streamText,
+  tool,
+} from 'ai';
 
 import { Autowired, Injectable } from '@opensumi/di';
 import { IAIBackServiceOption } from '@opensumi/ide-core-common';
@@ -48,6 +57,7 @@ export abstract class BaseLanguageModel {
       options.trimTexts,
       options.system,
       options.maxTokens,
+      options.images,
       cancellationToken,
     );
   }
@@ -77,6 +87,7 @@ export abstract class BaseLanguageModel {
     trimTexts?: [string, string],
     systemPrompt?: string,
     maxTokens?: number,
+    images?: string[],
     cancellationToken?: CancellationToken,
   ): Promise<any> {
     try {
@@ -89,7 +100,18 @@ export abstract class BaseLanguageModel {
         });
       }
 
-      const messages: CoreMessage[] = [...history, { role: 'user', content: request }];
+      const messages: CoreMessage[] = [
+        ...history,
+        {
+          role: 'user',
+          content: images?.length
+            ? [
+                { type: 'text', text: request } as TextPart,
+                ...images.map((image) => ({ type: 'image', image: new URL(image) } as ImagePart)),
+              ]
+            : request,
+        },
+      ];
       const modelInfo = modelId ? this.getModelInfo(modelId) : undefined;
       const stream = streamText({
         model: this.getModelIdentifier(provider, modelId),
@@ -101,9 +123,9 @@ export abstract class BaseLanguageModel {
         maxTokens,
         temperature: modelInfo?.temperature || 0,
         topP: modelInfo?.topP || 0.8,
-        topK: modelInfo?.topK || 1,
         system: systemPrompt,
         providerOptions,
+        ...(!images?.length && { topK: modelInfo?.topK || 1 }),
       });
 
       // 状态跟踪变量