@opensumi/ide-ai-native 3.8.3-next-1742194253.0 → 3.8.3-next-1742203639.0

package/src/browser/types.ts

@@ -1,3 +1,4 @@
+import { DataContent } from 'ai';
 import React from 'react';
 import { ZodSchema } from 'zod';
 
@@ -129,6 +130,7 @@ export interface IChatSlashCommandHandler {
 }
 
 export interface IChatFeatureRegistry {
+  registerImageUploadProvider(provider: IImageUploadProvider): void;
   registerWelcome(content: IChatWelcomeMessageContent | React.ReactNode, sampleQuestions?: ISampleQuestions[]): void;
   registerSlashCommand(command: IChatSlashCommandItem, handler: IChatSlashCommandHandler): void;
 }
@@ -140,13 +142,14 @@ export type ChatWelcomeRender = (props: {
 export type ChatAIRoleRender = (props: { content: string }) => React.ReactElement | React.JSX.Element;
 export type ChatUserRoleRender = (props: {
   content: string;
+  images?: string[];
   agentId?: string;
   command?: string;
 }) => React.ReactElement | React.JSX.Element;
 export type ChatThinkingRender = (props: { thinkingText?: string }) => React.ReactElement | React.JSX.Element;
 export type ChatThinkingResultRender = (props: { thinkingResult?: string }) => React.ReactElement | React.JSX.Element;
 export type ChatInputRender = (props: {
-  onSend: (value: string, agentId?: string, command?: string) => void;
+  onSend: (value: string, images?: string[], agentId?: string, command?: string) => void;
   onValueChange?: (value: string) => void;
   onExpand?: (value: boolean) => void;
   placeholder?: string;
@@ -289,6 +292,10 @@ export interface IProblemFixProviderRegistry {
   registerHoverFixProvider(handler: IHoverFixHandler): void;
 }
 
+export interface IImageUploadProvider {
+  imageUpload(file: File): Promise<DataContent | URL>;
+}
+
 export const AINativeCoreContribution = Symbol('AINativeCoreContribution');
 
 export interface AINativeCoreContribution {

package/src/common/index.ts

@@ -52,6 +52,10 @@ export interface IChatMessageStructure {
    * 用于 chat 面板展示
    */
   message: string;
+  /**
+   * 图片
+   */
+  images?: string[];
   /**
    * 实际调用的 prompt
    */
@@ -131,10 +135,10 @@ export const ChatProxyServiceToken = Symbol('ChatProxyServiceToken');
 export const TokenMCPServerProxyService = Symbol('TokenMCPServerProxyService');
 
 export interface ISumiMCPServerBackend {
-  initBuiltinMCPServer(enabled: boolean): void;
-  initExternalMCPServers(servers: MCPServerDescription[]): void;
-  getAllMCPTools(): Promise<MCPTool[]>;
-  getServers(): Promise<
+  $initBuiltinMCPServer(enabled: boolean): void;
+  $initExternalMCPServers(servers: MCPServerDescription[]): void;
+  $getAllMCPTools(): Promise<MCPTool[]>;
+  $getServers(): Promise<
     Array<{
       name: string;
       isStarted: boolean;
@@ -142,10 +146,10 @@ export interface ISumiMCPServerBackend {
       tools: MCPTool[];
     }>
   >;
-  startServer(serverName: string): Promise<void>;
-  stopServer(serverName: string): Promise<void>;
-  addOrUpdateServer(description: MCPServerDescription): void;
-  removeServer(name: string): void;
+  $startServer(serverName: string): Promise<void>;
+  $stopServer(serverName: string): Promise<void>;
+  $addOrUpdateServer(description: MCPServerDescription): void;
+  $removeServer(name: string): void;
 }
 
 export const SumiMCPServerProxyServicePath = 'SumiMCPServerProxyServicePath';
@@ -207,6 +211,7 @@ export interface IChatAgentRequest {
   requestId: string;
   command?: string;
   message: string;
+  images?: string[];
   regenerate?: boolean;
 }
 
@@ -244,6 +249,7 @@ export type IChatFollowup = IChatReplyFollowup | IChatResponseCommandFollowup;
 
 export interface IChatRequestMessage {
   prompt: string;
+  images?: string[];
   agentId: string;
   command?: string;
 }

package/src/common/llm-context.ts

@@ -1,3 +1,5 @@
+import { DataContent } from 'ai';
+
 import { Event, URI } from '@opensumi/ide-core-common/lib/utils';
 
 export interface LLMContextService {

package/src/node/base-language-model.ts

@@ -1,4 +1,13 @@
-import { CoreMessage, ToolExecutionOptions, jsonSchema, streamText, tool } from 'ai';
+import {
+  CoreMessage,
+  CoreUserMessage,
+  ImagePart,
+  TextPart,
+  ToolExecutionOptions,
+  jsonSchema,
+  streamText,
+  tool,
+} from 'ai';
 
 import { Autowired, Injectable } from '@opensumi/di';
 import { IAIBackServiceOption } from '@opensumi/ide-core-common';
@@ -48,6 +57,7 @@ export abstract class BaseLanguageModel {
       options.trimTexts,
       options.system,
       options.maxTokens,
+      options.images,
       cancellationToken,
     );
   }
@@ -77,6 +87,7 @@ export abstract class BaseLanguageModel {
     trimTexts?: [string, string],
     systemPrompt?: string,
     maxTokens?: number,
+    images?: string[],
     cancellationToken?: CancellationToken,
   ): Promise<any> {
     try {
@@ -89,7 +100,18 @@ export abstract class BaseLanguageModel {
         });
       }
 
-      const messages: CoreMessage[] = [...history, { role: 'user', content: request }];
+      const messages: CoreMessage[] = [
+        ...history,
+        {
+          role: 'user',
+          content: images?.length
+            ? [
+                { type: 'text', text: request } as TextPart,
+                ...images.map((image) => ({ type: 'image', image: new URL(image) } as ImagePart)),
+              ]
+            : request,
+        },
+      ];
       const modelInfo = modelId ? this.getModelInfo(modelId) : undefined;
       const stream = streamText({
         model: this.getModelIdentifier(provider, modelId),
@@ -101,9 +123,9 @@ export abstract class BaseLanguageModel {
         maxTokens,
         temperature: modelInfo?.temperature || 0,
         topP: modelInfo?.topP || 0.8,
-        topK: modelInfo?.topK || 1,
         system: systemPrompt,
         providerOptions,
+        ...(!images?.length && { topK: modelInfo?.topK || 1 }),
       });
 
       // 状态跟踪变量

package/src/node/mcp/sumi-mcp-server.ts

@@ -72,7 +72,7 @@ export class SumiMCPServerBackend extends RPCService<IMCPServerProxyService> imp
   }
 
   // TODO 这里涉及到 Chat Stream Call 中带上 ClientID，具体方案需要进一步讨论
-  async getAllMCPTools(): Promise<MCPTool[]> {
+  async $getAllMCPTools(): Promise<MCPTool[]> {
     const registry = this.toolInvocationRegistryManager.getRegistry(this.clientId);
     return registry.getAllFunctions().map((tool) => ({
       name: tool.name || 'no-name',
@@ -82,20 +82,20 @@ export class SumiMCPServerBackend extends RPCService<IMCPServerProxyService> imp
     }));
   }
 
-  public async initBuiltinMCPServer(enabled: boolean) {
+  public async $initBuiltinMCPServer(enabled: boolean) {
     const builtinMCPServer = new BuiltinMCPServer(this, this.logger);
     this.mcpServerManager.setClientId(this.clientId);
     await this.mcpServerManager.initBuiltinServer(builtinMCPServer, enabled);
     this.client?.$updateMCPServers();
   }
 
-  public async initExternalMCPServers(servers: MCPServerDescription[]) {
+  public async $initExternalMCPServers(servers: MCPServerDescription[]) {
     this.mcpServerManager.setClientId(this.clientId);
     await this.mcpServerManager.addExternalMCPServers(servers);
     this.client?.$updateMCPServers();
   }
 
-  async initExposedMCPServer() {
+  async $initExposedMCPServer() {
     // 初始化 MCP Server
     this.server = new Server(
       {
@@ -132,7 +132,7 @@ export class SumiMCPServerBackend extends RPCService<IMCPServerProxyService> imp
     return this.server;
   }
 
-  async getServers() {
+  async $getServers() {
     const servers = Array.from(this.mcpServerManager.getServers().entries());
     const serverInfos = await Promise.all(
       servers.map(async ([serverName, server]) => {
@@ -193,21 +193,21 @@ export class SumiMCPServerBackend extends RPCService<IMCPServerProxyService> imp
     return builtinServer ? [builtinServer, ...otherServers] : otherServers;
   }
 
-  async startServer(serverName: string) {
+  async $startServer(serverName: string) {
     await this.mcpServerManager.startServer(serverName);
     this.client?.$updateMCPServers();
   }
 
-  async stopServer(serverName: string) {
+  async $stopServer(serverName: string) {
     await this.mcpServerManager.stopServer(serverName);
     this.client?.$updateMCPServers();
   }
 
-  public addOrUpdateServer(description: MCPServerDescription) {
+  public $addOrUpdateServer(description: MCPServerDescription) {
     this.mcpServerManager.addOrUpdateServer(description);
   }
 
-  public removeServer(name: string) {
+  public $removeServer(name: string) {
     this.mcpServerManager.removeServer(name);
   }
 }

package/src/node/mcp-server-manager-impl.ts

@@ -1,6 +1,7 @@
 import { ToolExecutionOptions } from 'ai';
 
 import { ILogger } from '@opensumi/ide-core-common';
+import { getShellPath } from '@opensumi/ide-core-node';
 
 import { IMCPServer, MCPServerDescription, MCPServerManager, MCPTool } from '../common/mcp-server-manager';
 import { IToolInvocationRegistryManager, ToolRequest } from '../common/tool-invocation-registry';
@@ -17,6 +18,8 @@ export class MCPServerManagerImpl implements MCPServerManager {
   // 当前实例对应的 clientId
   private clientId: string;
 
+  private shellPath: string | undefined;
+
   getServers(): Map<string, IMCPServer> {
     return this.servers;
   }
@@ -26,6 +29,13 @@ export class MCPServerManagerImpl implements MCPServerManager {
     private readonly logger: ILogger,
   ) {}
 
+  async updateShellPath() {
+    const shellPath = await getShellPath();
+    if (shellPath) {
+      this.shellPath = shellPath;
+    }
+  }
+
   setClientId(clientId: string) {
     this.clientId = clientId;
   }
@@ -132,10 +142,14 @@ export class MCPServerManagerImpl implements MCPServerManager {
     const existingServer = this.servers.get(description.name);
     if (description.type === MCP_SERVER_TYPE.STDIO) {
       const { name, command, args, env } = description;
+      const envs = {
+        ...env,
+        PATH: this.shellPath || process.env.PATH || '',
+      };
       if (existingServer) {
-        existingServer.update(command, args, env);
+        existingServer.update(command, args, envs);
       } else {
-        const newServer = new StdioMCPServer(name, command, args, env, this.logger);
+        const newServer = new StdioMCPServer(name, command, args, envs, this.logger);
         this.servers.set(name, newServer);
       }
     } else if (description.type === MCP_SERVER_TYPE.SSE) {
@@ -163,6 +177,7 @@ export class MCPServerManagerImpl implements MCPServerManager {
   }
 
   async addExternalMCPServers(servers: MCPServerDescription[]): Promise<void> {
+    await this.updateShellPath();
     for (const server of servers) {
       this.addOrUpdateServer(server);
       if (!server.enabled) {