@opensumi/ide-ai-native 3.8.3-next-1741945847.0 → 3.8.3-next-1741949132.0

package/src/node/base-language-model.ts

@@ -1,4 +1,13 @@
-import { CoreMessage, ToolExecutionOptions, jsonSchema, streamText, tool } from 'ai';
+import {
+  CoreMessage,
+  CoreUserMessage,
+  ImagePart,
+  TextPart,
+  ToolExecutionOptions,
+  jsonSchema,
+  streamText,
+  tool,
+} from 'ai';
 
 import { Autowired, Injectable } from '@opensumi/di';
 import { IAIBackServiceOption } from '@opensumi/ide-core-common';
@@ -48,6 +57,7 @@ export abstract class BaseLanguageModel {
       options.trimTexts,
       options.system,
       options.maxTokens,
+      options.images,
       cancellationToken,
     );
   }
@@ -77,6 +87,7 @@ export abstract class BaseLanguageModel {
     trimTexts?: [string, string],
     systemPrompt?: string,
     maxTokens?: number,
+    images?: string[],
     cancellationToken?: CancellationToken,
   ): Promise<any> {
     try {
@@ -89,7 +100,18 @@ export abstract class BaseLanguageModel {
         });
       }
 
-      const messages: CoreMessage[] = [...history, { role: 'user', content: request }];
+      const messages: CoreMessage[] = [
+        ...history,
+        {
+          role: 'user',
+          content: images?.length
+            ? [
+                { type: 'text', text: request } as TextPart,
+                ...images.map((image) => ({ type: 'image', image: new URL(image) } as ImagePart)),
+              ]
+            : request,
+        },
+      ];
       const modelInfo = modelId ? this.getModelInfo(modelId) : undefined;
       const stream = streamText({
         model: this.getModelIdentifier(provider, modelId),
@@ -101,9 +123,9 @@ export abstract class BaseLanguageModel {
         maxTokens,
         temperature: modelInfo?.temperature || 0,
         topP: modelInfo?.topP || 0.8,
-        topK: modelInfo?.topK || 1,
         system: systemPrompt,
         providerOptions,
+        ...(!images?.length && { topK: modelInfo?.topK || 1 }),
       });
 
       // 状态跟踪变量