@lobehub/chat 1.99.5 → 1.99.6

package/src/services/chat.ts

@@ -15,6 +15,7 @@ import {
   ChatCompletionErrorPayload,
   ModelProvider,
 } from '@/libs/model-runtime';
+import { parseDataUri } from '@/libs/model-runtime/utils/uriParser';
 import { filesPrompts } from '@/prompts/files';
 import { BuiltinSystemRolePrompts } from '@/prompts/systemRole';
 import { getAgentStoreState } from '@/store/agent';
@@ -35,7 +36,7 @@ import {
 import { WebBrowsingManifest } from '@/tools/web-browsing';
 import { WorkingModel } from '@/types/agent';
 import { ChatErrorType } from '@/types/fetch';
-import { ChatMessage, MessageToolCall } from '@/types/message';
+import { ChatImageItem, ChatMessage, MessageToolCall } from '@/types/message';
 import type { ChatStreamPayload, OpenAIChatMessage } from '@/types/openai/chat';
 import { UserMessageContentPart } from '@/types/openai/chat';
 import { parsePlaceholderVariablesMessages } from '@/utils/client/parserPlaceholder';
@@ -46,8 +47,10 @@ import {
   getMessageError,
   standardizeAnimationStyle,
 } from '@/utils/fetch';
+import { imageUrlToBase64 } from '@/utils/imageToBase64';
 import { genToolCallingName } from '@/utils/toolCall';
 import { createTraceHeader, getTraceId } from '@/utils/trace';
+import { isLocalUrl } from '@/utils/url';
 
 import { createHeaderWithAuth, createPayloadWithKeyVaults } from './_auth';
 import { API_ENDPOINTS } from './_url';
@@ -61,6 +64,14 @@ const isCanUseFC = (model: string, provider: string) => {
   return aiModelSelectors.isModelSupportToolUse(model, provider)(getAiInfraStoreState());
 };
 
+const isCanUseVision = (model: string, provider: string) => {
+  // TODO: remove isDeprecatedEdition condition in V2.0
+  if (isDeprecatedEdition) {
+    return modelProviderSelectors.isModelEnabledVision(model)(getUserStoreState());
+  }
+  return aiModelSelectors.isModelSupportVision(model, provider)(getAiInfraStoreState());
+};
+
 /**
  * TODO: we need to update this function to auto find deploymentName with provider setting config
  */
@@ -205,7 +216,7 @@ class ChatService {
 
     // ============  2. preprocess messages   ============ //
 
-    const oaiMessages = this.processMessages(
+    const oaiMessages = await this.processMessages(
       {
         messages: parsedMessages,
         model: payload.model,
@@ -475,7 +486,7 @@ class ChatService {
     onLoadingChange?.(true);
 
     try {
-      const oaiMessages = this.processMessages({
+      const oaiMessages = await this.processMessages({
         messages: params.messages as any,
         model: params.model!,
         provider: params.provider!,
@@ -507,7 +518,7 @@ class ChatService {
     }
   };
 
-  private processMessages = (
+  private processMessages = async (
     {
       messages = [],
       tools,
@@ -520,29 +531,28 @@ class ChatService {
       tools?: string[];
     },
     options?: FetchOptions,
-  ): OpenAIChatMessage[] => {
+  ): Promise<OpenAIChatMessage[]> => {
     // handle content type for vision model
     // for the models with visual ability, add image url to content
     // refs: https://platform.openai.com/docs/guides/vision/quick-start
-    const getUserContent = (m: ChatMessage) => {
+    const getUserContent = async (m: ChatMessage) => {
       // only if message doesn't have images and files, then return the plain content
       if ((!m.imageList || m.imageList.length === 0) && (!m.fileList || m.fileList.length === 0))
         return m.content;
 
       const imageList = m.imageList || [];
+      const imageContentParts = await this.processImageList({ imageList, model, provider });
 
       const filesContext = isServerMode
         ? filesPrompts({ addUrl: !isDesktop, fileList: m.fileList, imageList })
         : '';
       return [
         { text: (m.content + '\n\n' + filesContext).trim(), type: 'text' },
-        ...imageList.map(
-          (i) => ({ image_url: { detail: 'auto', url: i.url }, type: 'image_url' }) as const,
-        ),
+        ...imageContentParts,
       ] as UserMessageContentPart[];
     };
 
-    const getAssistantContent = (m: ChatMessage) => {
+    const getAssistantContent = async (m: ChatMessage) => {
       // signature is a signal of anthropic thinking mode
       const shouldIncludeThinking = m.reasoning && !!m.reasoning?.signature;
 
@@ -559,65 +569,70 @@ class ChatService {
       // only if message doesn't have images and files, then return the plain content
 
       if (m.imageList && m.imageList.length > 0) {
+        const imageContentParts = await this.processImageList({
+          imageList: m.imageList,
+          model,
+          provider,
+        });
         return [
           !!m.content ? { text: m.content, type: 'text' } : undefined,
-          ...m.imageList.map(
-            (i) => ({ image_url: { detail: 'auto', url: i.url }, type: 'image_url' }) as const,
-          ),
+          ...imageContentParts,
         ].filter(Boolean) as UserMessageContentPart[];
       }
 
       return m.content;
     };
 
-    let postMessages = messages.map((m): OpenAIChatMessage => {
-      const supportTools = isCanUseFC(model, provider);
-      switch (m.role) {
-        case 'user': {
-          return { content: getUserContent(m), role: m.role };
-        }
-
-        case 'assistant': {
-          const content = getAssistantContent(m);
-
-          if (!supportTools) {
-            return { content, role: m.role };
+    let postMessages = await Promise.all(
+      messages.map(async (m): Promise<OpenAIChatMessage> => {
+        const supportTools = isCanUseFC(model, provider);
+        switch (m.role) {
+          case 'user': {
+            return { content: await getUserContent(m), role: m.role };
           }
 
-          return {
-            content,
-            role: m.role,
-            tool_calls: m.tools?.map(
-              (tool): MessageToolCall => ({
-                function: {
-                  arguments: tool.arguments,
-                  name: genToolCallingName(tool.identifier, tool.apiName, tool.type),
-                },
-                id: tool.id,
-                type: 'function',
-              }),
-            ),
-          };
-        }
-
-        case 'tool': {
-          if (!supportTools) {
-            return { content: m.content, role: 'user' };
+          case 'assistant': {
+            const content = await getAssistantContent(m);
+
+            if (!supportTools) {
+              return { content, role: m.role };
+            }
+
+            return {
+              content,
+              role: m.role,
+              tool_calls: m.tools?.map(
+                (tool): MessageToolCall => ({
+                  function: {
+                    arguments: tool.arguments,
+                    name: genToolCallingName(tool.identifier, tool.apiName, tool.type),
+                  },
+                  id: tool.id,
+                  type: 'function',
+                }),
+              ),
+            };
           }
 
-          return {
-            content: m.content,
-            name: genToolCallingName(m.plugin!.identifier, m.plugin!.apiName, m.plugin?.type),
-            role: m.role,
-            tool_call_id: m.tool_call_id,
-          };
-        }
+          case 'tool': {
+            if (!supportTools) {
+              return { content: m.content, role: 'user' };
+            }
+
+            return {
+              content: m.content,
+              name: genToolCallingName(m.plugin!.identifier, m.plugin!.apiName, m.plugin?.type),
+              role: m.role,
+              tool_call_id: m.tool_call_id,
+            };
+          }
 
-        default: {
-          return { content: m.content, role: m.role as any };
+          default: {
+            return { content: m.content, role: m.role as any };
+          }
         }
-      }
-    });
+      }),
+    );
 
     postMessages = produce(postMessages, (draft) => {
       // if it's a welcome question, inject InboxGuide SystemRole
@@ -657,6 +672,37 @@ class ChatService {
     return this.reorderToolMessages(postMessages);
   };
 
+  /**
+   * Process imageList: convert local URLs to base64 and format as UserMessageContentPart
+   */
+  private processImageList = async ({
+    model,
+    provider,
+    imageList,
+  }: {
+    imageList: ChatImageItem[];
+    model: string;
+    provider: string;
+  }) => {
+    if (!isCanUseVision(model, provider)) {
+      return [];
+    }
+
+    return Promise.all(
+      imageList.map(async (image) => {
+        const { type } = parseDataUri(image.url);
+
+        let processedUrl = image.url;
+        if (type === 'url' && isLocalUrl(image.url)) {
+          const { base64, mimeType } = await imageUrlToBase64(image.url);
+          processedUrl = `data:${mimeType};base64,${base64}`;
+        }
+
+        return { image_url: { detail: 'auto', url: processedUrl }, type: 'image_url' } as const;
+      }),
+    );
+  };
+
   private mapTrace = (trace?: TracePayload, tag?: TraceTagMap): TracePayload => {
     const tags = sessionMetaSelectors.currentAgentMeta(getSessionStoreState()).tags || [];
 
@@ -681,9 +727,6 @@ class ChatService {
     provider: string;
     signal?: AbortSignal;
   }) => {
-    const agentRuntime = await initializeWithClientStore(params.provider, params.payload);
-    const data = params.payload as ChatStreamPayload;
-
     /**
      * if enable login and not signed in, return unauthorized error
      */
@@ -692,6 +735,9 @@ class ChatService {
       throw AgentRuntimeError.createError(ChatErrorType.InvalidAccessCode);
     }
 
+    const agentRuntime = await initializeWithClientStore(params.provider, params.payload);
+    const data = params.payload as ChatStreamPayload;
+
     return agentRuntime.chat(data, { signal: params.signal });
   };
 

package/src/utils/url.test.ts

@@ -1,7 +1,7 @@
 import { vi } from 'vitest';
 
 import { pathString } from './url';
-import { inferContentTypeFromImageUrl, inferFileExtensionFromImageUrl } from './url';
+import { inferContentTypeFromImageUrl, inferFileExtensionFromImageUrl, isLocalUrl } from './url';
 
 describe('pathString', () => {
   it('should handle basic path', () => {
@@ -398,3 +398,44 @@ describe('inferFileExtensionFromImageUrl', () => {
     expect(result).toBe('gif');
   });
 });
+
+describe('isLocalUrl', () => {
+  it('should return true for URLs with 127.0.0.1 as hostname', () => {
+    expect(isLocalUrl('http://127.0.0.1')).toBe(true);
+    expect(isLocalUrl('https://127.0.0.1')).toBe(true);
+    expect(isLocalUrl('http://127.0.0.1:8080')).toBe(true);
+    expect(isLocalUrl('http://127.0.0.1/path/to/resource')).toBe(true);
+    expect(isLocalUrl('https://127.0.0.1/path?query=1#hash')).toBe(true);
+  });
+
+  it('should return false for URLs with "localhost" as hostname', () => {
+    expect(isLocalUrl('http://localhost')).toBe(false);
+    expect(isLocalUrl('http://localhost:3000')).toBe(false);
+  });
+
+  it('should return false for other IP addresses', () => {
+    expect(isLocalUrl('http://192.168.1.1')).toBe(false);
+    expect(isLocalUrl('http://0.0.0.0')).toBe(false);
+    expect(isLocalUrl('http://127.0.0.2')).toBe(false);
+  });
+
+  it('should return false for domain names', () => {
+    expect(isLocalUrl('https://example.com')).toBe(false);
+    expect(isLocalUrl('http://www.google.com')).toBe(false);
+  });
+
+  it('should return false for malformed URLs', () => {
+    expect(isLocalUrl('invalid-url')).toBe(false);
+    expect(isLocalUrl('http://')).toBe(false);
+    expect(isLocalUrl('a string but not a url')).toBe(false);
+  });
+
+  it('should return false for empty or nullish strings', () => {
+    expect(isLocalUrl('')).toBe(false);
+  });
+
+  it('should return false for relative URLs', () => {
+    expect(isLocalUrl('/path/to/file')).toBe(false);
+    expect(isLocalUrl('./relative/path')).toBe(false);
+  });
+});

package/src/utils/url.ts

@@ -123,3 +123,31 @@ export function inferContentTypeFromImageUrl(url: string) {
 
   return mimeType!; // Non-null assertion is safe due to whitelist validation
 }
+
+/**
+ * Check if a URL points to localhost (127.0.0.1)
+ *
+ * This function safely determines if the provided URL's hostname is '127.0.0.1'.
+ * It handles malformed URLs gracefully by returning false instead of throwing errors.
+ *
+ * @param url - The URL string to check
+ * @returns true if the URL's hostname is '127.0.0.1', false otherwise (including for malformed URLs)
+ *
+ * @example
+ * ```typescript
+ * isLocalUrl('http://127.0.0.1:8080/path') // true
+ * isLocalUrl('https://example.com') // false
+ * isLocalUrl('invalid-url') // false (instead of throwing)
+ * isLocalUrl('') // false (instead of throwing)
+ * ```
+ *
+ * check: apps/desktop/src/main/core/StaticFileServerManager.ts
+ */
+export function isLocalUrl(url: string) {
+  try {
+    return new URL(url).hostname === '127.0.0.1';
+  } catch {
+    // Return false for malformed URLs instead of throwing
+    return false;
+  }
+}