@lobehub/chat 1.99.5 → 1.100.0

package/src/services/chat.ts

@@ -15,6 +15,7 @@ import {
   ChatCompletionErrorPayload,
   ModelProvider,
 } from '@/libs/model-runtime';
+import { parseDataUri } from '@/libs/model-runtime/utils/uriParser';
 import { filesPrompts } from '@/prompts/files';
 import { BuiltinSystemRolePrompts } from '@/prompts/systemRole';
 import { getAgentStoreState } from '@/store/agent';
@@ -35,10 +36,11 @@ import {
 import { WebBrowsingManifest } from '@/tools/web-browsing';
 import { WorkingModel } from '@/types/agent';
 import { ChatErrorType } from '@/types/fetch';
-import { ChatMessage, MessageToolCall } from '@/types/message';
+import { ChatImageItem, ChatMessage, MessageToolCall } from '@/types/message';
 import type { ChatStreamPayload, OpenAIChatMessage } from '@/types/openai/chat';
 import { UserMessageContentPart } from '@/types/openai/chat';
 import { parsePlaceholderVariablesMessages } from '@/utils/client/parserPlaceholder';
+import { fetchWithInvokeStream } from '@/utils/electron/desktopRemoteRPCFetch';
 import { createErrorResponse } from '@/utils/errorResponse';
 import {
   FetchSSEOptions,
@@ -46,8 +48,10 @@ import {
   getMessageError,
   standardizeAnimationStyle,
 } from '@/utils/fetch';
+import { imageUrlToBase64 } from '@/utils/imageToBase64';
 import { genToolCallingName } from '@/utils/toolCall';
 import { createTraceHeader, getTraceId } from '@/utils/trace';
+import { isLocalUrl } from '@/utils/url';
 
 import { createHeaderWithAuth, createPayloadWithKeyVaults } from './_auth';
 import { API_ENDPOINTS } from './_url';
@@ -61,6 +65,14 @@ const isCanUseFC = (model: string, provider: string) => {
   return aiModelSelectors.isModelSupportToolUse(model, provider)(getAiInfraStoreState());
 };
 
+const isCanUseVision = (model: string, provider: string) => {
+  // TODO: remove isDeprecatedEdition condition in V2.0
+  if (isDeprecatedEdition) {
+    return modelProviderSelectors.isModelEnabledVision(model)(getUserStoreState());
+  }
+  return aiModelSelectors.isModelSupportVision(model, provider)(getAiInfraStoreState());
+};
+
 /**
  * TODO: we need to update this function to auto find deploymentName with provider setting config
  */
@@ -205,7 +217,7 @@ class ChatService {
 
     // ============  2. preprocess messages   ============ //
 
-    const oaiMessages = this.processMessages(
+    const oaiMessages = await this.processMessages(
       {
         messages: parsedMessages,
         model: payload.model,
@@ -350,7 +362,10 @@ class ChatService {
 
     let fetcher: typeof fetch | undefined = undefined;
 
-    if (enableFetchOnClient) {
+    // Add desktop remote RPC fetch support
+    if (isDesktop) {
+      fetcher = fetchWithInvokeStream;
+    } else if (enableFetchOnClient) {
       /**
        * Notes:
        * 1. Browser agent runtime will skip auth check if a key and endpoint provided by
@@ -475,7 +490,7 @@ class ChatService {
     onLoadingChange?.(true);
 
     try {
-      const oaiMessages = this.processMessages({
+      const oaiMessages = await this.processMessages({
         messages: params.messages as any,
         model: params.model!,
         provider: params.provider!,
@@ -507,7 +522,7 @@ class ChatService {
     }
   };
 
-  private processMessages = (
+  private processMessages = async (
     {
       messages = [],
       tools,
@@ -520,29 +535,28 @@ class ChatService {
       tools?: string[];
     },
     options?: FetchOptions,
-  ): OpenAIChatMessage[] => {
+  ): Promise<OpenAIChatMessage[]> => {
     // handle content type for vision model
     // for the models with visual ability, add image url to content
     // refs: https://platform.openai.com/docs/guides/vision/quick-start
-    const getUserContent = (m: ChatMessage) => {
+    const getUserContent = async (m: ChatMessage) => {
       // only if message doesn't have images and files, then return the plain content
       if ((!m.imageList || m.imageList.length === 0) && (!m.fileList || m.fileList.length === 0))
         return m.content;
 
       const imageList = m.imageList || [];
+      const imageContentParts = await this.processImageList({ imageList, model, provider });
 
       const filesContext = isServerMode
         ? filesPrompts({ addUrl: !isDesktop, fileList: m.fileList, imageList })
         : '';
       return [
         { text: (m.content + '\n\n' + filesContext).trim(), type: 'text' },
-        ...imageList.map(
-          (i) => ({ image_url: { detail: 'auto', url: i.url }, type: 'image_url' }) as const,
-        ),
+        ...imageContentParts,
       ] as UserMessageContentPart[];
     };
 
-    const getAssistantContent = (m: ChatMessage) => {
+    const getAssistantContent = async (m: ChatMessage) => {
       // signature is a signal of anthropic thinking mode
       const shouldIncludeThinking = m.reasoning && !!m.reasoning?.signature;
 
@@ -559,65 +573,70 @@ class ChatService {
       // only if message doesn't have images and files, then return the plain content
 
       if (m.imageList && m.imageList.length > 0) {
+        const imageContentParts = await this.processImageList({
+          imageList: m.imageList,
+          model,
+          provider,
+        });
         return [
           !!m.content ? { text: m.content, type: 'text' } : undefined,
-          ...m.imageList.map(
-            (i) => ({ image_url: { detail: 'auto', url: i.url }, type: 'image_url' }) as const,
-          ),
+          ...imageContentParts,
         ].filter(Boolean) as UserMessageContentPart[];
       }
 
       return m.content;
     };
 
-    let postMessages = messages.map((m): OpenAIChatMessage => {
-      const supportTools = isCanUseFC(model, provider);
-      switch (m.role) {
-        case 'user': {
-          return { content: getUserContent(m), role: m.role };
-        }
-
-        case 'assistant': {
-          const content = getAssistantContent(m);
-
-          if (!supportTools) {
-            return { content, role: m.role };
+    let postMessages = await Promise.all(
+      messages.map(async (m): Promise<OpenAIChatMessage> => {
+        const supportTools = isCanUseFC(model, provider);
+        switch (m.role) {
+          case 'user': {
+            return { content: await getUserContent(m), role: m.role };
           }
 
-          return {
-            content,
-            role: m.role,
-            tool_calls: m.tools?.map(
-              (tool): MessageToolCall => ({
-                function: {
-                  arguments: tool.arguments,
-                  name: genToolCallingName(tool.identifier, tool.apiName, tool.type),
-                },
-                id: tool.id,
-                type: 'function',
-              }),
-            ),
-          };
-        }
-
-        case 'tool': {
-          if (!supportTools) {
-            return { content: m.content, role: 'user' };
+          case 'assistant': {
+            const content = await getAssistantContent(m);
+
+            if (!supportTools) {
+              return { content, role: m.role };
+            }
+
+            return {
+              content,
+              role: m.role,
+              tool_calls: m.tools?.map(
+                (tool): MessageToolCall => ({
+                  function: {
+                    arguments: tool.arguments,
+                    name: genToolCallingName(tool.identifier, tool.apiName, tool.type),
+                  },
+                  id: tool.id,
+                  type: 'function',
+                }),
+              ),
+            };
           }
 
-          return {
-            content: m.content,
-            name: genToolCallingName(m.plugin!.identifier, m.plugin!.apiName, m.plugin?.type),
-            role: m.role,
-            tool_call_id: m.tool_call_id,
-          };
-        }
+          case 'tool': {
+            if (!supportTools) {
+              return { content: m.content, role: 'user' };
+            }
+
+            return {
+              content: m.content,
+              name: genToolCallingName(m.plugin!.identifier, m.plugin!.apiName, m.plugin?.type),
+              role: m.role,
+              tool_call_id: m.tool_call_id,
+            };
+          }
 
-        default: {
-          return { content: m.content, role: m.role as any };
+          default: {
+            return { content: m.content, role: m.role as any };
+          }
         }
-      }
-    });
+      }),
+    );
 
     postMessages = produce(postMessages, (draft) => {
       // if it's a welcome question, inject InboxGuide SystemRole
@@ -657,6 +676,37 @@ class ChatService {
     return this.reorderToolMessages(postMessages);
   };
 
+  /**
+   * Process imageList: convert local URLs to base64 and format as UserMessageContentPart
+   */
+  private processImageList = async ({
+    model,
+    provider,
+    imageList,
+  }: {
+    imageList: ChatImageItem[];
+    model: string;
+    provider: string;
+  }) => {
+    if (!isCanUseVision(model, provider)) {
+      return [];
+    }
+
+    return Promise.all(
+      imageList.map(async (image) => {
+        const { type } = parseDataUri(image.url);
+
+        let processedUrl = image.url;
+        if (type === 'url' && isLocalUrl(image.url)) {
+          const { base64, mimeType } = await imageUrlToBase64(image.url);
+          processedUrl = `data:${mimeType};base64,${base64}`;
+        }
+
+        return { image_url: { detail: 'auto', url: processedUrl }, type: 'image_url' } as const;
+      }),
+    );
+  };
+
   private mapTrace = (trace?: TracePayload, tag?: TraceTagMap): TracePayload => {
     const tags = sessionMetaSelectors.currentAgentMeta(getSessionStoreState()).tags || [];
 
@@ -681,9 +731,6 @@ class ChatService {
     provider: string;
     signal?: AbortSignal;
   }) => {
-    const agentRuntime = await initializeWithClientStore(params.provider, params.payload);
-    const data = params.payload as ChatStreamPayload;
-
     /**
      * if enable login and not signed in, return unauthorized error
      */
@@ -692,6 +739,9 @@ class ChatService {
       throw AgentRuntimeError.createError(ChatErrorType.InvalidAccessCode);
     }
 
+    const agentRuntime = await initializeWithClientStore(params.provider, params.payload);
+    const data = params.payload as ChatStreamPayload;
+
     return agentRuntime.chat(data, { signal: params.signal });
   };
 

package/src/services/electron/remoteServer.ts

@@ -28,13 +28,6 @@ class RemoteServerService {
   requestAuthorization = async (config: DataSyncConfig) => {
     return dispatch('requestAuthorization', config);
   };
-
-  /**
-   * 刷新访问令牌
-   */
-  refreshAccessToken = async () => {
-    return dispatch('refreshAccessToken');
-  };
 }
 
 export const remoteServerService = new RemoteServerService();

package/src/{libs/trpc/client/helpers → utils/electron}/desktopRemoteRPCFetch.ts

@@ -1,4 +1,4 @@
-import { ProxyTRPCRequestParams, dispatch } from '@lobechat/electron-client-ipc';
+import { ProxyTRPCRequestParams, dispatch, streamInvoke } from '@lobechat/electron-client-ipc';
 import debug from 'debug';
 
 import { isDesktop } from '@/const/version';
@@ -6,7 +6,7 @@ import { getElectronStoreState } from '@/store/electron';
 import { electronSyncSelectors } from '@/store/electron/selectors';
 import { getRequestBody, headersToRecord } from '@/utils/fetch';
 
-const log = debug('lobe-lambda:desktopRemoteRPCFetch');
+const log = debug('utils:desktopRemoteRPCFetch');
 
 // eslint-disable-next-line no-undef
 export const desktopRemoteRPCFetch = async (input: string, init?: RequestInit) => {
@@ -15,8 +15,8 @@ export const desktopRemoteRPCFetch = async (input: string, init?: RequestInit) =
 
   if (isSyncActive) {
     log('Using IPC proxy for tRPC request');
+    const url = input as string;
     try {
-      const url = input as string;
       const parsedUrl = new URL(url, window.location.origin);
       const urlPath = parsedUrl.pathname + parsedUrl.search;
       const method = init?.method?.toUpperCase() || 'GET';
@@ -32,7 +32,7 @@ export const desktopRemoteRPCFetch = async (input: string, init?: RequestInit) =
 
       const ipcResult = await dispatch('proxyTRPCRequest', params);
 
-      log('Received IPC proxy response:', { status: ipcResult.status });
+      log(`Received ${url} IPC proxy response:`, { status: ipcResult.status });
       const response = new Response(ipcResult.body, {
         headers: ipcResult.headers,
         status: ipcResult.status,
@@ -41,7 +41,7 @@ export const desktopRemoteRPCFetch = async (input: string, init?: RequestInit) =
 
       if (!response.ok) {
         console.warn(
-          '[lambda] IPC proxy response indicates an error:',
+          `[lambda] ${url} IPC proxy response indicates an error:`,
           response.status,
           response.statusText,
         );
@@ -49,7 +49,7 @@ export const desktopRemoteRPCFetch = async (input: string, init?: RequestInit) =
 
       return response;
     } catch (error) {
-      console.error('[lambda] Error during IPC proxy call:', error);
+      console.error(`[lambda] Error during ${url} IPC proxy call:`, error);
       return new Response(
         `IPC Proxy Error: ${error instanceof Error ? error.message : 'Unknown error'}`,
         {
@@ -62,11 +62,26 @@ export const desktopRemoteRPCFetch = async (input: string, init?: RequestInit) =
 };
 
 // eslint-disable-next-line no-undef
-export const fetchWithDesktopRemoteRPC = async (input: string, init?: RequestInit) => {
+export const fetchWithDesktopRemoteRPC = (async (input: RequestInfo | URL, init?: RequestInit) => {
   if (isDesktop) {
     const res = await desktopRemoteRPCFetch(input as string, init);
     if (res) return res;
   }
 
+  return fetch(input, init);
+}) as typeof fetch;
+
+// eslint-disable-next-line no-undef
+export const fetchWithInvokeStream = async (input: RequestInfo | URL, init?: RequestInit) => {
+  if (isDesktop) {
+    const isSyncActive = electronSyncSelectors.isSyncActive(getElectronStoreState());
+    log('isSyncActive:', isSyncActive);
+    if (isSyncActive) {
+      log('Using IPC stream proxy for request to:', input);
+
+      return streamInvoke(input, init);
+    }
+  }
+
   return fetch(input, init);
 };

package/src/utils/server/auth.ts

@@ -49,3 +49,25 @@ export const extractBearerToken = (authHeader?: string | null): string | null =>
   // Return the token only if it's not an empty string after trimming
   return token || null;
 };
+
+/**
+ * 从 Oidc-Auth header 中提取 JWT token
+ * @param authHeader - Oidc-Auth header 值 (例如 "Oidc-Auth xxx")
+ * @returns JWT token 或 null（如果授权头无效或不存在）
+ */
+export const extractOidcAuthToken = (authHeader?: string | null): string | null => {
+  if (!authHeader) return null;
+
+  const trimmedHeader = authHeader.trim(); // Trim leading/trailing spaces
+
+  // Check if it starts with 'Oidc-Auth ' (case-insensitive check)
+  if (!trimmedHeader.toLowerCase().startsWith('oidc-auth ')) {
+    return null;
+  }
+
+  // Extract the token part after "Oidc-Auth " and trim potential spaces around the token itself
+  const token = trimmedHeader.slice(10).trim(); // 'Oidc-Auth ' length is 10
+
+  // Return the token only if it's not an empty string after trimming
+  return token || null;
+};

package/src/utils/url.test.ts

@@ -1,7 +1,7 @@
 import { vi } from 'vitest';
 
 import { pathString } from './url';
-import { inferContentTypeFromImageUrl, inferFileExtensionFromImageUrl } from './url';
+import { inferContentTypeFromImageUrl, inferFileExtensionFromImageUrl, isLocalUrl } from './url';
 
 describe('pathString', () => {
   it('should handle basic path', () => {
@@ -398,3 +398,44 @@ describe('inferFileExtensionFromImageUrl', () => {
     expect(result).toBe('gif');
   });
 });
+
+describe('isLocalUrl', () => {
+  it('should return true for URLs with 127.0.0.1 as hostname', () => {
+    expect(isLocalUrl('http://127.0.0.1')).toBe(true);
+    expect(isLocalUrl('https://127.0.0.1')).toBe(true);
+    expect(isLocalUrl('http://127.0.0.1:8080')).toBe(true);
+    expect(isLocalUrl('http://127.0.0.1/path/to/resource')).toBe(true);
+    expect(isLocalUrl('https://127.0.0.1/path?query=1#hash')).toBe(true);
+  });
+
+  it('should return false for URLs with "localhost" as hostname', () => {
+    expect(isLocalUrl('http://localhost')).toBe(false);
+    expect(isLocalUrl('http://localhost:3000')).toBe(false);
+  });
+
+  it('should return false for other IP addresses', () => {
+    expect(isLocalUrl('http://192.168.1.1')).toBe(false);
+    expect(isLocalUrl('http://0.0.0.0')).toBe(false);
+    expect(isLocalUrl('http://127.0.0.2')).toBe(false);
+  });
+
+  it('should return false for domain names', () => {
+    expect(isLocalUrl('https://example.com')).toBe(false);
+    expect(isLocalUrl('http://www.google.com')).toBe(false);
+  });
+
+  it('should return false for malformed URLs', () => {
+    expect(isLocalUrl('invalid-url')).toBe(false);
+    expect(isLocalUrl('http://')).toBe(false);
+    expect(isLocalUrl('a string but not a url')).toBe(false);
+  });
+
+  it('should return false for empty or nullish strings', () => {
+    expect(isLocalUrl('')).toBe(false);
+  });
+
+  it('should return false for relative URLs', () => {
+    expect(isLocalUrl('/path/to/file')).toBe(false);
+    expect(isLocalUrl('./relative/path')).toBe(false);
+  });
+});

package/src/utils/url.ts

@@ -123,3 +123,31 @@ export function inferContentTypeFromImageUrl(url: string) {
 
   return mimeType!; // Non-null assertion is safe due to whitelist validation
 }
+
+/**
+ * Check if a URL points to localhost (127.0.0.1)
+ *
+ * This function safely determines if the provided URL's hostname is '127.0.0.1'.
+ * It handles malformed URLs gracefully by returning false instead of throwing errors.
+ *
+ * @param url - The URL string to check
+ * @returns true if the URL's hostname is '127.0.0.1', false otherwise (including for malformed URLs)
+ *
+ * @example
+ * ```typescript
+ * isLocalUrl('http://127.0.0.1:8080/path') // true
+ * isLocalUrl('https://example.com') // false
+ * isLocalUrl('invalid-url') // false (instead of throwing)
+ * isLocalUrl('') // false (instead of throwing)
+ * ```
+ *
+ * check: apps/desktop/src/main/core/StaticFileServerManager.ts
+ */
+export function isLocalUrl(url: string) {
+  try {
+    return new URL(url).hostname === '127.0.0.1';
+  } catch {
+    // Return false for malformed URLs instead of throwing
+    return false;
+  }
+}

package/src/app/[variants]/oauth/consent/[uid]/failed/page.tsx

@@ -1,36 +0,0 @@
-'use client';
-
-import { Button, Icon } from '@lobehub/ui';
-import { Card, Result } from 'antd';
-import { XCircle } from 'lucide-react';
-import Link from 'next/link';
-import React, { memo } from 'react';
-import { useTranslation } from 'react-i18next';
-import { Center } from 'react-layout-kit';
-
-const FailedPage = memo(() => {
-  const { t } = useTranslation('oauth');
-
-  return (
-    <Center height="100vh">
-      <Card style={{ maxWidth: 500, width: '100%' }}>
-        <Result
-          extra={
-            <Link href="/">
-              <Button type="primary">{t('failed.backToHome')}</Button>
-            </Link>
-          }
-          icon={<Icon icon={XCircle} />}
-          status="error"
-          style={{ padding: 0 }}
-          subTitle={t('failed.subTitle')}
-          title={t('failed.title')}
-        />
-      </Card>
-    </Center>
-  );
-});
-
-FailedPage.displayName = 'FailedPage';
-
-export default FailedPage;

package/src/app/[variants]/oauth/handoff/Client.tsx

@@ -1,98 +0,0 @@
-'use client';
-
-import { Text } from '@lobehub/ui';
-import { Spin } from 'antd';
-import { createStyles } from 'antd-style';
-import { useSearchParams } from 'next/navigation';
-import React, { useEffect, useState } from 'react';
-import { useTranslation } from 'react-i18next';
-import { Center, Flexbox } from 'react-layout-kit';
-
-const useStyles = createStyles(({ css, token }) => ({
-  container: css`
-    width: 100%;
-    min-height: 100vh;
-    padding-block: 40px;
-    padding-inline: 24px;
-
-    color: ${token.colorTextBase};
-
-    background-color: ${token.colorBgLayout};
-  `,
-  content: css`
-    max-width: 600px;
-    text-align: center;
-  `,
-  message: css`
-    margin-block-end: ${token.marginXL}px;
-    color: ${token.colorTextSecondary};
-  `,
-  title: css`
-    margin-block-end: ${token.marginLG}px;
-    font-size: 24px;
-  `,
-}));
-
-interface Status {
-  desc: string;
-  status: 'processing' | 'success';
-  title: string;
-}
-const AuthHandoffPage = () => {
-  const { styles } = useStyles();
-  const { t } = useTranslation('oauth'); // Assuming 'oauth' namespace exists
-  const searchParams = useSearchParams();
-
-  const [status, setStatus] = useState<Status>({
-    desc: t('handoff.desc.processing'),
-    status: 'processing',
-    title: t('handoff.title.processing'),
-  });
-
-  const [isError, setIsError] = useState<boolean>(false);
-
-  useEffect(() => {
-    const targetUrl = searchParams.get('target');
-
-    if (targetUrl) {
-      try {
-        const decodedTargetUrl = decodeURIComponent(targetUrl);
-        console.log(`Attempting redirect to: ${decodedTargetUrl}`);
-
-        window.location.href = decodedTargetUrl;
-
-        const url = new URL(decodedTargetUrl);
-        if (!url.pathname.startsWith('/oidc/auth')) {
-          setStatus({
-            desc: t('handoff.desc.success'),
-            status: 'success',
-            title: t('handoff.title.success'),
-          });
-        }
-      } catch (error) {
-        console.error('Error decoding or redirecting:', error);
-        // setMessage(
-        //   t('handoff.error', '无法自动打开桌面应用。请检查链接是否有效或尝试手动打开应用。'),
-        // );
-        setIsError(true);
-      }
-    } else {
-      console.error('Missing target URL for handoff.');
-      setIsError(true);
-    }
-  }, [searchParams]);
-
-  return (
-    <Center className={styles.container} gap={12}>
-      {!isError && <Spin size="large" />}
-      <Flexbox align="center" className={styles.content} gap={16}>
-        <Text as={'h3'} className={styles.title}>
-          {status.title}
-        </Text>
-        <Text className={styles.message}>{status.desc}</Text>
-      </Flexbox>
-    </Center>
-  );
-};
-
-export default AuthHandoffPage;

package/src/app/[variants]/oauth/handoff/page.tsx

@@ -1,13 +0,0 @@
-'use client';
-
-import React, { Suspense } from 'react';
-
-import Client from './Client';
-
-const AuthHandoffPage = () => (
-  <Suspense>
-    <Client />
-  </Suspense>
-);
-
-export default AuthHandoffPage;