@lobehub/chat 0.114.9 → 0.115.0

package/src/services/file.ts

@@ -1,5 +1,6 @@
 import { FileModel } from '@/database/models/file';
 import { DB_File } from '@/database/schemas/files';
+import { PROXY_URL } from '@/services/_url';
 import { FilePreview } from '@/types/files';
 import compressImage from '@/utils/compressImage';
 
@@ -14,6 +15,7 @@ class FileService {
     if (this.isImage(file.fileType) && !isTestData) {
       return this.uploadImageFile(file);
     }
+
     // save to local storage
     // we may want to save to a remote server later
     return FileModel.create(file);
@@ -22,6 +24,7 @@ class FileService {
   async uploadImageFile(file: DB_File) {
     // 加载图片
     const url = file.url || URL.createObjectURL(new Blob([file.data]));
+
     const img = new Image();
     img.src = url;
     await (() =>
@@ -31,10 +34,7 @@ class FileService {
 
     // 压缩图片
     const fileType = 'image/webp';
-    const base64String = compressImage({
-      img,
-      type: fileType,
-    });
+    const base64String = compressImage({ img, type: fileType });
     const binaryString = atob(base64String.split('base64,')[1]);
     const uint8Array = Uint8Array.from(binaryString, (char) => char.charCodeAt(0));
     file.data = uint8Array.buffer;
@@ -42,6 +42,21 @@ class FileService {
     return FileModel.create(file);
   }
 
+  async uploadImageByUrl(url: string, file: Pick<DB_File, 'name' | 'metadata'>) {
+    const res = await fetch(PROXY_URL, { body: url, method: 'POST' });
+    const data = await res.arrayBuffer();
+    const fileType = res.headers.get('content-type') || 'image/webp';
+
+    return this.uploadFile({
+      data,
+      fileType,
+      metadata: file.metadata,
+      name: file.name,
+      saveMode: 'local',
+      size: data.byteLength,
+    });
+  }
+
   async removeFile(id: string) {
     return FileModel.delete(id);
   }
@@ -57,7 +72,7 @@ class FileService {
     }
 
     // arrayBuffer to url
-    const url = URL.createObjectURL(new Blob([item.data]));
+    const url = URL.createObjectURL(new Blob([item.data], { type: item.fileType }));
     const base64 = Buffer.from(item.data).toString('base64');
 
     return {

package/src/services/imageGeneration.ts

@@ -0,0 +1,26 @@
+import { createHeaderWithOpenAI } from '@/services/_header';
+import { OPENAI_URLS } from '@/services/_url';
+import { OpenAIImagePayload } from '@/types/openai/image';
+
+interface FetchOptions {
+  signal?: AbortSignal | undefined;
+}
+
+class ImageGenerationService {
+  async generateImage(params: Omit<OpenAIImagePayload, 'model' | 'n'>, options?: FetchOptions) {
+    const payload: OpenAIImagePayload = { ...params, model: 'dall-e-3', n: 1 };
+
+    const res = await fetch(OPENAI_URLS.images, {
+      body: JSON.stringify(payload),
+      headers: createHeaderWithOpenAI({ 'Content-Type': 'application/json' }),
+      method: 'POST',
+      signal: options?.signal,
+    });
+
+    const urls = await res.json();
+
+    return urls[0] as string;
+  }
+}
+
+export const imageGenerationService = new ImageGenerationService();

package/src/store/chat/actions/enhance.ts

@@ -1,13 +1,17 @@
 import { produce } from 'immer';
+import pMap from 'p-map';
 import { StateCreator } from 'zustand/vanilla';
 
 import { chainLangDetect } from '@/chains/langDetect';
 import { chainTranslate } from '@/chains/translate';
 import { supportLocales } from '@/locales/options';
 import { chatService } from '@/services/chat';
+import { fileService } from '@/services/file';
+import { imageGenerationService } from '@/services/imageGeneration';
 import { messageService } from '@/services/message';
 import { chatSelectors } from '@/store/chat/selectors';
 import { ChatStore } from '@/store/chat/store';
+import { DallEImageItem } from '@/types/tool/dalle';
 import { setNamespace } from '@/utils/storeDebug';
 
 const n = setNamespace('enhance');
@@ -17,13 +21,16 @@ const n = setNamespace('enhance');
  */
 export interface ChatEnhanceAction {
   clearTTS: (id: string) => Promise<void>;
-
   clearTranslate: (id: string) => Promise<void>;
+  generateImageFromPrompts: (items: DallEImageItem[], id: string) => Promise<void>;
+  text2image: (id: string, data: DallEImageItem[]) => Promise<void>;
+  toggleDallEImageLoading: (key: string, value: boolean) => void;
   translateMessage: (id: string, targetLang: string) => Promise<void>;
   ttsMessage: (
     id: string,
     state?: { contentMd5?: string; file?: string; voice?: string },
   ) => Promise<void>;
+  updateImageItem: (id: string, updater: (data: DallEImageItem[]) => void) => Promise<void>;
 }
 
 export const chatEnhance: StateCreator<
@@ -42,6 +49,52 @@ export const chatEnhance: StateCreator<
     await get().refreshMessages();
   },
 
+  generateImageFromPrompts: async (items, messageId) => {
+    const { toggleDallEImageLoading, updateImageItem } = get();
+    // eslint-disable-next-line unicorn/consistent-function-scoping
+    const getMessageById = (id: string) => chatSelectors.getMessageById(id)(get());
+
+    const message = getMessageById(messageId);
+    const parent = getMessageById(message!.parentId!);
+    const originPrompt = parent?.content;
+
+    await pMap(items, async (params, index) => {
+      toggleDallEImageLoading(messageId + params.prompt, true);
+      const url = await imageGenerationService.generateImage(params);
+
+      await updateImageItem(messageId, (draft) => {
+        draft[index].previewUrl = url;
+      });
+
+      toggleDallEImageLoading(messageId + params.prompt, false);
+
+      fileService
+        .uploadImageByUrl(url, {
+          metadata: { ...params, originPrompt: originPrompt },
+          name: `${originPrompt || params.prompt}_${index}.png`,
+        })
+        .then(({ id }) => {
+          updateImageItem(messageId, (draft) => {
+            draft[index].imageId = id;
+            draft[index].previewUrl = undefined;
+          });
+        });
+    });
+  },
+  text2image: async (id, data) => {
+    // const isAutoGen = settingsSelectors.isDalleAutoGenerating(useGlobalStore.getState());
+    // if (!isAutoGen) return;
+
+    await get().generateImageFromPrompts(data, id);
+  },
+  toggleDallEImageLoading: (key, value) => {
+    set(
+      { dalleImageLoading: { ...get().dalleImageLoading, [key]: value } },
+      false,
+      n('toggleDallEImageLoading'),
+    );
+  },
+
   translateMessage: async (id, targetLang) => {
     const { toggleChatLoading, dispatchMessage, refreshMessages } = get();
 
@@ -95,4 +148,15 @@ export const chatEnhance: StateCreator<
     await messageService.updateMessageTTS(id, state);
     await get().refreshMessages();
   },
+
+  updateImageItem: async (id, updater) => {
+    const message = chatSelectors.getMessageById(id)(get());
+    if (!message) return;
+
+    const data: DallEImageItem[] = JSON.parse(message.content);
+
+    const nextContent = produce(data, updater);
+    await messageService.updateMessageContent(id, JSON.stringify(nextContent));
+    await get().refreshMessages();
+  },
 });

package/src/store/chat/actions/plugin.test.ts

@@ -86,7 +86,7 @@ describe('ChatPluginAction', () => {
       const { result } = renderHook(() => useChatStore());
 
       await act(async () => {
-        await result.current.runPluginDefaultType(messageId, pluginPayload);
+        await result.current.invokeDefaultTypePlugin(messageId, pluginPayload);
       });
 
       expect(initialState.toggleChatLoading).toHaveBeenCalledWith(
@@ -121,7 +121,7 @@ describe('ChatPluginAction', () => {
       const { result } = renderHook(() => useChatStore());
 
       await act(async () => {
-        await result.current.runPluginDefaultType(messageId, pluginPayload);
+        await result.current.invokeDefaultTypePlugin(messageId, pluginPayload);
       });
 
       expect(initialState.toggleChatLoading).toHaveBeenCalledWith(
@@ -150,7 +150,7 @@ describe('ChatPluginAction', () => {
 
       const initialState = {
         refreshMessages: vi.fn(),
-        runPluginDefaultType: vi.fn(),
+        invokeDefaultTypePlugin: vi.fn(),
       };
       useChatStore.setState(initialState);
 
@@ -172,7 +172,7 @@ describe('ChatPluginAction', () => {
         role: 'function',
       });
       expect(initialState.refreshMessages).toHaveBeenCalled();
-      expect(initialState.runPluginDefaultType).toHaveBeenCalledWith(
+      expect(initialState.invokeDefaultTypePlugin).toHaveBeenCalledWith(
         messageId,
         messagePluginPayload,
       );
@@ -191,7 +191,7 @@ describe('ChatPluginAction', () => {
 
       useToolStore.setState({ installedPlugins: [plugin] });
 
-      useChatStore.setState({ refreshMessages: vi.fn(), runPluginDefaultType: vi.fn() });
+      useChatStore.setState({ refreshMessages: vi.fn(), invokeDefaultTypePlugin: vi.fn() });
 
       (chatSelectors.getMessageById as Mock).mockImplementation(() => () => ({
         id: messageId,
@@ -212,7 +212,7 @@ describe('ChatPluginAction', () => {
           plugin: expect.objectContaining({ apiName }),
         }),
       );
-      expect(result.current.runPluginDefaultType).toHaveBeenCalledWith(
+      expect(result.current.invokeDefaultTypePlugin).toHaveBeenCalledWith(
         messageId,
         expect.objectContaining({
           apiName: apiName,
@@ -231,7 +231,7 @@ describe('ChatPluginAction', () => {
 
       useChatStore.setState({
         refreshMessages: vi.fn(),
-        runPluginDefaultType: vi.fn(),
+        invokeDefaultTypePlugin: vi.fn(),
       });
 
       (chatSelectors.getMessageById as Mock).mockImplementation(() => () => ({
@@ -248,8 +248,8 @@ describe('ChatPluginAction', () => {
       // 验证 refreshMessages 是否被调用
       expect(result.current.refreshMessages).toHaveBeenCalled();
 
-      // 验证 runPluginDefaultType 是否没有被调用，因为类型是 standalone
-      expect(result.current.runPluginDefaultType).not.toHaveBeenCalled();
+      // 验证 invokeDefaultTypePlugin 是否没有被调用，因为类型是 standalone
+      expect(result.current.invokeDefaultTypePlugin).not.toHaveBeenCalled();
     });
   });
 

package/src/store/chat/actions/plugin.ts

@@ -17,7 +17,8 @@ const n = setNamespace('plugin');
 
 export interface ChatPluginAction {
   fillPluginMessageContent: (id: string, content: string) => Promise<void>;
-  runPluginDefaultType: (id: string, payload: any) => Promise<void>;
+  invokeBuiltinTool: (id: string, payload: ChatPluginPayload) => Promise<void>;
+  invokeDefaultTypePlugin: (id: string, payload: any) => Promise<void>;
   triggerFunctionCall: (id: string) => Promise<void>;
   updatePluginState: (id: string, key: string, value: any) => Promise<void>;
 }
@@ -37,7 +38,26 @@ export const chatPlugin: StateCreator<
     const chats = chatSelectors.currentChats(get());
     await coreProcessMessage(chats, id);
   },
-  runPluginDefaultType: async (id, payload) => {
+  invokeBuiltinTool: async (id, payload) => {
+    const { toggleChatLoading, refreshMessages } = get();
+    const params = JSON.parse(payload.arguments);
+    toggleChatLoading(true, id, n('invokeBuiltinTool') as string);
+    const data = await useToolStore.getState().invokeBuiltinTool(payload.apiName, params);
+    toggleChatLoading(false);
+
+    if (data) {
+      await messageService.updateMessageContent(id, data);
+      await refreshMessages();
+    }
+
+    // postToolCalling
+    // @ts-ignore
+    const { [payload.apiName]: action } = get();
+    if (!action || !data) return;
+
+    await action(id, JSON.parse(data));
+  },
+  invokeDefaultTypePlugin: async (id, payload) => {
     const { refreshMessages, coreProcessMessage, toggleChatLoading } = get();
     let data: string;
 
@@ -65,8 +85,9 @@ export const chatPlugin: StateCreator<
     const chats = chatSelectors.currentChats(get());
     await coreProcessMessage(chats, id);
   },
+
   triggerFunctionCall: async (id) => {
-    const { runPluginDefaultType, refreshMessages } = get();
+    const { invokeDefaultTypePlugin, invokeBuiltinTool, refreshMessages } = get();
 
     const message = chatSelectors.getMessageById(id)(get());
     if (!message) return;
@@ -111,10 +132,21 @@ export const chatPlugin: StateCreator<
     });
     await refreshMessages();
 
-    if (payload.type === 'standalone') {
-      // TODO: need to auth user's settings
-    } else runPluginDefaultType(id, payload);
+    switch (payload.type) {
+      case 'standalone': {
+        // TODO: need to auth user's settings
+        break;
+      }
+      case 'builtin': {
+        await invokeBuiltinTool(id, payload);
+        break;
+      }
+      default: {
+        await invokeDefaultTypePlugin(id, payload);
+      }
+    }
   },
+
   updatePluginState: async (id, key, value) => {
     const { refreshMessages } = get();
 

package/src/store/chat/initialState.ts

@@ -13,6 +13,8 @@ export interface ChatStoreState {
    * @deprecated
    */
   chatLoadingId?: string;
+  dalleImageLoading: Record<string, boolean>;
+
   inputMessage: string;
   isSearchingTopic: boolean;
   messageLoadingIds: [];
@@ -36,6 +38,7 @@ export interface ChatStoreState {
 
 export const initialState: ChatStoreState = {
   activeId: 'inbox',
+  dalleImageLoading: {},
   inputMessage: '',
   isSearchingTopic: false,
   messageLoadingIds: [],

package/src/store/chat/selectors/enchance.ts

@@ -0,0 +1,11 @@
+import { ChatStoreState } from '@/store/chat';
+
+const isDallEImageGenerating = (id: string) => (s: ChatStoreState) => s.dalleImageLoading[id];
+
+const isGeneratingDallEImage = (s: ChatStoreState) =>
+  Object.values(s.dalleImageLoading).some(Boolean);
+
+export const chatEnhanceSelectors = {
+  isDallEImageGenerating,
+  isGeneratingDallEImage,
+};

package/src/store/chat/selectors/index.ts

@@ -1,2 +1,3 @@
 export { chatSelectors } from './chat';
+export { chatEnhanceSelectors } from './enchance';
 export { topicSelectors } from './topic';

package/src/store/global/selectors/__snapshots__/settings.test.ts.snap

@@ -147,6 +147,11 @@ exports[`settingsSelectors > currentSettings > should merge DEFAULT_SETTINGS and
   "password": "password123",
   "primaryColor": "blue",
   "themeMode": "light",
+  "tool": {
+    "dalle": {
+      "autoGenerate": false,
+    },
+  },
   "tts": {
     "openAI": {
       "sttModel": "whisper-1",

package/src/store/global/selectors/settings.ts

@@ -84,14 +84,19 @@ const currentLanguage = (s: GlobalStore) => {
   return locale;
 };
 
+const dalleConfig = (s: GlobalStore) => s.settings.tool?.dalle || {};
+const isDalleAutoGenerating = (s: GlobalStore) => s.settings.tool?.dalle?.autoGenerate;
+
 export const settingsSelectors = {
   currentLanguage,
   currentSettings,
   currentTTS,
+  dalleConfig,
   defaultAgent,
   defaultAgentConfig,
   defaultAgentMeta,
   exportSettings,
+  isDalleAutoGenerating,
   modelList: modelListSelectors,
   openAIAPI: openAIAPIKeySelectors,
   openAIProxyUrl: openAIProxyUrlSelectors,

package/src/store/tool/helpers.ts

@@ -1,14 +1,15 @@
-import { LobeChatPluginManifest, PluginSchema } from '@lobehub/chat-plugin-sdk';
+import { PluginSchema } from '@lobehub/chat-plugin-sdk';
 
+import { MetaData } from '@/types/meta';
 import { LobeTool } from '@/types/tool';
 
 const getPluginFormList = (list: LobeTool[], id: string) => list?.find((p) => p.identifier === id);
 
-const getPluginTitle = (meta?: LobeChatPluginManifest['meta']) => meta?.title;
-const getPluginDesc = (meta?: LobeChatPluginManifest['meta']) => meta?.description;
+const getPluginTitle = (meta?: MetaData) => meta?.title;
+const getPluginDesc = (meta?: MetaData) => meta?.description;
 
-const getPluginTags = (meta?: LobeChatPluginManifest['meta']) => meta?.tags;
-const getPluginAvatar = (meta?: LobeChatPluginManifest['meta']) => meta?.avatar || '🧩';
+const getPluginTags = (meta?: MetaData) => meta?.tags;
+const getPluginAvatar = (meta?: MetaData) => meta?.avatar || '🧩';
 
 const isCustomPlugin = (id: string, pluginList: LobeTool[]) =>
   pluginList.some((i) => i.identifier === id && i.type === 'customPlugin');

package/src/store/tool/initialState.ts

@@ -1,11 +1,13 @@
+import { BuiltinToolState, initialBuiltinToolState } from './slices/builtin';
 import { CustomPluginState, initialCustomPluginState } from './slices/customPlugin';
 import { PluginState, initialPluginState } from './slices/plugin';
 import { PluginStoreState, initialPluginStoreState } from './slices/store';
 
-export type ToolStoreState = PluginState & CustomPluginState & PluginStoreState;
+export type ToolStoreState = PluginState & CustomPluginState & PluginStoreState & BuiltinToolState;
 
 export const initialState: ToolStoreState = {
   ...initialPluginState,
   ...initialCustomPluginState,
   ...initialPluginStoreState,
+  ...initialBuiltinToolState,
 };

package/src/store/tool/selectors/index.ts

@@ -0,0 +1,5 @@
+export { builtinToolSelectors } from '../slices/builtin/selectors';
+export { customPluginSelectors } from '../slices/customPlugin/selectors';
+export { pluginSelectors } from '../slices/plugin/selectors';
+export { pluginStoreSelectors } from '../slices/store/selectors';
+export { toolSelectors } from './tool';

package/src/store/tool/selectors/tool.test.ts

@@ -0,0 +1,128 @@
+import { LobeChatPluginManifest, LobeChatPluginMeta } from '@lobehub/chat-plugin-sdk';
+import { describe, expect, it } from 'vitest';
+
+import { initialState } from '../initialState';
+import { ToolStoreState } from '../initialState';
+import { toolSelectors } from './tool';
+
+const mockState = {
+  ...initialState,
+  pluginInstallLoading: {
+    'plugin-1': false,
+    'plugin-2': true,
+  },
+  installedPlugins: [
+    {
+      identifier: 'plugin-1',
+      type: 'plugin',
+      manifest: {
+        identifier: 'plugin-1',
+        api: [{ name: 'api-1' }],
+        type: 'default',
+      } as LobeChatPluginManifest,
+      settings: { setting1: 'value1' },
+    },
+    {
+      identifier: 'plugin-2',
+      manifest: {
+        identifier: 'plugin-2',
+        api: [{ name: 'api-2' }],
+      },
+      type: 'plugin',
+    },
+    {
+      identifier: 'plugin-3',
+      manifest: {
+        identifier: 'plugin-3',
+        api: [{ name: 'api-3' }],
+      },
+      type: 'customPlugin',
+    },
+  ],
+  pluginStoreList: [
+    {
+      identifier: 'plugin-1',
+      author: 'Author 1',
+      createdAt: '2021-01-01',
+      meta: { avatar: 'avatar-url-1', title: 'Plugin 1' },
+      homepage: 'http://homepage-1.com',
+    } as LobeChatPluginMeta,
+    {
+      identifier: 'plugin-2',
+      author: 'Author 2',
+      createdAt: '2022-02-02',
+      meta: { avatar: 'avatar-url-2', title: 'Plugin 2' },
+      homepage: 'http://homepage-2.com',
+    },
+  ],
+} as ToolStoreState;
+
+describe('toolSelectors', () => {
+  describe('enabledSchema', () => {
+    it('enabledSchema should return correct ChatCompletionFunctions array', () => {
+      const result = toolSelectors.enabledSchema(['plugin-1', 'plugin-2'])(mockState);
+      expect(result).toEqual([{ name: 'plugin-1____api-1' }, { name: 'plugin-2____api-2' }]);
+    });
+
+    it('enabledSchema should return with standalone plugin', () => {
+      const result = toolSelectors.enabledSchema(['plugin-4'])({
+        ...mockState,
+        installedPlugins: [
+          ...mockState.installedPlugins,
+          {
+            identifier: 'plugin-4',
+            manifest: {
+              identifier: 'plugin-4',
+              api: [{ name: 'api-4' }],
+              type: 'standalone',
+            },
+            type: 'plugin',
+          },
+        ],
+      } as ToolStoreState);
+      expect(result).toEqual([{ name: 'plugin-4____api-4____standalone' }]);
+    });
+
+    it('enabledSchema should return md5 hash apiName', () => {
+      const result = toolSelectors.enabledSchema(['long-long-plugin-with-id'])({
+        ...mockState,
+        installedPlugins: [
+          ...mockState.installedPlugins,
+          {
+            identifier: 'long-long-plugin-with-id',
+            manifest: {
+              identifier: 'long-long-plugin-with-id',
+              api: [{ name: 'long-long-manifest-long-long-apiName' }],
+            },
+            type: 'plugin',
+          },
+        ],
+      } as ToolStoreState);
+      expect(result).toEqual([
+        { name: 'long-long-plugin-with-id____MD5HASH_396eae4c671da3fb642c49ad2b9e8790' },
+      ]);
+    });
+
+    it('enabledSchema should return empty', () => {
+      const result = toolSelectors.enabledSchema([])(mockState);
+      expect(result).toEqual([]);
+    });
+  });
+
+  describe('getPluginManifestLoadingStatus', () => {
+    it('should return "loading" if the plugin manifest is being loaded', () => {
+      const result = toolSelectors.getManifestLoadingStatus('plugin-2')(mockState);
+      expect(result).toBe('loading');
+    });
+
+    it('should return "error" if the plugin manifest is not found', () => {
+      const result = toolSelectors.getManifestLoadingStatus('non-existing-plugin')(mockState);
+      expect(result).toBe('error');
+    });
+
+    it('should return "success" if the plugin manifest is loaded', () => {
+      const result = toolSelectors.getManifestLoadingStatus('plugin-1')(mockState);
+      expect(result).toBe('success');
+    });
+  });
+});