@lobehub/chat 0.100.5 → 0.101.0

package/.eslintrc.js

@@ -13,5 +13,6 @@ config.rules['no-extra-boolean-cast'] = 0;
 config.rules['unicorn/no-useless-undefined'] = 0;
 config.rules['react/no-unknown-property'] = 0;
 config.rules['unicorn/prefer-ternary'] = 0;
+config.rules['unicorn/prefer-spread'] = 0;
 
 module.exports = config;

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 # Changelog
 
+## [Version 0.101.0](https://github.com/lobehub/lobe-chat/compare/v0.100.5...v0.101.0)
+
+<sup>Released on **2023-11-14**</sup>
+
+#### ✨ Features
+
+- **misc**: Support upload images to chat with gpt4-vision model.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's improved
+
+- **misc**: Support upload images to chat with gpt4-vision model, closes [#440](https://github.com/lobehub/lobe-chat/issues/440) ([858d047](https://github.com/lobehub/lobe-chat/commit/858d047))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 0.100.5](https://github.com/lobehub/lobe-chat/compare/v0.100.4...v0.100.5)
 
 <sup>Released on **2023-11-11**</sup>

package/README.md

@@ -126,12 +126,12 @@ In our agent market. We have accumulated a large number of practical, prompt age
 
 | Recent Submits                                                                                                                                                                                       | Description                                                                                                                                                                                                                           |
 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [Academic Writing Enhancement Bot](https://chat-preview.lobehub.com/market?agent=academic-writing-eb)<br/><sup>By **[Ruler27](https://github.com/Ruler27)** on **2023-11-11**</sup>                  | Academic English spelling and rhetoric refinement.<br/>`proofreading` `rhetoric` `academic` `research` `english` `editing`                                                                                                            |
 | [LobeChat Technical Documentation Expert](https://chat-preview.lobehub.com/market?agent=lobe-chat-develop-doc-writter)<br/><sup>By **[arvinxx](https://github.com/arvinxx)** on **2023-11-05**</sup> | LobeChat is an AI conversation application built with the Next.js framework. I will help you write the development documentation for LobeChat.<br/>`development-documentation` `technical-introduction` `next-js` `react` `lobe-chat` |
 | [Sketch Feature Summary Expert](https://chat-preview.lobehub.com/market?agent=sketch-changelog-highlighter)<br/><sup>By **[arvinxx](https://github.com/arvinxx)** on **2023-11-02**</sup>            | Skilled at extracting key changes from Sketch release notes<br/>`experience-design` `sketch` `update` `feature` `text-summary`                                                                                                        |
 | [Master of Debating](https://chat-preview.lobehub.com/market?agent=tqg-20231026)<br/><sup>By **[cake79](https://github.com/cake79)** on **2023-10-26**</sup>                                         | Simulates people who enjoy debating and can express counterarguments to any user input<br/>`writing` `dialogue`                                                                                                                       |
-| [Graph Generator](https://chat-preview.lobehub.com/market?agent=graph-generator)<br/><sup>By **[choldrim](https://github.com/choldrim)** on **2023-10-23**</sup>                                     | Automatic Graph Generator<br/>`graph`                                                                                                                                                                                                 |
 
-> 📊 Total agents: [<kbd>**38**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
+> 📊 Total agents: [<kbd>**39**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
 
  <!-- AGENT LIST -->
 

package/README.zh-CN.md

@@ -124,12 +124,12 @@ LobeChat 是一个开源的、可扩展的（[Function Calling][fc-link]）高
 
 | 最近新增                                                                                                                                                                           | 助手说明                                                                                                                                       |
 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
+| [学术写作增强机器人](https://chat-preview.lobehub.com/market?agent=academic-writing-eb)<br/><sup>By **[Ruler27](https://github.com/Ruler27)** on **2023-11-11**</sup>              | 学术英语拼写和修辞改进。<br/>`校对` `修辞` `学术` `研究` `英文` `润色`                                                                         |
 | [LobeChat 技术文档专家](https://chat-preview.lobehub.com/market?agent=lobe-chat-develop-doc-writter)<br/><sup>By **[arvinxx](https://github.com/arvinxx)** on **2023-11-05**</sup> | LobeChat 是一个 Next.js 框架构建的 AI 会话应用，我将会帮你一起书写 LobeChat 的开发文档<br/>`开发文档` `技术介绍` `next-js` `react` `lobe-chat` |
 | [Sketch 特性总结专家](https://chat-preview.lobehub.com/market?agent=sketch-changelog-highlighter)<br/><sup>By **[arvinxx](https://github.com/arvinxx)** on **2023-11-02**</sup>    | 擅长从 Sketch 的更新日志中提取关键变更要点<br/>`体验设计` `sketch` `更新` `特性` `文本总结`                                                    |
 | [抬杠高手](https://chat-preview.lobehub.com/market?agent=tqg-20231026)<br/><sup>By **[cake79](https://github.com/cake79)** on **2023-10-26**</sup>                                 | 模拟那些喜欢抬杠的人，能对用户输入的任何观点进行抬杠表达的角色<br/>`写作` `对话`                                                               |
-| [图表生成器](https://chat-preview.lobehub.com/market?agent=graph-generator)<br/><sup>By **[choldrim](https://github.com/choldrim)** on **2023-10-23**</sup>                        | 自动图表生成器<br/>`图表`                                                                                                                      |
 
-> 📊 Total agents: [<kbd>**38**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
+> 📊 Total agents: [<kbd>**39**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
 
  <!-- AGENT LIST -->
 

package/__mocks__/zustand/traditional.ts

@@ -0,0 +1,25 @@
+import { act } from 'react-dom/test-utils';
+import { beforeEach } from 'vitest';
+import { createWithEqualityFn as actualCreate } from 'zustand/traditional';
+
+// a variable to hold reset functions for all stores declared in the app
+const storeResetFns = new Set<() => void>();
+
+// when creating a store, we get its initial state, create a reset function and add it in the set
+const createImpl = (createState: any) => {
+  const store = actualCreate(createState, Object.is);
+  const initialState = store.getState();
+  storeResetFns.add(() => store.setState(initialState, true));
+  return store;
+};
+
+// Reset all stores after each test run
+beforeEach(() => {
+  act(() =>
+    { for (const resetFn of storeResetFns) {
+      resetFn();
+    } },
+  );
+});
+
+export const createWithEqualityFn = (f: any) => (f === undefined ? createImpl : createImpl(f));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "0.100.5",
+  "version": "0.101.0",
   "description": "An open-source, extensible (Function Calling), high-performance chatbot framework. It supports one-click free deployment of your private ChatGPT/LLM web application",
   "keywords": [
     "chatbot",
@@ -73,13 +73,14 @@
     "@lobehub/ui": "latest",
     "@vercel/analytics": "^1",
     "ahooks": "^3",
-    "ai": "2.2.20",
+    "ai": "^2.2.22",
     "antd": "^5",
     "antd-style": "^3",
     "brotli-wasm": "^1",
     "chroma-js": "^2",
     "copy-to-clipboard": "^3",
     "dayjs": "^1",
+    "dexie": "^3",
     "fast-deep-equal": "^3",
     "gpt-tokenizer": "^2",
     "i18next": "^23",
@@ -93,7 +94,7 @@
     "modern-screenshot": "^4",
     "nanoid": "^5",
     "next": "14.0.1",
-    "openai": "~4.15.0",
+    "openai": "^4.17.3",
     "polished": "^4",
     "posthog-js": "^1",
     "query-string": "^8",
@@ -104,6 +105,7 @@
     "react-intersection-observer": "^9",
     "react-layout-kit": "^1",
     "react-lazy-load": "^4",
+    "react-spring-lightbox": "^1",
     "remark": "^14",
     "remark-gfm": "^3",
     "remark-html": "^15",
@@ -116,6 +118,7 @@
     "use-merge-value": "^1",
     "utility-types": "^3",
     "uuid": "^9",
+    "zod": "^3",
     "zustand": "^4.4",
     "zustand-utils": "^1"
   },
@@ -144,6 +147,7 @@
     "consola": "^3",
     "dpdm": "^3",
     "eslint": "^8",
+    "fake-indexeddb": "^5",
     "husky": "^8",
     "jsdom": "^22",
     "lint-staged": "^15",

package/src/app/api/files/image/imgur.ts

@@ -0,0 +1,72 @@
+import { getServerConfig } from '@/config/server';
+
+interface UploadResponse {
+  data: UploadData;
+  status: number;
+  success: boolean;
+}
+
+interface UploadData {
+  account_id: any;
+  account_url: any;
+  ad_type: any;
+  ad_url: any;
+  animated: boolean;
+  bandwidth: number;
+  datetime: number;
+  deletehash: string;
+  description: any;
+  favorite: boolean;
+  has_sound: boolean;
+  height: number;
+  hls: string;
+  id: string;
+  in_gallery: boolean;
+  in_most_viral: boolean;
+  is_ad: boolean;
+  link: string;
+  mp4: string;
+  name: string;
+  nsfw: any;
+  section: any;
+  size: number;
+  tags: any[];
+  title: any;
+  type: string;
+  views: number;
+  vote: any;
+  width: number;
+}
+
+export class Imgur {
+  clientId: string;
+  api = 'https://api.imgur.com/3';
+
+  constructor() {
+    this.clientId = getServerConfig().IMGUR_CLIENT_ID;
+  }
+
+  async upload(image: Blob) {
+    const formData = new FormData();
+
+    formData.append('image', image, 'image.png');
+
+    const res = await fetch(`${this.api}/upload`, {
+      body: formData,
+      headers: {
+        Authorization: `Client-ID ${this.clientId}`,
+      },
+      method: 'POST',
+    });
+
+    if (!res.ok) {
+      console.log(await res.text());
+    }
+
+    const data: UploadResponse = await res.json();
+    if (data.success) {
+      return data.data.link;
+    }
+    return undefined;
+  }
+}

package/src/app/api/files/image/route.ts

@@ -0,0 +1,42 @@
+import { Imgur } from './imgur';
+
+const updateByImgur = async ({ url, blob }: { blob?: Blob; url?: string }) => {
+  let imageBlob: Blob;
+
+  if (url) {
+    const res = await fetch(url);
+    imageBlob = await res.blob();
+  } else if (blob) {
+    imageBlob = blob;
+  } else {
+    // TODO: error handle
+    return;
+  }
+
+  const imgur = new Imgur();
+
+  return await imgur.upload(imageBlob);
+};
+
+export const POST = async (req: Request) => {
+  const { url } = await req.json();
+
+  const cdnUrl = await updateByImgur({ url });
+
+  return new Response(JSON.stringify({ url: cdnUrl }));
+};
+// import { Imgur } from './imgur';
+
+export const runtime = 'edge';
+
+// export const POST = async (req: Request) => {
+//   const { url } = await req.json();
+//
+//   const imgur = new Imgur();
+//
+//   const image = await fetch(url);
+//
+//   const cdnUrl = await imgur.upload(await image.blob());
+//
+//   return new Response(JSON.stringify({ url: cdnUrl }));
+// };

package/src/app/api/openai/chat/createChatCompletion.ts

@@ -14,11 +14,12 @@ export const createChatCompletion = async ({ payload, openai }: CreateChatComple
   // ============  1. preprocess messages   ============ //
   const { messages, ...params } = payload;
 
+  // remove unnecessary fields like `plugins` or `files` by lobe-chat
   const formatMessages = messages.map((m) => ({
     content: m.content,
     name: m.name,
     role: m.role,
-  }));
+  })) as OpenAI.ChatCompletionMessageParam[];
 
   // ============  2. send api   ============ //
 

package/src/app/chat/(desktop)/features/ChatInput/DragUpload.tsx

@@ -0,0 +1,166 @@
+import { Icon } from '@lobehub/ui';
+import { createStyles } from 'antd-style';
+import { FileImage, FileText, FileUpIcon } from 'lucide-react';
+import { rgba } from 'polished';
+import { memo, useEffect, useRef, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Center, Flexbox } from 'react-layout-kit';
+
+import { useFileStore } from '@/store/files';
+
+const useStyles = createStyles(({ css, token, stylish }) => {
+  return {
+    container: css`
+      width: 300px;
+      height: 300px;
+      padding: 16px;
+
+      color: ${token.colorWhite};
+
+      background: ${token.geekblue};
+      border-radius: 16px;
+      box-shadow:
+        ${rgba(token.geekblue, 0.1)} 0 1px 1px 0 inset,
+        ${rgba(token.geekblue, 0.1)} 0 50px 100px -20px,
+        ${rgba(token.geekblue, 0.3)} 0 30px 60px -30px;
+    `,
+    content: css`
+      width: 100%;
+      height: 100%;
+      padding: 16px;
+
+      border: 2px dashed ${token.colorWhite};
+      border-radius: 12px;
+    `,
+    desc: css`
+      color: ${rgba(token.colorTextLightSolid, 0.6)};
+    `,
+    title: css`
+      font-size: 24px;
+      font-weight: bold;
+    `,
+    wrapper: css`
+      position: fixed;
+      z-index: 10000000;
+      top: 0;
+      left: 0;
+
+      width: 100%;
+      height: 100%;
+
+      transition: all 0.3s ease-in-out;
+
+      background: ${token.colorBgMask};
+      ${stylish.blur};
+    `,
+  };
+});
+
+const handleDragOver = (e: DragEvent) => {
+  e.preventDefault();
+};
+
+const DragUpload = memo(() => {
+  const { styles } = useStyles();
+  const { t } = useTranslation('chat');
+  const [isDragging, setIsDragging] = useState(false);
+  // When a file is dragged to a different area, the 'dragleave' event may be triggered,
+  // causing isDragging to be mistakenly set to false.
+  // to fix this issue, use a counter to ensure the status change only when drag event left the browser window .
+  const dragCounter = useRef(0);
+
+  const uploadFile = useFileStore((s) => s.uploadFile);
+
+  const uploadImages = async (fileList: FileList | undefined) => {
+    if (!fileList || fileList.length === 0) return;
+
+    const pools = Array.from(fileList).map(async (file) => {
+      // skip none-file items
+      if (!file.type.startsWith('image')) return;
+      await uploadFile(file);
+    });
+
+    await Promise.all(pools);
+  };
+
+  const handleDragEnter = (e: DragEvent) => {
+    e.preventDefault();
+
+    dragCounter.current += 1;
+    if (e.dataTransfer?.items && e.dataTransfer.items.length > 0) {
+      setIsDragging(true);
+    }
+  };
+
+  const handleDragLeave = (e: DragEvent) => {
+    e.preventDefault();
+
+    // reset counter
+    dragCounter.current -= 1;
+
+    if (dragCounter.current === 0) {
+      setIsDragging(false);
+    }
+  };
+
+  const handleDrop = async (e: DragEvent) => {
+    e.preventDefault();
+    // reset counter
+    dragCounter.current = 0;
+
+    setIsDragging(false);
+
+    // get filesList
+    // TODO: support folder files upload
+    const files = e.dataTransfer?.files;
+
+    // upload files
+    uploadImages(files);
+  };
+
+  const handlePaste = (event: ClipboardEvent) => {
+    // get files from clipboard
+
+    const files = event.clipboardData?.files;
+
+    uploadImages(files);
+  };
+
+  useEffect(() => {
+    window.addEventListener('dragenter', handleDragEnter);
+    window.addEventListener('dragover', handleDragOver);
+    window.addEventListener('dragleave', handleDragLeave);
+    window.addEventListener('drop', handleDrop);
+    window.addEventListener('paste', handlePaste);
+
+    return () => {
+      window.removeEventListener('dragenter', handleDragEnter);
+      window.removeEventListener('dragover', handleDragOver);
+      window.removeEventListener('dragleave', handleDragLeave);
+      window.removeEventListener('drop', handleDrop);
+      window.removeEventListener('paste', handlePaste);
+    };
+  }, []);
+
+  return (
+    isDragging && (
+      <Center className={styles.wrapper}>
+        <div className={styles.container}>
+          <Center className={styles.content} gap={40}>
+            <Flexbox horizontal>
+              <Icon icon={FileImage} size={{ fontSize: 64, strokeWidth: 1 }} />
+              <Icon icon={FileUpIcon} size={{ fontSize: 64, strokeWidth: 1 }} />
+              <Icon icon={FileText} size={{ fontSize: 64, strokeWidth: 1 }} />
+            </Flexbox>
+            <Flexbox align={'center'} gap={8} style={{ textAlign: 'center' }}>
+              <Flexbox className={styles.title}>{t('upload.dragTitle')}</Flexbox>
+              <Flexbox className={styles.desc}>{t('upload.dragDesc')}</Flexbox>
+            </Flexbox>
+          </Center>
+        </div>
+      </Center>
+    )
+  );
+});
+
+export default DragUpload;

package/src/app/chat/(desktop)/features/ChatInput/Footer/LocalFiles.tsx

@@ -0,0 +1,10 @@
+import { memo } from 'react';
+
+import FileList from '@/app/chat/components/FileList';
+import { useFileStore } from '@/store/files';
+
+export const LocalFiles = memo(() => {
+  const inputFilesList = useFileStore((s) => s.inputFilesList);
+
+  return <FileList items={inputFilesList} />;
+});

package/src/app/chat/(desktop)/features/ChatInput/Footer/index.tsx

@@ -0,0 +1,64 @@
+import { Icon } from '@lobehub/ui';
+import { Button } from 'antd';
+import { useTheme } from 'antd-style';
+import { ArrowBigUp, CornerDownLeft, Loader2 } from 'lucide-react';
+import { memo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Flexbox } from 'react-layout-kit';
+
+import SaveTopic from '@/app/chat/features/ChatInput/Topic';
+import { useSendMessage } from '@/app/chat/features/ChatInput/useSend';
+import { useSessionStore } from '@/store/session';
+import { agentSelectors } from '@/store/session/selectors';
+
+import { LocalFiles } from './LocalFiles';
+
+const Footer = memo(() => {
+  const { t } = useTranslation('chat');
+  const theme = useTheme();
+  const [loading, onStop] = useSessionStore((s) => [!!s.chatLoadingId, s.stopGenerateMessage]);
+
+  const onSend = useSendMessage();
+  const canUpload = useSessionStore(agentSelectors.modelHasVisionAbility);
+
+  return (
+    <Flexbox
+      align={'end'}
+      distribution={'space-between'}
+      flex={'none'}
+      gap={8}
+      horizontal
+      padding={'0 24px'}
+    >
+      <Flexbox>{canUpload && <LocalFiles />}</Flexbox>
+      <Flexbox align={'center'} gap={8} horizontal>
+        <Flexbox
+          gap={4}
+          horizontal
+          style={{ color: theme.colorTextDescription, fontSize: 12, marginRight: 12 }}
+        >
+          <Icon icon={CornerDownLeft} />
+          <span>{t('send')}</span>
+          <span>/</span>
+          <Flexbox horizontal>
+            <Icon icon={ArrowBigUp} />
+            <Icon icon={CornerDownLeft} />
+          </Flexbox>
+          <span>{t('warp')}</span>
+        </Flexbox>
+        <SaveTopic />
+        {loading ? (
+          <Button icon={loading && <Icon icon={Loader2} spin />} onClick={onStop}>
+            {t('stop')}
+          </Button>
+        ) : (
+          <Button onClick={() => onSend()} type={'primary'}>
+            {t('send')}
+          </Button>
+        )}
+      </Flexbox>
+    </Flexbox>
+  );
+});
+
+export default Footer;

package/src/app/chat/(desktop)/features/ChatInput/InputArea.tsx

@@ -5,7 +5,7 @@ import { useTranslation } from 'react-i18next';
 
 import { useSessionStore } from '@/store/session';
 
-import { useSendMessage } from './useSend';
+import { useSendMessage } from '../../../features/ChatInput/useSend';
 
 const useStyles = createStyles(({ css }) => {
   return {
@@ -23,10 +23,10 @@ const useStyles = createStyles(({ css }) => {
 
 const InputArea = memo(() => {
   const { t } = useTranslation('common');
-  const isChineseInput = useRef(false);
-
   const { cx, styles } = useStyles();
 
+  const isChineseInput = useRef(false);
+
   const [loading, message, updateInputMessage] = useSessionStore((s) => [
     !!s.chatLoadingId,
     s.inputMessage,

package/src/app/chat/(desktop)/features/ChatInput/index.tsx

@@ -3,11 +3,14 @@ import { createStyles } from 'antd-style';
 import { Maximize2, Minimize2 } from 'lucide-react';
 import { memo, useState } from 'react';
 
-import Footer from '@/app/chat/(desktop)/features/ChatInput/Footer';
 import ActionBar from '@/app/chat/features/ChatInput/ActionBar';
 import { CHAT_TEXTAREA_HEIGHT, HEADER_HEIGHT } from '@/const/layoutTokens';
 import { useGlobalStore } from '@/store/global';
+import { useSessionStore } from '@/store/session';
+import { agentSelectors } from '@/store/session/selectors';
 
+import DragUpload from './DragUpload';
+import Footer from './Footer';
 import InputArea from './InputArea';
 
 const useStyles = createStyles(({ css }) => {
@@ -34,37 +37,42 @@ const ChatInputDesktopLayout = memo(() => {
     s.updatePreference,
   ]);
 
+  const canUpload = useSessionStore(agentSelectors.modelHasVisionAbility);
   return (
-    <DraggablePanel
-      fullscreen={expand}
-      headerHeight={HEADER_HEIGHT}
-      minHeight={CHAT_TEXTAREA_HEIGHT}
-      onSizeChange={(_, size) => {
-        if (!size) return;
+    <>
+      {canUpload && <DragUpload />}
+      <DraggablePanel
+        fullscreen={expand}
+        headerHeight={HEADER_HEIGHT}
+        minHeight={CHAT_TEXTAREA_HEIGHT}
+        onSizeChange={(_, size) => {
+          if (!size) return;
 
-        updatePreference({
-          inputHeight: typeof size.height === 'string' ? Number.parseInt(size.height) : size.height,
-        });
-      }}
-      placement="bottom"
-      size={{ height: inputHeight, width: '100%' }}
-      style={{ zIndex: 10 }}
-    >
-      <section className={styles.container} style={{ minHeight: CHAT_TEXTAREA_HEIGHT }}>
-        <ActionBar
-          rightAreaEndRender={
-            <ActionIcon
-              icon={expand ? Minimize2 : Maximize2}
-              onClick={() => {
-                setExpand(!expand);
-              }}
-            />
-          }
-        />
-        <InputArea />
-        <Footer />
-      </section>
-    </DraggablePanel>
+          updatePreference({
+            inputHeight:
+              typeof size.height === 'string' ? Number.parseInt(size.height) : size.height,
+          });
+        }}
+        placement="bottom"
+        size={{ height: inputHeight, width: '100%' }}
+        style={{ zIndex: 10 }}
+      >
+        <section className={styles.container} style={{ minHeight: CHAT_TEXTAREA_HEIGHT }}>
+          <ActionBar
+            rightAreaEndRender={
+              <ActionIcon
+                icon={expand ? Minimize2 : Maximize2}
+                onClick={() => {
+                  setExpand(!expand);
+                }}
+              />
+            }
+          />
+          <InputArea />
+          <Footer />
+        </section>
+      </DraggablePanel>
+    </>
   );
 });