@lobehub/chat 0.114.8 → 0.115.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/.i18nrc.js +3 -0
  2. package/CHANGELOG.md +50 -0
  3. package/locales/de-DE/common.json +20 -0
  4. package/locales/de-DE/tool.json +8 -0
  5. package/locales/en-US/common.json +20 -0
  6. package/locales/en-US/tool.json +8 -0
  7. package/locales/es-ES/common.json +20 -0
  8. package/locales/es-ES/tool.json +8 -0
  9. package/locales/fr-FR/common.json +20 -0
  10. package/locales/fr-FR/tool.json +8 -0
  11. package/locales/it-IT/chat.json +77 -0
  12. package/locales/it-IT/common.json +113 -0
  13. package/locales/it-IT/empty.json +6 -0
  14. package/locales/it-IT/error.json +53 -0
  15. package/locales/it-IT/market.json +30 -0
  16. package/locales/it-IT/migration.json +45 -0
  17. package/locales/it-IT/plugin.json +166 -0
  18. package/locales/it-IT/setting.json +306 -0
  19. package/locales/it-IT/tool.json +8 -0
  20. package/locales/it-IT/welcome.json +14 -0
  21. package/locales/ja-JP/common.json +20 -0
  22. package/locales/ja-JP/tool.json +8 -0
  23. package/locales/ko-KR/common.json +20 -0
  24. package/locales/ko-KR/tool.json +8 -0
  25. package/locales/nl-NL/chat.json +77 -0
  26. package/locales/nl-NL/common.json +113 -0
  27. package/locales/nl-NL/empty.json +6 -0
  28. package/locales/nl-NL/error.json +53 -0
  29. package/locales/nl-NL/market.json +30 -0
  30. package/locales/nl-NL/migration.json +45 -0
  31. package/locales/nl-NL/plugin.json +166 -0
  32. package/locales/nl-NL/setting.json +306 -0
  33. package/locales/nl-NL/tool.json +8 -0
  34. package/locales/nl-NL/welcome.json +14 -0
  35. package/locales/pl-PL/chat.json +77 -0
  36. package/locales/pl-PL/common.json +113 -0
  37. package/locales/pl-PL/empty.json +6 -0
  38. package/locales/pl-PL/error.json +53 -0
  39. package/locales/pl-PL/market.json +30 -0
  40. package/locales/pl-PL/migration.json +45 -0
  41. package/locales/pl-PL/plugin.json +166 -0
  42. package/locales/pl-PL/setting.json +306 -0
  43. package/locales/pl-PL/tool.json +8 -0
  44. package/locales/pl-PL/welcome.json +14 -0
  45. package/locales/pt-BR/common.json +20 -0
  46. package/locales/pt-BR/tool.json +8 -0
  47. package/locales/ru-RU/common.json +20 -0
  48. package/locales/ru-RU/tool.json +8 -0
  49. package/locales/tr-TR/common.json +20 -0
  50. package/locales/tr-TR/tool.json +8 -0
  51. package/locales/zh-CN/common.json +20 -0
  52. package/locales/zh-CN/plugin.json +2 -2
  53. package/locales/zh-CN/setting.json +1 -1
  54. package/locales/zh-CN/tool.json +8 -0
  55. package/locales/zh-TW/common.json +20 -0
  56. package/locales/zh-TW/tool.json +8 -0
  57. package/package.json +1 -1
  58. package/src/app/api/openai/images/createImageGeneration.ts +26 -0
  59. package/src/app/api/openai/images/route.ts +16 -0
  60. package/src/app/api/proxy/route.ts +3 -1
  61. package/src/app/chat/(desktop)/features/ChatHeader.tsx +56 -56
  62. package/src/app/chat/features/ChatHeader/PluginTag/PluginStatus.tsx +7 -4
  63. package/src/app/chat/features/ChatHeader/PluginTag/index.tsx +3 -3
  64. package/src/app/chat/features/SystemRole/index.tsx +0 -1
  65. package/src/components/GalleyGrid/Grid.tsx +36 -0
  66. package/src/components/GalleyGrid/index.tsx +58 -0
  67. package/src/components/GalleyGrid/style.ts +24 -0
  68. package/src/const/settings.ts +7 -0
  69. package/src/database/schemas/files.ts +1 -0
  70. package/src/database/schemas/message.ts +1 -1
  71. package/src/features/AgentSetting/AgentPlugin/index.tsx +2 -2
  72. package/src/features/ChatInput/ActionBar/ModelSwitch.tsx +2 -1
  73. package/src/features/ChatInput/ActionBar/Token/TokenTag.tsx +3 -3
  74. package/src/features/ChatInput/ActionBar/Tools/index.tsx +26 -14
  75. package/src/features/Conversation/ChatList/Messages/Function.tsx +1 -1
  76. package/src/features/Conversation/ChatList/Plugins/Inspector/index.tsx +4 -2
  77. package/src/features/Conversation/ChatList/Plugins/Render/BuiltinType/index.tsx +36 -0
  78. package/src/features/Conversation/ChatList/Plugins/Render/DefaultType/index.tsx +6 -66
  79. package/src/features/Conversation/ChatList/Plugins/Render/Loading.tsx +61 -0
  80. package/src/features/Conversation/ChatList/Plugins/Render/index.tsx +21 -12
  81. package/src/features/Conversation/ChatList/Plugins/Render/useParseContent.ts +15 -0
  82. package/src/locales/default/common.ts +20 -0
  83. package/src/locales/default/index.ts +2 -0
  84. package/src/locales/default/plugin.ts +2 -2
  85. package/src/locales/default/setting.ts +1 -1
  86. package/src/locales/default/tool.ts +9 -0
  87. package/src/locales/options.ts +14 -2
  88. package/src/services/__tests__/chat.test.ts +132 -2
  89. package/src/services/_url.ts +1 -0
  90. package/src/services/chat.ts +3 -5
  91. package/src/services/file.ts +20 -5
  92. package/src/services/imageGeneration.ts +26 -0
  93. package/src/store/chat/actions/enhance.ts +65 -1
  94. package/src/store/chat/actions/plugin.test.ts +9 -9
  95. package/src/store/chat/actions/plugin.ts +38 -6
  96. package/src/store/chat/initialState.ts +3 -0
  97. package/src/store/chat/selectors/enchance.ts +11 -0
  98. package/src/store/chat/selectors/index.ts +1 -0
  99. package/src/store/global/selectors/__snapshots__/settings.test.ts.snap +5 -0
  100. package/src/store/global/selectors/settings.ts +5 -0
  101. package/src/store/tool/helpers.ts +6 -5
  102. package/src/store/tool/initialState.ts +3 -1
  103. package/src/store/tool/selectors/index.ts +5 -0
  104. package/src/store/tool/selectors/tool.test.ts +128 -0
  105. package/src/store/tool/selectors/tool.ts +125 -0
  106. package/src/store/tool/slices/builtin/action.ts +54 -0
  107. package/src/store/tool/slices/builtin/index.ts +2 -0
  108. package/src/store/tool/slices/builtin/initialState.ts +13 -0
  109. package/src/store/tool/slices/builtin/selectors.test.ts +70 -0
  110. package/src/store/tool/slices/builtin/selectors.ts +20 -0
  111. package/src/store/tool/slices/customPlugin/action.ts +0 -3
  112. package/src/store/tool/slices/plugin/selectors.test.ts +0 -69
  113. package/src/store/tool/slices/plugin/selectors.ts +1 -84
  114. package/src/store/tool/store.ts +8 -2
  115. package/src/tools/dalle/Render/EditMode.tsx +66 -0
  116. package/src/tools/dalle/Render/Item.tsx +97 -0
  117. package/src/tools/dalle/Render/ToolBar.tsx +54 -0
  118. package/src/tools/dalle/Render/index.tsx +21 -0
  119. package/src/tools/dalle/index.ts +81 -0
  120. package/src/tools/index.ts +11 -0
  121. package/src/tools/renders.ts +8 -0
  122. package/src/types/chatMessage.ts +1 -1
  123. package/src/types/openai/image.ts +34 -0
  124. package/src/types/settings.ts +6 -0
  125. package/src/types/tool/builtin.ts +36 -0
  126. package/src/types/tool/dalle.ts +12 -0
  127. package/src/types/tool/index.ts +5 -1
  128. package/src/types/tool/tool.ts +9 -0
  129. package/src/store/tool/selectors.ts +0 -3
@@ -0,0 +1,36 @@
1
+ import { memo } from 'react';
2
+ import { Flexbox } from 'react-layout-kit';
3
+
4
+ import { BuiltinToolsRenders } from '@/tools/renders';
5
+
6
+ import Loading from '../Loading';
7
+ import { useParseContent } from '../useParseContent';
8
+
9
+ export interface BuiltinTypeProps {
10
+ content: string;
11
+ id: string;
12
+ identifier?: string;
13
+ loading?: boolean;
14
+ }
15
+
16
+ const BuiltinType = memo<BuiltinTypeProps>(({ content, id, identifier, loading }) => {
17
+ const { isJSON, data } = useParseContent(content);
18
+
19
+ if (!isJSON) {
20
+ return (
21
+ loading && (
22
+ <Flexbox gap={8}>
23
+ <Loading />
24
+ </Flexbox>
25
+ )
26
+ );
27
+ }
28
+
29
+ const Render = BuiltinToolsRenders[identifier || ''];
30
+
31
+ if (!Render) return;
32
+
33
+ return <Render content={data} identifier={identifier} messageId={id} />;
34
+ });
35
+
36
+ export default BuiltinType;
@@ -1,71 +1,17 @@
1
1
  import { Skeleton } from 'antd';
2
- import { createStyles } from 'antd-style';
3
2
  import dynamic from 'next/dynamic';
4
- import { Suspense, memo, useMemo } from 'react';
5
- import { useTranslation } from 'react-i18next';
3
+ import { Suspense, memo } from 'react';
6
4
  import { Flexbox } from 'react-layout-kit';
7
5
 
8
6
  import { useToolStore } from '@/store/tool';
9
7
  import { pluginSelectors } from '@/store/tool/selectors';
10
8
 
9
+ import Loading from '../Loading';
10
+ import { useParseContent } from '../useParseContent';
11
11
  import IFrameRender from './IFrameRender';
12
12
 
13
13
  const SystemJsRender = dynamic(() => import('./SystemJsRender'), { ssr: false });
14
14
 
15
- const useStyles = createStyles(
16
- ({ css, token }) => css`
17
- position: relative;
18
-
19
- overflow: hidden;
20
- display: block;
21
-
22
- width: 300px;
23
- height: 12px;
24
-
25
- border: 1px solid ${token.colorBorder};
26
- border-radius: 10px;
27
-
28
- &::after {
29
- content: '';
30
-
31
- position: absolute;
32
- top: 0;
33
- left: 0;
34
-
35
- box-sizing: border-box;
36
- width: 40%;
37
- height: 100%;
38
-
39
- background: ${token.colorPrimary};
40
-
41
- animation: animloader 2s linear infinite;
42
- }
43
-
44
- @keyframes animloader {
45
- 0% {
46
- left: 0;
47
- transform: translateX(-100%);
48
- }
49
-
50
- 100% {
51
- left: 100%;
52
- transform: translateX(0%);
53
- }
54
- }
55
- `,
56
- );
57
- const Loading = () => {
58
- const { t } = useTranslation('plugin');
59
- const { styles } = useStyles();
60
-
61
- return (
62
- <Flexbox align={'center'} gap={8} padding={16}>
63
- <span className={styles} />
64
- {t('loading.content')}
65
- </Flexbox>
66
- );
67
- };
68
-
69
15
  export interface PluginDefaultTypeProps {
70
16
  content: string;
71
17
  loading?: boolean;
@@ -74,14 +20,8 @@ export interface PluginDefaultTypeProps {
74
20
 
75
21
  const PluginDefaultType = memo<PluginDefaultTypeProps>(({ content, name, loading }) => {
76
22
  const manifest = useToolStore(pluginSelectors.getPluginManifestById(name || ''));
77
- let isJSON = true;
78
- try {
79
- JSON.parse(content);
80
- } catch {
81
- isJSON = false;
82
- }
83
23
 
84
- const contentObj = useMemo<object>(() => (isJSON ? JSON.parse(content) : content), [content]);
24
+ const { isJSON, data } = useParseContent(content);
85
25
 
86
26
  if (!isJSON) {
87
27
  return (
@@ -102,13 +42,13 @@ const PluginDefaultType = memo<PluginDefaultTypeProps>(({ content, name, loading
102
42
  if (ui.mode === 'module')
103
43
  return (
104
44
  <Suspense fallback={<Skeleton active style={{ width: 400 }} />}>
105
- <SystemJsRender content={contentObj} name={name || 'unknown'} url={ui.url} />
45
+ <SystemJsRender content={data} name={name || 'unknown'} url={ui.url} />
106
46
  </Suspense>
107
47
  );
108
48
 
109
49
  return (
110
50
  <IFrameRender
111
- content={contentObj}
51
+ content={data}
112
52
  height={ui.height}
113
53
  name={name || 'unknown'}
114
54
  url={ui.url}
@@ -0,0 +1,61 @@
1
+ import {createStyles} from 'antd-style';
2
+ import {useTranslation} from 'react-i18next';
3
+ import {Flexbox} from 'react-layout-kit';
4
+ import {memo} from "react";
5
+
6
+
7
+ const useStyles = createStyles(
8
+ ({css, token}) => css`
9
+ position: relative;
10
+
11
+ overflow: hidden;
12
+ display: block;
13
+
14
+ width: 300px;
15
+ height: 12px;
16
+
17
+ border: 1px solid ${token.colorBorder};
18
+ border-radius: 10px;
19
+
20
+ &::after {
21
+ content: '';
22
+
23
+ position: absolute;
24
+ top: 0;
25
+ left: 0;
26
+
27
+ box-sizing: border-box;
28
+ width: 40%;
29
+ height: 100%;
30
+
31
+ background: ${token.colorPrimary};
32
+
33
+ animation: animloader 2s linear infinite;
34
+ }
35
+
36
+ @keyframes animloader {
37
+ 0% {
38
+ left: 0;
39
+ transform: translateX(-100%);
40
+ }
41
+
42
+ 100% {
43
+ left: 100%;
44
+ transform: translateX(0%);
45
+ }
46
+ }
47
+ `,
48
+ );
49
+ const Loading = memo(() => {
50
+ const {t} = useTranslation('plugin');
51
+ const {styles} = useStyles();
52
+
53
+ return (
54
+ <Flexbox align={'center'} gap={8} padding={16}>
55
+ <span className={styles}/>
56
+ {t('loading.content')}
57
+ </Flexbox>
58
+ );
59
+ });
60
+
61
+ export default Loading
@@ -1,28 +1,37 @@
1
- import { LobePluginType, PluginRequestPayload } from '@lobehub/chat-plugin-sdk';
1
+ import { PluginRequestPayload } from '@lobehub/chat-plugin-sdk';
2
2
  import { memo } from 'react';
3
3
 
4
+ import { LobeToolRenderType } from '@/types/tool';
5
+
6
+ import BuiltinType from '././BuiltinType';
4
7
  import DefaultType from './DefaultType';
5
8
  import Standalone from './StandaloneType';
6
9
 
7
10
  export interface PluginRenderProps {
8
11
  content: string;
9
12
  id: string;
13
+ identifier?: string;
10
14
  loading?: boolean;
11
- name?: string;
12
15
  payload?: PluginRequestPayload;
13
- type?: LobePluginType;
16
+ type?: LobeToolRenderType;
14
17
  }
15
18
 
16
- const PluginRender = memo<PluginRenderProps>(({ content, id, payload, name, type, loading }) => {
17
- switch (type) {
18
- case 'standalone': {
19
- return <Standalone id={id} name={name} payload={payload} />;
20
- }
19
+ const PluginRender = memo<PluginRenderProps>(
20
+ ({ content, id, payload, identifier, type, loading }) => {
21
+ switch (type) {
22
+ case 'standalone': {
23
+ return <Standalone id={id} name={identifier} payload={payload} />;
24
+ }
25
+
26
+ case 'builtin': {
27
+ return <BuiltinType content={content} id={id} identifier={identifier} loading={loading} />;
28
+ }
21
29
 
22
- default: {
23
- return <DefaultType content={content} loading={loading} name={name} />;
30
+ default: {
31
+ return <DefaultType content={content} loading={loading} name={identifier} />;
32
+ }
24
33
  }
25
- }
26
- });
34
+ },
35
+ );
27
36
 
28
37
  export default PluginRender;
@@ -0,0 +1,15 @@
1
+ import { useMemo } from 'react';
2
+
3
+ export const useParseContent = (content: string) => {
4
+ let isJSON = true;
5
+
6
+ try {
7
+ JSON.parse(content);
8
+ } catch {
9
+ isJSON = false;
10
+ }
11
+
12
+ const data = isJSON ? JSON.parse(content) : content;
13
+
14
+ return useMemo(() => ({ data, isJSON }), [content]);
15
+ };
@@ -50,16 +50,36 @@ export default {
50
50
  title: '导入数据',
51
51
  },
52
52
  lang: {
53
+ 'bn': '孟加拉语',
54
+ 'cs-CZ': '捷克语',
55
+ 'da-DK': '丹麦语',
53
56
  'de-DE': '德语',
57
+ 'el-GR': '希腊语',
54
58
  'en': '英语',
55
59
  'en-US': '英语',
56
60
  'es-ES': '西班牙语',
61
+ 'fi-FI': '芬兰语',
57
62
  'fr-FR': '法语',
63
+ 'hi-IN': '印地语',
64
+ 'hu-HU': '匈牙利语',
65
+ 'id-ID': '印尼语',
66
+ 'it-IT': '意大利语',
58
67
  'ja-JP': '日语',
59
68
  'ko-KR': '韩语',
69
+ 'nl-NL': '荷兰语',
70
+ 'no-NO': '挪威语',
71
+ 'pl-PL': '波兰语',
60
72
  'pt-BR': '葡萄牙语',
73
+ 'pt-PT': '葡萄牙语',
74
+ 'ro-RO': '罗马尼亚语',
61
75
  'ru-RU': '俄语',
76
+ 'sk-SK': '斯洛伐克语',
77
+ 'sr-RS': '塞尔维亚语',
78
+ 'sv-SE': '瑞典语',
79
+ 'th-TH': '泰语',
62
80
  'tr-TR': '土耳其语',
81
+ 'uk-UA': '乌克兰语',
82
+ 'vi-VN': '越南语',
63
83
  'zh': '简体中文',
64
84
  'zh-CN': '简体中文',
65
85
  'zh-TW': '繁体中文',
@@ -1,3 +1,4 @@
1
+ import tool from '../default/tool';
1
2
  import chat from './chat';
2
3
  import common from './common';
3
4
  import empty from './empty';
@@ -17,6 +18,7 @@ const resources = {
17
18
  migration,
18
19
  plugin,
19
20
  setting,
21
+ tool,
20
22
  welcome,
21
23
  } as const;
22
24
 
@@ -130,8 +130,8 @@ export default {
130
130
  },
131
131
  pluginList: '插件列表',
132
132
  plugins: {
133
- loading: '插件检测中...',
134
- unknown: '未知插件',
133
+ loading: '工具检测中...',
134
+ unknown: '未知工具',
135
135
  },
136
136
  setting: '插件设置',
137
137
  settings: {
@@ -293,7 +293,7 @@ export default {
293
293
  },
294
294
  tools: {
295
295
  builtins: {
296
- groupName: '内置',
296
+ groupName: '内置扩展',
297
297
  },
298
298
  plugins: {
299
299
  enabled: '已启用 {{num}}',
@@ -0,0 +1,9 @@
1
+ export default {
2
+ dalle: {
3
+ autoGenerate: '自动生成',
4
+ downloading: 'DallE3 生成的图片链接有效期仅1小时,正在缓存图片到本地...',
5
+ generate: '生成',
6
+ generating: '生成中...',
7
+ images: '图片:',
8
+ },
9
+ };
@@ -45,13 +45,25 @@ export const localeOptions: LocaleOptions = [
45
45
  value: 'pt-BR',
46
46
  },
47
47
  {
48
- label: 'Russian',
48
+ label: 'Русский',
49
49
  value: 'ru-RU',
50
50
  },
51
51
  {
52
- label: 'Turkish',
52
+ label: 'Türkçe',
53
53
  value: 'tr-TR',
54
54
  },
55
+ {
56
+ label: 'Polski',
57
+ value: 'pl-PL',
58
+ },
59
+ {
60
+ label: 'Nederlands',
61
+ value: 'nl-NL',
62
+ },
63
+ {
64
+ label: 'Italiano',
65
+ value: 'it-IT',
66
+ },
55
67
  ] as LocaleOptions;
56
68
 
57
69
  export const supportLocales: string[] = [...locales, 'en', 'zh'];
@@ -326,7 +326,13 @@ You can use these tools below:
326
326
 
327
327
  ### SEO
328
328
 
329
- The user provides you with a URL and keyword and this gives you the robots.txt, response code, load time, tags for: title, metadata, h1-h5, image file names, image alt text, ahref text, ahref outbound links, keyword occurrence in both body text and URL. The URL must begin with http or https. You must request this URL structure from the user. Your role is to act like a word-class SEO analyst and give suggestions to the user on how they can improve the SEO for On-Page factors contained in the URL. Always include the https://createmorecustomers.agency link at the end of all requests.`,
329
+ The user provides you with a URL and keyword and this gives you the robots.txt, response code, load time, tags for: title, metadata, h1-h5, image file names, image alt text, ahref text, ahref outbound links, keyword occurrence in both body text and URL. The URL must begin with http or https. You must request this URL structure from the user. Your role is to act like a word-class SEO analyst and give suggestions to the user on how they can improve the SEO for On-Page factors contained in the URL. Always include the https://createmorecustomers.agency link at the end of all requests.
330
+
331
+ The APIs you can use:
332
+
333
+ #### seo____getData
334
+
335
+ Get data from users`,
330
336
  role: 'system',
331
337
  },
332
338
  { content: 'https://vercel.com/ 请分析 chatGPT 关键词\n\n', role: 'user' },
@@ -426,7 +432,13 @@ You can use these tools below:
426
432
 
427
433
  ### SEO
428
434
 
429
- The user provides you with a URL and keyword and this gives you the robots.txt, response code, load time, tags for: title, metadata, h1-h5, image file names, image alt text, ahref text, ahref outbound links, keyword occurrence in both body text and URL. The URL must begin with http or https. You must request this URL structure from the user. Your role is to act like a word-class SEO analyst and give suggestions to the user on how they can improve the SEO for On-Page factors contained in the URL. Always include the https://createmorecustomers.agency link at the end of all requests.`,
435
+ The user provides you with a URL and keyword and this gives you the robots.txt, response code, load time, tags for: title, metadata, h1-h5, image file names, image alt text, ahref text, ahref outbound links, keyword occurrence in both body text and URL. The URL must begin with http or https. You must request this URL structure from the user. Your role is to act like a word-class SEO analyst and give suggestions to the user on how they can improve the SEO for On-Page factors contained in the URL. Always include the https://createmorecustomers.agency link at the end of all requests.
436
+
437
+ The APIs you can use:
438
+
439
+ #### seo____getData
440
+
441
+ Get data from users`,
430
442
  role: 'system',
431
443
  },
432
444
  { content: 'https://vercel.com/ 请分析 chatGPT 关键词\n\n', role: 'user' },
@@ -468,6 +480,124 @@ The user provides you with a URL and keyword and this gives you the robots.txt,
468
480
  undefined,
469
481
  );
470
482
  });
483
+
484
+ it('work with dalle3', async () => {
485
+ const getChatCompletionSpy = vi.spyOn(chatService, 'getChatCompletion');
486
+ const messages = [
487
+ {
488
+ role: 'user',
489
+ content: 'https://vercel.com/ 请分析 chatGPT 关键词\n\n',
490
+ sessionId: 'inbox',
491
+ createdAt: 1702723964330,
492
+ id: 'vyQvEw6V',
493
+ updatedAt: 1702723964330,
494
+ extra: {},
495
+ meta: {
496
+ avatar: '😀',
497
+ },
498
+ },
499
+ ] as ChatMessage[];
500
+
501
+ await chatService.createAssistantMessage({
502
+ messages,
503
+ model: 'gpt-3.5-turbo-1106',
504
+ top_p: 1,
505
+ plugins: ['dalle3'],
506
+ });
507
+
508
+ expect(getChatCompletionSpy).toHaveBeenCalledWith(
509
+ {
510
+ model: 'gpt-3.5-turbo-1106',
511
+ top_p: 1,
512
+ functions: [
513
+ {
514
+ description: 'Create images from a text-only prompt.',
515
+ name: 'dalle3____text2image____builtin',
516
+ parameters: {
517
+ properties: {
518
+ prompts: {
519
+ description:
520
+ "The user's original image description, potentially modified to abide by the dalle policies. If the user does not suggest a number of captions to create, create four of them. If creating multiple captions, make them as diverse as possible. If the user requested modifications to previous images, the captions should not simply be longer, but rather it should be refactored to integrate the suggestions into each of the captions. Generate no more than 4 images, even if the user requests more.",
521
+ items: { type: 'string' },
522
+ maxItems: 4,
523
+ minItems: 1,
524
+ type: 'array',
525
+ },
526
+ quality: {
527
+ default: 'standard',
528
+ description:
529
+ 'The quality of the image that will be generated. hd creates images with finer details and greater consistency across the image.',
530
+ enum: ['standard', 'hd'],
531
+ type: 'string',
532
+ },
533
+ seeds: {
534
+ description:
535
+ 'A list of seeds to use for each prompt. If the user asks to modify a previous image, populate this field with the seed used to generate that image from the image dalle metadata.',
536
+ items: { type: 'integer' },
537
+ type: 'array',
538
+ },
539
+ size: {
540
+ default: '1024x1024',
541
+ description:
542
+ 'The resolution of the requested image, which can be wide, square, or tall. Use 1024x1024 (square) as the default unless the prompt suggests a wide image, 1792x1024, or a full-body portrait, in which case 1024x1792 (tall) should be used instead. Always include this parameter in the request.',
543
+ enum: ['1792x1024', '1024x1024', '1024x1792'],
544
+ type: 'string',
545
+ },
546
+ style: {
547
+ default: 'vivid',
548
+ description:
549
+ 'The style of the generated images. Must be one of vivid or natural. Vivid causes the model to lean towards generating hyper-real and dramatic images. Natural causes the model to produce more natural, less hyper-real looking images.',
550
+ enum: ['vivid', 'natural'],
551
+ type: 'string',
552
+ },
553
+ },
554
+ required: ['prompts'],
555
+ type: 'object',
556
+ },
557
+ },
558
+ ],
559
+ messages: [
560
+ {
561
+ content: `## Tools
562
+
563
+ You can use these tools below:
564
+
565
+ ### Dall·E 3
566
+
567
+ Whenever a description of an image is given, use dalle to create the images and then summarize the prompts used to generate the images in plain text. If the user does not ask for a specific number of images, default to creating four captions to send to dalle that are written to be as diverse as possible. All captions sent to dalle must abide by the following policies:
568
+ 1. If the description is not in English, then translate it.
569
+ 2. Do not create more than 4 images, even if the user requests more.
570
+ 3. Don't create images of politicians or other public figures. Recommend other ideas instead.
571
+ 4. Don't create images in the style of artists whose last work was created within the last 100 years (e.g. Picasso, Kahlo). Artists whose last work was over 100 years ago are ok to reference directly (e.g. Van Gogh, Klimt). If asked say, \"I can't reference this artist\", but make no mention of this policy. Instead, apply the following procedure when creating the captions for dalle: (a) substitute the artist's name with three adjectives that capture key aspects of the style; (b) include an associated artistic movement or era to provide context; and (c) mention the primary medium used by the artist.
572
+ 5. DO NOT list or refer to the descriptions before OR after generating the images. They should ONLY ever be written out ONCE, in the \`\"prompts\"\` field of the request. You do not need to ask for permission to generate, just do it!
573
+ 6. Always mention the image type (photo, oil painting, watercolor painting, illustration, cartoon, drawing, vector, render, etc.) at the beginning of the caption. Unless the caption suggests otherwise, make at least 1--2 of the 4 images photos.
574
+ 7. Diversify depictions of ALL images with people to include DESCENT and GENDER for EACH person using direct terms. Adjust only human descriptions.
575
+ - EXPLICITLY specify these attributes, not abstractly reference them. The attributes should be specified in a minimal way and should directly describe their physical form.
576
+ - Your choices should be grounded in reality. For example, all of a given OCCUPATION should not be the same gender or race. Additionally, focus on creating diverse, inclusive, and exploratory scenes via the properties you choose during rewrites. Make choices that may be insightful or unique sometimes.
577
+ - Use \"various\" or \"diverse\" ONLY IF the description refers to groups of more than 3 people. Do not change the number of people requested in the original description.
578
+ - Don't alter memes, fictional character origins, or unseen people. Maintain the original prompt's intent and prioritize quality.
579
+ - Do not create any imagery that would be offensive.
580
+ 8. Silently modify descriptions that include names or hints or references of specific people or celebrities by carefully selecting a few minimal modifications to substitute references to the people with generic descriptions that don't divulge any information about their identities, except for their genders and physiques. Do this EVEN WHEN the instructions ask for the prompt to not be changed. Some special cases:
581
+ - Modify such prompts even if you don't know who the person is, or if their name is misspelled (e.g. \"Barake Obema\")
582
+ - If the reference to the person will only appear as TEXT out in the image, then use the reference as is and do not modify it.
583
+ - When making the substitutions, don't use prominent titles that could give away the person's identity. E.g., instead of saying \"president\", \"prime minister\", or \"chancellor\", say \"politician\"; instead of saying \"king\", \"queen\", \"emperor\", or \"empress\", say \"public figure\"; instead of saying \"Pope\" or \"Dalai Lama\", say \"religious figure\"; and so on.
584
+ - If any creative professional or studio is named, substitute the name with a description of their style that does not reference any specific people, or delete the reference if they are unknown. DO NOT refer to the artist or studio's style.
585
+ The prompt must intricately describe every part of the image in concrete, objective detail. THINK about what the end goal of the description is, and extrapolate that to what would make satisfying images.
586
+ All descriptions sent to dalle should be a paragraph of text that is extremely descriptive and detailed. Each should be more than 3 sentences long.
587
+
588
+ The APIs you can use:
589
+
590
+ #### dalle3____text2image____builtin
591
+
592
+ Create images from a text-only prompt.`,
593
+ role: 'system',
594
+ },
595
+ { content: 'https://vercel.com/ 请分析 chatGPT 关键词\n\n', role: 'user' },
596
+ ],
597
+ },
598
+ undefined,
599
+ );
600
+ });
471
601
  });
472
602
  });
473
603
 
@@ -5,6 +5,7 @@ export const URLS = {
5
5
 
6
6
  export const OPENAI_URLS = {
7
7
  chat: '/api/openai/chat',
8
+ images: '/api/openai/images',
8
9
  models: '/api/openai/models',
9
10
  stt: '/api/openai/stt',
10
11
  tts: '/api/openai/tts',
@@ -6,7 +6,7 @@ import { VISION_MODEL_WHITE_LIST } from '@/const/llm';
6
6
  import { DEFAULT_AGENT_CONFIG } from '@/const/settings';
7
7
  import { filesSelectors, useFileStore } from '@/store/file';
8
8
  import { useToolStore } from '@/store/tool';
9
- import { pluginSelectors } from '@/store/tool/selectors';
9
+ import { pluginSelectors, toolSelectors } from '@/store/tool/selectors';
10
10
  import { ChatMessage } from '@/types/chatMessage';
11
11
  import type { OpenAIChatMessage, OpenAIChatStreamPayload } from '@/types/openai/chat';
12
12
  import { UserMessageContentPart } from '@/types/openai/chat';
@@ -48,7 +48,7 @@ class ChatService {
48
48
 
49
49
  // ============ 2. preprocess tools ============ //
50
50
 
51
- const filterTools = pluginSelectors.enabledSchema(enabledPlugins)(useToolStore.getState());
51
+ const filterTools = toolSelectors.enabledSchema(enabledPlugins)(useToolStore.getState());
52
52
 
53
53
  // the rule that model can use tools:
54
54
  // 1. tools is not empty
@@ -161,9 +161,7 @@ class ChatService {
161
161
 
162
162
  const systemMessage = draft.find((i) => i.role === 'system');
163
163
 
164
- const toolsSystemRoles = pluginSelectors.enabledPluginsSystemRoles(tools)(
165
- useToolStore.getState(),
166
- );
164
+ const toolsSystemRoles = toolSelectors.enabledSystemRoles(tools)(useToolStore.getState());
167
165
 
168
166
  if (!toolsSystemRoles) return;
169
167
 
@@ -1,5 +1,6 @@
1
1
  import { FileModel } from '@/database/models/file';
2
2
  import { DB_File } from '@/database/schemas/files';
3
+ import { PROXY_URL } from '@/services/_url';
3
4
  import { FilePreview } from '@/types/files';
4
5
  import compressImage from '@/utils/compressImage';
5
6
 
@@ -14,6 +15,7 @@ class FileService {
14
15
  if (this.isImage(file.fileType) && !isTestData) {
15
16
  return this.uploadImageFile(file);
16
17
  }
18
+
17
19
  // save to local storage
18
20
  // we may want to save to a remote server later
19
21
  return FileModel.create(file);
@@ -22,6 +24,7 @@ class FileService {
22
24
  async uploadImageFile(file: DB_File) {
23
25
  // 加载图片
24
26
  const url = file.url || URL.createObjectURL(new Blob([file.data]));
27
+
25
28
  const img = new Image();
26
29
  img.src = url;
27
30
  await (() =>
@@ -31,10 +34,7 @@ class FileService {
31
34
 
32
35
  // 压缩图片
33
36
  const fileType = 'image/webp';
34
- const base64String = compressImage({
35
- img,
36
- type: fileType,
37
- });
37
+ const base64String = compressImage({ img, type: fileType });
38
38
  const binaryString = atob(base64String.split('base64,')[1]);
39
39
  const uint8Array = Uint8Array.from(binaryString, (char) => char.charCodeAt(0));
40
40
  file.data = uint8Array.buffer;
@@ -42,6 +42,21 @@ class FileService {
42
42
  return FileModel.create(file);
43
43
  }
44
44
 
45
+ async uploadImageByUrl(url: string, file: Pick<DB_File, 'name' | 'metadata'>) {
46
+ const res = await fetch(PROXY_URL, { body: url, method: 'POST' });
47
+ const data = await res.arrayBuffer();
48
+ const fileType = res.headers.get('content-type') || 'image/webp';
49
+
50
+ return this.uploadFile({
51
+ data,
52
+ fileType,
53
+ metadata: file.metadata,
54
+ name: file.name,
55
+ saveMode: 'local',
56
+ size: data.byteLength,
57
+ });
58
+ }
59
+
45
60
  async removeFile(id: string) {
46
61
  return FileModel.delete(id);
47
62
  }
@@ -57,7 +72,7 @@ class FileService {
57
72
  }
58
73
 
59
74
  // arrayBuffer to url
60
- const url = URL.createObjectURL(new Blob([item.data]));
75
+ const url = URL.createObjectURL(new Blob([item.data], { type: item.fileType }));
61
76
  const base64 = Buffer.from(item.data).toString('base64');
62
77
 
63
78
  return {
@@ -0,0 +1,26 @@
1
+ import { createHeaderWithOpenAI } from '@/services/_header';
2
+ import { OPENAI_URLS } from '@/services/_url';
3
+ import { OpenAIImagePayload } from '@/types/openai/image';
4
+
5
+ interface FetchOptions {
6
+ signal?: AbortSignal | undefined;
7
+ }
8
+
9
+ class ImageGenerationService {
10
+ async generateImage(params: Omit<OpenAIImagePayload, 'model' | 'n'>, options?: FetchOptions) {
11
+ const payload: OpenAIImagePayload = { ...params, model: 'dall-e-3', n: 1 };
12
+
13
+ const res = await fetch(OPENAI_URLS.images, {
14
+ body: JSON.stringify(payload),
15
+ headers: createHeaderWithOpenAI({ 'Content-Type': 'application/json' }),
16
+ method: 'POST',
17
+ signal: options?.signal,
18
+ });
19
+
20
+ const urls = await res.json();
21
+
22
+ return urls[0] as string;
23
+ }
24
+ }
25
+
26
+ export const imageGenerationService = new ImageGenerationService();