@lobehub/lobehub 2.0.0-next.45 → 2.0.0-next.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/changelog/v1.json +9 -0
- package/package.json +1 -1
- package/src/app/[variants]/(main)/chat/components/conversation/features/ChatList/ChatItem/Thread.tsx +1 -1
- package/src/app/[variants]/(main)/chat/components/conversation/features/ChatList/ChatItem/index.tsx +9 -16
- package/src/features/Conversation/Messages/Assistant/Actions/index.tsx +3 -5
- package/src/features/Conversation/Messages/Assistant/Extra/index.test.tsx +3 -3
- package/src/features/Conversation/Messages/Assistant/Extra/index.tsx +8 -5
- package/src/features/Conversation/Messages/Assistant/index.tsx +29 -15
- package/src/features/Conversation/Messages/Group/Actions/WithContentId.tsx +3 -5
- package/src/features/Conversation/Messages/Group/index.tsx +12 -20
- package/src/features/Conversation/Messages/Supervisor/index.tsx +14 -5
- package/src/features/Conversation/Messages/User/index.tsx +14 -8
- package/src/features/Conversation/Messages/index.tsx +16 -26
- package/src/features/Conversation/components/Extras/Usage/UsageDetail/index.tsx +7 -6
- package/src/features/Conversation/components/Extras/Usage/UsageDetail/tokens.ts +2 -5
- package/src/features/Conversation/components/Extras/Usage/index.tsx +13 -6
- package/src/server/modules/ContentChunk/index.test.ts +372 -0
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,31 @@
|
|
|
2
2
|
|
|
3
3
|
# Changelog
|
|
4
4
|
|
|
5
|
+
## [Version 2.0.0-next.46](https://github.com/lobehub/lobe-chat/compare/v2.0.0-next.45...v2.0.0-next.46)
|
|
6
|
+
|
|
7
|
+
<sup>Released on **2025-11-11**</sup>
|
|
8
|
+
|
|
9
|
+
#### ♻ Code Refactoring
|
|
10
|
+
|
|
11
|
+
- **misc**: Fix thread display.
|
|
12
|
+
|
|
13
|
+
<br/>
|
|
14
|
+
|
|
15
|
+
<details>
|
|
16
|
+
<summary><kbd>Improvements and Fixes</kbd></summary>
|
|
17
|
+
|
|
18
|
+
#### Code refactoring
|
|
19
|
+
|
|
20
|
+
- **misc**: Fix thread display, closes [#10153](https://github.com/lobehub/lobe-chat/issues/10153) ([8fda83e](https://github.com/lobehub/lobe-chat/commit/8fda83e))
|
|
21
|
+
|
|
22
|
+
</details>
|
|
23
|
+
|
|
24
|
+
<div align="right">
|
|
25
|
+
|
|
26
|
+
[](#readme-top)
|
|
27
|
+
|
|
28
|
+
</div>
|
|
29
|
+
|
|
5
30
|
## [Version 2.0.0-next.45](https://github.com/lobehub/lobe-chat/compare/v2.0.0-next.44...v2.0.0-next.45)
|
|
6
31
|
|
|
7
32
|
<sup>Released on **2025-11-10**</sup>
|
package/changelog/v1.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lobehub/lobehub",
|
|
3
|
-
"version": "2.0.0-next.
|
|
3
|
+
"version": "2.0.0-next.46",
|
|
4
4
|
"description": "LobeHub - an open-source,comprehensive AI Agent framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"framework",
|
package/src/app/[variants]/(main)/chat/components/conversation/features/ChatList/ChatItem/Thread.tsx
CHANGED
|
@@ -39,7 +39,7 @@ const Thread = memo<ThreadProps>(({ id, placement, style }) => {
|
|
|
39
39
|
direction={placement === 'end' ? 'horizontal-reverse' : 'horizontal'}
|
|
40
40
|
gap={12}
|
|
41
41
|
paddingInline={16}
|
|
42
|
-
style={{ paddingBottom: 16, ...style }}
|
|
42
|
+
style={{ marginTop: -12, paddingBottom: 16, ...style }}
|
|
43
43
|
>
|
|
44
44
|
<div style={{ width: 40 }} />
|
|
45
45
|
<Flexbox className={styles.container} gap={4} padding={4} style={{ width: 'fit-content' }}>
|
package/src/app/[variants]/(main)/chat/components/conversation/features/ChatList/ChatItem/index.tsx
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { createStyles } from 'antd-style';
|
|
2
2
|
import React, { memo } from 'react';
|
|
3
3
|
|
|
4
|
-
import SupervisorThinkingTag from '@/app/[variants]/(main)/chat/components/conversation/features/ChatList/ChatItem/OrchestratorThinking';
|
|
5
4
|
import { ChatItem } from '@/features/Conversation';
|
|
6
5
|
import { useAgentStore } from '@/store/agent';
|
|
7
6
|
import { agentChatConfigSelectors } from '@/store/agent/selectors';
|
|
8
7
|
import { useChatStore } from '@/store/chat';
|
|
9
|
-
import {
|
|
8
|
+
import { displayMessageSelectors, threadSelectors } from '@/store/chat/selectors';
|
|
10
9
|
|
|
10
|
+
import SupervisorThinkingTag from './OrchestratorThinking';
|
|
11
11
|
import Thread from './Thread';
|
|
12
12
|
|
|
13
13
|
const useStyles = createStyles(({ css, token, isDarkMode }) => {
|
|
@@ -26,15 +26,16 @@ const useStyles = createStyles(({ css, token, isDarkMode }) => {
|
|
|
26
26
|
content: '';
|
|
27
27
|
|
|
28
28
|
position: absolute;
|
|
29
|
-
inset-block:
|
|
29
|
+
inset-block-end: 60px;
|
|
30
30
|
|
|
31
|
-
width:
|
|
31
|
+
width: 38px;
|
|
32
|
+
height: 53px;
|
|
32
33
|
border-block-end: 2px solid ${borderColor};
|
|
33
34
|
}
|
|
34
35
|
`,
|
|
35
36
|
start: css`
|
|
36
37
|
&::after {
|
|
37
|
-
inset-inline-start:
|
|
38
|
+
inset-inline-start: 30px;
|
|
38
39
|
border-inline-start: 2px solid ${borderColor};
|
|
39
40
|
border-end-start-radius: 8px;
|
|
40
41
|
}
|
|
@@ -52,7 +53,7 @@ const MainChatItem = memo<ThreadChatItemProps>(({ id, index }) => {
|
|
|
52
53
|
|
|
53
54
|
const [showThread, historyLength] = useChatStore((s) => [
|
|
54
55
|
threadSelectors.hasThreadBySourceMsgId(id)(s),
|
|
55
|
-
|
|
56
|
+
displayMessageSelectors.mainDisplayChatIDs(s).length,
|
|
56
57
|
]);
|
|
57
58
|
|
|
58
59
|
const [displayMode, enableHistoryDivider] = useAgentStore((s) => [
|
|
@@ -60,7 +61,7 @@ const MainChatItem = memo<ThreadChatItemProps>(({ id, index }) => {
|
|
|
60
61
|
agentChatConfigSelectors.enableHistoryDivider(historyLength, index)(s),
|
|
61
62
|
]);
|
|
62
63
|
|
|
63
|
-
const userRole = useChatStore((s) =>
|
|
64
|
+
const userRole = useChatStore((s) => displayMessageSelectors.getDisplayMessageById(id)(s)?.role);
|
|
64
65
|
|
|
65
66
|
const placement = displayMode === 'chat' && userRole === 'user' ? 'end' : 'start';
|
|
66
67
|
|
|
@@ -71,15 +72,7 @@ const MainChatItem = memo<ThreadChatItemProps>(({ id, index }) => {
|
|
|
71
72
|
<ChatItem
|
|
72
73
|
className={showThread ? cx(styles.line, styles[placement]) : ''}
|
|
73
74
|
enableHistoryDivider={enableHistoryDivider}
|
|
74
|
-
endRender={
|
|
75
|
-
showThread && (
|
|
76
|
-
<Thread
|
|
77
|
-
id={id}
|
|
78
|
-
placement={placement}
|
|
79
|
-
style={{ marginTop: displayMode === 'docs' ? 12 : undefined }}
|
|
80
|
-
/>
|
|
81
|
-
)
|
|
82
|
-
}
|
|
75
|
+
endRender={showThread && <Thread id={id} placement={placement} />}
|
|
83
76
|
id={id}
|
|
84
77
|
index={index}
|
|
85
78
|
/>
|
|
@@ -53,11 +53,9 @@ export const AssistantActionsBar = memo<AssistantActionsProps>(({ id, data, inde
|
|
|
53
53
|
const items = useMemo(() => {
|
|
54
54
|
if (hasTools) return [delAndRegenerate, copy];
|
|
55
55
|
|
|
56
|
-
return [
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
// inThread || isGroupSession ? null : branching
|
|
60
|
-
].filter(Boolean) as ActionIconGroupItemType[];
|
|
56
|
+
return [edit, copy, inThread || isGroupSession ? null : branching].filter(
|
|
57
|
+
Boolean,
|
|
58
|
+
) as ActionIconGroupItemType[];
|
|
61
59
|
}, [inThread, hasTools, isGroupSession, delAndRegenerate, copy, edit, branching]);
|
|
62
60
|
|
|
63
61
|
const { t } = useTranslation('common');
|
|
@@ -24,7 +24,7 @@ vi.mock('@/store/chat', () => ({
|
|
|
24
24
|
useChatStore: vi.fn(),
|
|
25
25
|
}));
|
|
26
26
|
|
|
27
|
-
const mockData
|
|
27
|
+
const mockData = {
|
|
28
28
|
content: 'test-content',
|
|
29
29
|
createdAt: 0,
|
|
30
30
|
id: 'abc',
|
|
@@ -53,8 +53,8 @@ describe('AssistantMessageExtra', () => {
|
|
|
53
53
|
expect(screen.queryByText('Translate Component')).toBeNull();
|
|
54
54
|
});
|
|
55
55
|
|
|
56
|
-
it('should render Usage component if
|
|
57
|
-
render(<AssistantMessageExtra {...mockData}
|
|
56
|
+
it('should render Usage component if model prop exists', async () => {
|
|
57
|
+
render(<AssistantMessageExtra {...mockData} model="gpt-4" provider="openai" />);
|
|
58
58
|
|
|
59
59
|
expect(screen.getByText('Usage Component')).toBeInTheDocument();
|
|
60
60
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ModelPerformance, ModelUsage } from '@lobechat/types';
|
|
2
2
|
import { memo } from 'react';
|
|
3
3
|
import { Flexbox } from 'react-layout-kit';
|
|
4
4
|
|
|
@@ -14,18 +14,21 @@ interface AssistantMessageExtraProps {
|
|
|
14
14
|
content: string;
|
|
15
15
|
extra?: any;
|
|
16
16
|
id: string;
|
|
17
|
-
|
|
17
|
+
model?: string;
|
|
18
|
+
performance?: ModelPerformance;
|
|
19
|
+
provider?: string;
|
|
18
20
|
tools?: any[];
|
|
21
|
+
usage?: ModelUsage;
|
|
19
22
|
}
|
|
20
23
|
|
|
21
24
|
export const AssistantMessageExtra = memo<AssistantMessageExtraProps>(
|
|
22
|
-
({ extra, id, content,
|
|
25
|
+
({ extra, id, content, performance, usage, tools, provider, model }) => {
|
|
23
26
|
const loading = useChatStore(messageStateSelectors.isMessageGenerating(id));
|
|
24
27
|
|
|
25
28
|
return (
|
|
26
29
|
<Flexbox gap={8} style={{ marginTop: !!tools?.length ? 8 : 4 }}>
|
|
27
|
-
{content !== LOADING_FLAT &&
|
|
28
|
-
<Usage
|
|
30
|
+
{content !== LOADING_FLAT && model && (
|
|
31
|
+
<Usage model={model} performance={performance} provider={provider!} usage={usage} />
|
|
29
32
|
)}
|
|
30
33
|
<>
|
|
31
34
|
{!!extra?.tts && (
|
|
@@ -20,7 +20,7 @@ import { useOpenChatSettings } from '@/hooks/useInterceptingRoutes';
|
|
|
20
20
|
import { useAgentStore } from '@/store/agent';
|
|
21
21
|
import { agentChatConfigSelectors } from '@/store/agent/selectors';
|
|
22
22
|
import { useChatStore } from '@/store/chat';
|
|
23
|
-
import { messageStateSelectors } from '@/store/chat/
|
|
23
|
+
import { displayMessageSelectors, messageStateSelectors } from '@/store/chat/selectors';
|
|
24
24
|
import { chatGroupSelectors, useChatGroupStore } from '@/store/chatGroup';
|
|
25
25
|
import { useGlobalStore } from '@/store/global';
|
|
26
26
|
import { useSessionStore } from '@/store/session';
|
|
@@ -48,28 +48,38 @@ const isHtmlCode = (content: string, language: string) => {
|
|
|
48
48
|
};
|
|
49
49
|
const MOBILE_AVATAR_SIZE = 32;
|
|
50
50
|
|
|
51
|
-
interface AssistantMessageProps
|
|
51
|
+
interface AssistantMessageProps {
|
|
52
52
|
disableEditing?: boolean;
|
|
53
|
+
id: string;
|
|
53
54
|
index: number;
|
|
54
|
-
showTitle?: boolean;
|
|
55
55
|
}
|
|
56
|
-
|
|
56
|
+
|
|
57
|
+
const AssistantMessage = memo<AssistantMessageProps>(({ id, index, disableEditing }) => {
|
|
58
|
+
const item = useChatStore(
|
|
59
|
+
displayMessageSelectors.getDisplayMessageById(id),
|
|
60
|
+
isEqual,
|
|
61
|
+
) as UIChatMessage;
|
|
62
|
+
|
|
57
63
|
const {
|
|
58
64
|
error,
|
|
59
|
-
showTitle,
|
|
60
|
-
id,
|
|
61
65
|
role,
|
|
62
66
|
search,
|
|
63
|
-
disableEditing,
|
|
64
|
-
index,
|
|
65
67
|
content,
|
|
66
68
|
createdAt,
|
|
67
69
|
tools,
|
|
68
70
|
extra,
|
|
69
|
-
|
|
71
|
+
model,
|
|
72
|
+
provider,
|
|
70
73
|
meta,
|
|
71
74
|
targetId,
|
|
72
|
-
|
|
75
|
+
groupId,
|
|
76
|
+
performance,
|
|
77
|
+
usage,
|
|
78
|
+
metadata,
|
|
79
|
+
} = item;
|
|
80
|
+
|
|
81
|
+
const showTitle = !!groupId;
|
|
82
|
+
|
|
73
83
|
const avatar = meta;
|
|
74
84
|
const { t } = useTranslation('chat');
|
|
75
85
|
const { mobile } = useResponsive();
|
|
@@ -199,11 +209,12 @@ const AssistantMessage = memo<AssistantMessageProps>((props) => {
|
|
|
199
209
|
|
|
200
210
|
const renderMessage = useCallback(
|
|
201
211
|
(editableContent: ReactNode) => (
|
|
202
|
-
<AssistantMessageContent {...
|
|
212
|
+
<AssistantMessageContent {...item} editableContent={editableContent} />
|
|
203
213
|
),
|
|
204
|
-
[
|
|
214
|
+
[item],
|
|
205
215
|
);
|
|
206
|
-
const errorMessage = <ErrorMessageExtra data={
|
|
216
|
+
const errorMessage = <ErrorMessageExtra data={item} />;
|
|
217
|
+
|
|
207
218
|
return (
|
|
208
219
|
<Flexbox className={styles.container} gap={mobile ? 6 : 12}>
|
|
209
220
|
<Flexbox gap={4} horizontal>
|
|
@@ -254,8 +265,11 @@ const AssistantMessage = memo<AssistantMessageProps>((props) => {
|
|
|
254
265
|
content={content}
|
|
255
266
|
extra={extra}
|
|
256
267
|
id={id}
|
|
257
|
-
|
|
268
|
+
model={model!}
|
|
269
|
+
performance={performance! || metadata}
|
|
270
|
+
provider={provider!}
|
|
258
271
|
tools={tools}
|
|
272
|
+
usage={usage! || metadata}
|
|
259
273
|
/>
|
|
260
274
|
</>
|
|
261
275
|
}
|
|
@@ -268,7 +282,7 @@ const AssistantMessage = memo<AssistantMessageProps>((props) => {
|
|
|
268
282
|
</Flexbox>
|
|
269
283
|
{!disableEditing && !editing && (
|
|
270
284
|
<Flexbox align={'flex-start'} className={styles.actions} role="menubar">
|
|
271
|
-
<AssistantActionsBar data={
|
|
285
|
+
<AssistantActionsBar data={item} id={id} index={index} />
|
|
272
286
|
</Flexbox>
|
|
273
287
|
)}
|
|
274
288
|
</Flexbox>
|
|
@@ -46,11 +46,9 @@ const WithContentId = memo<GroupActionsProps>(({ id, data, index, contentBlock }
|
|
|
46
46
|
const items = useMemo(() => {
|
|
47
47
|
if (hasTools) return [delAndRegenerate, copy];
|
|
48
48
|
|
|
49
|
-
return [
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
// inThread || isGroupSession ? null : branching
|
|
53
|
-
].filter(Boolean) as ActionIconGroupItemType[];
|
|
49
|
+
return [edit, copy, inThread || isGroupSession ? null : branching].filter(
|
|
50
|
+
Boolean,
|
|
51
|
+
) as ActionIconGroupItemType[];
|
|
54
52
|
}, [inThread, hasTools, isGroupSession, delAndRegenerate, copy, edit, branching]);
|
|
55
53
|
|
|
56
54
|
const { t } = useTranslation('common');
|
|
@@ -29,28 +29,20 @@ import EditState from './EditState';
|
|
|
29
29
|
|
|
30
30
|
const MOBILE_AVATAR_SIZE = 32;
|
|
31
31
|
|
|
32
|
-
interface GroupMessageProps
|
|
32
|
+
interface GroupMessageProps {
|
|
33
33
|
disableEditing?: boolean;
|
|
34
|
+
id: string;
|
|
34
35
|
index: number;
|
|
35
|
-
showTitle?: boolean;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
const GroupMessage = memo<GroupMessageProps>((
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
index,
|
|
45
|
-
createdAt,
|
|
46
|
-
meta,
|
|
47
|
-
children,
|
|
48
|
-
performance,
|
|
49
|
-
model,
|
|
50
|
-
provider,
|
|
51
|
-
} = props;
|
|
38
|
+
const GroupMessage = memo<GroupMessageProps>(({ id, index, disableEditing }) => {
|
|
39
|
+
const item = useChatStore(
|
|
40
|
+
displayMessageSelectors.getDisplayMessageById(id),
|
|
41
|
+
isEqual,
|
|
42
|
+
) as UIChatMessage;
|
|
43
|
+
const { usage, createdAt, meta, children, performance, model, provider } = item;
|
|
52
44
|
const avatar = meta;
|
|
53
|
-
|
|
45
|
+
|
|
54
46
|
const { mobile } = useResponsive();
|
|
55
47
|
const placement = 'left';
|
|
56
48
|
const type = useAgentStore(agentChatConfigSelectors.displayMode);
|
|
@@ -60,7 +52,7 @@ const GroupMessage = memo<GroupMessageProps>((props) => {
|
|
|
60
52
|
editing: false,
|
|
61
53
|
placement,
|
|
62
54
|
primary: false,
|
|
63
|
-
showTitle,
|
|
55
|
+
showTitle: true,
|
|
64
56
|
time: createdAt,
|
|
65
57
|
title: avatar.title,
|
|
66
58
|
variant,
|
|
@@ -129,14 +121,14 @@ const GroupMessage = memo<GroupMessageProps>((props) => {
|
|
|
129
121
|
)}
|
|
130
122
|
|
|
131
123
|
{model && (
|
|
132
|
-
<Usage
|
|
124
|
+
<Usage model={model} performance={performance} provider={provider!} usage={usage} />
|
|
133
125
|
)}
|
|
134
126
|
{!disableEditing && (
|
|
135
127
|
<Flexbox align={'flex-start'} className={styles.actions} role="menubar">
|
|
136
128
|
<GroupActionsBar
|
|
137
129
|
contentBlock={lastAssistantMsg}
|
|
138
130
|
contentId={contentId}
|
|
139
|
-
data={
|
|
131
|
+
data={item}
|
|
140
132
|
id={id}
|
|
141
133
|
index={index}
|
|
142
134
|
/>
|
|
@@ -4,6 +4,7 @@ import { UIChatMessage } from '@lobechat/types';
|
|
|
4
4
|
import { ModelIcon } from '@lobehub/icons';
|
|
5
5
|
import { Button, Text } from '@lobehub/ui';
|
|
6
6
|
import { createStyles, useTheme } from 'antd-style';
|
|
7
|
+
import isEqual from 'fast-deep-equal';
|
|
7
8
|
import { LucideRefreshCw } from 'lucide-react';
|
|
8
9
|
import { memo, useCallback } from 'react';
|
|
9
10
|
import { useTranslation } from 'react-i18next';
|
|
@@ -12,6 +13,7 @@ import { Flexbox } from 'react-layout-kit';
|
|
|
12
13
|
import { DEFAULT_SUPERVISOR_AVATAR } from '@/const/meta';
|
|
13
14
|
import { ChatItem } from '@/features/ChatItem';
|
|
14
15
|
import { useChatStore } from '@/store/chat';
|
|
16
|
+
import { displayMessageSelectors } from '@/store/chat/slices/message/selectors';
|
|
15
17
|
import { ChatErrorType } from '@/types/fetch';
|
|
16
18
|
|
|
17
19
|
import TodoList, { TodoData } from './TodoList';
|
|
@@ -54,13 +56,20 @@ const parseMarkdownTodos = (content: string): TodoData => {
|
|
|
54
56
|
};
|
|
55
57
|
};
|
|
56
58
|
|
|
57
|
-
interface SupervisorMessageProps
|
|
59
|
+
interface SupervisorMessageProps {
|
|
58
60
|
disableEditing?: boolean;
|
|
61
|
+
id: string;
|
|
59
62
|
index: number;
|
|
60
63
|
}
|
|
61
64
|
|
|
62
|
-
const SupervisorMessage = memo<SupervisorMessageProps>((
|
|
63
|
-
const
|
|
65
|
+
const SupervisorMessage = memo<SupervisorMessageProps>(({ id }) => {
|
|
66
|
+
const item = useChatStore(
|
|
67
|
+
displayMessageSelectors.getDisplayMessageById(id),
|
|
68
|
+
isEqual,
|
|
69
|
+
) as UIChatMessage;
|
|
70
|
+
|
|
71
|
+
const { content, error, groupId, role, updatedAt, createdAt } = item;
|
|
72
|
+
|
|
64
73
|
const { t } = useTranslation('chat');
|
|
65
74
|
const theme = useTheme();
|
|
66
75
|
const { styles } = useStyles();
|
|
@@ -128,8 +137,8 @@ const SupervisorMessage = memo<SupervisorMessageProps>((props) => {
|
|
|
128
137
|
|
|
129
138
|
// Render todo message with dedicated component
|
|
130
139
|
if (isTodoMessage && todoData) {
|
|
131
|
-
const model =
|
|
132
|
-
const provider =
|
|
140
|
+
const model = item.extra?.model;
|
|
141
|
+
const provider = item.extra?.provider;
|
|
133
142
|
const hasModelInfo = model || provider;
|
|
134
143
|
|
|
135
144
|
return (
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { UIChatMessage } from '@lobechat/types';
|
|
2
2
|
import { Tag } from '@lobehub/ui';
|
|
3
3
|
import { useResponsive } from 'antd-style';
|
|
4
|
+
import isEqual from 'fast-deep-equal';
|
|
4
5
|
import { ReactNode, memo, useCallback, useMemo } from 'react';
|
|
5
6
|
import { useTranslation } from 'react-i18next';
|
|
6
7
|
import { Flexbox } from 'react-layout-kit';
|
|
@@ -15,7 +16,7 @@ import { useUserAvatar } from '@/hooks/useUserAvatar';
|
|
|
15
16
|
import { useAgentStore } from '@/store/agent';
|
|
16
17
|
import { agentChatConfigSelectors } from '@/store/agent/selectors';
|
|
17
18
|
import { useChatStore } from '@/store/chat';
|
|
18
|
-
import { messageStateSelectors } from '@/store/chat/selectors';
|
|
19
|
+
import { displayMessageSelectors, messageStateSelectors } from '@/store/chat/selectors';
|
|
19
20
|
import { useSessionStore } from '@/store/session';
|
|
20
21
|
import { sessionSelectors } from '@/store/session/selectors';
|
|
21
22
|
import { useUserStore } from '@/store/user';
|
|
@@ -28,8 +29,9 @@ import { UserMessageExtra } from './Extra';
|
|
|
28
29
|
import { MarkdownRender as UserMarkdownRender } from './MarkdownRender';
|
|
29
30
|
import { UserMessageContent } from './MessageContent';
|
|
30
31
|
|
|
31
|
-
interface UserMessageProps
|
|
32
|
+
interface UserMessageProps {
|
|
32
33
|
disableEditing?: boolean;
|
|
34
|
+
id: string;
|
|
33
35
|
index: number;
|
|
34
36
|
}
|
|
35
37
|
|
|
@@ -43,9 +45,13 @@ const remarkPlugins = markdownElements
|
|
|
43
45
|
.map((element) => element.remarkPlugin)
|
|
44
46
|
.filter(Boolean);
|
|
45
47
|
|
|
46
|
-
const UserMessage = memo<UserMessageProps>((
|
|
47
|
-
const
|
|
48
|
-
|
|
48
|
+
const UserMessage = memo<UserMessageProps>(({ id, disableEditing, index }) => {
|
|
49
|
+
const item = useChatStore(
|
|
50
|
+
displayMessageSelectors.getDisplayMessageById(id),
|
|
51
|
+
isEqual,
|
|
52
|
+
) as UIChatMessage;
|
|
53
|
+
|
|
54
|
+
const { ragQuery, content, createdAt, error, role, extra, targetId } = item;
|
|
49
55
|
|
|
50
56
|
const { t } = useTranslation('chat');
|
|
51
57
|
const { mobile } = useResponsive();
|
|
@@ -97,9 +103,9 @@ const UserMessage = memo<UserMessageProps>((props) => {
|
|
|
97
103
|
|
|
98
104
|
const renderMessage = useCallback(
|
|
99
105
|
(editableContent: ReactNode) => (
|
|
100
|
-
<UserMessageContent {...
|
|
106
|
+
<UserMessageContent {...item} editableContent={editableContent} />
|
|
101
107
|
),
|
|
102
|
-
[
|
|
108
|
+
[item],
|
|
103
109
|
);
|
|
104
110
|
|
|
105
111
|
const components = useMemo(
|
|
@@ -178,7 +184,7 @@ const UserMessage = memo<UserMessageProps>((props) => {
|
|
|
178
184
|
</Flexbox>
|
|
179
185
|
|
|
180
186
|
<Flexbox direction={'horizontal-reverse'}>
|
|
181
|
-
<Actions data={
|
|
187
|
+
<Actions data={item} disableEditing={disableEditing} id={id} index={index} />
|
|
182
188
|
</Flexbox>
|
|
183
189
|
</Flexbox>
|
|
184
190
|
);
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
removeVirtuosoVisibleItem,
|
|
11
11
|
upsertVirtuosoVisibleItem,
|
|
12
12
|
} from '@/features/Conversation/components/VirtualizedList/VirtuosoContext';
|
|
13
|
-
import { useChatStore } from '@/store/chat';
|
|
13
|
+
import { getChatStoreState, useChatStore } from '@/store/chat';
|
|
14
14
|
import { displayMessageSelectors, messageStateSelectors } from '@/store/chat/selectors';
|
|
15
15
|
|
|
16
16
|
import History from '../components/History';
|
|
@@ -56,9 +56,10 @@ const Item = memo<ChatListItemProps>(
|
|
|
56
56
|
const { styles, cx } = useStyles();
|
|
57
57
|
const containerRef = useRef<HTMLDivElement | null>(null);
|
|
58
58
|
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
59
|
+
const [isMessageLoading, role] = useChatStore((s) => [
|
|
60
|
+
messageStateSelectors.isMessageLoading(id)(s),
|
|
61
|
+
displayMessageSelectors.getDisplayMessageById(id)(s)?.role,
|
|
62
|
+
]);
|
|
62
63
|
|
|
63
64
|
// ======================= Performance Optimization ======================= //
|
|
64
65
|
// these useMemo/useCallback are all for the performance optimization
|
|
@@ -104,6 +105,8 @@ const Item = memo<ChatListItemProps>(
|
|
|
104
105
|
}, [index]);
|
|
105
106
|
|
|
106
107
|
const onContextMenu = useCallback(async () => {
|
|
108
|
+
const item = displayMessageSelectors.getDisplayMessageById(id)(getChatStoreState());
|
|
109
|
+
|
|
107
110
|
if (isDesktop && item) {
|
|
108
111
|
const { electronSystemService } = await import('@/services/electron/system');
|
|
109
112
|
|
|
@@ -114,45 +117,31 @@ const Item = memo<ChatListItemProps>(
|
|
|
114
117
|
role: item.role,
|
|
115
118
|
});
|
|
116
119
|
}
|
|
117
|
-
}, [id
|
|
120
|
+
}, [id]);
|
|
118
121
|
|
|
119
122
|
const renderContent = useMemo(() => {
|
|
120
|
-
switch (
|
|
123
|
+
switch (role) {
|
|
121
124
|
case 'user': {
|
|
122
|
-
return <UserMessage {
|
|
125
|
+
return <UserMessage disableEditing={disableEditing} id={id} index={index} />;
|
|
123
126
|
}
|
|
124
127
|
|
|
125
128
|
case 'assistant': {
|
|
126
|
-
return
|
|
127
|
-
<AssistantMessage
|
|
128
|
-
{...item}
|
|
129
|
-
disableEditing={disableEditing}
|
|
130
|
-
index={index}
|
|
131
|
-
showTitle={item.groupId ? true : false}
|
|
132
|
-
/>
|
|
133
|
-
);
|
|
129
|
+
return <AssistantMessage disableEditing={disableEditing} id={id} index={index} />;
|
|
134
130
|
}
|
|
135
131
|
|
|
136
132
|
case 'assistantGroup': {
|
|
137
|
-
return
|
|
138
|
-
<GroupMessage
|
|
139
|
-
{...item}
|
|
140
|
-
disableEditing={disableEditing}
|
|
141
|
-
index={index}
|
|
142
|
-
showTitle={item.groupId ? true : false}
|
|
143
|
-
/>
|
|
144
|
-
);
|
|
133
|
+
return <GroupMessage disableEditing={disableEditing} id={id} index={index} />;
|
|
145
134
|
}
|
|
146
135
|
|
|
147
136
|
case 'supervisor': {
|
|
148
|
-
return <SupervisorMessage {
|
|
137
|
+
return <SupervisorMessage disableEditing={disableEditing} id={id} index={index} />;
|
|
149
138
|
}
|
|
150
139
|
}
|
|
151
140
|
|
|
152
141
|
return null;
|
|
153
|
-
}, [
|
|
142
|
+
}, [role, disableEditing, id, index]);
|
|
154
143
|
|
|
155
|
-
if (!
|
|
144
|
+
if (!role) return;
|
|
156
145
|
|
|
157
146
|
return (
|
|
158
147
|
<InPortalThreadContext.Provider value={inPortalThread}>
|
|
@@ -169,6 +158,7 @@ const Item = memo<ChatListItemProps>(
|
|
|
169
158
|
</InPortalThreadContext.Provider>
|
|
170
159
|
);
|
|
171
160
|
},
|
|
161
|
+
isEqual,
|
|
172
162
|
);
|
|
173
163
|
|
|
174
164
|
Item.displayName = 'ChatItem';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ModelPerformance, ModelUsage } from '@lobechat/types';
|
|
2
2
|
import { Icon } from '@lobehub/ui';
|
|
3
3
|
import { Divider, Popover } from 'antd';
|
|
4
4
|
import { useTheme } from 'antd-style';
|
|
@@ -20,12 +20,13 @@ import TokenProgress, { TokenProgressItem } from './TokenProgress';
|
|
|
20
20
|
import { getDetailsToken } from './tokens';
|
|
21
21
|
|
|
22
22
|
interface TokenDetailProps {
|
|
23
|
-
meta: MessageMetadata;
|
|
24
23
|
model: string;
|
|
24
|
+
performance?: ModelPerformance;
|
|
25
25
|
provider: string;
|
|
26
|
+
usage: ModelUsage;
|
|
26
27
|
}
|
|
27
28
|
|
|
28
|
-
const TokenDetail = memo<TokenDetailProps>(({
|
|
29
|
+
const TokenDetail = memo<TokenDetailProps>(({ usage, performance, model, provider }) => {
|
|
29
30
|
const { t } = useTranslation('chat');
|
|
30
31
|
const theme = useTheme();
|
|
31
32
|
const isMobile = useIsMobile();
|
|
@@ -37,7 +38,7 @@ const TokenDetail = memo<TokenDetailProps>(({ meta, model, provider }) => {
|
|
|
37
38
|
const modelCard = useAiInfraStore(aiModelSelectors.getModelCard(model, provider));
|
|
38
39
|
const isShowCredit = useGlobalStore(systemStatusSelectors.isShowCredit) && !!modelCard?.pricing;
|
|
39
40
|
|
|
40
|
-
const detailTokens = getDetailsToken(
|
|
41
|
+
const detailTokens = getDetailsToken(usage, modelCard);
|
|
41
42
|
const inputDetails = [
|
|
42
43
|
!!detailTokens.inputAudio && {
|
|
43
44
|
color: theme.cyan9,
|
|
@@ -130,8 +131,8 @@ const TokenDetail = memo<TokenDetailProps>(({ meta, model, provider }) => {
|
|
|
130
131
|
2,
|
|
131
132
|
);
|
|
132
133
|
|
|
133
|
-
const tps =
|
|
134
|
-
const ttft =
|
|
134
|
+
const tps = performance?.tps ? formatNumber(performance.tps, 2) : undefined;
|
|
135
|
+
const ttft = performance?.ttft ? formatNumber(performance.ttft / 1000, 2) : undefined;
|
|
135
136
|
|
|
136
137
|
return (
|
|
137
138
|
<Popover
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { ModelUsage } from '@lobechat/types';
|
|
2
2
|
import { LobeDefaultAiModelListItem } from 'model-bank';
|
|
3
3
|
|
|
4
4
|
import { getAudioInputUnitRate, getAudioOutputUnitRate } from '@/utils/pricing';
|
|
@@ -11,10 +11,7 @@ const calcCredit = (token: number, pricing?: number) => {
|
|
|
11
11
|
return parseInt((token * pricing).toFixed(0));
|
|
12
12
|
};
|
|
13
13
|
|
|
14
|
-
export const getDetailsToken = (
|
|
15
|
-
usage: ModelTokensUsage,
|
|
16
|
-
modelCard?: LobeDefaultAiModelListItem,
|
|
17
|
-
) => {
|
|
14
|
+
export const getDetailsToken = (usage: ModelUsage, modelCard?: LobeDefaultAiModelListItem) => {
|
|
18
15
|
const inputTextTokens = usage.inputTextTokens || (usage as any).inputTokens || 0;
|
|
19
16
|
const totalInputTokens = usage.totalInputTokens || (usage as any).inputTokens || 0;
|
|
20
17
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ModelPerformance, ModelUsage } from '@lobechat/types';
|
|
2
2
|
import { ModelIcon } from '@lobehub/icons';
|
|
3
3
|
import { createStyles } from 'antd-style';
|
|
4
|
+
import isEqual from 'fast-deep-equal';
|
|
4
5
|
import { memo } from 'react';
|
|
5
6
|
import { Center, Flexbox } from 'react-layout-kit';
|
|
6
7
|
|
|
@@ -14,12 +15,13 @@ export const useStyles = createStyles(({ token, css, cx }) => ({
|
|
|
14
15
|
}));
|
|
15
16
|
|
|
16
17
|
interface UsageProps {
|
|
17
|
-
metadata: MessageMetadata;
|
|
18
18
|
model: string;
|
|
19
|
+
performance?: ModelPerformance;
|
|
19
20
|
provider: string;
|
|
21
|
+
usage?: ModelUsage;
|
|
20
22
|
}
|
|
21
23
|
|
|
22
|
-
const Usage = memo<UsageProps>(({ model,
|
|
24
|
+
const Usage = memo<UsageProps>(({ model, usage, performance, provider }) => {
|
|
23
25
|
const { styles } = useStyles();
|
|
24
26
|
|
|
25
27
|
return (
|
|
@@ -35,11 +37,16 @@ const Usage = memo<UsageProps>(({ model, metadata, provider }) => {
|
|
|
35
37
|
{model}
|
|
36
38
|
</Center>
|
|
37
39
|
|
|
38
|
-
{!!
|
|
39
|
-
<TokenDetail
|
|
40
|
+
{!!usage?.totalTokens && (
|
|
41
|
+
<TokenDetail
|
|
42
|
+
model={model as string}
|
|
43
|
+
performance={performance}
|
|
44
|
+
provider={provider}
|
|
45
|
+
usage={usage}
|
|
46
|
+
/>
|
|
40
47
|
)}
|
|
41
48
|
</Flexbox>
|
|
42
49
|
);
|
|
43
|
-
});
|
|
50
|
+
}, isEqual);
|
|
44
51
|
|
|
45
52
|
export default Usage;
|
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
|
2
|
+
|
|
3
|
+
import type { NewChunkItem, NewUnstructuredChunkItem } from '@/database/schemas';
|
|
4
|
+
import { knowledgeEnv } from '@/envs/knowledge';
|
|
5
|
+
import { ChunkingLoader } from '@/libs/langchain';
|
|
6
|
+
import { ChunkingStrategy, Unstructured } from '@/libs/unstructured';
|
|
7
|
+
|
|
8
|
+
import { ContentChunk } from './index';
|
|
9
|
+
|
|
10
|
+
// Mock the dependencies
|
|
11
|
+
vi.mock('@/libs/unstructured');
|
|
12
|
+
vi.mock('@/libs/langchain');
|
|
13
|
+
vi.mock('@/envs/knowledge', () => ({
|
|
14
|
+
knowledgeEnv: {
|
|
15
|
+
FILE_TYPE_CHUNKING_RULES: '',
|
|
16
|
+
UNSTRUCTURED_API_KEY: 'test-api-key',
|
|
17
|
+
UNSTRUCTURED_SERVER_URL: 'https://test.unstructured.io',
|
|
18
|
+
},
|
|
19
|
+
}));
|
|
20
|
+
|
|
21
|
+
describe('ContentChunk', () => {
|
|
22
|
+
let contentChunk: ContentChunk;
|
|
23
|
+
let mockUnstructuredPartition: ReturnType<typeof vi.fn>;
|
|
24
|
+
let mockLangChainPartition: ReturnType<typeof vi.fn>;
|
|
25
|
+
|
|
26
|
+
beforeEach(() => {
|
|
27
|
+
vi.clearAllMocks();
|
|
28
|
+
|
|
29
|
+
// Setup Unstructured mock
|
|
30
|
+
mockUnstructuredPartition = vi.fn();
|
|
31
|
+
(Unstructured as unknown as ReturnType<typeof vi.fn>).mockImplementation(() => ({
|
|
32
|
+
partition: mockUnstructuredPartition,
|
|
33
|
+
}));
|
|
34
|
+
|
|
35
|
+
// Setup LangChain mock
|
|
36
|
+
mockLangChainPartition = vi.fn();
|
|
37
|
+
(ChunkingLoader as unknown as ReturnType<typeof vi.fn>).mockImplementation(() => ({
|
|
38
|
+
partitionContent: mockLangChainPartition,
|
|
39
|
+
}));
|
|
40
|
+
|
|
41
|
+
contentChunk = new ContentChunk();
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
describe('constructor', () => {
|
|
45
|
+
it('should initialize with Unstructured and LangChain clients', () => {
|
|
46
|
+
expect(Unstructured).toHaveBeenCalledTimes(1);
|
|
47
|
+
expect(ChunkingLoader).toHaveBeenCalledTimes(1);
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
describe('chunkContent', () => {
|
|
52
|
+
const mockFileContent = new Uint8Array([1, 2, 3, 4, 5]);
|
|
53
|
+
const mockFilename = 'test-document.pdf';
|
|
54
|
+
|
|
55
|
+
it('should use default langchain service when no rules are configured', async () => {
|
|
56
|
+
const mockLangChainResult = [
|
|
57
|
+
{
|
|
58
|
+
id: 'chunk-1',
|
|
59
|
+
metadata: { source: 'test' },
|
|
60
|
+
pageContent: 'Test content chunk 1',
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
id: 'chunk-2',
|
|
64
|
+
metadata: { source: 'test' },
|
|
65
|
+
pageContent: 'Test content chunk 2',
|
|
66
|
+
},
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
mockLangChainPartition.mockResolvedValue(mockLangChainResult);
|
|
70
|
+
|
|
71
|
+
const result = await contentChunk.chunkContent({
|
|
72
|
+
content: mockFileContent,
|
|
73
|
+
fileType: 'application/pdf',
|
|
74
|
+
filename: mockFilename,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
expect(mockLangChainPartition).toHaveBeenCalledWith(mockFilename, mockFileContent);
|
|
78
|
+
expect(result.chunks).toHaveLength(2);
|
|
79
|
+
expect(result.chunks[0]).toMatchObject({
|
|
80
|
+
id: 'chunk-1',
|
|
81
|
+
index: 0,
|
|
82
|
+
metadata: { source: 'test' },
|
|
83
|
+
text: 'Test content chunk 1',
|
|
84
|
+
type: 'LangChainElement',
|
|
85
|
+
});
|
|
86
|
+
expect(result.unstructuredChunks).toBeUndefined();
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('should use langchain when unstructured is not configured', async () => {
|
|
90
|
+
// Temporarily mock env to disable unstructured
|
|
91
|
+
vi.mocked(knowledgeEnv).UNSTRUCTURED_API_KEY = '';
|
|
92
|
+
|
|
93
|
+
const mockLangChainResult = [
|
|
94
|
+
{
|
|
95
|
+
id: 'chunk-1',
|
|
96
|
+
metadata: { source: 'test' },
|
|
97
|
+
pageContent: 'LangChain content',
|
|
98
|
+
},
|
|
99
|
+
];
|
|
100
|
+
|
|
101
|
+
mockLangChainPartition.mockResolvedValue(mockLangChainResult);
|
|
102
|
+
|
|
103
|
+
const result = await contentChunk.chunkContent({
|
|
104
|
+
content: mockFileContent,
|
|
105
|
+
fileType: 'application/pdf',
|
|
106
|
+
filename: mockFilename,
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
expect(mockLangChainPartition).toHaveBeenCalledWith(mockFilename, mockFileContent);
|
|
110
|
+
expect(result.chunks).toHaveLength(1);
|
|
111
|
+
expect(result.chunks[0].text).toBe('LangChain content');
|
|
112
|
+
expect(result.unstructuredChunks).toBeUndefined();
|
|
113
|
+
|
|
114
|
+
// Restore mock
|
|
115
|
+
vi.mocked(knowledgeEnv).UNSTRUCTURED_API_KEY = 'test-api-key';
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
it('should handle langchain results with metadata', async () => {
|
|
119
|
+
const mockLangChainResult = [
|
|
120
|
+
{
|
|
121
|
+
id: 'chunk-1',
|
|
122
|
+
metadata: {
|
|
123
|
+
source: 'test-document.pdf',
|
|
124
|
+
page: 1,
|
|
125
|
+
loc: { lines: { from: 1, to: 10 } },
|
|
126
|
+
},
|
|
127
|
+
pageContent: 'First paragraph content',
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
id: 'chunk-2',
|
|
131
|
+
metadata: {
|
|
132
|
+
source: 'test-document.pdf',
|
|
133
|
+
page: 2,
|
|
134
|
+
},
|
|
135
|
+
pageContent: 'Second paragraph content',
|
|
136
|
+
},
|
|
137
|
+
];
|
|
138
|
+
|
|
139
|
+
mockLangChainPartition.mockResolvedValue(mockLangChainResult);
|
|
140
|
+
|
|
141
|
+
const result = await contentChunk.chunkContent({
|
|
142
|
+
content: mockFileContent,
|
|
143
|
+
fileType: 'application/pdf',
|
|
144
|
+
filename: mockFilename,
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
expect(result.chunks).toHaveLength(2);
|
|
148
|
+
expect(result.chunks[0]).toMatchObject({
|
|
149
|
+
id: 'chunk-1',
|
|
150
|
+
index: 0,
|
|
151
|
+
metadata: {
|
|
152
|
+
source: 'test-document.pdf',
|
|
153
|
+
page: 1,
|
|
154
|
+
loc: { lines: { from: 1, to: 10 } },
|
|
155
|
+
},
|
|
156
|
+
text: 'First paragraph content',
|
|
157
|
+
type: 'LangChainElement',
|
|
158
|
+
});
|
|
159
|
+
expect(result.chunks[1]).toMatchObject({
|
|
160
|
+
id: 'chunk-2',
|
|
161
|
+
index: 1,
|
|
162
|
+
text: 'Second paragraph content',
|
|
163
|
+
type: 'LangChainElement',
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
it('should handle different file types', async () => {
|
|
168
|
+
const mockLangChainResult = [
|
|
169
|
+
{
|
|
170
|
+
id: 'docx-chunk-1',
|
|
171
|
+
metadata: { source: 'test.docx' },
|
|
172
|
+
pageContent: 'Word document content',
|
|
173
|
+
},
|
|
174
|
+
];
|
|
175
|
+
|
|
176
|
+
mockLangChainPartition.mockResolvedValue(mockLangChainResult);
|
|
177
|
+
|
|
178
|
+
const result = await contentChunk.chunkContent({
|
|
179
|
+
content: mockFileContent,
|
|
180
|
+
fileType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
181
|
+
filename: 'test.docx',
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
expect(mockLangChainPartition).toHaveBeenCalledWith('test.docx', mockFileContent);
|
|
185
|
+
expect(result.chunks).toHaveLength(1);
|
|
186
|
+
expect(result.chunks[0].text).toBe('Word document content');
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it('should throw error when all services fail and its the last service', async () => {
|
|
190
|
+
mockLangChainPartition.mockRejectedValue(new Error('LangChain error'));
|
|
191
|
+
|
|
192
|
+
await expect(
|
|
193
|
+
contentChunk.chunkContent({
|
|
194
|
+
content: mockFileContent,
|
|
195
|
+
fileType: 'application/pdf',
|
|
196
|
+
filename: mockFilename,
|
|
197
|
+
}),
|
|
198
|
+
).rejects.toThrow('LangChain error');
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it('should handle empty langchain results', async () => {
|
|
202
|
+
mockLangChainPartition.mockResolvedValue([]);
|
|
203
|
+
|
|
204
|
+
const result = await contentChunk.chunkContent({
|
|
205
|
+
content: mockFileContent,
|
|
206
|
+
fileType: 'application/pdf',
|
|
207
|
+
filename: mockFilename,
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
expect(result.chunks).toHaveLength(0);
|
|
211
|
+
expect(result.unstructuredChunks).toBeUndefined();
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
it('should extract file extension correctly from MIME type', async () => {
|
|
215
|
+
const mockLangChainResult = [
|
|
216
|
+
{
|
|
217
|
+
id: 'chunk-1',
|
|
218
|
+
metadata: {},
|
|
219
|
+
pageContent: 'Content',
|
|
220
|
+
},
|
|
221
|
+
];
|
|
222
|
+
|
|
223
|
+
mockLangChainPartition.mockResolvedValue(mockLangChainResult);
|
|
224
|
+
|
|
225
|
+
await contentChunk.chunkContent({
|
|
226
|
+
content: mockFileContent,
|
|
227
|
+
fileType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
228
|
+
filename: 'test.docx',
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
expect(mockLangChainPartition).toHaveBeenCalledWith('test.docx', mockFileContent);
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
it('should handle langchain results with minimal metadata', async () => {
|
|
235
|
+
const mockLangChainResult = [
|
|
236
|
+
{
|
|
237
|
+
id: 'chunk-minimal',
|
|
238
|
+
metadata: {},
|
|
239
|
+
pageContent: 'Content with no metadata',
|
|
240
|
+
},
|
|
241
|
+
];
|
|
242
|
+
|
|
243
|
+
mockLangChainPartition.mockResolvedValue(mockLangChainResult);
|
|
244
|
+
|
|
245
|
+
const result = await contentChunk.chunkContent({
|
|
246
|
+
content: mockFileContent,
|
|
247
|
+
fileType: 'text/plain',
|
|
248
|
+
filename: 'test.txt',
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
expect(result.chunks[0]).toMatchObject({
|
|
252
|
+
id: 'chunk-minimal',
|
|
253
|
+
index: 0,
|
|
254
|
+
metadata: {},
|
|
255
|
+
text: 'Content with no metadata',
|
|
256
|
+
type: 'LangChainElement',
|
|
257
|
+
});
|
|
258
|
+
});
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
describe('canUseUnstructured', () => {
|
|
262
|
+
it('should return true when API key and server URL are configured', () => {
|
|
263
|
+
const result = contentChunk['canUseUnstructured']();
|
|
264
|
+
expect(result).toBe(true);
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
it('should return false when API key is missing', () => {
|
|
268
|
+
const originalKey = knowledgeEnv.UNSTRUCTURED_API_KEY;
|
|
269
|
+
vi.mocked(knowledgeEnv).UNSTRUCTURED_API_KEY = '';
|
|
270
|
+
|
|
271
|
+
const result = contentChunk['canUseUnstructured']();
|
|
272
|
+
expect(result).toBe(false);
|
|
273
|
+
|
|
274
|
+
// Restore
|
|
275
|
+
vi.mocked(knowledgeEnv).UNSTRUCTURED_API_KEY = originalKey;
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
it('should return false when server URL is missing', () => {
|
|
279
|
+
const originalUrl = knowledgeEnv.UNSTRUCTURED_SERVER_URL;
|
|
280
|
+
vi.mocked(knowledgeEnv).UNSTRUCTURED_SERVER_URL = '';
|
|
281
|
+
|
|
282
|
+
const result = contentChunk['canUseUnstructured']();
|
|
283
|
+
expect(result).toBe(false);
|
|
284
|
+
|
|
285
|
+
// Restore
|
|
286
|
+
vi.mocked(knowledgeEnv).UNSTRUCTURED_SERVER_URL = originalUrl;
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
describe('getChunkingServices', () => {
|
|
291
|
+
it('should return default service for unknown file type', () => {
|
|
292
|
+
const services = contentChunk['getChunkingServices']('application/unknown');
|
|
293
|
+
expect(services).toEqual(['default']);
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
it('should extract extension from MIME type correctly', () => {
|
|
297
|
+
const services = contentChunk['getChunkingServices']('application/pdf');
|
|
298
|
+
expect(services).toEqual(['default']);
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
it('should handle MIME types with multiple slashes', () => {
|
|
302
|
+
const services = contentChunk['getChunkingServices'](
|
|
303
|
+
'application/vnd.openxmlformats-officedocument/wordprocessingml.document',
|
|
304
|
+
);
|
|
305
|
+
expect(services).toEqual(['default']);
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
it('should convert extension to lowercase', () => {
|
|
309
|
+
const services = contentChunk['getChunkingServices']('application/PDF');
|
|
310
|
+
expect(services).toEqual(['default']);
|
|
311
|
+
});
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
describe('integration scenarios', () => {
|
|
315
|
+
it('should handle multiple chunk items with correct indices', async () => {
|
|
316
|
+
const mockLangChainResult = Array.from({ length: 5 }, (_, i) => ({
|
|
317
|
+
id: `chunk-${i}`,
|
|
318
|
+
metadata: { index: i },
|
|
319
|
+
pageContent: `Content ${i}`,
|
|
320
|
+
}));
|
|
321
|
+
|
|
322
|
+
mockLangChainPartition.mockResolvedValue(mockLangChainResult);
|
|
323
|
+
|
|
324
|
+
const result = await contentChunk.chunkContent({
|
|
325
|
+
content: new Uint8Array([1, 2, 3]),
|
|
326
|
+
fileType: 'text/plain',
|
|
327
|
+
filename: 'test.txt',
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
expect(result.chunks).toHaveLength(5);
|
|
331
|
+
result.chunks.forEach((chunk, index) => {
|
|
332
|
+
expect(chunk.index).toBe(index);
|
|
333
|
+
expect(chunk.text).toBe(`Content ${index}`);
|
|
334
|
+
});
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
it('should preserve order of chunks from langchain response', async () => {
|
|
338
|
+
const mockLangChainResult = [
|
|
339
|
+
{
|
|
340
|
+
id: 'elem-3',
|
|
341
|
+
metadata: { source: 'test.txt' },
|
|
342
|
+
pageContent: 'Third',
|
|
343
|
+
},
|
|
344
|
+
{
|
|
345
|
+
id: 'elem-1',
|
|
346
|
+
metadata: { source: 'test.txt' },
|
|
347
|
+
pageContent: 'First',
|
|
348
|
+
},
|
|
349
|
+
{
|
|
350
|
+
id: 'elem-2',
|
|
351
|
+
metadata: { source: 'test.txt' },
|
|
352
|
+
pageContent: 'Second',
|
|
353
|
+
},
|
|
354
|
+
];
|
|
355
|
+
|
|
356
|
+
mockLangChainPartition.mockResolvedValue(mockLangChainResult);
|
|
357
|
+
|
|
358
|
+
const result = await contentChunk.chunkContent({
|
|
359
|
+
content: new Uint8Array([1, 2, 3]),
|
|
360
|
+
fileType: 'text/plain',
|
|
361
|
+
filename: 'test.txt',
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
expect(result.chunks[0].text).toBe('Third');
|
|
365
|
+
expect(result.chunks[1].text).toBe('First');
|
|
366
|
+
expect(result.chunks[2].text).toBe('Second');
|
|
367
|
+
expect(result.chunks[0].index).toBe(0);
|
|
368
|
+
expect(result.chunks[1].index).toBe(1);
|
|
369
|
+
expect(result.chunks[2].index).toBe(2);
|
|
370
|
+
});
|
|
371
|
+
});
|
|
372
|
+
});
|