@lobehub/lobehub 2.0.0-next.115 → 2.0.0-next.117
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/changelog/v1.json +18 -0
- package/package.json +1 -1
- package/packages/context-engine/src/processors/MessageContent.ts +100 -6
- package/packages/context-engine/src/processors/__tests__/MessageContent.test.ts +239 -0
- package/packages/fetch-sse/src/fetchSSE.ts +30 -0
- package/packages/model-bank/src/aiModels/bedrock.ts +30 -2
- package/packages/model-runtime/src/const/models.ts +62 -24
- package/packages/model-runtime/src/core/contextBuilders/anthropic.ts +14 -0
- package/packages/model-runtime/src/core/contextBuilders/google.test.ts +78 -24
- package/packages/model-runtime/src/core/contextBuilders/google.ts +10 -2
- package/packages/model-runtime/src/core/parameterResolver.test.ts +34 -50
- package/packages/model-runtime/src/core/parameterResolver.ts +0 -41
- package/packages/model-runtime/src/core/streams/google/google-ai.test.ts +451 -20
- package/packages/model-runtime/src/core/streams/google/index.ts +113 -3
- package/packages/model-runtime/src/core/streams/protocol.ts +19 -0
- package/packages/model-runtime/src/index.ts +1 -0
- package/packages/model-runtime/src/providers/anthropic/index.ts +20 -32
- package/packages/model-runtime/src/providers/anthropic/resolveMaxTokens.ts +35 -0
- package/packages/model-runtime/src/providers/bedrock/index.test.ts +5 -7
- package/packages/model-runtime/src/providers/bedrock/index.ts +50 -11
- package/packages/types/src/message/common/base.ts +26 -0
- package/packages/types/src/message/common/metadata.ts +7 -0
- package/packages/utils/src/index.ts +1 -0
- package/packages/utils/src/multimodalContent.ts +25 -0
- package/src/components/Thinking/index.tsx +3 -3
- package/src/features/ChatList/Messages/Assistant/DisplayContent.tsx +44 -0
- package/src/features/ChatList/Messages/Assistant/MessageBody.tsx +96 -0
- package/src/features/ChatList/Messages/Assistant/Reasoning/index.tsx +26 -13
- package/src/features/ChatList/Messages/Assistant/index.tsx +8 -6
- package/src/features/ChatList/Messages/Default.tsx +4 -7
- package/src/features/ChatList/components/RichContentRenderer.tsx +35 -0
- package/src/store/agent/slices/chat/selectors/chatConfig.ts +4 -3
- package/src/store/chat/slices/aiChat/actions/streamingExecutor.ts +244 -17
- package/packages/const/src/models.ts +0 -93
- package/src/features/ChatList/Messages/Assistant/MessageContent.tsx +0 -78
|
@@ -77,6 +77,10 @@ export interface StreamProtocolChunk {
|
|
|
77
77
|
| 'reasoning_signature'
|
|
78
78
|
// flagged reasoning signature
|
|
79
79
|
| 'flagged_reasoning_signature'
|
|
80
|
+
// multimodal content part in reasoning
|
|
81
|
+
| 'reasoning_part'
|
|
82
|
+
// multimodal content part in content
|
|
83
|
+
| 'content_part'
|
|
80
84
|
// Search or Grounding
|
|
81
85
|
| 'grounding'
|
|
82
86
|
// stop signal
|
|
@@ -91,6 +95,21 @@ export interface StreamProtocolChunk {
|
|
|
91
95
|
| 'data';
|
|
92
96
|
}
|
|
93
97
|
|
|
98
|
+
/**
|
|
99
|
+
* Stream content part chunk data for multimodal support
|
|
100
|
+
*/
|
|
101
|
+
export interface StreamPartChunkData {
|
|
102
|
+
content: string;
|
|
103
|
+
// whether this part is in reasoning or regular content
|
|
104
|
+
inReasoning: boolean;
|
|
105
|
+
// image MIME type
|
|
106
|
+
mimeType?: string;
|
|
107
|
+
// text content or base64 image data
|
|
108
|
+
partType: 'text' | 'image';
|
|
109
|
+
// Optional signature for reasoning verification (Google Gemini feature)
|
|
110
|
+
thoughtSignature?: string;
|
|
111
|
+
}
|
|
112
|
+
|
|
94
113
|
export interface StreamToolCallChunkData {
|
|
95
114
|
function?: {
|
|
96
115
|
arguments?: string;
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import Anthropic, { ClientOptions } from '@anthropic-ai/sdk';
|
|
2
2
|
import { ModelProvider } from 'model-bank';
|
|
3
3
|
|
|
4
|
+
import { hasTemperatureTopPConflict } from '../../const/models';
|
|
4
5
|
import { LobeRuntimeAI } from '../../core/BaseAI';
|
|
5
|
-
import {
|
|
6
|
-
|
|
6
|
+
import {
|
|
7
|
+
buildAnthropicMessages,
|
|
8
|
+
buildAnthropicTools,
|
|
9
|
+
buildSearchTool,
|
|
10
|
+
} from '../../core/contextBuilders/anthropic';
|
|
11
|
+
import { resolveParameters } from '../../core/parameterResolver';
|
|
7
12
|
import { AnthropicStream } from '../../core/streams';
|
|
8
13
|
import {
|
|
9
14
|
type ChatCompletionErrorPayload,
|
|
@@ -22,6 +27,7 @@ import { StreamingResponse } from '../../utils/response';
|
|
|
22
27
|
import { createAnthropicGenerateObject } from './generateObject';
|
|
23
28
|
import { handleAnthropicError } from './handleAnthropicError';
|
|
24
29
|
import { resolveCacheTTL } from './resolveCacheTTL';
|
|
30
|
+
import { resolveMaxTokens } from './resolveMaxTokens';
|
|
25
31
|
|
|
26
32
|
export interface AnthropicModelCard {
|
|
27
33
|
created_at: string;
|
|
@@ -31,8 +37,6 @@ export interface AnthropicModelCard {
|
|
|
31
37
|
|
|
32
38
|
type anthropicTools = Anthropic.Tool | Anthropic.WebSearchTool20250305;
|
|
33
39
|
|
|
34
|
-
const modelsWithSmallContextWindow = new Set(['claude-3-opus-20240229', 'claude-3-haiku-20240307']);
|
|
35
|
-
|
|
36
40
|
const DEFAULT_BASE_URL = 'https://api.anthropic.com';
|
|
37
41
|
|
|
38
42
|
interface AnthropicAIParams extends ClientOptions {
|
|
@@ -140,15 +144,13 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
|
|
|
140
144
|
} = payload;
|
|
141
145
|
|
|
142
146
|
const { anthropic: anthropicModels } = await import('model-bank');
|
|
143
|
-
const modelConfig = anthropicModels.find((m) => m.id === model);
|
|
144
|
-
const defaultMaxOutput = modelConfig?.maxOutput;
|
|
145
147
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
};
|
|
148
|
+
const resolvedMaxTokens = await resolveMaxTokens({
|
|
149
|
+
max_tokens,
|
|
150
|
+
model,
|
|
151
|
+
providerModels: anthropicModels,
|
|
152
|
+
thinking,
|
|
153
|
+
});
|
|
152
154
|
|
|
153
155
|
const system_message = messages.find((m) => m.role === 'system');
|
|
154
156
|
const user_messages = messages.filter((m) => m.role !== 'system');
|
|
@@ -170,20 +172,8 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
|
|
|
170
172
|
});
|
|
171
173
|
|
|
172
174
|
if (enabledSearch) {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
const webSearchTool: Anthropic.WebSearchTool20250305 = {
|
|
177
|
-
name: 'web_search',
|
|
178
|
-
type: 'web_search_20250305',
|
|
179
|
-
...(maxUses &&
|
|
180
|
-
Number.isInteger(Number(maxUses)) &&
|
|
181
|
-
Number(maxUses) > 0 && {
|
|
182
|
-
max_uses: Number(maxUses),
|
|
183
|
-
}),
|
|
184
|
-
};
|
|
185
|
-
|
|
186
|
-
// 如果已有工具,则添加到现有工具列表中;否则创建新的工具列表
|
|
175
|
+
const webSearchTool = buildSearchTool();
|
|
176
|
+
|
|
187
177
|
if (postTools && postTools.length > 0) {
|
|
188
178
|
postTools = [...postTools, webSearchTool];
|
|
189
179
|
} else {
|
|
@@ -192,19 +182,17 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
|
|
|
192
182
|
}
|
|
193
183
|
|
|
194
184
|
if (!!thinking && thinking.type === 'enabled') {
|
|
195
|
-
const maxTokens = getMaxTokens() || 32_000; // Claude Opus 4 has minimum maxOutput
|
|
196
|
-
|
|
197
185
|
// `temperature` may only be set to 1 when thinking is enabled.
|
|
198
186
|
// `top_p` must be unset when thinking is enabled.
|
|
199
187
|
return {
|
|
200
|
-
max_tokens:
|
|
188
|
+
max_tokens: resolvedMaxTokens,
|
|
201
189
|
messages: postMessages,
|
|
202
190
|
model,
|
|
203
191
|
system: systemPrompts,
|
|
204
192
|
thinking: {
|
|
205
193
|
...thinking,
|
|
206
194
|
budget_tokens: thinking?.budget_tokens
|
|
207
|
-
? Math.min(thinking.budget_tokens,
|
|
195
|
+
? Math.min(thinking.budget_tokens, resolvedMaxTokens - 1) // `max_tokens` must be greater than `thinking.budget_tokens`.
|
|
208
196
|
: 1024,
|
|
209
197
|
},
|
|
210
198
|
tools: postTools,
|
|
@@ -212,7 +200,7 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
|
|
|
212
200
|
}
|
|
213
201
|
|
|
214
202
|
// Resolve temperature and top_p parameters based on model constraints
|
|
215
|
-
const hasConflict =
|
|
203
|
+
const hasConflict = hasTemperatureTopPConflict(model);
|
|
216
204
|
const resolvedParams = resolveParameters(
|
|
217
205
|
{ temperature, top_p },
|
|
218
206
|
{ hasConflict, normalizeTemperature: true, preferTemperature: true },
|
|
@@ -221,7 +209,7 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
|
|
|
221
209
|
return {
|
|
222
210
|
// claude 3 series model hax max output token of 4096, 3.x series has 8192
|
|
223
211
|
// https://docs.anthropic.com/en/docs/about-claude/models/all-models#:~:text=200K-,Max%20output,-Normal%3A
|
|
224
|
-
max_tokens:
|
|
212
|
+
max_tokens: resolvedMaxTokens,
|
|
225
213
|
messages: postMessages,
|
|
226
214
|
model,
|
|
227
215
|
system: systemPrompts,
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { ChatStreamPayload } from '../../types';
|
|
2
|
+
|
|
3
|
+
const smallContextWindowPatterns = [
|
|
4
|
+
/claude-3-opus-20240229/,
|
|
5
|
+
/claude-3-haiku-20240307/,
|
|
6
|
+
/claude-v2(:1)?$/,
|
|
7
|
+
];
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Resolve the max_tokens value to align Anthropic and Bedrock behavior.
|
|
11
|
+
* Priority: user input > model-bank default maxOutput > hardcoded fallback (context-window aware).
|
|
12
|
+
*/
|
|
13
|
+
export const resolveMaxTokens = async ({
|
|
14
|
+
max_tokens,
|
|
15
|
+
model,
|
|
16
|
+
thinking,
|
|
17
|
+
providerModels,
|
|
18
|
+
}: {
|
|
19
|
+
max_tokens?: number;
|
|
20
|
+
model: string;
|
|
21
|
+
providerModels: { id: string; maxOutput?: number }[];
|
|
22
|
+
thinking?: ChatStreamPayload['thinking'];
|
|
23
|
+
}) => {
|
|
24
|
+
const defaultMaxOutput = providerModels.find((m) => m.id === model)?.maxOutput;
|
|
25
|
+
|
|
26
|
+
const preferredMaxTokens = max_tokens ?? defaultMaxOutput;
|
|
27
|
+
|
|
28
|
+
if (preferredMaxTokens) return preferredMaxTokens;
|
|
29
|
+
|
|
30
|
+
if (thinking?.type === 'enabled') return 32_000;
|
|
31
|
+
|
|
32
|
+
const hasSmallContextWindow = smallContextWindowPatterns.some((pattern) => pattern.test(model));
|
|
33
|
+
|
|
34
|
+
return hasSmallContextWindow ? 4096 : 8192;
|
|
35
|
+
};
|
|
@@ -10,8 +10,6 @@ import { AgentRuntimeErrorType } from '../../types/error';
|
|
|
10
10
|
import * as debugStreamModule from '../../utils/debugStream';
|
|
11
11
|
import { LobeBedrockAI, experimental_buildLlama2Prompt } from './index';
|
|
12
12
|
|
|
13
|
-
const provider = 'bedrock';
|
|
14
|
-
|
|
15
13
|
// Mock the console.error to avoid polluting test output
|
|
16
14
|
vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
17
15
|
|
|
@@ -478,7 +476,7 @@ describe('LobeBedrockAI', () => {
|
|
|
478
476
|
accept: 'application/json',
|
|
479
477
|
body: JSON.stringify({
|
|
480
478
|
anthropic_version: 'bedrock-2023-05-31',
|
|
481
|
-
max_tokens:
|
|
479
|
+
max_tokens: 8192,
|
|
482
480
|
messages: [
|
|
483
481
|
{
|
|
484
482
|
content: [
|
|
@@ -521,7 +519,7 @@ describe('LobeBedrockAI', () => {
|
|
|
521
519
|
accept: 'application/json',
|
|
522
520
|
body: JSON.stringify({
|
|
523
521
|
anthropic_version: 'bedrock-2023-05-31',
|
|
524
|
-
max_tokens:
|
|
522
|
+
max_tokens: 8192,
|
|
525
523
|
messages: [
|
|
526
524
|
{
|
|
527
525
|
content: [
|
|
@@ -565,7 +563,7 @@ describe('LobeBedrockAI', () => {
|
|
|
565
563
|
accept: 'application/json',
|
|
566
564
|
body: JSON.stringify({
|
|
567
565
|
anthropic_version: 'bedrock-2023-05-31',
|
|
568
|
-
max_tokens:
|
|
566
|
+
max_tokens: 8192,
|
|
569
567
|
messages: [
|
|
570
568
|
{
|
|
571
569
|
content: [
|
|
@@ -610,7 +608,7 @@ describe('LobeBedrockAI', () => {
|
|
|
610
608
|
accept: 'application/json',
|
|
611
609
|
body: JSON.stringify({
|
|
612
610
|
anthropic_version: 'bedrock-2023-05-31',
|
|
613
|
-
max_tokens:
|
|
611
|
+
max_tokens: 64_000,
|
|
614
612
|
messages: [
|
|
615
613
|
{
|
|
616
614
|
content: [
|
|
@@ -654,7 +652,7 @@ describe('LobeBedrockAI', () => {
|
|
|
654
652
|
accept: 'application/json',
|
|
655
653
|
body: JSON.stringify({
|
|
656
654
|
anthropic_version: 'bedrock-2023-05-31',
|
|
657
|
-
max_tokens:
|
|
655
|
+
max_tokens: 8192,
|
|
658
656
|
messages: [
|
|
659
657
|
{
|
|
660
658
|
content: [
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type Anthropic from '@anthropic-ai/sdk';
|
|
1
2
|
import {
|
|
2
3
|
BedrockRuntimeClient,
|
|
3
4
|
InvokeModelCommand,
|
|
@@ -5,9 +6,10 @@ import {
|
|
|
5
6
|
} from '@aws-sdk/client-bedrock-runtime';
|
|
6
7
|
import { ModelProvider } from 'model-bank';
|
|
7
8
|
|
|
9
|
+
import { hasTemperatureTopPConflict } from '../../const/models';
|
|
8
10
|
import { LobeRuntimeAI } from '../../core/BaseAI';
|
|
9
11
|
import { buildAnthropicMessages, buildAnthropicTools } from '../../core/contextBuilders/anthropic';
|
|
10
|
-
import {
|
|
12
|
+
import { resolveParameters } from '../../core/parameterResolver';
|
|
11
13
|
import {
|
|
12
14
|
AWSBedrockClaudeStream,
|
|
13
15
|
AWSBedrockLlamaStream,
|
|
@@ -26,6 +28,7 @@ import { debugStream } from '../../utils/debugStream';
|
|
|
26
28
|
import { getModelPricing } from '../../utils/getModelPricing';
|
|
27
29
|
import { StreamingResponse } from '../../utils/response';
|
|
28
30
|
import { resolveCacheTTL } from '../anthropic/resolveCacheTTL';
|
|
31
|
+
import { resolveMaxTokens } from '../anthropic/resolveMaxTokens';
|
|
29
32
|
|
|
30
33
|
/**
|
|
31
34
|
* A prompt constructor for HuggingFace LLama 2 chat models.
|
|
@@ -62,19 +65,24 @@ export function experimental_buildLlama2Prompt(messages: { content: string; role
|
|
|
62
65
|
export interface LobeBedrockAIParams {
|
|
63
66
|
accessKeyId?: string;
|
|
64
67
|
accessKeySecret?: string;
|
|
68
|
+
id?: string;
|
|
65
69
|
region?: string;
|
|
66
70
|
sessionToken?: string;
|
|
67
71
|
}
|
|
68
72
|
|
|
69
73
|
export class LobeBedrockAI implements LobeRuntimeAI {
|
|
70
74
|
private client: BedrockRuntimeClient;
|
|
75
|
+
private id: string;
|
|
71
76
|
|
|
72
77
|
region: string;
|
|
73
78
|
|
|
74
|
-
constructor(
|
|
79
|
+
constructor(options: LobeBedrockAIParams = {}) {
|
|
80
|
+
const { id, region, accessKeyId, accessKeySecret, sessionToken } = options;
|
|
81
|
+
|
|
75
82
|
if (!(accessKeyId && accessKeySecret))
|
|
76
83
|
throw AgentRuntimeError.createError(AgentRuntimeErrorType.InvalidBedrockCredentials);
|
|
77
84
|
this.region = region ?? 'us-east-1';
|
|
85
|
+
this.id = id ?? ModelProvider.Bedrock;
|
|
78
86
|
this.client = new BedrockRuntimeClient({
|
|
79
87
|
credentials: {
|
|
80
88
|
accessKeyId: accessKeyId,
|
|
@@ -158,18 +166,28 @@ export class LobeBedrockAI implements LobeRuntimeAI {
|
|
|
158
166
|
temperature,
|
|
159
167
|
top_p,
|
|
160
168
|
tools,
|
|
169
|
+
thinking,
|
|
161
170
|
} = payload;
|
|
162
171
|
const inputStartAt = Date.now();
|
|
163
172
|
const system_message = messages.find((m) => m.role === 'system');
|
|
164
173
|
const user_messages = messages.filter((m) => m.role !== 'system');
|
|
165
174
|
|
|
166
175
|
// Resolve temperature and top_p parameters based on model constraints
|
|
167
|
-
const hasConflict =
|
|
176
|
+
const hasConflict = hasTemperatureTopPConflict(model);
|
|
168
177
|
const resolvedParams = resolveParameters(
|
|
169
178
|
{ temperature, top_p },
|
|
170
179
|
{ hasConflict, normalizeTemperature: true, preferTemperature: true },
|
|
171
180
|
);
|
|
172
181
|
|
|
182
|
+
const { bedrock: bedrockModels } = await import('model-bank');
|
|
183
|
+
|
|
184
|
+
const resolvedMaxTokens = await resolveMaxTokens({
|
|
185
|
+
max_tokens,
|
|
186
|
+
model,
|
|
187
|
+
providerModels: bedrockModels,
|
|
188
|
+
thinking,
|
|
189
|
+
});
|
|
190
|
+
|
|
173
191
|
const systemPrompts = !!system_message?.content
|
|
174
192
|
? ([
|
|
175
193
|
{
|
|
@@ -177,19 +195,40 @@ export class LobeBedrockAI implements LobeRuntimeAI {
|
|
|
177
195
|
text: system_message.content as string,
|
|
178
196
|
type: 'text',
|
|
179
197
|
},
|
|
180
|
-
] as
|
|
198
|
+
] as Anthropic.TextBlockParam[])
|
|
181
199
|
: undefined;
|
|
182
200
|
|
|
183
|
-
const
|
|
201
|
+
const postTools = buildAnthropicTools(tools, {
|
|
202
|
+
enabledContextCaching,
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
const anthropicBase = {
|
|
184
206
|
anthropic_version: 'bedrock-2023-05-31',
|
|
185
|
-
max_tokens:
|
|
207
|
+
max_tokens: resolvedMaxTokens,
|
|
186
208
|
messages: await buildAnthropicMessages(user_messages, { enabledContextCaching }),
|
|
187
209
|
system: systemPrompts,
|
|
188
|
-
|
|
189
|
-
tools: buildAnthropicTools(tools, { enabledContextCaching }),
|
|
190
|
-
top_p: resolvedParams.top_p,
|
|
210
|
+
tools: postTools,
|
|
191
211
|
};
|
|
192
212
|
|
|
213
|
+
const anthropicPayload =
|
|
214
|
+
thinking?.type === 'enabled'
|
|
215
|
+
? {
|
|
216
|
+
...anthropicBase,
|
|
217
|
+
thinking: {
|
|
218
|
+
...thinking,
|
|
219
|
+
// `max_tokens` must be greater than `budget_tokens`
|
|
220
|
+
budget_tokens: Math.max(
|
|
221
|
+
1,
|
|
222
|
+
Math.min(thinking.budget_tokens || 1024, resolvedMaxTokens - 1),
|
|
223
|
+
),
|
|
224
|
+
},
|
|
225
|
+
}
|
|
226
|
+
: {
|
|
227
|
+
...anthropicBase,
|
|
228
|
+
temperature: resolvedParams.temperature,
|
|
229
|
+
top_p: resolvedParams.top_p,
|
|
230
|
+
};
|
|
231
|
+
|
|
193
232
|
const command = new InvokeModelWithResponseStreamCommand({
|
|
194
233
|
accept: 'application/json',
|
|
195
234
|
body: JSON.stringify(anthropicPayload),
|
|
@@ -209,7 +248,7 @@ export class LobeBedrockAI implements LobeRuntimeAI {
|
|
|
209
248
|
debugStream(debug).catch(console.error);
|
|
210
249
|
}
|
|
211
250
|
|
|
212
|
-
const pricing = await getModelPricing(payload.model,
|
|
251
|
+
const pricing = await getModelPricing(payload.model, this.id);
|
|
213
252
|
const cacheTTL = resolveCacheTTL({ ...payload, enabledContextCaching }, anthropicPayload);
|
|
214
253
|
const pricingOptions = cacheTTL ? { lookupParams: { ttl: cacheTTL } } : undefined;
|
|
215
254
|
|
|
@@ -218,7 +257,7 @@ export class LobeBedrockAI implements LobeRuntimeAI {
|
|
|
218
257
|
AWSBedrockClaudeStream(prod, {
|
|
219
258
|
callbacks: options?.callback,
|
|
220
259
|
inputStartAt,
|
|
221
|
-
payload: { model, pricing, pricingOptions, provider:
|
|
260
|
+
payload: { model, pricing, pricingOptions, provider: this.id },
|
|
222
261
|
}),
|
|
223
262
|
{
|
|
224
263
|
headers: options?.headers,
|
|
@@ -26,14 +26,40 @@ export interface ChatCitationItem {
|
|
|
26
26
|
url: string;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
/**
|
|
30
|
+
* Message content part types for multimodal content support
|
|
31
|
+
*/
|
|
32
|
+
export interface MessageContentPartText {
|
|
33
|
+
text: string;
|
|
34
|
+
thoughtSignature?: string;
|
|
35
|
+
type: 'text';
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface MessageContentPartImage {
|
|
39
|
+
image: string;
|
|
40
|
+
thoughtSignature?: string;
|
|
41
|
+
type: 'image';
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export type MessageContentPart = MessageContentPartText | MessageContentPartImage;
|
|
45
|
+
|
|
29
46
|
export interface ModelReasoning {
|
|
47
|
+
/**
|
|
48
|
+
* Reasoning content, can be plain string or serialized JSON array of MessageContentPart[]
|
|
49
|
+
*/
|
|
30
50
|
content?: string;
|
|
31
51
|
duration?: number;
|
|
52
|
+
/**
|
|
53
|
+
* Flag indicating if content is multimodal (serialized MessageContentPart[])
|
|
54
|
+
*/
|
|
55
|
+
isMultimodal?: boolean;
|
|
32
56
|
signature?: string;
|
|
57
|
+
tempDisplayContent?: MessageContentPart[];
|
|
33
58
|
}
|
|
34
59
|
|
|
35
60
|
export const ModelReasoningSchema = z.object({
|
|
36
61
|
content: z.string().optional(),
|
|
37
62
|
duration: z.number().optional(),
|
|
63
|
+
isMultimodal: z.boolean().optional(),
|
|
38
64
|
signature: z.string().optional(),
|
|
39
65
|
});
|
|
@@ -78,6 +78,7 @@ export const ModelPerformanceSchema = z.object({
|
|
|
78
78
|
export const MessageMetadataSchema = ModelUsageSchema.merge(ModelPerformanceSchema).extend({
|
|
79
79
|
collapsed: z.boolean().optional(),
|
|
80
80
|
inspectExpanded: z.boolean().optional(),
|
|
81
|
+
isMultimodal: z.boolean().optional(),
|
|
81
82
|
});
|
|
82
83
|
|
|
83
84
|
export interface ModelUsage extends ModelTokensUsage {
|
|
@@ -123,4 +124,10 @@ export interface MessageMetadata extends ModelUsage, ModelPerformance {
|
|
|
123
124
|
compare?: boolean;
|
|
124
125
|
usage?: ModelUsage;
|
|
125
126
|
performance?: ModelPerformance;
|
|
127
|
+
/**
|
|
128
|
+
* Flag indicating if message content is multimodal (serialized MessageContentPart[])
|
|
129
|
+
*/
|
|
130
|
+
isMultimodal?: boolean;
|
|
131
|
+
// message content is multimodal, display content in the streaming, won't save to db
|
|
132
|
+
tempDisplayContent?: string;
|
|
126
133
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { MessageContentPart } from '@lobechat/types';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Serialize message content parts to JSON string for storage
|
|
5
|
+
*/
|
|
6
|
+
export function serializePartsForStorage(parts: MessageContentPart[]): string {
|
|
7
|
+
return JSON.stringify(parts);
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Deserialize content string to message content parts
|
|
12
|
+
* Returns null if content is not valid JSON array of parts
|
|
13
|
+
*/
|
|
14
|
+
export function deserializeParts(content: string): MessageContentPart[] | null {
|
|
15
|
+
try {
|
|
16
|
+
const parsed = JSON.parse(content);
|
|
17
|
+
// Validate it's an array with valid part structure
|
|
18
|
+
if (Array.isArray(parsed) && parsed.length > 0 && parsed[0]?.type) {
|
|
19
|
+
return parsed as MessageContentPart[];
|
|
20
|
+
}
|
|
21
|
+
} catch {
|
|
22
|
+
// Not JSON, treat as plain text
|
|
23
|
+
}
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
@@ -4,7 +4,7 @@ import { createStyles } from 'antd-style';
|
|
|
4
4
|
import { AnimatePresence, motion } from 'framer-motion';
|
|
5
5
|
import { AtomIcon } from 'lucide-react';
|
|
6
6
|
import { rgba } from 'polished';
|
|
7
|
-
import { CSSProperties, RefObject, memo, useEffect, useRef, useState } from 'react';
|
|
7
|
+
import { CSSProperties, ReactNode, RefObject, memo, useEffect, useRef, useState } from 'react';
|
|
8
8
|
import { useTranslation } from 'react-i18next';
|
|
9
9
|
import { Flexbox } from 'react-layout-kit';
|
|
10
10
|
|
|
@@ -76,7 +76,7 @@ const useStyles = createStyles(({ css, token }) => ({
|
|
|
76
76
|
|
|
77
77
|
interface ThinkingProps {
|
|
78
78
|
citations?: ChatCitationItem[];
|
|
79
|
-
content?: string;
|
|
79
|
+
content?: string | ReactNode;
|
|
80
80
|
duration?: number;
|
|
81
81
|
style?: CSSProperties;
|
|
82
82
|
thinking?: boolean;
|
|
@@ -158,7 +158,7 @@ const Thinking = memo<ThinkingProps>((props) => {
|
|
|
158
158
|
</Flexbox>
|
|
159
159
|
)}
|
|
160
160
|
<Flexbox gap={4} horizontal>
|
|
161
|
-
{showDetail && content && (
|
|
161
|
+
{showDetail && content && typeof content === 'string' && (
|
|
162
162
|
<div
|
|
163
163
|
onClick={(event) => {
|
|
164
164
|
event.stopPropagation();
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { deserializeParts } from '@lobechat/utils';
|
|
2
|
+
import { Markdown, MarkdownProps } from '@lobehub/ui';
|
|
3
|
+
import { memo } from 'react';
|
|
4
|
+
|
|
5
|
+
import BubblesLoading from '@/components/BubblesLoading';
|
|
6
|
+
import { LOADING_FLAT } from '@/const/message';
|
|
7
|
+
import { RichContentRenderer } from '@/features/ChatList/components/RichContentRenderer';
|
|
8
|
+
import { normalizeThinkTags, processWithArtifact } from '@/features/ChatList/utils/markdown';
|
|
9
|
+
|
|
10
|
+
const MessageContent = memo<{
|
|
11
|
+
addIdOnDOM?: boolean;
|
|
12
|
+
content: string;
|
|
13
|
+
hasImages?: boolean;
|
|
14
|
+
isMultimodal?: boolean;
|
|
15
|
+
isToolCallGenerating?: boolean;
|
|
16
|
+
markdownProps?: Omit<MarkdownProps, 'className' | 'style' | 'children'>;
|
|
17
|
+
tempDisplayContent?: string;
|
|
18
|
+
}>(
|
|
19
|
+
({
|
|
20
|
+
markdownProps,
|
|
21
|
+
content,
|
|
22
|
+
isToolCallGenerating,
|
|
23
|
+
hasImages,
|
|
24
|
+
isMultimodal,
|
|
25
|
+
tempDisplayContent,
|
|
26
|
+
}) => {
|
|
27
|
+
const message = normalizeThinkTags(processWithArtifact(content));
|
|
28
|
+
if (isToolCallGenerating) return;
|
|
29
|
+
|
|
30
|
+
if ((!content && !hasImages) || content === LOADING_FLAT) return <BubblesLoading />;
|
|
31
|
+
|
|
32
|
+
const contentParts = isMultimodal ? deserializeParts(tempDisplayContent || content) : null;
|
|
33
|
+
|
|
34
|
+
return contentParts ? (
|
|
35
|
+
<RichContentRenderer parts={contentParts} />
|
|
36
|
+
) : (
|
|
37
|
+
<Markdown {...markdownProps} variant={'chat'}>
|
|
38
|
+
{message}
|
|
39
|
+
</Markdown>
|
|
40
|
+
);
|
|
41
|
+
},
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
export default MessageContent;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { LOADING_FLAT } from '@lobechat/const';
|
|
2
|
+
import { UIChatMessage } from '@lobechat/types';
|
|
3
|
+
import { MarkdownProps } from '@lobehub/ui';
|
|
4
|
+
import { ReactNode, memo } from 'react';
|
|
5
|
+
import { Flexbox } from 'react-layout-kit';
|
|
6
|
+
|
|
7
|
+
import { useChatStore } from '@/store/chat';
|
|
8
|
+
import { aiChatSelectors, messageStateSelectors } from '@/store/chat/selectors';
|
|
9
|
+
|
|
10
|
+
import { DefaultMessage } from '../Default';
|
|
11
|
+
import ImageFileListViewer from '../User/ImageFileListViewer';
|
|
12
|
+
import { CollapsedMessage } from './CollapsedMessage';
|
|
13
|
+
import MessageContent from './DisplayContent';
|
|
14
|
+
import FileChunks from './FileChunks';
|
|
15
|
+
import IntentUnderstanding from './IntentUnderstanding';
|
|
16
|
+
import Reasoning from './Reasoning';
|
|
17
|
+
import SearchGrounding from './SearchGrounding';
|
|
18
|
+
|
|
19
|
+
export const AssistantMessageBody = memo<
|
|
20
|
+
UIChatMessage & {
|
|
21
|
+
editableContent: ReactNode;
|
|
22
|
+
markdownProps?: Omit<MarkdownProps, 'className' | 'style' | 'children'>;
|
|
23
|
+
}
|
|
24
|
+
>(
|
|
25
|
+
({
|
|
26
|
+
id,
|
|
27
|
+
tools,
|
|
28
|
+
content,
|
|
29
|
+
chunksList,
|
|
30
|
+
search,
|
|
31
|
+
imageList,
|
|
32
|
+
metadata,
|
|
33
|
+
editableContent,
|
|
34
|
+
markdownProps,
|
|
35
|
+
...props
|
|
36
|
+
}) => {
|
|
37
|
+
const [editing, generating, isCollapsed] = useChatStore((s) => [
|
|
38
|
+
messageStateSelectors.isMessageEditing(id)(s),
|
|
39
|
+
messageStateSelectors.isMessageGenerating(id)(s),
|
|
40
|
+
messageStateSelectors.isMessageCollapsed(id)(s),
|
|
41
|
+
]);
|
|
42
|
+
|
|
43
|
+
const isToolCallGenerating = generating && (content === LOADING_FLAT || !content) && !!tools;
|
|
44
|
+
|
|
45
|
+
const isReasoning = useChatStore(aiChatSelectors.isMessageInReasoning(id));
|
|
46
|
+
|
|
47
|
+
const isIntentUnderstanding = useChatStore(aiChatSelectors.isIntentUnderstanding(id));
|
|
48
|
+
|
|
49
|
+
const showSearch = !!search && !!search.citations?.length;
|
|
50
|
+
const showImageItems = !!imageList && imageList.length > 0;
|
|
51
|
+
|
|
52
|
+
// remove \n to avoid empty content
|
|
53
|
+
// refs: https://github.com/lobehub/lobe-chat/pull/6153
|
|
54
|
+
const showReasoning =
|
|
55
|
+
(!!props.reasoning && props.reasoning.content?.trim() !== '') ||
|
|
56
|
+
(!props.reasoning && isReasoning);
|
|
57
|
+
|
|
58
|
+
const showFileChunks = !!chunksList && chunksList.length > 0;
|
|
59
|
+
|
|
60
|
+
if (editing)
|
|
61
|
+
return (
|
|
62
|
+
<DefaultMessage
|
|
63
|
+
content={content}
|
|
64
|
+
editableContent={editableContent}
|
|
65
|
+
id={id}
|
|
66
|
+
isToolCallGenerating={isToolCallGenerating}
|
|
67
|
+
{...props}
|
|
68
|
+
/>
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
if (isCollapsed) return <CollapsedMessage content={content} id={id} />;
|
|
72
|
+
|
|
73
|
+
return (
|
|
74
|
+
<Flexbox gap={8} id={id}>
|
|
75
|
+
{showSearch && (
|
|
76
|
+
<SearchGrounding citations={search?.citations} searchQueries={search?.searchQueries} />
|
|
77
|
+
)}
|
|
78
|
+
{showFileChunks && <FileChunks data={chunksList} />}
|
|
79
|
+
{showReasoning && <Reasoning {...props.reasoning} id={id} />}
|
|
80
|
+
{isIntentUnderstanding ? (
|
|
81
|
+
<IntentUnderstanding />
|
|
82
|
+
) : (
|
|
83
|
+
<MessageContent
|
|
84
|
+
content={content}
|
|
85
|
+
hasImages={showImageItems}
|
|
86
|
+
isMultimodal={metadata?.isMultimodal}
|
|
87
|
+
isToolCallGenerating={isToolCallGenerating}
|
|
88
|
+
markdownProps={markdownProps}
|
|
89
|
+
tempDisplayContent={metadata?.tempDisplayContent}
|
|
90
|
+
/>
|
|
91
|
+
)}
|
|
92
|
+
{showImageItems && <ImageFileListViewer items={imageList} />}
|
|
93
|
+
</Flexbox>
|
|
94
|
+
);
|
|
95
|
+
},
|
|
96
|
+
);
|