@just-every/ensemble 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +245 -0
- package/dist/cost_tracker.d.ts +2 -0
- package/dist/cost_tracker.d.ts.map +1 -0
- package/dist/cost_tracker.js +2 -0
- package/dist/cost_tracker.js.map +1 -0
- package/dist/errors.d.ts +55 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +134 -0
- package/dist/errors.js.map +1 -0
- package/dist/external_models.d.ts +10 -0
- package/dist/external_models.d.ts.map +1 -0
- package/dist/external_models.js +36 -0
- package/dist/external_models.js.map +1 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +47 -0
- package/dist/index.js.map +1 -0
- package/dist/model_data.d.ts +63 -0
- package/dist/model_data.d.ts.map +1 -0
- package/dist/model_data.js +1070 -0
- package/dist/model_data.js.map +1 -0
- package/dist/model_providers/base_provider.d.ts +37 -0
- package/dist/model_providers/base_provider.d.ts.map +1 -0
- package/dist/model_providers/base_provider.js +146 -0
- package/dist/model_providers/base_provider.js.map +1 -0
- package/dist/model_providers/claude.d.ts +11 -0
- package/dist/model_providers/claude.d.ts.map +1 -0
- package/dist/model_providers/claude.js +788 -0
- package/dist/model_providers/claude.js.map +1 -0
- package/dist/model_providers/deepseek.d.ts +8 -0
- package/dist/model_providers/deepseek.d.ts.map +1 -0
- package/dist/model_providers/deepseek.js +136 -0
- package/dist/model_providers/deepseek.js.map +1 -0
- package/dist/model_providers/gemini.d.ts +11 -0
- package/dist/model_providers/gemini.d.ts.map +1 -0
- package/dist/model_providers/gemini.js +711 -0
- package/dist/model_providers/gemini.js.map +1 -0
- package/dist/model_providers/grok.d.ts +8 -0
- package/dist/model_providers/grok.d.ts.map +1 -0
- package/dist/model_providers/grok.js +22 -0
- package/dist/model_providers/grok.js.map +1 -0
- package/dist/model_providers/model_provider.d.ts +11 -0
- package/dist/model_providers/model_provider.d.ts.map +1 -0
- package/dist/model_providers/model_provider.js +170 -0
- package/dist/model_providers/model_provider.js.map +1 -0
- package/dist/model_providers/openai.d.ts +13 -0
- package/dist/model_providers/openai.d.ts.map +1 -0
- package/dist/model_providers/openai.js +822 -0
- package/dist/model_providers/openai.js.map +1 -0
- package/dist/model_providers/openai_chat.d.ts +14 -0
- package/dist/model_providers/openai_chat.d.ts.map +1 -0
- package/dist/model_providers/openai_chat.js +719 -0
- package/dist/model_providers/openai_chat.js.map +1 -0
- package/dist/model_providers/openrouter.d.ts +6 -0
- package/dist/model_providers/openrouter.d.ts.map +1 -0
- package/dist/model_providers/openrouter.js +18 -0
- package/dist/model_providers/openrouter.js.map +1 -0
- package/dist/model_providers/refactored_openai.d.ts +22 -0
- package/dist/model_providers/refactored_openai.d.ts.map +1 -0
- package/dist/model_providers/refactored_openai.js +310 -0
- package/dist/model_providers/refactored_openai.js.map +1 -0
- package/dist/model_providers/test_provider.d.ts +27 -0
- package/dist/model_providers/test_provider.d.ts.map +1 -0
- package/dist/model_providers/test_provider.js +185 -0
- package/dist/model_providers/test_provider.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/dist/types/api_types.d.ts +249 -0
- package/dist/types/api_types.d.ts.map +1 -0
- package/dist/types/api_types.js +2 -0
- package/dist/types/api_types.js.map +1 -0
- package/dist/types/extended_types.d.ts +43 -0
- package/dist/types/extended_types.d.ts.map +1 -0
- package/dist/types/extended_types.js +2 -0
- package/dist/types/extended_types.js.map +1 -0
- package/dist/types.d.ts +301 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/async_queue.d.ts +14 -0
- package/dist/utils/async_queue.d.ts.map +1 -0
- package/dist/utils/async_queue.js +68 -0
- package/dist/utils/async_queue.js.map +1 -0
- package/dist/utils/cache.d.ts +60 -0
- package/dist/utils/cache.d.ts.map +1 -0
- package/dist/utils/cache.js +205 -0
- package/dist/utils/cache.js.map +1 -0
- package/dist/utils/communication.d.ts +3 -0
- package/dist/utils/communication.d.ts.map +1 -0
- package/dist/utils/communication.js +8 -0
- package/dist/utils/communication.js.map +1 -0
- package/dist/utils/cost_tracker.d.ts +26 -0
- package/dist/utils/cost_tracker.d.ts.map +1 -0
- package/dist/utils/cost_tracker.js +177 -0
- package/dist/utils/cost_tracker.js.map +1 -0
- package/dist/utils/delta_buffer.d.ts +14 -0
- package/dist/utils/delta_buffer.d.ts.map +1 -0
- package/dist/utils/delta_buffer.js +60 -0
- package/dist/utils/delta_buffer.js.map +1 -0
- package/dist/utils/image_to_text.d.ts +3 -0
- package/dist/utils/image_to_text.d.ts.map +1 -0
- package/dist/utils/image_to_text.js +81 -0
- package/dist/utils/image_to_text.js.map +1 -0
- package/dist/utils/image_utils.d.ts +18 -0
- package/dist/utils/image_utils.d.ts.map +1 -0
- package/dist/utils/image_utils.js +132 -0
- package/dist/utils/image_utils.js.map +1 -0
- package/dist/utils/llm_logger.d.ts +8 -0
- package/dist/utils/llm_logger.d.ts.map +1 -0
- package/dist/utils/llm_logger.js +24 -0
- package/dist/utils/llm_logger.js.map +1 -0
- package/dist/utils/quota_tracker.d.ts +22 -0
- package/dist/utils/quota_tracker.d.ts.map +1 -0
- package/dist/utils/quota_tracker.js +338 -0
- package/dist/utils/quota_tracker.js.map +1 -0
- package/dist/utils/stream_converter.d.ts +19 -0
- package/dist/utils/stream_converter.d.ts.map +1 -0
- package/dist/utils/stream_converter.js +172 -0
- package/dist/utils/stream_converter.js.map +1 -0
- package/dist/validation.d.ts +1789 -0
- package/dist/validation.d.ts.map +1 -0
- package/dist/validation.js +289 -0
- package/dist/validation.js.map +1 -0
- package/dist/vitest.config.d.ts +3 -0
- package/dist/vitest.config.d.ts.map +1 -0
- package/dist/vitest.config.js +34 -0
- package/dist/vitest.config.js.map +1 -0
- package/package.json +86 -0
|
@@ -0,0 +1,788 @@
|
|
|
1
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
3
|
+
function formatWebSearchResults(results) {
|
|
4
|
+
if (!Array.isArray(results))
|
|
5
|
+
return '';
|
|
6
|
+
return results
|
|
7
|
+
.filter(r => r.type === 'web_search_result')
|
|
8
|
+
.map((r, i) => `${i + 1}. ${r.title || 'Untitled'} – ${r.url}`)
|
|
9
|
+
.join('\n');
|
|
10
|
+
}
|
|
11
|
+
function createCitationTracker() {
|
|
12
|
+
return {
|
|
13
|
+
citations: new Map(),
|
|
14
|
+
lastIndex: 0,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
function formatCitation(tracker, citation) {
|
|
18
|
+
const url = citation.url;
|
|
19
|
+
let index;
|
|
20
|
+
if (tracker.citations.has(url)) {
|
|
21
|
+
const entries = Array.from(tracker.citations.entries());
|
|
22
|
+
index = entries.findIndex(([k]) => k === url) + 1;
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
tracker.lastIndex++;
|
|
26
|
+
index = tracker.lastIndex;
|
|
27
|
+
tracker.citations.set(url, {
|
|
28
|
+
title: citation.title,
|
|
29
|
+
url: citation.url,
|
|
30
|
+
citedText: citation.cited_text,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return ` [${index}]`;
|
|
34
|
+
}
|
|
35
|
+
function generateFootnotes(tracker) {
|
|
36
|
+
if (tracker.citations.size === 0)
|
|
37
|
+
return '';
|
|
38
|
+
const footnotes = Array.from(tracker.citations.values())
|
|
39
|
+
.map((citation, i) => `[${i + 1}] ${citation.title} – ${citation.url}`)
|
|
40
|
+
.join('\n');
|
|
41
|
+
return '\n\nReferences:\n' + footnotes;
|
|
42
|
+
}
|
|
43
|
+
import { costTracker } from '@just-every/ensemble/cost_tracker';
|
|
44
|
+
import { log_llm_error, log_llm_request, log_llm_response, } from '../utils/llm_logger.js';
|
|
45
|
+
import { isPaused } from '../utils/communication.js';
|
|
46
|
+
import { extractBase64Image, resizeAndTruncateForClaude, } from '../utils/image_utils.js';
|
|
47
|
+
import { convertImageToTextIfNeeded } from '../utils/image_to_text.js';
|
|
48
|
+
import { bufferDelta, flushBufferedDeltas, } from '../utils/delta_buffer.js';
|
|
49
|
+
function convertToClaudeTools(tools) {
|
|
50
|
+
return tools.map(tool => {
|
|
51
|
+
if (tool.definition.function.name === 'claude_web_search') {
|
|
52
|
+
return {
|
|
53
|
+
type: 'web_search_20250305',
|
|
54
|
+
name: 'web_search',
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
name: tool.definition.function.name,
|
|
59
|
+
description: tool.definition.function.description,
|
|
60
|
+
input_schema: tool.definition.function.parameters,
|
|
61
|
+
};
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
function getImageMediaType(imageData) {
|
|
65
|
+
if (imageData.includes('data:image/jpeg'))
|
|
66
|
+
return 'image/jpeg';
|
|
67
|
+
if (imageData.includes('data:image/png'))
|
|
68
|
+
return 'image/png';
|
|
69
|
+
if (imageData.includes('data:image/gif'))
|
|
70
|
+
return 'image/gif';
|
|
71
|
+
if (imageData.includes('data:image/webp'))
|
|
72
|
+
return 'image/webp';
|
|
73
|
+
return 'image/jpeg';
|
|
74
|
+
}
|
|
75
|
+
function cleanBase64Data(imageData) {
|
|
76
|
+
return imageData.replace(/^data:image\/[a-z]+;base64,/, '');
|
|
77
|
+
}
|
|
78
|
+
async function convertToClaudeMessage(role, content, msg, result) {
|
|
79
|
+
if (!msg)
|
|
80
|
+
return null;
|
|
81
|
+
if (msg.type === 'function_call') {
|
|
82
|
+
let inputArgs = {};
|
|
83
|
+
try {
|
|
84
|
+
inputArgs = JSON.parse(msg.arguments || '{}');
|
|
85
|
+
}
|
|
86
|
+
catch (e) {
|
|
87
|
+
console.error(`Error parsing function call arguments for ${msg.name}: ${msg.arguments}`, e);
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
const toolUseBlock = {
|
|
91
|
+
type: 'tool_use',
|
|
92
|
+
id: msg.call_id,
|
|
93
|
+
name: msg.name,
|
|
94
|
+
input: inputArgs,
|
|
95
|
+
};
|
|
96
|
+
return { role: 'assistant', content: [toolUseBlock] };
|
|
97
|
+
}
|
|
98
|
+
else if (msg.type === 'function_call_output') {
|
|
99
|
+
if (typeof msg.output === 'string') {
|
|
100
|
+
const extracted = extractBase64Image(msg.output);
|
|
101
|
+
if (extracted.found && extracted.image_id !== null) {
|
|
102
|
+
const image_id = extracted.image_id;
|
|
103
|
+
try {
|
|
104
|
+
const originalImageData = extracted.images[image_id];
|
|
105
|
+
const processedImageData = await resizeAndTruncateForClaude(originalImageData);
|
|
106
|
+
const mediaType = getImageMediaType(processedImageData);
|
|
107
|
+
const cleanedImageData = cleanBase64Data(processedImageData);
|
|
108
|
+
const toolResultBlock = {
|
|
109
|
+
type: 'tool_result',
|
|
110
|
+
tool_use_id: msg.call_id,
|
|
111
|
+
content: extracted.replaceContent.trim() || '',
|
|
112
|
+
...(msg.status === 'incomplete'
|
|
113
|
+
? { is_error: true }
|
|
114
|
+
: {}),
|
|
115
|
+
};
|
|
116
|
+
const contentBlocks = [
|
|
117
|
+
toolResultBlock,
|
|
118
|
+
{
|
|
119
|
+
type: 'image',
|
|
120
|
+
source: {
|
|
121
|
+
type: 'base64',
|
|
122
|
+
media_type: mediaType,
|
|
123
|
+
data: cleanedImageData,
|
|
124
|
+
},
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
type: 'text',
|
|
128
|
+
text: `This is [image #${image_id}] from the function call output of ${msg.name}`,
|
|
129
|
+
},
|
|
130
|
+
];
|
|
131
|
+
return { role: 'user', content: contentBlocks };
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
console.error('Error processing image in function call output:', error);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
const toolResultBlock = {
|
|
139
|
+
type: 'tool_result',
|
|
140
|
+
tool_use_id: msg.call_id,
|
|
141
|
+
content: msg.output || '',
|
|
142
|
+
...(msg.status === 'incomplete' ? { is_error: true } : {}),
|
|
143
|
+
};
|
|
144
|
+
return { role: 'user', content: [toolResultBlock] };
|
|
145
|
+
}
|
|
146
|
+
else if (msg.type === 'thinking') {
|
|
147
|
+
if (!content) {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
if ('signature' in msg && msg.signature) {
|
|
151
|
+
return {
|
|
152
|
+
role: 'assistant',
|
|
153
|
+
content: [
|
|
154
|
+
{
|
|
155
|
+
type: 'thinking',
|
|
156
|
+
thinking: content.trim(),
|
|
157
|
+
signature: msg.signature,
|
|
158
|
+
},
|
|
159
|
+
],
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
return { role: 'assistant', content: 'Thinking: ' + content.trim() };
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
if (!content) {
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
168
|
+
let messageRole = role === 'assistant'
|
|
169
|
+
? 'assistant'
|
|
170
|
+
: (role === 'system' || role === 'developer') && !result?.length
|
|
171
|
+
? 'system'
|
|
172
|
+
: 'user';
|
|
173
|
+
if (messageRole !== 'system') {
|
|
174
|
+
const extracted = extractBase64Image(content);
|
|
175
|
+
if (extracted.found && extracted.image_id !== null) {
|
|
176
|
+
messageRole = 'user';
|
|
177
|
+
try {
|
|
178
|
+
const image_id = extracted.image_id;
|
|
179
|
+
const originalImageData = extracted.images[image_id];
|
|
180
|
+
const processedImageData = await resizeAndTruncateForClaude(originalImageData);
|
|
181
|
+
const mediaType = getImageMediaType(processedImageData);
|
|
182
|
+
const cleanedImageData = cleanBase64Data(processedImageData);
|
|
183
|
+
const contentBlocks = [];
|
|
184
|
+
contentBlocks.push({
|
|
185
|
+
type: 'image',
|
|
186
|
+
source: {
|
|
187
|
+
type: 'base64',
|
|
188
|
+
media_type: mediaType,
|
|
189
|
+
data: cleanedImageData,
|
|
190
|
+
},
|
|
191
|
+
});
|
|
192
|
+
if (extracted.replaceContent.trim()) {
|
|
193
|
+
contentBlocks.push({
|
|
194
|
+
type: 'text',
|
|
195
|
+
text: extracted.replaceContent.trim(),
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
return {
|
|
199
|
+
role: messageRole,
|
|
200
|
+
content: contentBlocks,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
console.error('Error processing image in message:', error);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return {
|
|
209
|
+
role: messageRole,
|
|
210
|
+
content: content.trim(),
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
export class ClaudeProvider {
|
|
215
|
+
client;
|
|
216
|
+
constructor(apiKey) {
|
|
217
|
+
this.client = new Anthropic({
|
|
218
|
+
apiKey: apiKey || process.env.ANTHROPIC_API_KEY,
|
|
219
|
+
});
|
|
220
|
+
if (!this.client) {
|
|
221
|
+
throw new Error('Failed to initialize Claude client. Make sure ANTHROPIC_API_KEY is set.');
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
async preprocessMessagesForImageSupport(messages, modelId) {
|
|
225
|
+
const processedMessages = [...messages];
|
|
226
|
+
for (let i = 0; i < processedMessages.length; i++) {
|
|
227
|
+
const msg = processedMessages[i];
|
|
228
|
+
if (!this.isMessageWithStringContent(msg)) {
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
if (typeof msg.content === 'string') {
|
|
232
|
+
const extracted = extractBase64Image(msg.content);
|
|
233
|
+
if (extracted.found && extracted.image_id !== null) {
|
|
234
|
+
try {
|
|
235
|
+
const image_id = extracted.image_id;
|
|
236
|
+
const imageData = extracted.images[image_id];
|
|
237
|
+
const processedImageData = await convertImageToTextIfNeeded(imageData, modelId);
|
|
238
|
+
if (!processedImageData.startsWith('data:image/')) {
|
|
239
|
+
const newContent = extracted.replaceContent.trim()
|
|
240
|
+
? extracted.replaceContent.trim() +
|
|
241
|
+
' ' +
|
|
242
|
+
processedImageData
|
|
243
|
+
: processedImageData;
|
|
244
|
+
processedMessages[i] = {
|
|
245
|
+
...msg,
|
|
246
|
+
content: newContent,
|
|
247
|
+
};
|
|
248
|
+
console.log(`Converted image to text description for model ${modelId}`);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
console.error('Error converting image to text:', error);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
else if (Array.isArray(msg.content)) {
|
|
257
|
+
let hasChanges = false;
|
|
258
|
+
const newContentItems = [...msg.content];
|
|
259
|
+
for (let j = 0; j < newContentItems.length; j++) {
|
|
260
|
+
const item = newContentItems[j];
|
|
261
|
+
if (item.type === 'input_text' &&
|
|
262
|
+
typeof item.text === 'string') {
|
|
263
|
+
const extracted = extractBase64Image(item.text);
|
|
264
|
+
if (extracted.found && extracted.image_id !== null) {
|
|
265
|
+
try {
|
|
266
|
+
const image_id = extracted.image_id;
|
|
267
|
+
const imageData = extracted.images[image_id];
|
|
268
|
+
const processedImageData = await convertImageToTextIfNeeded(imageData, modelId);
|
|
269
|
+
if (!processedImageData.startsWith('data:image/')) {
|
|
270
|
+
const newText = extracted.replaceContent.trim()
|
|
271
|
+
? extracted.replaceContent.trim() +
|
|
272
|
+
' ' +
|
|
273
|
+
processedImageData
|
|
274
|
+
: processedImageData;
|
|
275
|
+
newContentItems[j] = {
|
|
276
|
+
...item,
|
|
277
|
+
text: newText,
|
|
278
|
+
};
|
|
279
|
+
hasChanges = true;
|
|
280
|
+
console.log(`Converted image to text description in array content for model ${modelId}`);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
catch (error) {
|
|
284
|
+
console.error('Error converting image to text in array content:', error);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
if (hasChanges) {
|
|
290
|
+
processedMessages[i] = {
|
|
291
|
+
...msg,
|
|
292
|
+
content: newContentItems,
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return processedMessages;
|
|
298
|
+
}
|
|
299
|
+
async prepareClaudeMessages(messages, modelId) {
|
|
300
|
+
const result = [];
|
|
301
|
+
for (const originalMsg of messages) {
|
|
302
|
+
let msg = originalMsg;
|
|
303
|
+
if (this.isMessageWithStringContent(msg)) {
|
|
304
|
+
if (typeof msg.content === 'string') {
|
|
305
|
+
const extracted = extractBase64Image(msg.content);
|
|
306
|
+
if (extracted.found && extracted.image_id !== null) {
|
|
307
|
+
try {
|
|
308
|
+
const image_id = extracted.image_id;
|
|
309
|
+
const imageData = extracted.images[image_id];
|
|
310
|
+
const processedImageData = await convertImageToTextIfNeeded(imageData, modelId);
|
|
311
|
+
if (processedImageData &&
|
|
312
|
+
!processedImageData.startsWith('data:image/')) {
|
|
313
|
+
const newContent = extracted.replaceContent.trim()
|
|
314
|
+
? extracted.replaceContent.trim() +
|
|
315
|
+
' ' +
|
|
316
|
+
processedImageData
|
|
317
|
+
: processedImageData;
|
|
318
|
+
msg = {
|
|
319
|
+
...msg,
|
|
320
|
+
content: newContent,
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
catch (error) {
|
|
325
|
+
console.error('Error converting image to text:', error);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
else if (Array.isArray(msg.content)) {
|
|
330
|
+
let hasChanges = false;
|
|
331
|
+
const newContentItems = [...msg.content];
|
|
332
|
+
for (let j = 0; j < newContentItems.length; j++) {
|
|
333
|
+
const item = newContentItems[j];
|
|
334
|
+
if (item.type === 'input_text' &&
|
|
335
|
+
typeof item.text === 'string') {
|
|
336
|
+
const extracted = extractBase64Image(item.text);
|
|
337
|
+
if (extracted.found &&
|
|
338
|
+
extracted.image_id !== null) {
|
|
339
|
+
try {
|
|
340
|
+
const image_id = extracted.image_id;
|
|
341
|
+
const imageData = extracted.images[image_id];
|
|
342
|
+
const processedImageData = await convertImageToTextIfNeeded(imageData, modelId);
|
|
343
|
+
if (processedImageData &&
|
|
344
|
+
!processedImageData.startsWith('data:image/')) {
|
|
345
|
+
const newText = extracted.replaceContent.trim()
|
|
346
|
+
? extracted.replaceContent.trim() +
|
|
347
|
+
' ' +
|
|
348
|
+
processedImageData
|
|
349
|
+
: processedImageData;
|
|
350
|
+
newContentItems[j] = {
|
|
351
|
+
...item,
|
|
352
|
+
text: newText,
|
|
353
|
+
};
|
|
354
|
+
hasChanges = true;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
catch (error) {
|
|
358
|
+
console.error('Error converting image to text in array content:', error);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
if (hasChanges) {
|
|
364
|
+
msg = {
|
|
365
|
+
...msg,
|
|
366
|
+
content: newContentItems,
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
const role = 'role' in msg && msg.role !== 'developer' ? msg.role : 'system';
|
|
372
|
+
let content = '';
|
|
373
|
+
if ('content' in msg) {
|
|
374
|
+
if (typeof msg.content === 'string') {
|
|
375
|
+
content = msg.content;
|
|
376
|
+
}
|
|
377
|
+
else if (msg.content.text &&
|
|
378
|
+
typeof msg.content.text === 'string') {
|
|
379
|
+
content = msg.content.text;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
const structuredMsg = await convertToClaudeMessage(role, content, msg, result);
|
|
383
|
+
if (structuredMsg) {
|
|
384
|
+
result.push(structuredMsg);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
return result;
|
|
388
|
+
}
|
|
389
|
+
isMessageWithStringContent(msg) {
|
|
390
|
+
if (!('content' in msg))
|
|
391
|
+
return false;
|
|
392
|
+
if (!msg.content)
|
|
393
|
+
return false;
|
|
394
|
+
if (msg.type &&
|
|
395
|
+
['function_call', 'function_call_output'].includes(msg.type))
|
|
396
|
+
return false;
|
|
397
|
+
if (typeof msg.content === 'string') {
|
|
398
|
+
return true;
|
|
399
|
+
}
|
|
400
|
+
else if (Array.isArray(msg.content)) {
|
|
401
|
+
return msg.content.some(item => item.type === 'input_text' && typeof item.text === 'string');
|
|
402
|
+
}
|
|
403
|
+
return false;
|
|
404
|
+
}
|
|
405
|
+
async *createResponseStream(model, messages, agent) {
|
|
406
|
+
let totalInputTokens = 0;
|
|
407
|
+
let totalOutputTokens = 0;
|
|
408
|
+
let totalCacheCreationInputTokens = 0;
|
|
409
|
+
let totalCacheReadInputTokens = 0;
|
|
410
|
+
let streamCompletedSuccessfully = false;
|
|
411
|
+
let messageCompleteYielded = false;
|
|
412
|
+
let requestId;
|
|
413
|
+
try {
|
|
414
|
+
const tools = agent
|
|
415
|
+
? await agent.getTools()
|
|
416
|
+
: [];
|
|
417
|
+
const settings = agent?.modelSettings;
|
|
418
|
+
const modelClass = agent?.modelClass;
|
|
419
|
+
let thinking = undefined;
|
|
420
|
+
let max_tokens = settings?.max_tokens || 64000;
|
|
421
|
+
switch (modelClass) {
|
|
422
|
+
case 'monologue':
|
|
423
|
+
case 'reasoning':
|
|
424
|
+
case 'code':
|
|
425
|
+
if (model === 'claude-3-7-sonnet-latest') {
|
|
426
|
+
thinking = {
|
|
427
|
+
type: 'enabled',
|
|
428
|
+
budget_tokens: 16000,
|
|
429
|
+
};
|
|
430
|
+
max_tokens = Math.min(max_tokens, 64000);
|
|
431
|
+
}
|
|
432
|
+
else {
|
|
433
|
+
max_tokens = Math.min(max_tokens, 8192);
|
|
434
|
+
}
|
|
435
|
+
break;
|
|
436
|
+
case 'standard':
|
|
437
|
+
max_tokens = Math.min(max_tokens, 8192);
|
|
438
|
+
break;
|
|
439
|
+
default:
|
|
440
|
+
max_tokens = Math.min(max_tokens, 4096);
|
|
441
|
+
}
|
|
442
|
+
if (settings?.json_schema) {
|
|
443
|
+
messages.push({
|
|
444
|
+
type: 'message',
|
|
445
|
+
role: 'system',
|
|
446
|
+
content: `Your response MUST be a valid JSON object that conforms to this schema:\n${JSON.stringify(settings.json_schema, null, 2)}`,
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
const claudeMessages = await this.prepareClaudeMessages(messages, model);
|
|
450
|
+
const systemPrompt = claudeMessages.reduce((acc, msg) => {
|
|
451
|
+
if (msg.role === 'system' &&
|
|
452
|
+
msg.content &&
|
|
453
|
+
typeof msg.content === 'string') {
|
|
454
|
+
return acc + msg.content + '\n';
|
|
455
|
+
}
|
|
456
|
+
return acc;
|
|
457
|
+
}, '');
|
|
458
|
+
const requestParams = {
|
|
459
|
+
model: model,
|
|
460
|
+
messages: claudeMessages.filter(m => m.role === 'user' || m.role === 'assistant'),
|
|
461
|
+
...(systemPrompt ? { system: systemPrompt } : {}),
|
|
462
|
+
stream: true,
|
|
463
|
+
max_tokens,
|
|
464
|
+
...(thinking ? { thinking } : {}),
|
|
465
|
+
...(settings?.temperature !== undefined
|
|
466
|
+
? { temperature: settings.temperature }
|
|
467
|
+
: {}),
|
|
468
|
+
};
|
|
469
|
+
if (tools && tools.length > 0) {
|
|
470
|
+
requestParams.tools = convertToClaudeTools(tools);
|
|
471
|
+
}
|
|
472
|
+
if (!requestParams.messages ||
|
|
473
|
+
requestParams.messages.length === 0) {
|
|
474
|
+
console.warn('Claude API Warning: No user or assistant messages provided after filtering. Adding default message.');
|
|
475
|
+
requestParams.messages = [
|
|
476
|
+
{
|
|
477
|
+
role: 'user',
|
|
478
|
+
content: "Let's think this through step by step.",
|
|
479
|
+
},
|
|
480
|
+
];
|
|
481
|
+
}
|
|
482
|
+
requestId = log_llm_request(agent.agent_id, 'anthropic', model, requestParams, new Date());
|
|
483
|
+
let currentToolCall = null;
|
|
484
|
+
let accumulatedSignature = '';
|
|
485
|
+
let accumulatedThinking = '';
|
|
486
|
+
let accumulatedContent = '';
|
|
487
|
+
const messageId = uuidv4();
|
|
488
|
+
let deltaPosition = 0;
|
|
489
|
+
const deltaBuffers = new Map();
|
|
490
|
+
const citationTracker = createCitationTracker();
|
|
491
|
+
const stream = await this.client.messages.create(requestParams);
|
|
492
|
+
const events = [];
|
|
493
|
+
try {
|
|
494
|
+
for await (const event of stream) {
|
|
495
|
+
events.push(event);
|
|
496
|
+
if (isPaused()) {
|
|
497
|
+
console.log(`[Claude] System paused during stream for model ${model}. Aborting processing.`);
|
|
498
|
+
yield {
|
|
499
|
+
type: 'message_delta',
|
|
500
|
+
content: '\n⏸️ Stream paused by user.',
|
|
501
|
+
message_id: messageId,
|
|
502
|
+
order: 999,
|
|
503
|
+
};
|
|
504
|
+
streamCompletedSuccessfully = false;
|
|
505
|
+
break;
|
|
506
|
+
}
|
|
507
|
+
if (event.type === 'message_start' &&
|
|
508
|
+
event.message?.usage) {
|
|
509
|
+
const usage = event.message.usage;
|
|
510
|
+
totalInputTokens += usage.input_tokens || 0;
|
|
511
|
+
totalOutputTokens += usage.output_tokens || 0;
|
|
512
|
+
totalCacheCreationInputTokens +=
|
|
513
|
+
usage.cache_creation_input_tokens || 0;
|
|
514
|
+
totalCacheReadInputTokens +=
|
|
515
|
+
usage.cache_read_input_tokens || 0;
|
|
516
|
+
}
|
|
517
|
+
else if (event.type === 'message_delta' && event.usage) {
|
|
518
|
+
const usage = event.usage;
|
|
519
|
+
totalInputTokens += usage.input_tokens || 0;
|
|
520
|
+
totalOutputTokens += usage.output_tokens || 0;
|
|
521
|
+
totalCacheCreationInputTokens +=
|
|
522
|
+
usage.cache_creation_input_tokens || 0;
|
|
523
|
+
totalCacheReadInputTokens +=
|
|
524
|
+
usage.cache_read_input_tokens || 0;
|
|
525
|
+
}
|
|
526
|
+
if (event.type === 'content_block_delta') {
|
|
527
|
+
if (event.delta.type === 'signature_delta' &&
|
|
528
|
+
event.delta.signature) {
|
|
529
|
+
accumulatedSignature += event.delta.signature;
|
|
530
|
+
}
|
|
531
|
+
else if (event.delta.type === 'thinking_delta' &&
|
|
532
|
+
event.delta.thinking) {
|
|
533
|
+
yield {
|
|
534
|
+
type: 'message_delta',
|
|
535
|
+
content: '',
|
|
536
|
+
thinking_content: event.delta.thinking,
|
|
537
|
+
message_id: messageId,
|
|
538
|
+
order: deltaPosition++,
|
|
539
|
+
};
|
|
540
|
+
accumulatedThinking += event.delta.thinking;
|
|
541
|
+
}
|
|
542
|
+
else if (event.delta.type === 'text_delta' &&
|
|
543
|
+
event.delta.text) {
|
|
544
|
+
for (const ev of bufferDelta(deltaBuffers, messageId, event.delta.text, content => ({
|
|
545
|
+
type: 'message_delta',
|
|
546
|
+
content,
|
|
547
|
+
message_id: messageId,
|
|
548
|
+
order: deltaPosition++,
|
|
549
|
+
}))) {
|
|
550
|
+
yield ev;
|
|
551
|
+
}
|
|
552
|
+
accumulatedContent += event.delta.text;
|
|
553
|
+
}
|
|
554
|
+
else if (event.delta.type === 'input_json_delta' &&
|
|
555
|
+
currentToolCall &&
|
|
556
|
+
event.delta.partial_json) {
|
|
557
|
+
try {
|
|
558
|
+
if (!currentToolCall.function._partialArguments) {
|
|
559
|
+
currentToolCall.function._partialArguments =
|
|
560
|
+
'';
|
|
561
|
+
}
|
|
562
|
+
currentToolCall.function._partialArguments +=
|
|
563
|
+
event.delta.partial_json;
|
|
564
|
+
currentToolCall.function.arguments =
|
|
565
|
+
currentToolCall.function._partialArguments;
|
|
566
|
+
yield {
|
|
567
|
+
type: 'tool_delta',
|
|
568
|
+
tool_calls: [currentToolCall],
|
|
569
|
+
};
|
|
570
|
+
}
|
|
571
|
+
catch (err) {
|
|
572
|
+
console.error('Error processing tool_use delta (input_json_delta):', err, event);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
else if (event.delta.type === 'citations_delta' &&
|
|
576
|
+
event.delta.citation) {
|
|
577
|
+
const citationMarker = formatCitation(citationTracker, event.delta.citation);
|
|
578
|
+
yield {
|
|
579
|
+
type: 'message_delta',
|
|
580
|
+
content: citationMarker,
|
|
581
|
+
message_id: messageId,
|
|
582
|
+
order: deltaPosition++,
|
|
583
|
+
};
|
|
584
|
+
accumulatedContent += citationMarker;
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
else if (event.type === 'content_block_start' &&
|
|
588
|
+
event.content_block?.type === 'text') {
|
|
589
|
+
if (event.content_block.text) {
|
|
590
|
+
for (const ev of bufferDelta(deltaBuffers, messageId, event.content_block.text, content => ({
|
|
591
|
+
type: 'message_delta',
|
|
592
|
+
content,
|
|
593
|
+
message_id: messageId,
|
|
594
|
+
order: deltaPosition++,
|
|
595
|
+
}))) {
|
|
596
|
+
yield ev;
|
|
597
|
+
}
|
|
598
|
+
accumulatedContent += event.content_block.text;
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
else if (event.type === 'content_block_stop' &&
|
|
602
|
+
event.content_block?.type === 'text') {
|
|
603
|
+
}
|
|
604
|
+
else if (event.type === 'content_block_start' &&
|
|
605
|
+
event.content_block?.type === 'web_search_tool_result') {
|
|
606
|
+
if (event.content_block.content) {
|
|
607
|
+
const formatted = formatWebSearchResults(event.content_block.content);
|
|
608
|
+
if (formatted) {
|
|
609
|
+
yield {
|
|
610
|
+
type: 'message_delta',
|
|
611
|
+
content: '\n\nSearch Results:\n' +
|
|
612
|
+
formatted +
|
|
613
|
+
'\n',
|
|
614
|
+
message_id: messageId,
|
|
615
|
+
order: deltaPosition++,
|
|
616
|
+
};
|
|
617
|
+
accumulatedContent +=
|
|
618
|
+
'\n\nSearch Results:\n' + formatted + '\n';
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
else if (event.type === 'content_block_start' &&
|
|
623
|
+
event.content_block?.type === 'tool_use') {
|
|
624
|
+
const toolUse = event.content_block;
|
|
625
|
+
const toolId = toolUse.id || `call_${Date.now()}`;
|
|
626
|
+
const toolName = toolUse.name;
|
|
627
|
+
const toolInput = toolUse.input !== undefined ? toolUse.input : {};
|
|
628
|
+
currentToolCall = {
|
|
629
|
+
id: toolId,
|
|
630
|
+
type: 'function',
|
|
631
|
+
function: {
|
|
632
|
+
name: toolName,
|
|
633
|
+
arguments: typeof toolInput === 'string'
|
|
634
|
+
? toolInput
|
|
635
|
+
: JSON.stringify(toolInput),
|
|
636
|
+
},
|
|
637
|
+
};
|
|
638
|
+
}
|
|
639
|
+
else if (event.type === 'content_block_stop' &&
|
|
640
|
+
event.content_block?.type === 'tool_use' &&
|
|
641
|
+
currentToolCall) {
|
|
642
|
+
try {
|
|
643
|
+
if (currentToolCall.function._partialArguments) {
|
|
644
|
+
currentToolCall.function.arguments =
|
|
645
|
+
currentToolCall.function._partialArguments;
|
|
646
|
+
delete currentToolCall.function
|
|
647
|
+
._partialArguments;
|
|
648
|
+
}
|
|
649
|
+
yield {
|
|
650
|
+
type: 'tool_start',
|
|
651
|
+
tool_calls: [currentToolCall],
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
catch (err) {
|
|
655
|
+
console.error('Error finalizing tool call:', err, event);
|
|
656
|
+
}
|
|
657
|
+
finally {
|
|
658
|
+
currentToolCall = null;
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
else if (event.type === 'message_stop') {
|
|
662
|
+
if (event['amazon-bedrock-invocationMetrics']) {
|
|
663
|
+
const metrics = event['amazon-bedrock-invocationMetrics'];
|
|
664
|
+
totalInputTokens += metrics.inputTokenCount || 0;
|
|
665
|
+
totalOutputTokens += metrics.outputTokenCount || 0;
|
|
666
|
+
}
|
|
667
|
+
else if (event.usage) {
|
|
668
|
+
const usage = event.usage;
|
|
669
|
+
totalInputTokens += usage.input_tokens || 0;
|
|
670
|
+
totalOutputTokens += usage.output_tokens || 0;
|
|
671
|
+
totalCacheCreationInputTokens +=
|
|
672
|
+
usage.cache_creation_input_tokens || 0;
|
|
673
|
+
totalCacheReadInputTokens +=
|
|
674
|
+
usage.cache_read_input_tokens || 0;
|
|
675
|
+
}
|
|
676
|
+
if (currentToolCall) {
|
|
677
|
+
console.warn('Tool call was still active at message_stop:', currentToolCall);
|
|
678
|
+
yield {
|
|
679
|
+
type: 'tool_start',
|
|
680
|
+
tool_calls: [currentToolCall],
|
|
681
|
+
};
|
|
682
|
+
currentToolCall = null;
|
|
683
|
+
}
|
|
684
|
+
for (const ev of flushBufferedDeltas(deltaBuffers, (_id, content) => ({
|
|
685
|
+
type: 'message_delta',
|
|
686
|
+
content,
|
|
687
|
+
message_id: messageId,
|
|
688
|
+
order: deltaPosition++,
|
|
689
|
+
}))) {
|
|
690
|
+
yield ev;
|
|
691
|
+
}
|
|
692
|
+
if (accumulatedContent || accumulatedThinking) {
|
|
693
|
+
if (citationTracker.citations.size > 0) {
|
|
694
|
+
const footnotes = generateFootnotes(citationTracker);
|
|
695
|
+
accumulatedContent += footnotes;
|
|
696
|
+
}
|
|
697
|
+
yield {
|
|
698
|
+
type: 'message_complete',
|
|
699
|
+
message_id: messageId,
|
|
700
|
+
content: accumulatedContent,
|
|
701
|
+
thinking_content: accumulatedThinking,
|
|
702
|
+
thinking_signature: accumulatedSignature,
|
|
703
|
+
};
|
|
704
|
+
messageCompleteYielded = true;
|
|
705
|
+
}
|
|
706
|
+
streamCompletedSuccessfully = true;
|
|
707
|
+
}
|
|
708
|
+
else if (event.type === 'error') {
|
|
709
|
+
log_llm_error(requestId, event);
|
|
710
|
+
console.error('Claude API error event:', event.error);
|
|
711
|
+
yield {
|
|
712
|
+
type: 'error',
|
|
713
|
+
error: 'Claude API error: ' +
|
|
714
|
+
(event.error
|
|
715
|
+
? event.error.message ||
|
|
716
|
+
JSON.stringify(event.error)
|
|
717
|
+
: 'Unknown error'),
|
|
718
|
+
};
|
|
719
|
+
streamCompletedSuccessfully = false;
|
|
720
|
+
break;
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
if (streamCompletedSuccessfully &&
|
|
724
|
+
(accumulatedContent || accumulatedThinking) &&
|
|
725
|
+
!messageCompleteYielded) {
|
|
726
|
+
console.warn('Stream finished successfully but message_stop might not have triggered message_complete emission. Emitting now.');
|
|
727
|
+
for (const ev of flushBufferedDeltas(deltaBuffers, (_id, content) => ({
|
|
728
|
+
type: 'message_delta',
|
|
729
|
+
content,
|
|
730
|
+
message_id: messageId,
|
|
731
|
+
order: deltaPosition++,
|
|
732
|
+
}))) {
|
|
733
|
+
yield ev;
|
|
734
|
+
}
|
|
735
|
+
if (citationTracker.citations.size > 0) {
|
|
736
|
+
const footnotes = generateFootnotes(citationTracker);
|
|
737
|
+
accumulatedContent += footnotes;
|
|
738
|
+
}
|
|
739
|
+
yield {
|
|
740
|
+
type: 'message_complete',
|
|
741
|
+
message_id: messageId,
|
|
742
|
+
content: accumulatedContent,
|
|
743
|
+
thinking_content: accumulatedThinking,
|
|
744
|
+
thinking_signature: accumulatedSignature,
|
|
745
|
+
};
|
|
746
|
+
messageCompleteYielded = true;
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
catch (streamError) {
|
|
750
|
+
log_llm_error(requestId, streamError);
|
|
751
|
+
console.error('Error processing Claude stream:', streamError);
|
|
752
|
+
yield {
|
|
753
|
+
type: 'error',
|
|
754
|
+
error: `Claude stream error (${model}): ${streamError}`,
|
|
755
|
+
};
|
|
756
|
+
}
|
|
757
|
+
finally {
|
|
758
|
+
log_llm_response(requestId, events);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
catch (error) {
|
|
762
|
+
log_llm_error(requestId, error);
|
|
763
|
+
console.error('Error in Claude streaming completion setup:', error);
|
|
764
|
+
yield {
|
|
765
|
+
type: 'error',
|
|
766
|
+
error: `Claude request error (${model}): ${error}`,
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
finally {
|
|
770
|
+
if (totalInputTokens > 0 || totalOutputTokens > 0) {
|
|
771
|
+
const cachedTokens = totalCacheCreationInputTokens + totalCacheReadInputTokens;
|
|
772
|
+
costTracker.addUsage({
|
|
773
|
+
model,
|
|
774
|
+
input_tokens: totalInputTokens,
|
|
775
|
+
output_tokens: totalOutputTokens,
|
|
776
|
+
cached_tokens: cachedTokens,
|
|
777
|
+
metadata: {
|
|
778
|
+
cache_creation_input_tokens: totalCacheCreationInputTokens,
|
|
779
|
+
cache_read_input_tokens: totalCacheReadInputTokens,
|
|
780
|
+
total_tokens: totalInputTokens + totalOutputTokens,
|
|
781
|
+
},
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
export const claudeProvider = new ClaudeProvider();
|
|
788
|
+
//# sourceMappingURL=claude.js.map
|