@jsonstudio/llms 0.6.230 → 0.6.375
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/conversion/codecs/gemini-openai-codec.js +9 -1
- package/dist/conversion/compat/actions/gemini-web-search.d.ts +17 -0
- package/dist/conversion/compat/actions/gemini-web-search.js +68 -0
- package/dist/conversion/compat/actions/glm-image-content.d.ts +2 -0
- package/dist/conversion/compat/actions/glm-image-content.js +83 -0
- package/dist/conversion/compat/actions/glm-vision-prompt.d.ts +11 -0
- package/dist/conversion/compat/actions/glm-vision-prompt.js +177 -0
- package/dist/conversion/compat/actions/glm-web-search.js +25 -28
- package/dist/conversion/compat/actions/universal-shape-filter.js +11 -0
- package/dist/conversion/compat/profiles/chat-gemini.json +17 -0
- package/dist/conversion/compat/profiles/chat-glm.json +190 -184
- package/dist/conversion/compat/profiles/chat-iflow.json +195 -195
- package/dist/conversion/compat/profiles/chat-lmstudio.json +43 -43
- package/dist/conversion/compat/profiles/chat-qwen.json +20 -20
- package/dist/conversion/compat/profiles/responses-c4m.json +42 -42
- package/dist/conversion/config/sample-config.json +1 -1
- package/dist/conversion/hub/pipeline/compat/compat-pipeline-executor.js +18 -0
- package/dist/conversion/hub/pipeline/compat/compat-types.d.ts +6 -0
- package/dist/conversion/hub/pipeline/hub-pipeline.js +28 -1
- package/dist/conversion/hub/pipeline/target-utils.js +6 -0
- package/dist/conversion/hub/process/chat-process.js +100 -18
- package/dist/conversion/hub/response/provider-response.d.ts +13 -1
- package/dist/conversion/hub/response/provider-response.js +84 -35
- package/dist/conversion/hub/response/server-side-tools.js +61 -4
- package/dist/conversion/hub/semantic-mappers/gemini-mapper.js +123 -3
- package/dist/conversion/hub/semantic-mappers/responses-mapper.js +17 -1
- package/dist/conversion/hub/standardized-bridge.js +14 -0
- package/dist/conversion/responses/responses-openai-bridge.js +35 -2
- package/dist/conversion/shared/anthropic-message-utils.js +92 -3
- package/dist/conversion/shared/bridge-message-utils.js +137 -10
- package/dist/conversion/shared/responses-output-builder.js +43 -2
- package/dist/conversion/shared/tool-filter-pipeline.js +1 -0
- package/dist/router/virtual-router/bootstrap.js +44 -12
- package/dist/router/virtual-router/classifier.js +11 -17
- package/dist/router/virtual-router/engine.d.ts +9 -0
- package/dist/router/virtual-router/engine.js +160 -18
- package/dist/router/virtual-router/features.js +1 -1
- package/dist/router/virtual-router/message-utils.js +36 -24
- package/dist/router/virtual-router/provider-registry.js +2 -1
- package/dist/router/virtual-router/token-counter.js +14 -3
- package/dist/router/virtual-router/types.d.ts +45 -0
- package/dist/router/virtual-router/types.js +2 -1
- package/dist/servertool/engine.d.ts +27 -0
- package/dist/servertool/engine.js +60 -0
- package/dist/servertool/flow-types.d.ts +40 -0
- package/dist/servertool/flow-types.js +1 -0
- package/dist/servertool/handlers/vision.d.ts +1 -0
- package/dist/servertool/handlers/vision.js +194 -0
- package/dist/servertool/handlers/web-search.d.ts +1 -0
- package/dist/servertool/handlers/web-search.js +638 -0
- package/dist/servertool/orchestration-types.d.ts +33 -0
- package/dist/servertool/orchestration-types.js +1 -0
- package/dist/servertool/registry.d.ts +18 -0
- package/dist/servertool/registry.js +27 -0
- package/dist/servertool/server-side-tools.d.ts +8 -0
- package/dist/servertool/server-side-tools.js +208 -0
- package/dist/servertool/types.d.ts +88 -0
- package/dist/servertool/types.js +1 -0
- package/dist/servertool/vision-tool.d.ts +2 -0
- package/dist/servertool/vision-tool.js +185 -0
- package/dist/sse/sse-to-json/builders/response-builder.js +6 -3
- package/package.json +1 -1
|
@@ -171,6 +171,117 @@ function collectParameters(payload) {
|
|
|
171
171
|
}
|
|
172
172
|
return Object.keys(params).length ? params : undefined;
|
|
173
173
|
}
|
|
174
|
+
function appendChatContentToGeminiParts(message, targetParts) {
|
|
175
|
+
const content = message.content;
|
|
176
|
+
if (typeof content === 'string') {
|
|
177
|
+
const text = content.trim();
|
|
178
|
+
if (text.length) {
|
|
179
|
+
targetParts.push({ text });
|
|
180
|
+
}
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
if (!Array.isArray(content)) {
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
const items = content;
|
|
187
|
+
for (const block of items) {
|
|
188
|
+
if (block == null)
|
|
189
|
+
continue;
|
|
190
|
+
if (typeof block === 'string') {
|
|
191
|
+
const text = block.trim();
|
|
192
|
+
if (text.length) {
|
|
193
|
+
targetParts.push({ text });
|
|
194
|
+
}
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
if (typeof block !== 'object') {
|
|
198
|
+
const text = String(block);
|
|
199
|
+
if (text.trim().length) {
|
|
200
|
+
targetParts.push({ text: text.trim() });
|
|
201
|
+
}
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
const record = block;
|
|
205
|
+
const rawType = record.type;
|
|
206
|
+
const type = typeof rawType === 'string' ? rawType.toLowerCase() : '';
|
|
207
|
+
// Text-style blocks
|
|
208
|
+
if (!type || type === 'text') {
|
|
209
|
+
const textValue = typeof record.text === 'string'
|
|
210
|
+
? record.text
|
|
211
|
+
: typeof record.content === 'string'
|
|
212
|
+
? record.content
|
|
213
|
+
: '';
|
|
214
|
+
const text = textValue.trim();
|
|
215
|
+
if (text.length) {
|
|
216
|
+
targetParts.push({ text });
|
|
217
|
+
}
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
// Image-style blocks -> Gemini inlineData
|
|
221
|
+
if (type === 'image' || type === 'image_url') {
|
|
222
|
+
// Prefer OpenAI-style image_url.url, but also accept uri/url/data.
|
|
223
|
+
let url;
|
|
224
|
+
const imageUrlRaw = record.image_url;
|
|
225
|
+
if (typeof imageUrlRaw === 'string') {
|
|
226
|
+
url = imageUrlRaw;
|
|
227
|
+
}
|
|
228
|
+
else if (imageUrlRaw && typeof imageUrlRaw === 'object' && typeof imageUrlRaw.url === 'string') {
|
|
229
|
+
url = imageUrlRaw.url;
|
|
230
|
+
}
|
|
231
|
+
else if (typeof record.uri === 'string') {
|
|
232
|
+
url = record.uri;
|
|
233
|
+
}
|
|
234
|
+
else if (typeof record.url === 'string') {
|
|
235
|
+
url = record.url;
|
|
236
|
+
}
|
|
237
|
+
else if (typeof record.data === 'string') {
|
|
238
|
+
url = record.data;
|
|
239
|
+
}
|
|
240
|
+
const trimmed = (url ?? '').trim();
|
|
241
|
+
if (!trimmed.length) {
|
|
242
|
+
// Fallback: at least emit a textual marker so内容不会完全丢失
|
|
243
|
+
targetParts.push({ text: '[image]' });
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
let mimeType;
|
|
247
|
+
let data;
|
|
248
|
+
// data:URL → inlineData { mimeType, data }
|
|
249
|
+
if (trimmed.startsWith('data:')) {
|
|
250
|
+
const match = /^data:([^;,]+)?(?:;base64)?,(.*)$/s.exec(trimmed);
|
|
251
|
+
if (match) {
|
|
252
|
+
mimeType = (match[1] || '').trim() || undefined;
|
|
253
|
+
data = match[2] || '';
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if (data && data.trim().length) {
|
|
257
|
+
const inline = {
|
|
258
|
+
inlineData: {
|
|
259
|
+
data: data.trim()
|
|
260
|
+
}
|
|
261
|
+
};
|
|
262
|
+
if (mimeType && mimeType.length) {
|
|
263
|
+
inline.inlineData.mimeType = mimeType;
|
|
264
|
+
}
|
|
265
|
+
targetParts.push(inline);
|
|
266
|
+
}
|
|
267
|
+
else {
|
|
268
|
+
// 非 data: URL 暂时作为文本 URL 传递,保持语义可见
|
|
269
|
+
targetParts.push({ text: trimmed });
|
|
270
|
+
}
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
// 默认:回退为文本 JSON 表示,避免静默丢失内容
|
|
274
|
+
try {
|
|
275
|
+
const jsonText = JSON.stringify(record);
|
|
276
|
+
if (jsonText.trim().length) {
|
|
277
|
+
targetParts.push({ text: jsonText });
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
catch {
|
|
281
|
+
// ignore malformed block
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
174
285
|
function buildGeminiRequestFromChat(chat, metadata) {
|
|
175
286
|
const contents = [];
|
|
176
287
|
const emittedToolOutputs = new Set();
|
|
@@ -191,9 +302,7 @@ function buildGeminiRequestFromChat(chat, metadata) {
|
|
|
191
302
|
role: mapChatRoleToGemini(message.role),
|
|
192
303
|
parts: []
|
|
193
304
|
};
|
|
194
|
-
|
|
195
|
-
entry.parts.push({ text: message.content });
|
|
196
|
-
}
|
|
305
|
+
appendChatContentToGeminiParts(message, entry.parts);
|
|
197
306
|
const toolCalls = Array.isArray(message.tool_calls) ? message.tool_calls : [];
|
|
198
307
|
for (const tc of toolCalls) {
|
|
199
308
|
if (!tc || typeof tc !== 'object')
|
|
@@ -335,7 +444,18 @@ function safeParseJson(value) {
|
|
|
335
444
|
}
|
|
336
445
|
}
|
|
337
446
|
function ensureFunctionResponsePayload(value) {
|
|
447
|
+
// Gemini function_response.response 字段在 CloudCode/Gemini CLI 协议里对应的是
|
|
448
|
+
// protobuf Struct(JSON object),而不是顶层数组。
|
|
449
|
+
// 这里做一层规范化:
|
|
450
|
+
// - 对象:直接透传;
|
|
451
|
+
// - 数组:包一层 { result: [...] } 避免把数组作为 Struct 根节点;
|
|
452
|
+
// - 原始值:包一层 { result: value },并把 undefined 映射为 null。
|
|
338
453
|
if (value && typeof value === 'object') {
|
|
454
|
+
if (Array.isArray(value)) {
|
|
455
|
+
return {
|
|
456
|
+
result: value
|
|
457
|
+
};
|
|
458
|
+
}
|
|
339
459
|
return value;
|
|
340
460
|
}
|
|
341
461
|
return {
|
|
@@ -148,6 +148,20 @@ function serializeSystemContent(message) {
|
|
|
148
148
|
}
|
|
149
149
|
return undefined;
|
|
150
150
|
}
|
|
151
|
+
function mergeMetadata(a, b) {
|
|
152
|
+
if (!a && !b) {
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
if (!a && b) {
|
|
156
|
+
return jsonClone(b);
|
|
157
|
+
}
|
|
158
|
+
if (a && !b) {
|
|
159
|
+
return jsonClone(a);
|
|
160
|
+
}
|
|
161
|
+
const left = jsonClone(a);
|
|
162
|
+
const right = jsonClone(b);
|
|
163
|
+
return { ...left, ...right };
|
|
164
|
+
}
|
|
151
165
|
export class ResponsesSemanticMapper {
|
|
152
166
|
async toChat(format, ctx) {
|
|
153
167
|
const payload = format.payload || {};
|
|
@@ -209,13 +223,15 @@ export class ResponsesSemanticMapper {
|
|
|
209
223
|
.map(message => serializeSystemContent(message))
|
|
210
224
|
.filter((content) => typeof content === 'string' && content.length > 0);
|
|
211
225
|
const capturedContext = chat.metadata?.responsesContext;
|
|
226
|
+
const envelopeMetadata = chat.metadata && isJsonObject(chat.metadata) ? chat.metadata : undefined;
|
|
212
227
|
const responsesContext = isJsonObject(capturedContext)
|
|
213
228
|
? {
|
|
214
229
|
...capturedContext,
|
|
230
|
+
metadata: mergeMetadata(capturedContext.metadata, envelopeMetadata),
|
|
215
231
|
originalSystemMessages
|
|
216
232
|
}
|
|
217
233
|
: {
|
|
218
|
-
metadata:
|
|
234
|
+
metadata: envelopeMetadata,
|
|
219
235
|
originalSystemMessages
|
|
220
236
|
};
|
|
221
237
|
const responsesResult = buildResponsesRequestFromChat(requestShape, responsesContext);
|
|
@@ -60,6 +60,20 @@ export function standardizedToChatEnvelope(request, options) {
|
|
|
60
60
|
const metadata = {
|
|
61
61
|
context: adapterContext
|
|
62
62
|
};
|
|
63
|
+
const sourceMeta = (request.metadata && typeof request.metadata === 'object'
|
|
64
|
+
? request.metadata
|
|
65
|
+
: undefined);
|
|
66
|
+
if (sourceMeta) {
|
|
67
|
+
if (sourceMeta.webSearch && typeof sourceMeta.webSearch === 'object') {
|
|
68
|
+
metadata.webSearch = sourceMeta.webSearch;
|
|
69
|
+
}
|
|
70
|
+
if (sourceMeta.forceWebSearch === true) {
|
|
71
|
+
metadata.forceWebSearch = true;
|
|
72
|
+
}
|
|
73
|
+
if (sourceMeta.forceVision === true) {
|
|
74
|
+
metadata.forceVision = true;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
63
77
|
if (typeof adapterContext.toolCallIdStyle === 'string' && adapterContext.toolCallIdStyle.length) {
|
|
64
78
|
metadata.toolCallIdStyle = adapterContext.toolCallIdStyle;
|
|
65
79
|
}
|
|
@@ -190,17 +190,50 @@ function mergeResponsesTools(originalTools, fromChat) {
|
|
|
190
190
|
export function buildResponsesRequestFromChat(payload, ctx, extras) {
|
|
191
191
|
const chat = unwrapData(payload);
|
|
192
192
|
const out = {};
|
|
193
|
+
const forceWebSearch = !!ctx &&
|
|
194
|
+
isObject(ctx.metadata) &&
|
|
195
|
+
isObject(ctx.metadata.webSearch) &&
|
|
196
|
+
ctx.metadata.webSearch.force === true;
|
|
193
197
|
// 基本字段
|
|
194
198
|
out.model = chat.model;
|
|
195
199
|
// tools: 反向映射为 ResponsesToolDefinition 形状
|
|
196
|
-
const
|
|
200
|
+
const chatTools = Array.isArray(chat.tools) ? chat.tools : [];
|
|
201
|
+
// 对于 openai-responses upstream,内建 web_search 由官方服务器处理。
|
|
202
|
+
// Chat 侧注入的 server-side web_search 函数(带 engine/query/recency/count)
|
|
203
|
+
// 仅用于非 Responses provider 的 server-tool 回环;在这里构造真正的
|
|
204
|
+
// `/v1/responses` 请求时,需要:
|
|
205
|
+
// 1) 不再把函数版 web_search 透传上游;
|
|
206
|
+
// 2) 若检测到 Chat 侧启用了 web_search 且原始请求中没有 builtin web_search,
|
|
207
|
+
// 则补一个 `{ type: "web_search" }` 内建工具给 OpenAI Responses。
|
|
208
|
+
const hasServerSideWebSearch = !forceWebSearch && chatTools.some((tool) => {
|
|
209
|
+
const fn = tool && typeof tool === 'object' ? tool.function : undefined;
|
|
210
|
+
const name = typeof fn?.name === 'string' ? fn.name.trim().toLowerCase() : '';
|
|
211
|
+
return name === 'web_search';
|
|
212
|
+
});
|
|
213
|
+
const toolsForBridge = hasServerSideWebSearch
|
|
214
|
+
? chatTools.filter((tool) => {
|
|
215
|
+
const fn = tool && typeof tool === 'object' ? tool.function : undefined;
|
|
216
|
+
const name = typeof fn?.name === 'string' ? fn.name.trim().toLowerCase() : '';
|
|
217
|
+
return name !== 'web_search';
|
|
218
|
+
})
|
|
219
|
+
: chatTools;
|
|
220
|
+
const responsesToolsFromChat = mapChatToolsToBridge(toolsForBridge, {
|
|
197
221
|
sanitizeName: sanitizeResponsesFunctionName
|
|
198
222
|
});
|
|
199
223
|
// Prefer Chat‑normalized tools, but if the original Responses payload carried
|
|
200
224
|
// non‑function tools (such as builtin `web_search`), merge them back so that
|
|
201
225
|
// upstream `/v1/responses` providers see their original tool definitions.
|
|
202
226
|
const originalTools = Array.isArray(ctx?.toolsRaw) ? ctx.toolsRaw : undefined;
|
|
203
|
-
|
|
227
|
+
let mergedTools = mergeResponsesTools(originalTools, responsesToolsFromChat);
|
|
228
|
+
if (hasServerSideWebSearch) {
|
|
229
|
+
const normalizeType = (value) => typeof value === 'string' ? value.trim().toLowerCase() : '';
|
|
230
|
+
const hasBuiltinWebSearch = (mergedTools && mergedTools.some((tool) => normalizeType(tool.type) === 'web_search')) ||
|
|
231
|
+
(originalTools && originalTools.some((tool) => normalizeType(tool.type) === 'web_search'));
|
|
232
|
+
if (!hasBuiltinWebSearch) {
|
|
233
|
+
const injected = { type: 'web_search' };
|
|
234
|
+
mergedTools = mergedTools ? [...mergedTools, injected] : [injected];
|
|
235
|
+
}
|
|
236
|
+
}
|
|
204
237
|
if (mergedTools?.length) {
|
|
205
238
|
out.tools = mergedTools;
|
|
206
239
|
}
|
|
@@ -266,6 +266,7 @@ export function buildOpenAIChatFromAnthropic(payload) {
|
|
|
266
266
|
continue;
|
|
267
267
|
}
|
|
268
268
|
const textParts = [];
|
|
269
|
+
const imageBlocks = [];
|
|
269
270
|
const toolCalls = [];
|
|
270
271
|
const reasoningParts = [];
|
|
271
272
|
const toolResults = [];
|
|
@@ -284,6 +285,29 @@ export function buildOpenAIChatFromAnthropic(payload) {
|
|
|
284
285
|
reasoningParts.push(thinkingText);
|
|
285
286
|
}
|
|
286
287
|
}
|
|
288
|
+
else if (t === 'image') {
|
|
289
|
+
const source = block.source;
|
|
290
|
+
if (source && typeof source === 'object') {
|
|
291
|
+
const s = source;
|
|
292
|
+
const srcType = typeof s.type === 'string' ? s.type.toLowerCase() : '';
|
|
293
|
+
let url;
|
|
294
|
+
if (srcType === 'url' && typeof s.url === 'string') {
|
|
295
|
+
url = s.url;
|
|
296
|
+
}
|
|
297
|
+
else if (srcType === 'base64' && typeof s.data === 'string') {
|
|
298
|
+
const mediaType = typeof s.media_type === 'string' && s.media_type.trim().length
|
|
299
|
+
? s.media_type.trim()
|
|
300
|
+
: 'image/png';
|
|
301
|
+
url = `data:${mediaType};base64,${s.data}`;
|
|
302
|
+
}
|
|
303
|
+
if (url && url.trim().length) {
|
|
304
|
+
imageBlocks.push({
|
|
305
|
+
type: 'image_url',
|
|
306
|
+
image_url: { url: url.trim() }
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
287
311
|
else if (t === 'tool_use') {
|
|
288
312
|
const name = requireTrimmedString(block.name, 'tool_use.name');
|
|
289
313
|
const id = requireTrimmedString(block.id, 'tool_use.id');
|
|
@@ -310,10 +334,22 @@ export function buildOpenAIChatFromAnthropic(payload) {
|
|
|
310
334
|
}
|
|
311
335
|
const hasText = typeof normalized.contentText === 'string' && normalized.contentText.length > 0;
|
|
312
336
|
const hasReasoning = mergedReasoning.length > 0;
|
|
313
|
-
if (hasText || hasRawText || toolCalls.length > 0 || hasReasoning) {
|
|
337
|
+
if (hasText || hasRawText || toolCalls.length > 0 || hasReasoning || imageBlocks.length > 0) {
|
|
338
|
+
let contentNode = (hasText ? normalized.contentText : undefined) ?? combinedText ?? '';
|
|
339
|
+
if (imageBlocks.length > 0) {
|
|
340
|
+
const blocks = [];
|
|
341
|
+
const textPayload = (hasText ? normalized.contentText : undefined) ?? combinedText ?? '';
|
|
342
|
+
if (typeof textPayload === 'string' && textPayload.trim().length) {
|
|
343
|
+
blocks.push({ type: 'text', text: textPayload.trim() });
|
|
344
|
+
}
|
|
345
|
+
for (const img of imageBlocks) {
|
|
346
|
+
blocks.push(jsonClone(img));
|
|
347
|
+
}
|
|
348
|
+
contentNode = blocks;
|
|
349
|
+
}
|
|
314
350
|
const msg = {
|
|
315
351
|
role,
|
|
316
|
-
content:
|
|
352
|
+
content: contentNode
|
|
317
353
|
};
|
|
318
354
|
if (toolCalls.length)
|
|
319
355
|
msg.tool_calls = toolCalls;
|
|
@@ -690,7 +726,8 @@ export function buildAnthropicRequestFromOpenAIChat(chatReq) {
|
|
|
690
726
|
targetShape = mirrorShapes[mirrorIndex];
|
|
691
727
|
mirrorIndex += 1;
|
|
692
728
|
}
|
|
693
|
-
const
|
|
729
|
+
const contentNode = m.content;
|
|
730
|
+
const text = collectText(contentNode).trim();
|
|
694
731
|
if (role === 'system') {
|
|
695
732
|
if (!text) {
|
|
696
733
|
throw new Error('Anthropic bridge constraint violated: Chat system message must contain text');
|
|
@@ -715,6 +752,58 @@ export function buildAnthropicRequestFromOpenAIChat(chatReq) {
|
|
|
715
752
|
continue;
|
|
716
753
|
}
|
|
717
754
|
const blocks = [];
|
|
755
|
+
if (Array.isArray(contentNode)) {
|
|
756
|
+
// Preserve or synthesize image blocks where possible, and fall back to text for the rest.
|
|
757
|
+
for (const entry of contentNode) {
|
|
758
|
+
if (!entry || typeof entry !== 'object')
|
|
759
|
+
continue;
|
|
760
|
+
const node = entry;
|
|
761
|
+
const t = typeof node.type === 'string' ? node.type.toLowerCase() : '';
|
|
762
|
+
if (t === 'image' && node.source && typeof node.source === 'object') {
|
|
763
|
+
// Pass-through Anthropic image block as-is.
|
|
764
|
+
blocks.push({
|
|
765
|
+
type: 'image',
|
|
766
|
+
source: jsonClone(node.source)
|
|
767
|
+
});
|
|
768
|
+
continue;
|
|
769
|
+
}
|
|
770
|
+
if (t === 'image_url') {
|
|
771
|
+
let url = '';
|
|
772
|
+
const imageUrl = node.image_url;
|
|
773
|
+
if (typeof imageUrl === 'string') {
|
|
774
|
+
url = imageUrl;
|
|
775
|
+
}
|
|
776
|
+
else if (imageUrl && typeof imageUrl === 'object' && typeof imageUrl.url === 'string') {
|
|
777
|
+
url = imageUrl.url;
|
|
778
|
+
}
|
|
779
|
+
const trimmed = url.trim();
|
|
780
|
+
if (!trimmed.length)
|
|
781
|
+
continue;
|
|
782
|
+
const source = {};
|
|
783
|
+
if (trimmed.startsWith('data:')) {
|
|
784
|
+
const match = /^data:([^;,]+)?(?:;base64)?,(.*)$/s.exec(trimmed);
|
|
785
|
+
if (match) {
|
|
786
|
+
const mediaType = (match[1] || '').trim() || 'image/png';
|
|
787
|
+
source.type = 'base64';
|
|
788
|
+
source.media_type = mediaType;
|
|
789
|
+
source.data = match[2] || '';
|
|
790
|
+
}
|
|
791
|
+
else {
|
|
792
|
+
source.type = 'url';
|
|
793
|
+
source.url = trimmed;
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
else {
|
|
797
|
+
source.type = 'url';
|
|
798
|
+
source.url = trimmed;
|
|
799
|
+
}
|
|
800
|
+
blocks.push({
|
|
801
|
+
type: 'image',
|
|
802
|
+
source
|
|
803
|
+
});
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
}
|
|
718
807
|
if (text) {
|
|
719
808
|
blocks.push({ type: 'text', text });
|
|
720
809
|
}
|
|
@@ -66,6 +66,59 @@ function collectText(value) {
|
|
|
66
66
|
}
|
|
67
67
|
return '';
|
|
68
68
|
}
|
|
69
|
+
function extractImageBlocksFromContent(content) {
|
|
70
|
+
const images = [];
|
|
71
|
+
const visit = (value) => {
|
|
72
|
+
if (!value)
|
|
73
|
+
return;
|
|
74
|
+
if (Array.isArray(value)) {
|
|
75
|
+
for (const entry of value)
|
|
76
|
+
visit(entry);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
if (typeof value !== 'object') {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const record = value;
|
|
83
|
+
const typeValue = typeof record.type === 'string' ? record.type.toLowerCase() : '';
|
|
84
|
+
if (typeValue === 'image' || typeValue === 'image_url' || typeValue === 'input_image') {
|
|
85
|
+
let url = '';
|
|
86
|
+
const imageUrl = record.image_url;
|
|
87
|
+
if (typeof imageUrl === 'string') {
|
|
88
|
+
url = imageUrl;
|
|
89
|
+
}
|
|
90
|
+
else if (imageUrl && typeof imageUrl === 'object' && typeof imageUrl.url === 'string') {
|
|
91
|
+
url = imageUrl.url;
|
|
92
|
+
}
|
|
93
|
+
else if (typeof record.url === 'string') {
|
|
94
|
+
url = record.url;
|
|
95
|
+
}
|
|
96
|
+
else if (typeof record.uri === 'string') {
|
|
97
|
+
url = record.uri;
|
|
98
|
+
}
|
|
99
|
+
else if (typeof record.data === 'string') {
|
|
100
|
+
url = record.data;
|
|
101
|
+
}
|
|
102
|
+
const trimmed = url.trim();
|
|
103
|
+
if (trimmed.length) {
|
|
104
|
+
let detail;
|
|
105
|
+
if (imageUrl && typeof imageUrl === 'object' && typeof imageUrl.detail === 'string') {
|
|
106
|
+
detail = imageUrl.detail.trim() || undefined;
|
|
107
|
+
}
|
|
108
|
+
else if (typeof record.detail === 'string') {
|
|
109
|
+
detail = record.detail.trim() || undefined;
|
|
110
|
+
}
|
|
111
|
+
images.push({ url: trimmed, detail });
|
|
112
|
+
}
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
if (Array.isArray(record.content)) {
|
|
116
|
+
visit(record.content);
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
visit(content);
|
|
120
|
+
return images;
|
|
121
|
+
}
|
|
69
122
|
function extractUserTextFromEntry(entry) {
|
|
70
123
|
if (!entry || typeof entry !== 'object')
|
|
71
124
|
return '';
|
|
@@ -94,6 +147,7 @@ export function convertMessagesToBridgeInput(options) {
|
|
|
94
147
|
const role = coerceBridgeRole(m.role || 'user');
|
|
95
148
|
const content = m.content;
|
|
96
149
|
const collectedText = collectText(content);
|
|
150
|
+
const imageBlocks = extractImageBlocksFromContent(content);
|
|
97
151
|
const text = role === 'system' ? collectedText : collectedText.trim();
|
|
98
152
|
if (role === 'system') {
|
|
99
153
|
if (collectedText && collectedText.length) {
|
|
@@ -170,13 +224,29 @@ export function convertMessagesToBridgeInput(options) {
|
|
|
170
224
|
}
|
|
171
225
|
continue;
|
|
172
226
|
}
|
|
173
|
-
if (typeof text === 'string') {
|
|
227
|
+
if (typeof text === 'string' || imageBlocks.length) {
|
|
174
228
|
const tRole = role === 'assistant' ? 'output_text' : 'input_text';
|
|
175
|
-
const
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
}
|
|
179
|
-
|
|
229
|
+
const blocks = [];
|
|
230
|
+
if (typeof text === 'string' && text.length) {
|
|
231
|
+
blocks.push({ type: tRole, text });
|
|
232
|
+
}
|
|
233
|
+
for (const img of imageBlocks) {
|
|
234
|
+
const block = {
|
|
235
|
+
type: 'input_image',
|
|
236
|
+
image_url: img.url
|
|
237
|
+
};
|
|
238
|
+
if (img.detail) {
|
|
239
|
+
block.detail = img.detail;
|
|
240
|
+
}
|
|
241
|
+
blocks.push(block);
|
|
242
|
+
}
|
|
243
|
+
if (blocks.length) {
|
|
244
|
+
const entry = {
|
|
245
|
+
role,
|
|
246
|
+
content: blocks
|
|
247
|
+
};
|
|
248
|
+
input.push(entry);
|
|
249
|
+
}
|
|
180
250
|
if (role === 'user') {
|
|
181
251
|
const trimmed = typeof text === 'string' ? text.trim() : '';
|
|
182
252
|
if (trimmed.length) {
|
|
@@ -260,6 +330,7 @@ function processMessageBlocks(blocks, normalizeFunctionName, tools, toolNameById
|
|
|
260
330
|
const toolMessages = [];
|
|
261
331
|
let currentLastCall = lastToolCallId;
|
|
262
332
|
const reasoningSegments = [];
|
|
333
|
+
const images = [];
|
|
263
334
|
for (const block of blocks) {
|
|
264
335
|
if (!block || typeof block !== 'object')
|
|
265
336
|
continue;
|
|
@@ -282,6 +353,18 @@ function processMessageBlocks(blocks, normalizeFunctionName, tools, toolNameById
|
|
|
282
353
|
toolMessages.push(tm);
|
|
283
354
|
currentLastCall = nested.lastCallId;
|
|
284
355
|
reasoningSegments.push(...nested.reasoningSegments);
|
|
356
|
+
if (nested.images.length)
|
|
357
|
+
images.push(...nested.images);
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
if (type === 'input_image') {
|
|
361
|
+
const url = typeof block.image_url === 'string' ? block.image_url.trim() : '';
|
|
362
|
+
if (url) {
|
|
363
|
+
const detail = typeof block.detail === 'string' && block.detail.trim()
|
|
364
|
+
? block.detail.trim()
|
|
365
|
+
: undefined;
|
|
366
|
+
images.push({ url, detail });
|
|
367
|
+
}
|
|
285
368
|
continue;
|
|
286
369
|
}
|
|
287
370
|
if (type === 'function_call') {
|
|
@@ -344,7 +427,7 @@ function processMessageBlocks(blocks, normalizeFunctionName, tools, toolNameById
|
|
|
344
427
|
}
|
|
345
428
|
}
|
|
346
429
|
const text = textParts.length ? textParts.join('\n').trim() : null;
|
|
347
|
-
return { text, toolCalls, toolMessages, lastCallId: currentLastCall, reasoningSegments };
|
|
430
|
+
return { text, images, toolCalls, toolMessages, lastCallId: currentLastCall, reasoningSegments };
|
|
348
431
|
}
|
|
349
432
|
export function convertBridgeInputToChatMessages(options) {
|
|
350
433
|
const { input, tools, normalizeFunctionName, toolResultFallbackText } = options;
|
|
@@ -470,7 +553,29 @@ export function convertBridgeInputToChatMessages(options) {
|
|
|
470
553
|
for (const msg of nested.toolMessages)
|
|
471
554
|
messages.push(msg);
|
|
472
555
|
const normalizedRole = coerceBridgeRole((explicit.role ?? entry.role) || 'user');
|
|
473
|
-
if (
|
|
556
|
+
if (nested.images.length) {
|
|
557
|
+
const contentBlocks = [];
|
|
558
|
+
if (typeof nested.text === 'string' && nested.text.trim().length) {
|
|
559
|
+
contentBlocks.push({ type: 'text', text: nested.text });
|
|
560
|
+
}
|
|
561
|
+
for (const img of nested.images) {
|
|
562
|
+
const imgBlock = { type: 'image_url', image_url: { url: img.url } };
|
|
563
|
+
if (img.detail) {
|
|
564
|
+
imgBlock.image_url.detail = img.detail;
|
|
565
|
+
}
|
|
566
|
+
contentBlocks.push(imgBlock);
|
|
567
|
+
}
|
|
568
|
+
const msg = {
|
|
569
|
+
role: normalizedRole,
|
|
570
|
+
content: contentBlocks
|
|
571
|
+
};
|
|
572
|
+
const combinedReasoning = combineReasoningSegments(consumeEntryReasoning(), nested.reasoningSegments);
|
|
573
|
+
if (combinedReasoning.length) {
|
|
574
|
+
msg.reasoning_content = combinedReasoning.join('\n');
|
|
575
|
+
}
|
|
576
|
+
messages.push(msg);
|
|
577
|
+
}
|
|
578
|
+
else if (typeof nested.text === 'string') {
|
|
474
579
|
pushNormalizedChatMessage(messages, normalizedRole, nested.text, {
|
|
475
580
|
reasoningSegments: combineReasoningSegments(consumeEntryReasoning(), nested.reasoningSegments)
|
|
476
581
|
});
|
|
@@ -491,9 +596,31 @@ export function convertBridgeInputToChatMessages(options) {
|
|
|
491
596
|
for (const msg of nested.toolMessages)
|
|
492
597
|
messages.push(msg);
|
|
493
598
|
const normalizedRole = coerceBridgeRole(entry.role || 'user');
|
|
494
|
-
if (
|
|
599
|
+
if (nested.images.length) {
|
|
600
|
+
const contentBlocks = [];
|
|
601
|
+
if (typeof nested.text === 'string' && nested.text.trim().length) {
|
|
602
|
+
contentBlocks.push({ type: 'text', text: nested.text });
|
|
603
|
+
}
|
|
604
|
+
for (const img of nested.images) {
|
|
605
|
+
const imgBlock = { type: 'image_url', image_url: { url: img.url } };
|
|
606
|
+
if (img.detail) {
|
|
607
|
+
imgBlock.image_url.detail = img.detail;
|
|
608
|
+
}
|
|
609
|
+
contentBlocks.push(imgBlock);
|
|
610
|
+
}
|
|
611
|
+
const msg = {
|
|
612
|
+
role: normalizedRole,
|
|
613
|
+
content: contentBlocks
|
|
614
|
+
};
|
|
615
|
+
const combinedReasoning = combineReasoningSegments(consumeEntryReasoning(), nested.reasoningSegments);
|
|
616
|
+
if (combinedReasoning.length) {
|
|
617
|
+
msg.reasoning_content = combinedReasoning.join('\n');
|
|
618
|
+
}
|
|
619
|
+
messages.push(msg);
|
|
620
|
+
}
|
|
621
|
+
else if (typeof nested.text === 'string') {
|
|
495
622
|
pushNormalizedChatMessage(messages, normalizedRole, nested.text, {
|
|
496
|
-
reasoningSegments:
|
|
623
|
+
reasoningSegments: consumeEntryReasoning()
|
|
497
624
|
});
|
|
498
625
|
}
|
|
499
626
|
lastToolCallId = nested.lastCallId;
|
|
@@ -1,6 +1,41 @@
|
|
|
1
1
|
import { normalizeFunctionCallId } from './bridge-id-utils.js';
|
|
2
2
|
import { normalizeContentPart } from './output-content-normalizer.js';
|
|
3
3
|
import { expandResponsesMessageItem } from '../../sse/shared/responses-output-normalizer.js';
|
|
4
|
+
function buildToolOutputIndex(response) {
|
|
5
|
+
const ids = new Set();
|
|
6
|
+
try {
|
|
7
|
+
const primary = Array.isArray(response.tool_outputs)
|
|
8
|
+
? response.tool_outputs
|
|
9
|
+
: [];
|
|
10
|
+
for (const entry of primary) {
|
|
11
|
+
if (!entry || typeof entry !== 'object')
|
|
12
|
+
continue;
|
|
13
|
+
const raw = entry.tool_call_id ||
|
|
14
|
+
entry.call_id ||
|
|
15
|
+
entry.id;
|
|
16
|
+
if (typeof raw === 'string' && raw.trim().length) {
|
|
17
|
+
const trimmed = raw.trim();
|
|
18
|
+
// 记录原始 ID(例如 OpenAI 的 toolu_ 前缀),以兼容直接使用
|
|
19
|
+
// tool_call_id 的客户端;同时记录归一化后的 fc_ 形式,保证与
|
|
20
|
+
// buildFunctionCallOutput 中 normalizeFunctionCallId 的结果对齐。
|
|
21
|
+
ids.add(trimmed);
|
|
22
|
+
try {
|
|
23
|
+
const normalized = normalizeFunctionCallId({ callId: trimmed, fallback: trimmed });
|
|
24
|
+
if (normalized && normalized !== trimmed) {
|
|
25
|
+
ids.add(normalized);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
// 归一化失败不应影响主流程
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
// best-effort: 不因索引构建失败影响主流程
|
|
36
|
+
}
|
|
37
|
+
return ids;
|
|
38
|
+
}
|
|
4
39
|
function appendReasoningSegments(target, raw) {
|
|
5
40
|
if (typeof raw !== 'string' || !raw.length) {
|
|
6
41
|
return;
|
|
@@ -91,7 +126,13 @@ export function buildResponsesOutputFromChat(options) {
|
|
|
91
126
|
const usage = normalizeUsage(response.usage);
|
|
92
127
|
const outputTextMeta = response?.__responses_output_text_meta;
|
|
93
128
|
const outputText = resolveOutputText(convertedContent, outputTextMeta);
|
|
94
|
-
|
|
129
|
+
// 如果顶层 tool_outputs 已经为所有 tool_calls 提供了结果,说明这些函数调用
|
|
130
|
+
// 已在服务端(例如 server-side web_search)完成,不应再对客户端暴露
|
|
131
|
+
// required_action/submit_tool_outputs。此时只需返回 completed 状态即可,避免
|
|
132
|
+
// 再触发一轮工具回合。
|
|
133
|
+
const executedIds = buildToolOutputIndex(response);
|
|
134
|
+
const pendingToolCalls = normalizedToolCalls.filter((entry) => !executedIds.has(entry.id));
|
|
135
|
+
const hasNormalizedToolCalls = pendingToolCalls.length > 0;
|
|
95
136
|
if (hasNormalizedToolCalls) {
|
|
96
137
|
for (const item of outputItems) {
|
|
97
138
|
if (item.type === 'message') {
|
|
@@ -100,7 +141,7 @@ export function buildResponsesOutputFromChat(options) {
|
|
|
100
141
|
}
|
|
101
142
|
}
|
|
102
143
|
const requiredAction = hasNormalizedToolCalls
|
|
103
|
-
? buildRequiredActionFromNormalized(
|
|
144
|
+
? buildRequiredActionFromNormalized(pendingToolCalls)
|
|
104
145
|
: undefined;
|
|
105
146
|
const status = hasNormalizedToolCalls ? 'requires_action' : 'completed';
|
|
106
147
|
return {
|
|
@@ -111,6 +111,7 @@ function applyLocalToolGovernance(chatRequest, rawPayload) {
|
|
|
111
111
|
}
|
|
112
112
|
const hasImageHint = detectImageHint(messages, rawPayload);
|
|
113
113
|
if (hasImageHint) {
|
|
114
|
+
// 有图片线索时不干预工具列表,保持由上游(Codex 等)决定 view_image 的暴露与使用。
|
|
114
115
|
return chatRequest;
|
|
115
116
|
}
|
|
116
117
|
const filteredTools = tools.filter((tool) => {
|