@hebo-ai/gateway 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config.js +2 -0
- package/dist/endpoints/chat-completions/converters.d.ts +5 -1
- package/dist/endpoints/chat-completions/converters.js +61 -12
- package/dist/endpoints/chat-completions/schema.d.ts +54 -9
- package/dist/endpoints/chat-completions/schema.js +20 -13
- package/dist/models/anthropic/middleware.js +14 -13
- package/dist/providers/bedrock/middleware.d.ts +2 -1
- package/dist/providers/bedrock/middleware.js +29 -9
- package/dist/telemetry/ai-sdk.d.ts +2 -0
- package/dist/telemetry/ai-sdk.js +31 -0
- package/package.json +1 -1
- package/src/config.ts +3 -0
- package/src/endpoints/chat-completions/converters.test.ts +111 -0
- package/src/endpoints/chat-completions/converters.ts +71 -13
- package/src/endpoints/chat-completions/handler.ts +10 -3
- package/src/endpoints/chat-completions/schema.ts +22 -14
- package/src/endpoints/embeddings/handler.ts +5 -3
- package/src/middleware/debug.ts +37 -0
- package/src/middleware/matcher.ts +4 -0
- package/src/models/anthropic/middleware.test.ts +5 -1
- package/src/models/anthropic/middleware.ts +17 -13
- package/src/providers/bedrock/middleware.test.ts +118 -8
- package/src/providers/bedrock/middleware.ts +34 -9
- package/src/telemetry/ai-sdk.ts +46 -0
package/README.md
CHANGED
|
@@ -660,9 +660,9 @@ const gw = gateway({
|
|
|
660
660
|
// ...
|
|
661
661
|
telemetry: {
|
|
662
662
|
enabled: true,
|
|
663
|
-
tracer
|
|
663
|
+
tracer: new BasicTracerProvider({
|
|
664
664
|
spanProcessors: [new LangfuseSpanProcessor()],
|
|
665
|
-
}).getTracer("hebo")
|
|
665
|
+
}).getTracer("hebo"),
|
|
666
666
|
},
|
|
667
667
|
});
|
|
668
668
|
```
|
package/dist/config.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isLogger, logger, setLoggerInstance } from "./logger";
|
|
2
2
|
import { createDefaultLogger } from "./logger/default";
|
|
3
|
+
import { installAiSdkWarningLogger } from "./telemetry/ai-sdk";
|
|
3
4
|
import { kParsed, } from "./types";
|
|
4
5
|
export const parseConfig = (config) => {
|
|
5
6
|
// If it has been parsed before, just return.
|
|
@@ -64,6 +65,7 @@ export const parseConfig = (config) => {
|
|
|
64
65
|
gen_ai: "off",
|
|
65
66
|
hebo: "off",
|
|
66
67
|
};
|
|
68
|
+
installAiSdkWarningLogger(telemetrySignals.gen_ai);
|
|
67
69
|
// Return parsed config.
|
|
68
70
|
return {
|
|
69
71
|
...config,
|
|
@@ -7,6 +7,7 @@ export type TextCallOptions = {
|
|
|
7
7
|
messages: ModelMessage[];
|
|
8
8
|
tools?: ToolSet;
|
|
9
9
|
toolChoice?: ToolChoice<ToolSet>;
|
|
10
|
+
activeTools?: Array<keyof ToolSet>;
|
|
10
11
|
output?: Output.Output;
|
|
11
12
|
temperature?: number;
|
|
12
13
|
maxOutputTokens?: number;
|
|
@@ -24,7 +25,10 @@ export declare function fromChatCompletionsAssistantMessage(message: ChatComplet
|
|
|
24
25
|
export declare function fromChatCompletionsToolResultMessage(message: ChatCompletionsAssistantMessage, toolById: Map<string, ChatCompletionsToolMessage>): ToolModelMessage | undefined;
|
|
25
26
|
export declare function fromChatCompletionsContent(content: ChatCompletionsContentPart[]): UserContent;
|
|
26
27
|
export declare const convertToToolSet: (tools: ChatCompletionsTool[] | undefined) => ToolSet | undefined;
|
|
27
|
-
export declare const
|
|
28
|
+
export declare const convertToToolChoiceOptions: (toolChoice: ChatCompletionsToolChoice | undefined) => {
|
|
29
|
+
toolChoice?: ToolChoice<ToolSet>;
|
|
30
|
+
activeTools?: Array<keyof ToolSet>;
|
|
31
|
+
};
|
|
28
32
|
export declare function toChatCompletions(result: GenerateTextResult<ToolSet, Output.Output>, model: string): ChatCompletions;
|
|
29
33
|
export declare function toChatCompletionsResponse(result: GenerateTextResult<ToolSet, Output.Output>, model: string, responseInit?: ResponseInit): Response;
|
|
30
34
|
export declare function toChatCompletionsStream<E extends boolean = false>(result: StreamTextResult<ToolSet, Output.Output>, model: string, wrapErrors?: E): ReadableStream<ChatCompletionsChunk | (E extends true ? OpenAIError : Error)>;
|
|
@@ -7,10 +7,12 @@ import { toResponse } from "../../utils/response";
|
|
|
7
7
|
export function convertToTextCallOptions(params) {
|
|
8
8
|
const { messages, tools, tool_choice, temperature, max_tokens, max_completion_tokens, response_format, reasoning_effort, reasoning, frequency_penalty, presence_penalty, seed, stop, top_p, ...rest } = params;
|
|
9
9
|
Object.assign(rest, parseReasoningOptions(reasoning_effort, reasoning));
|
|
10
|
+
const { toolChoice, activeTools } = convertToToolChoiceOptions(tool_choice);
|
|
10
11
|
return {
|
|
11
12
|
messages: convertToModelMessages(messages),
|
|
12
13
|
tools: convertToToolSet(tools),
|
|
13
|
-
toolChoice
|
|
14
|
+
toolChoice,
|
|
15
|
+
activeTools,
|
|
14
16
|
output: convertToOutput(response_format),
|
|
15
17
|
temperature,
|
|
16
18
|
maxOutputTokens: max_completion_tokens ?? max_tokens,
|
|
@@ -207,24 +209,33 @@ export const convertToToolSet = (tools) => {
|
|
|
207
209
|
toolSet[t.function.name] = tool({
|
|
208
210
|
description: t.function.description,
|
|
209
211
|
inputSchema: jsonSchema(t.function.parameters),
|
|
212
|
+
strict: t.function.strict,
|
|
210
213
|
});
|
|
211
214
|
}
|
|
212
215
|
return toolSet;
|
|
213
216
|
};
|
|
214
|
-
export const
|
|
217
|
+
export const convertToToolChoiceOptions = (toolChoice) => {
|
|
215
218
|
if (!toolChoice) {
|
|
216
|
-
return
|
|
219
|
+
return {};
|
|
217
220
|
}
|
|
218
221
|
if (toolChoice === "none" || toolChoice === "auto" || toolChoice === "required") {
|
|
219
|
-
return toolChoice;
|
|
222
|
+
return { toolChoice };
|
|
220
223
|
}
|
|
221
224
|
// FUTURE: this is right now google specific, which is not supported by AI SDK, until then, we temporarily map it to auto for now https://docs.cloud.google.com/vertex-ai/generative-ai/docs/migrate/openai/overview
|
|
222
225
|
if (toolChoice === "validated") {
|
|
223
|
-
return "auto";
|
|
226
|
+
return { toolChoice: "auto" };
|
|
227
|
+
}
|
|
228
|
+
if (toolChoice.type === "allowed_tools") {
|
|
229
|
+
return {
|
|
230
|
+
toolChoice: toolChoice.allowed_tools.mode,
|
|
231
|
+
activeTools: toolChoice.allowed_tools.tools.map((toolRef) => toolRef.function.name),
|
|
232
|
+
};
|
|
224
233
|
}
|
|
225
234
|
return {
|
|
226
|
-
|
|
227
|
-
|
|
235
|
+
toolChoice: {
|
|
236
|
+
type: "tool",
|
|
237
|
+
toolName: toolChoice.function.name,
|
|
238
|
+
},
|
|
228
239
|
};
|
|
229
240
|
};
|
|
230
241
|
function parseToolResult(content) {
|
|
@@ -409,9 +420,12 @@ export const toChatCompletionsAssistantMessage = (result) => {
|
|
|
409
420
|
if (part.type === "text") {
|
|
410
421
|
if (message.content === null) {
|
|
411
422
|
message.content = part.text;
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
423
|
+
}
|
|
424
|
+
else {
|
|
425
|
+
message.content += part.text;
|
|
426
|
+
}
|
|
427
|
+
if (part.providerMetadata) {
|
|
428
|
+
message.extra_content = part.providerMetadata;
|
|
415
429
|
}
|
|
416
430
|
}
|
|
417
431
|
else if (part.type === "reasoning") {
|
|
@@ -427,6 +441,10 @@ export const toChatCompletionsAssistantMessage = (result) => {
|
|
|
427
441
|
if (reasoningDetails.length > 0) {
|
|
428
442
|
message.reasoning_details = reasoningDetails;
|
|
429
443
|
}
|
|
444
|
+
if (!message.content && !message.tool_calls) {
|
|
445
|
+
// some models return just reasoning without tool calls or content
|
|
446
|
+
message.content = "";
|
|
447
|
+
}
|
|
430
448
|
return message;
|
|
431
449
|
};
|
|
432
450
|
export function toReasoningDetail(reasoning, id, index) {
|
|
@@ -485,8 +503,8 @@ export function toChatCompletionsToolCall(id, name, args, providerMetadata) {
|
|
|
485
503
|
id,
|
|
486
504
|
type: "function",
|
|
487
505
|
function: {
|
|
488
|
-
name,
|
|
489
|
-
arguments: typeof args === "string" ? args : JSON.stringify(args),
|
|
506
|
+
name: normalizeToolName(name),
|
|
507
|
+
arguments: typeof args === "string" ? args : JSON.stringify(stripEmptyKeys(args)),
|
|
490
508
|
},
|
|
491
509
|
};
|
|
492
510
|
if (providerMetadata) {
|
|
@@ -494,6 +512,37 @@ export function toChatCompletionsToolCall(id, name, args, providerMetadata) {
|
|
|
494
512
|
}
|
|
495
513
|
return out;
|
|
496
514
|
}
|
|
515
|
+
function normalizeToolName(name) {
|
|
516
|
+
// some models hallucinate invalid characters
|
|
517
|
+
// normalize to valid characters [^A-Za-z0-9_-.] (non regex for perf)
|
|
518
|
+
// https://modelcontextprotocol.io/specification/draft/server/tools#tool-names
|
|
519
|
+
let out = "";
|
|
520
|
+
for (let i = 0; i < name.length; i++) {
|
|
521
|
+
if (out.length === 128)
|
|
522
|
+
break;
|
|
523
|
+
// eslint-disable-next-line unicorn/prefer-code-point
|
|
524
|
+
const c = name.charCodeAt(i);
|
|
525
|
+
if ((c >= 48 && c <= 57) ||
|
|
526
|
+
(c >= 65 && c <= 90) ||
|
|
527
|
+
(c >= 97 && c <= 122) ||
|
|
528
|
+
c === 95 ||
|
|
529
|
+
c === 45 ||
|
|
530
|
+
c === 46) {
|
|
531
|
+
out += name[i];
|
|
532
|
+
}
|
|
533
|
+
else {
|
|
534
|
+
out += "_";
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
return out;
|
|
538
|
+
}
|
|
539
|
+
function stripEmptyKeys(obj) {
|
|
540
|
+
if (!obj || typeof obj !== "object" || Array.isArray(obj))
|
|
541
|
+
return obj;
|
|
542
|
+
// some models hallucinate empty parameters
|
|
543
|
+
delete obj[""];
|
|
544
|
+
return obj;
|
|
545
|
+
}
|
|
497
546
|
export const toChatCompletionsFinishReason = (finishReason) => {
|
|
498
547
|
if (finishReason === "error" || finishReason === "other") {
|
|
499
548
|
return "stop";
|
|
@@ -281,20 +281,35 @@ export declare const ChatCompletionsToolSchema: z.ZodObject<{
|
|
|
281
281
|
name: z.ZodString;
|
|
282
282
|
description: z.ZodOptional<z.ZodString>;
|
|
283
283
|
parameters: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
284
|
+
strict: z.ZodOptional<z.ZodBoolean>;
|
|
284
285
|
}, z.core.$strip>;
|
|
285
286
|
}, z.core.$strip>;
|
|
286
287
|
export type ChatCompletionsTool = z.infer<typeof ChatCompletionsToolSchema>;
|
|
287
288
|
export declare const ChatCompletionsToolChoiceSchema: z.ZodUnion<readonly [z.ZodEnum<{
|
|
288
289
|
auto: "auto";
|
|
289
|
-
none: "none";
|
|
290
290
|
required: "required";
|
|
291
|
+
none: "none";
|
|
291
292
|
validated: "validated";
|
|
292
|
-
}>, z.ZodObject<{
|
|
293
|
+
}>, z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
293
294
|
type: z.ZodLiteral<"function">;
|
|
294
295
|
function: z.ZodObject<{
|
|
295
296
|
name: z.ZodString;
|
|
296
297
|
}, z.core.$strip>;
|
|
297
|
-
}, z.core.$strip
|
|
298
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
299
|
+
type: z.ZodLiteral<"allowed_tools">;
|
|
300
|
+
allowed_tools: z.ZodObject<{
|
|
301
|
+
mode: z.ZodEnum<{
|
|
302
|
+
auto: "auto";
|
|
303
|
+
required: "required";
|
|
304
|
+
}>;
|
|
305
|
+
tools: z.ZodArray<z.ZodObject<{
|
|
306
|
+
type: z.ZodLiteral<"function">;
|
|
307
|
+
function: z.ZodObject<{
|
|
308
|
+
name: z.ZodString;
|
|
309
|
+
}, z.core.$strip>;
|
|
310
|
+
}, z.core.$strip>>;
|
|
311
|
+
}, z.core.$strip>;
|
|
312
|
+
}, z.core.$strip>], "type">]>;
|
|
298
313
|
export type ChatCompletionsToolChoice = z.infer<typeof ChatCompletionsToolChoiceSchema>;
|
|
299
314
|
export declare const ChatCompletionsReasoningEffortSchema: z.ZodEnum<{
|
|
300
315
|
low: "low";
|
|
@@ -434,19 +449,34 @@ declare const ChatCompletionsInputsSchema: z.ZodObject<{
|
|
|
434
449
|
name: z.ZodString;
|
|
435
450
|
description: z.ZodOptional<z.ZodString>;
|
|
436
451
|
parameters: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
452
|
+
strict: z.ZodOptional<z.ZodBoolean>;
|
|
437
453
|
}, z.core.$strip>;
|
|
438
454
|
}, z.core.$strip>>>;
|
|
439
455
|
tool_choice: z.ZodOptional<z.ZodUnion<readonly [z.ZodEnum<{
|
|
440
456
|
auto: "auto";
|
|
441
|
-
none: "none";
|
|
442
457
|
required: "required";
|
|
458
|
+
none: "none";
|
|
443
459
|
validated: "validated";
|
|
444
|
-
}>, z.ZodObject<{
|
|
460
|
+
}>, z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
445
461
|
type: z.ZodLiteral<"function">;
|
|
446
462
|
function: z.ZodObject<{
|
|
447
463
|
name: z.ZodString;
|
|
448
464
|
}, z.core.$strip>;
|
|
449
|
-
}, z.core.$strip
|
|
465
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
466
|
+
type: z.ZodLiteral<"allowed_tools">;
|
|
467
|
+
allowed_tools: z.ZodObject<{
|
|
468
|
+
mode: z.ZodEnum<{
|
|
469
|
+
auto: "auto";
|
|
470
|
+
required: "required";
|
|
471
|
+
}>;
|
|
472
|
+
tools: z.ZodArray<z.ZodObject<{
|
|
473
|
+
type: z.ZodLiteral<"function">;
|
|
474
|
+
function: z.ZodObject<{
|
|
475
|
+
name: z.ZodString;
|
|
476
|
+
}, z.core.$strip>;
|
|
477
|
+
}, z.core.$strip>>;
|
|
478
|
+
}, z.core.$strip>;
|
|
479
|
+
}, z.core.$strip>], "type">]>>;
|
|
450
480
|
temperature: z.ZodOptional<z.ZodNumber>;
|
|
451
481
|
max_tokens: z.ZodOptional<z.ZodInt>;
|
|
452
482
|
max_completion_tokens: z.ZodOptional<z.ZodInt>;
|
|
@@ -580,19 +610,34 @@ export declare const ChatCompletionsBodySchema: z.ZodObject<{
|
|
|
580
610
|
name: z.ZodString;
|
|
581
611
|
description: z.ZodOptional<z.ZodString>;
|
|
582
612
|
parameters: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
613
|
+
strict: z.ZodOptional<z.ZodBoolean>;
|
|
583
614
|
}, z.core.$strip>;
|
|
584
615
|
}, z.core.$strip>>>;
|
|
585
616
|
tool_choice: z.ZodOptional<z.ZodUnion<readonly [z.ZodEnum<{
|
|
586
617
|
auto: "auto";
|
|
587
|
-
none: "none";
|
|
588
618
|
required: "required";
|
|
619
|
+
none: "none";
|
|
589
620
|
validated: "validated";
|
|
590
|
-
}>, z.ZodObject<{
|
|
621
|
+
}>, z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
591
622
|
type: z.ZodLiteral<"function">;
|
|
592
623
|
function: z.ZodObject<{
|
|
593
624
|
name: z.ZodString;
|
|
594
625
|
}, z.core.$strip>;
|
|
595
|
-
}, z.core.$strip
|
|
626
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
627
|
+
type: z.ZodLiteral<"allowed_tools">;
|
|
628
|
+
allowed_tools: z.ZodObject<{
|
|
629
|
+
mode: z.ZodEnum<{
|
|
630
|
+
auto: "auto";
|
|
631
|
+
required: "required";
|
|
632
|
+
}>;
|
|
633
|
+
tools: z.ZodArray<z.ZodObject<{
|
|
634
|
+
type: z.ZodLiteral<"function">;
|
|
635
|
+
function: z.ZodObject<{
|
|
636
|
+
name: z.ZodString;
|
|
637
|
+
}, z.core.$strip>;
|
|
638
|
+
}, z.core.$strip>>;
|
|
639
|
+
}, z.core.$strip>;
|
|
640
|
+
}, z.core.$strip>], "type">]>>;
|
|
596
641
|
temperature: z.ZodOptional<z.ZodNumber>;
|
|
597
642
|
max_tokens: z.ZodOptional<z.ZodInt>;
|
|
598
643
|
max_completion_tokens: z.ZodOptional<z.ZodInt>;
|
|
@@ -113,18 +113,29 @@ export const ChatCompletionsToolSchema = z.object({
|
|
|
113
113
|
name: z.string(),
|
|
114
114
|
description: z.string().optional(),
|
|
115
115
|
parameters: z.record(z.string(), z.unknown()),
|
|
116
|
-
|
|
116
|
+
strict: z.boolean().optional(),
|
|
117
|
+
}),
|
|
118
|
+
});
|
|
119
|
+
const ChatCompletionsNamedFunctionToolChoiceSchema = z.object({
|
|
120
|
+
type: z.literal("function"),
|
|
121
|
+
function: z.object({
|
|
122
|
+
name: z.string(),
|
|
123
|
+
}),
|
|
124
|
+
});
|
|
125
|
+
const ChatCompletionsAllowedFunctionToolChoiceSchema = z.object({
|
|
126
|
+
type: z.literal("allowed_tools"),
|
|
127
|
+
allowed_tools: z.object({
|
|
128
|
+
mode: z.enum(["auto", "required"]),
|
|
129
|
+
tools: z.array(ChatCompletionsNamedFunctionToolChoiceSchema).nonempty(),
|
|
117
130
|
}),
|
|
118
131
|
});
|
|
119
132
|
export const ChatCompletionsToolChoiceSchema = z.union([
|
|
120
133
|
z.enum(["none", "auto", "required", "validated"]),
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}),
|
|
127
|
-
}),
|
|
134
|
+
z.discriminatedUnion("type", [
|
|
135
|
+
ChatCompletionsNamedFunctionToolChoiceSchema,
|
|
136
|
+
ChatCompletionsAllowedFunctionToolChoiceSchema,
|
|
137
|
+
]),
|
|
138
|
+
// FUTURE: Missing CustomTool
|
|
128
139
|
]);
|
|
129
140
|
export const ChatCompletionsReasoningEffortSchema = z.enum([
|
|
130
141
|
"none",
|
|
@@ -161,11 +172,7 @@ export const ChatCompletionsResponseFormatSchema = z.discriminatedUnion("type",
|
|
|
161
172
|
]);
|
|
162
173
|
const ChatCompletionsInputsSchema = z.object({
|
|
163
174
|
messages: z.array(ChatCompletionsMessageSchema),
|
|
164
|
-
tools: z
|
|
165
|
-
.array(
|
|
166
|
-
// FUTURE: Missing CustomTool
|
|
167
|
-
ChatCompletionsToolSchema)
|
|
168
|
-
.optional(),
|
|
175
|
+
tools: z.array(ChatCompletionsToolSchema).optional(),
|
|
169
176
|
tool_choice: ChatCompletionsToolChoiceSchema.optional(),
|
|
170
177
|
temperature: z.number().min(0).max(2).optional(),
|
|
171
178
|
max_tokens: z.int().nonnegative().optional(),
|
|
@@ -5,11 +5,11 @@ const isClaude = (family, version) => {
|
|
|
5
5
|
return (modelId) => modelId.includes(`claude-${family}-${version}`) ||
|
|
6
6
|
modelId.includes(`claude-${family}-${dashed}`);
|
|
7
7
|
};
|
|
8
|
+
const isClaude4 = (modelId) => modelId.includes("claude-") && modelId.includes("-4");
|
|
8
9
|
const isOpus46 = isClaude("opus", "4.6");
|
|
9
10
|
const isOpus45 = isClaude("opus", "4.5");
|
|
10
11
|
const isOpus4 = isClaude("opus", "4");
|
|
11
12
|
const isSonnet46 = isClaude("sonnet", "4.6");
|
|
12
|
-
const isSonnet45 = isClaude("sonnet", "4.5");
|
|
13
13
|
export function mapClaudeReasoningEffort(effort, modelId) {
|
|
14
14
|
if (isOpus46(modelId)) {
|
|
15
15
|
switch (effort) {
|
|
@@ -48,7 +48,10 @@ function getMaxOutputTokens(modelId) {
|
|
|
48
48
|
return 32_000;
|
|
49
49
|
return 64_000;
|
|
50
50
|
}
|
|
51
|
+
// Documentation:
|
|
51
52
|
// https://platform.claude.com/docs/en/build-with-claude/effort
|
|
53
|
+
// https://platform.claude.com/docs/en/build-with-claude/extended-thinking
|
|
54
|
+
// https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking
|
|
52
55
|
export const claudeReasoningMiddleware = {
|
|
53
56
|
specificationVersion: "v3",
|
|
54
57
|
// eslint-disable-next-line require-await
|
|
@@ -66,30 +69,28 @@ export const claudeReasoningMiddleware = {
|
|
|
66
69
|
target["thinking"] = { type: "disabled" };
|
|
67
70
|
}
|
|
68
71
|
else if (reasoning.effort) {
|
|
72
|
+
if (isClaude4(modelId)) {
|
|
73
|
+
target["effort"] = mapClaudeReasoningEffort(reasoning.effort, modelId);
|
|
74
|
+
}
|
|
69
75
|
if (isOpus46(modelId)) {
|
|
70
76
|
target["thinking"] = clampedMaxTokens
|
|
71
77
|
? { type: "adaptive", budgetTokens: clampedMaxTokens }
|
|
72
78
|
: { type: "adaptive" };
|
|
73
|
-
target["effort"] = mapClaudeReasoningEffort(reasoning.effort, modelId);
|
|
74
79
|
}
|
|
75
80
|
else if (isSonnet46(modelId)) {
|
|
76
81
|
target["thinking"] = clampedMaxTokens
|
|
77
82
|
? { type: "enabled", budgetTokens: clampedMaxTokens }
|
|
78
83
|
: { type: "adaptive" };
|
|
79
|
-
target["effort"] = mapClaudeReasoningEffort(reasoning.effort, modelId);
|
|
80
84
|
}
|
|
81
|
-
else
|
|
85
|
+
else {
|
|
82
86
|
target["thinking"] = { type: "enabled" };
|
|
83
|
-
if (clampedMaxTokens)
|
|
87
|
+
if (clampedMaxTokens) {
|
|
84
88
|
target["thinking"]["budgetTokens"] = clampedMaxTokens;
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
type: "enabled",
|
|
91
|
-
budgetTokens: calculateReasoningBudgetFromEffort(reasoning.effort, params.maxOutputTokens ?? getMaxOutputTokens(modelId), 1024),
|
|
92
|
-
};
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
// FUTURE: warn that reasoning.max_tokens was computed
|
|
92
|
+
target["thinking"]["budgetTokens"] = calculateReasoningBudgetFromEffort(reasoning.effort, params.maxOutputTokens ?? getMaxOutputTokens(modelId), 1024);
|
|
93
|
+
}
|
|
93
94
|
}
|
|
94
95
|
}
|
|
95
96
|
else if (clampedMaxTokens) {
|
|
@@ -1,2 +1,3 @@
|
|
|
1
1
|
import type { LanguageModelMiddleware } from "ai";
|
|
2
|
-
export declare const
|
|
2
|
+
export declare const bedrockGptReasoningMiddleware: LanguageModelMiddleware;
|
|
3
|
+
export declare const bedrockClaudeReasoningMiddleware: LanguageModelMiddleware;
|
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
import { modelMiddlewareMatcher } from "../../middleware/matcher";
|
|
2
|
-
|
|
2
|
+
const isClaude46 = (modelId) => modelId.includes("-4-6");
|
|
3
|
+
export const bedrockGptReasoningMiddleware = {
|
|
4
|
+
specificationVersion: "v3",
|
|
5
|
+
// eslint-disable-next-line require-await
|
|
6
|
+
transformParams: async ({ params, model }) => {
|
|
7
|
+
if (!model.modelId.includes("gpt"))
|
|
8
|
+
return params;
|
|
9
|
+
const bedrock = params.providerOptions?.["bedrock"];
|
|
10
|
+
if (!bedrock || typeof bedrock !== "object")
|
|
11
|
+
return params;
|
|
12
|
+
const effort = bedrock["reasoningEffort"];
|
|
13
|
+
if (effort === undefined)
|
|
14
|
+
return params;
|
|
15
|
+
const target = (bedrock["reasoningConfig"] ??= {});
|
|
16
|
+
target["maxReasoningEffort"] = effort;
|
|
17
|
+
delete bedrock["reasoningEffort"];
|
|
18
|
+
return params;
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
export const bedrockClaudeReasoningMiddleware = {
|
|
3
22
|
specificationVersion: "v3",
|
|
4
23
|
// eslint-disable-next-line require-await
|
|
5
24
|
transformParams: async ({ params, model }) => {
|
|
@@ -8,12 +27,11 @@ export const bedrockAnthropicReasoningMiddleware = {
|
|
|
8
27
|
const bedrock = params.providerOptions?.["bedrock"];
|
|
9
28
|
if (!bedrock || typeof bedrock !== "object")
|
|
10
29
|
return params;
|
|
11
|
-
const
|
|
12
|
-
const
|
|
13
|
-
const effort = bedrockOptions["effort"];
|
|
30
|
+
const thinking = bedrock["thinking"];
|
|
31
|
+
const effort = bedrock["effort"];
|
|
14
32
|
if (!thinking && effort === undefined)
|
|
15
33
|
return params;
|
|
16
|
-
const target = (
|
|
34
|
+
const target = (bedrock["reasoningConfig"] ??= {});
|
|
17
35
|
if (thinking && typeof thinking === "object") {
|
|
18
36
|
const thinkingOptions = thinking;
|
|
19
37
|
if (thinkingOptions["type"] !== undefined) {
|
|
@@ -23,13 +41,15 @@ export const bedrockAnthropicReasoningMiddleware = {
|
|
|
23
41
|
target["budgetTokens"] = thinkingOptions["budgetTokens"];
|
|
24
42
|
}
|
|
25
43
|
}
|
|
26
|
-
|
|
44
|
+
// FUTURE: bedrock currently does not support "effort" for other 4.x models
|
|
45
|
+
if (effort !== undefined && isClaude46(model.modelId)) {
|
|
27
46
|
target["maxReasoningEffort"] = effort;
|
|
28
|
-
|
|
29
|
-
delete
|
|
47
|
+
}
|
|
48
|
+
delete bedrock["thinking"];
|
|
49
|
+
delete bedrock["effort"];
|
|
30
50
|
return params;
|
|
31
51
|
},
|
|
32
52
|
};
|
|
33
53
|
modelMiddlewareMatcher.useForProvider("amazon-bedrock", {
|
|
34
|
-
language: [
|
|
54
|
+
language: [bedrockGptReasoningMiddleware, bedrockClaudeReasoningMiddleware],
|
|
35
55
|
});
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { logger } from "../logger";
|
|
2
|
+
import { addSpanEvent, setSpanAttributes } from "./span";
|
|
3
|
+
export const installAiSdkWarningLogger = (genAiSignalLevel) => {
|
|
4
|
+
const logWarnings = ({ warnings, provider, model }) => {
|
|
5
|
+
if (warnings.length === 0)
|
|
6
|
+
return;
|
|
7
|
+
for (const warning of warnings) {
|
|
8
|
+
logger.warn({
|
|
9
|
+
provider,
|
|
10
|
+
model,
|
|
11
|
+
warning,
|
|
12
|
+
}, `[ai-sdk] ${warning.type}`);
|
|
13
|
+
}
|
|
14
|
+
if (!(genAiSignalLevel === "recommended" || genAiSignalLevel === "full"))
|
|
15
|
+
return;
|
|
16
|
+
setSpanAttributes({
|
|
17
|
+
"gen_ai.response.warning_count": warnings.length,
|
|
18
|
+
});
|
|
19
|
+
for (const warning of warnings) {
|
|
20
|
+
addSpanEvent("gen_ai.warning", {
|
|
21
|
+
"gen_ai.provider.name": provider,
|
|
22
|
+
"gen_ai.response.model": model,
|
|
23
|
+
"gen_ai.warning.type": warning.type,
|
|
24
|
+
"gen_ai.warning.feature": "feature" in warning ? warning.feature : undefined,
|
|
25
|
+
"gen_ai.warning.details": "details" in warning ? warning.details : undefined,
|
|
26
|
+
"gen_ai.warning.message": "message" in warning ? warning.message : undefined,
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
globalThis.AI_SDK_LOG_WARNINGS = logWarnings;
|
|
31
|
+
};
|
package/package.json
CHANGED
package/src/config.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isLogger, logger, setLoggerInstance } from "./logger";
|
|
2
2
|
import { createDefaultLogger } from "./logger/default";
|
|
3
|
+
import { installAiSdkWarningLogger } from "./telemetry/ai-sdk";
|
|
3
4
|
import {
|
|
4
5
|
kParsed,
|
|
5
6
|
type GatewayConfig,
|
|
@@ -80,6 +81,8 @@ export const parseConfig = (config: GatewayConfig): GatewayConfigParsed => {
|
|
|
80
81
|
hebo: "off",
|
|
81
82
|
};
|
|
82
83
|
|
|
84
|
+
installAiSdkWarningLogger(telemetrySignals.gen_ai);
|
|
85
|
+
|
|
83
86
|
// Return parsed config.
|
|
84
87
|
return {
|
|
85
88
|
...config,
|
|
@@ -7,6 +7,7 @@ import type { ChatCompletionsToolMessage } from "./schema";
|
|
|
7
7
|
import {
|
|
8
8
|
convertToTextCallOptions,
|
|
9
9
|
toChatCompletionsAssistantMessage,
|
|
10
|
+
toChatCompletionsToolCall,
|
|
10
11
|
fromChatCompletionsAssistantMessage,
|
|
11
12
|
fromChatCompletionsToolResultMessage,
|
|
12
13
|
} from "./converters";
|
|
@@ -455,5 +456,115 @@ describe("Chat Completions Converters", () => {
|
|
|
455
456
|
});
|
|
456
457
|
expect(result.toolChoice).toBe("auto");
|
|
457
458
|
});
|
|
459
|
+
|
|
460
|
+
test("should map allowed_tools to activeTools and auto mode", () => {
|
|
461
|
+
const result = convertToTextCallOptions({
|
|
462
|
+
messages: [{ role: "user", content: "hi" }],
|
|
463
|
+
tool_choice: {
|
|
464
|
+
type: "allowed_tools",
|
|
465
|
+
allowed_tools: {
|
|
466
|
+
mode: "auto",
|
|
467
|
+
tools: [
|
|
468
|
+
{
|
|
469
|
+
type: "function",
|
|
470
|
+
function: { name: "get_weather" },
|
|
471
|
+
},
|
|
472
|
+
],
|
|
473
|
+
},
|
|
474
|
+
},
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
expect(result.toolChoice).toBe("auto");
|
|
478
|
+
expect(result.activeTools).toEqual(["get_weather"]);
|
|
479
|
+
});
|
|
480
|
+
|
|
481
|
+
test("should map allowed_tools required mode to required", () => {
|
|
482
|
+
const result = convertToTextCallOptions({
|
|
483
|
+
messages: [{ role: "user", content: "hi" }],
|
|
484
|
+
tool_choice: {
|
|
485
|
+
type: "allowed_tools",
|
|
486
|
+
allowed_tools: {
|
|
487
|
+
mode: "required",
|
|
488
|
+
tools: [
|
|
489
|
+
{
|
|
490
|
+
type: "function",
|
|
491
|
+
function: { name: "get_weather" },
|
|
492
|
+
},
|
|
493
|
+
],
|
|
494
|
+
},
|
|
495
|
+
},
|
|
496
|
+
});
|
|
497
|
+
|
|
498
|
+
expect(result.toolChoice).toBe("required");
|
|
499
|
+
expect(result.activeTools).toEqual(["get_weather"]);
|
|
500
|
+
});
|
|
501
|
+
|
|
502
|
+
test("should convert function tools into tool set entries", () => {
|
|
503
|
+
const result = convertToTextCallOptions({
|
|
504
|
+
messages: [{ role: "user", content: "hi" }],
|
|
505
|
+
tools: [
|
|
506
|
+
{
|
|
507
|
+
type: "function",
|
|
508
|
+
function: {
|
|
509
|
+
name: "get_weather",
|
|
510
|
+
description: "Get weather",
|
|
511
|
+
parameters: {
|
|
512
|
+
type: "object",
|
|
513
|
+
properties: {},
|
|
514
|
+
},
|
|
515
|
+
},
|
|
516
|
+
},
|
|
517
|
+
],
|
|
518
|
+
});
|
|
519
|
+
|
|
520
|
+
expect(result.tools).toBeDefined();
|
|
521
|
+
expect(Object.keys(result.tools!)).toEqual(["get_weather"]);
|
|
522
|
+
});
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
describe("toChatCompletionsToolCall", () => {
|
|
526
|
+
test("should filter top-level empty-string keys from object arguments", () => {
|
|
527
|
+
const call = toChatCompletionsToolCall("call_1", "my_tool", {
|
|
528
|
+
"": {},
|
|
529
|
+
city: "San Francisco",
|
|
530
|
+
nested: {
|
|
531
|
+
"": {},
|
|
532
|
+
country: "US",
|
|
533
|
+
},
|
|
534
|
+
});
|
|
535
|
+
|
|
536
|
+
expect(call.function.arguments).toBe(
|
|
537
|
+
JSON.stringify({
|
|
538
|
+
city: "San Francisco",
|
|
539
|
+
nested: {
|
|
540
|
+
"": {},
|
|
541
|
+
country: "US",
|
|
542
|
+
},
|
|
543
|
+
}),
|
|
544
|
+
);
|
|
545
|
+
});
|
|
546
|
+
|
|
547
|
+
test("should pass through JSON string arguments unchanged", () => {
|
|
548
|
+
const call = toChatCompletionsToolCall(
|
|
549
|
+
"call_1",
|
|
550
|
+
"my_tool",
|
|
551
|
+
'{"":{},"city":"San Francisco","nested":{"":{},"country":"US"}}',
|
|
552
|
+
);
|
|
553
|
+
|
|
554
|
+
expect(call.function.arguments).toBe(
|
|
555
|
+
'{"":{},"city":"San Francisco","nested":{"":{},"country":"US"}}',
|
|
556
|
+
);
|
|
557
|
+
});
|
|
558
|
+
|
|
559
|
+
test("should normalize invalid tool names", () => {
|
|
560
|
+
const call = toChatCompletionsToolCall("call_1", "bad. Tool- name1!@", {});
|
|
561
|
+
expect(call.function.name).toBe("bad._Tool-_name1__");
|
|
562
|
+
});
|
|
563
|
+
|
|
564
|
+
test("should truncate tool names longer than 128 chars", () => {
|
|
565
|
+
const call = toChatCompletionsToolCall("call_1", "a".repeat(200), {});
|
|
566
|
+
expect(call.function.name).toHaveLength(128);
|
|
567
|
+
expect(call.function.name).toBe("a".repeat(128));
|
|
568
|
+
});
|
|
458
569
|
});
|
|
459
570
|
});
|