@velum-labs/routekit-gateway 1.0.5 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/providers/bedrock-source.d.ts +14 -3
- package/dist/providers/bedrock-source.js +249 -26
- package/dist/routing/router.js +1 -1
- package/dist/test/bedrock-source.test.js +193 -1
- package/dist/test/chat.test.js +1 -1
- package/dist/test/router.test.js +50 -0
- package/package.json +7 -7
package/dist/index.d.ts
CHANGED
|
@@ -19,8 +19,8 @@ export type { EvalSessionAdmission, GatewayPrincipal, WorkloadJwtPrincipalPolicy
|
|
|
19
19
|
export { authorizedRequest, createWorkloadJwtVerifier, parsePrincipalHeader, presentedCredential, ROUTEKIT_PRINCIPAL_HEADER, resolvePrincipal } from "./http/auth.js";
|
|
20
20
|
export type { Backend, BackendLifecyclePort, BackendModelPort, BackendModelRoute, BackendPorts, BackendRequest, BackendRequestOptions, BackendResponseMode, BackendResponsesPort, ModelRoutedBackendOptions, RequestAttributionUpdate } from "./providers/backend.js";
|
|
21
21
|
export { borrowedBackendPorts, joinPath, ModelRoutedBackend, staticBackendModelPort } from "./providers/backend.js";
|
|
22
|
-
export type { BedrockControlClient, BedrockProviderSourceOptions, BedrockRuntime } from "./providers/bedrock-source.js";
|
|
23
|
-
export { BedrockProviderSource, fromBedrockConverseOutput, toBedrockConverseInput } from "./providers/bedrock-source.js";
|
|
22
|
+
export type { BedrockControlClient, BedrockMantleBackend, BedrockProviderSourceOptions, BedrockRuntime } from "./providers/bedrock-source.js";
|
|
23
|
+
export { BEDROCK_OPENAI_ALLOWLIST, BedrockProviderSource, fromBedrockConverseOutput, isBedrockOpenAiModel, toBedrockConverseInput } from "./providers/bedrock-source.js";
|
|
24
24
|
export type { CapacityLease, CapacityPoolMember, CapacityPoolOptions, CapacityPoolStrategy } from "./capacity-pool.js";
|
|
25
25
|
export { CapacityPool } from "./capacity-pool.js";
|
|
26
26
|
export type { OpenRouterModelMetadata, OpenRouterModelMetadataClientOptions, ResolvedCodexStartupSelection } from "./providers/codex-model-selection.js";
|
package/dist/index.js
CHANGED
|
@@ -10,7 +10,7 @@ export { chatToResponses, handleResponses, openAiSseToResponses, responsesToChat
|
|
|
10
10
|
export { MAX_WEB_SEARCHES_PER_TURN, resolveWebSearchExecutor } from "./adapters/web-search.js";
|
|
11
11
|
export { authorizedRequest, createWorkloadJwtVerifier, parsePrincipalHeader, presentedCredential, ROUTEKIT_PRINCIPAL_HEADER, resolvePrincipal } from "./http/auth.js";
|
|
12
12
|
export { borrowedBackendPorts, joinPath, ModelRoutedBackend, staticBackendModelPort } from "./providers/backend.js";
|
|
13
|
-
export { BedrockProviderSource, fromBedrockConverseOutput, toBedrockConverseInput } from "./providers/bedrock-source.js";
|
|
13
|
+
export { BEDROCK_OPENAI_ALLOWLIST, BedrockProviderSource, fromBedrockConverseOutput, isBedrockOpenAiModel, toBedrockConverseInput } from "./providers/bedrock-source.js";
|
|
14
14
|
export { CapacityPool } from "./capacity-pool.js";
|
|
15
15
|
export { OpenRouterModelMetadataClient, resolveCodexStartupModel } from "./providers/codex-model-selection.js";
|
|
16
16
|
export { CompositionalRoutingError, routeCompositionalRequest } from "./routing/compositional.js";
|
|
@@ -1,19 +1,30 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Amazon Bedrock provider source.
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Amazon Bedrock provider source. Anthropic models use Bedrock Converse;
|
|
3
|
+
* allowlisted OpenAI frontier models use the regional bedrock-mantle
|
|
4
|
+
* OpenAI-compatible API.
|
|
5
5
|
*/
|
|
6
6
|
import { BedrockClient } from "@aws-sdk/client-bedrock";
|
|
7
7
|
import { BedrockRuntimeClient } from "@aws-sdk/client-bedrock-runtime";
|
|
8
8
|
import { fromBedrockConverseOutput, toBedrockConverseInput } from "./bedrock-codec.js";
|
|
9
|
+
import { OpenAiBackend } from "./openai-backend.js";
|
|
9
10
|
import type { ProviderSource } from "./source.js";
|
|
10
11
|
export type BedrockControlClient = Pick<BedrockClient, "send">;
|
|
11
12
|
export type BedrockRuntime = Pick<BedrockRuntimeClient, "send">;
|
|
13
|
+
export type BedrockMantleBackend = Pick<OpenAiBackend, "chat" | "responses">;
|
|
12
14
|
export type BedrockProviderSourceOptions = {
|
|
13
15
|
controlClient?: BedrockControlClient;
|
|
14
16
|
runtimeClient?: BedrockRuntime;
|
|
17
|
+
env?: Readonly<Record<string, string | undefined>>;
|
|
18
|
+
mantleBackend?: BedrockMantleBackend;
|
|
15
19
|
};
|
|
16
20
|
export { fromBedrockConverseOutput, toBedrockConverseInput };
|
|
21
|
+
export declare const BEDROCK_OPENAI_ALLOWLIST: readonly ["openai.gpt-5.6-sol", "openai.gpt-5.6-terra", "openai.gpt-5.6-luna"];
|
|
22
|
+
export declare function isBedrockOpenAiModel(modelId: string): boolean;
|
|
23
|
+
/**
|
|
24
|
+
* Remove OpenAI/Codex protocol fields that bedrock-mantle rejects while
|
|
25
|
+
* leaving native OpenAI egress unchanged.
|
|
26
|
+
*/
|
|
27
|
+
export declare function sanitizeBedrockMantleRequestBody(body: unknown): unknown;
|
|
17
28
|
export declare class BedrockProviderSource implements ProviderSource {
|
|
18
29
|
#private;
|
|
19
30
|
readonly sourceId: "bedrock";
|
|
@@ -1,20 +1,171 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Amazon Bedrock provider source.
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Amazon Bedrock provider source. Anthropic models use Bedrock Converse;
|
|
3
|
+
* allowlisted OpenAI frontier models use the regional bedrock-mantle
|
|
4
|
+
* OpenAI-compatible API.
|
|
5
5
|
*/
|
|
6
6
|
import { BedrockClient, ListFoundationModelsCommand, ListInferenceProfilesCommand } from "@aws-sdk/client-bedrock";
|
|
7
7
|
import { BedrockRuntimeClient, ConverseCommand, ConverseStreamCommand } from "@aws-sdk/client-bedrock-runtime";
|
|
8
8
|
import { RouteKitFailure } from "@velum-labs/routekit-runtime/effect";
|
|
9
9
|
import { Effect } from "effect";
|
|
10
10
|
import { errorResponse, fromBedrockConverseOutput, isOpusFiveModel, streamResponse, toBedrockConverseInput } from "./bedrock-codec.js";
|
|
11
|
+
import { OpenAiBackend } from "./openai-backend.js";
|
|
11
12
|
import { gatewayTry, gatewayTryPromise } from "../effect/gateway.js";
|
|
12
13
|
export { fromBedrockConverseOutput, toBedrockConverseInput };
|
|
14
|
+
export const BEDROCK_OPENAI_ALLOWLIST = [
|
|
15
|
+
"openai.gpt-5.6-sol",
|
|
16
|
+
"openai.gpt-5.6-terra",
|
|
17
|
+
"openai.gpt-5.6-luna"
|
|
18
|
+
];
|
|
19
|
+
const BEDROCK_OPENAI_MODEL = /^(?:(?:us|eu|global)\.)?openai\.gpt-/;
|
|
20
|
+
export function isBedrockOpenAiModel(modelId) {
|
|
21
|
+
return BEDROCK_OPENAI_MODEL.test(modelId);
|
|
22
|
+
}
|
|
13
23
|
function record(value) {
|
|
14
24
|
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
15
25
|
? value
|
|
16
26
|
: undefined;
|
|
17
27
|
}
|
|
28
|
+
function recognizedEncryptedPrefix(value) {
|
|
29
|
+
return typeof value === "string" && (value.startsWith("rsn_") || value.startsWith("smry_"));
|
|
30
|
+
}
|
|
31
|
+
function sanitizeBedrockMantleContent(content) {
|
|
32
|
+
if (!Array.isArray(content))
|
|
33
|
+
return { content, changed: false };
|
|
34
|
+
const parts = content.filter((part) => record(part)?.type !== "encrypted_content");
|
|
35
|
+
return { content: parts, changed: parts.length !== content.length };
|
|
36
|
+
}
|
|
37
|
+
function sanitizeBedrockMantleInputItem(item) {
|
|
38
|
+
const entry = record(item);
|
|
39
|
+
if (entry === undefined)
|
|
40
|
+
return item;
|
|
41
|
+
const type = typeof entry.type === "string" ? entry.type : "";
|
|
42
|
+
if (type === "compaction" &&
|
|
43
|
+
"encrypted_content" in entry &&
|
|
44
|
+
!recognizedEncryptedPrefix(entry.encrypted_content)) {
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
if (type === "agent_message") {
|
|
48
|
+
const { content } = sanitizeBedrockMantleContent(entry.content);
|
|
49
|
+
return { type: "message", role: "user", content };
|
|
50
|
+
}
|
|
51
|
+
let next = entry;
|
|
52
|
+
let changed = false;
|
|
53
|
+
if (Array.isArray(entry.content)) {
|
|
54
|
+
const sanitized = sanitizeBedrockMantleContent(entry.content);
|
|
55
|
+
if (sanitized.changed) {
|
|
56
|
+
next = { ...next, content: sanitized.content };
|
|
57
|
+
changed = true;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (type === "reasoning" &&
|
|
61
|
+
"encrypted_content" in next &&
|
|
62
|
+
!recognizedEncryptedPrefix(next.encrypted_content)) {
|
|
63
|
+
const { encrypted_content: _ignored, ...rest } = next;
|
|
64
|
+
next = rest;
|
|
65
|
+
changed = true;
|
|
66
|
+
}
|
|
67
|
+
return changed ? next : item;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Remove OpenAI/Codex protocol fields that bedrock-mantle rejects while
|
|
71
|
+
* leaving native OpenAI egress unchanged.
|
|
72
|
+
*/
|
|
73
|
+
export function sanitizeBedrockMantleRequestBody(body) {
|
|
74
|
+
const payload = record(body);
|
|
75
|
+
if (payload === undefined)
|
|
76
|
+
return body;
|
|
77
|
+
let next = payload;
|
|
78
|
+
let changed = false;
|
|
79
|
+
if (Array.isArray(payload.tools)) {
|
|
80
|
+
const tools = payload.tools.map((tool) => {
|
|
81
|
+
const entry = record(tool);
|
|
82
|
+
if (entry === undefined)
|
|
83
|
+
return tool;
|
|
84
|
+
const type = typeof entry.type === "string" ? entry.type : "";
|
|
85
|
+
if (!type.startsWith("web_search") || !("search_content_types" in entry))
|
|
86
|
+
return tool;
|
|
87
|
+
changed = true;
|
|
88
|
+
const { search_content_types: _ignored, ...rest } = entry;
|
|
89
|
+
return rest;
|
|
90
|
+
});
|
|
91
|
+
if (changed)
|
|
92
|
+
next = { ...next, tools };
|
|
93
|
+
}
|
|
94
|
+
if (Array.isArray(payload.input)) {
|
|
95
|
+
const input = [];
|
|
96
|
+
let inputChanged = false;
|
|
97
|
+
for (const item of payload.input) {
|
|
98
|
+
const sanitized = sanitizeBedrockMantleInputItem(item);
|
|
99
|
+
if (sanitized === undefined) {
|
|
100
|
+
inputChanged = true;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (sanitized !== item)
|
|
104
|
+
inputChanged = true;
|
|
105
|
+
input.push(sanitized);
|
|
106
|
+
}
|
|
107
|
+
if (inputChanged) {
|
|
108
|
+
changed = true;
|
|
109
|
+
next = { ...next, input };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return changed ? next : body;
|
|
113
|
+
}
|
|
114
|
+
function mantleApiKey(env) {
|
|
115
|
+
const key = env.AWS_BEARER_TOKEN_BEDROCK ?? env.BEDROCK_API_KEY;
|
|
116
|
+
return typeof key === "string" && key.length > 0 ? key : undefined;
|
|
117
|
+
}
|
|
118
|
+
function mantleRegion(env) {
|
|
119
|
+
const region = env.AWS_REGION ?? env.AWS_DEFAULT_REGION;
|
|
120
|
+
return typeof region === "string" && region.length > 0 ? region : undefined;
|
|
121
|
+
}
|
|
122
|
+
function mantleBaseUrl(region) {
|
|
123
|
+
return `https://bedrock-mantle.${region}.api.aws/openai/v1`;
|
|
124
|
+
}
|
|
125
|
+
function bedrockOpenAiNativeId(modelId) {
|
|
126
|
+
return modelId.replace(/^(?:us|eu|global)\./, "");
|
|
127
|
+
}
|
|
128
|
+
function bedrockOpenAiReasoning(modelId) {
|
|
129
|
+
const native = bedrockOpenAiNativeId(modelId).replace(/^openai\./, "");
|
|
130
|
+
if (/^gpt-5\.6(?:-(?:sol|terra|luna))?(?:-\d{4}-\d{2}-\d{2})?$/.test(native)) {
|
|
131
|
+
return {
|
|
132
|
+
status: "supported",
|
|
133
|
+
efforts: ["none", "low", "medium", "high", "xhigh", "max"].map((id) => ({ id })),
|
|
134
|
+
defaultEffort: "medium",
|
|
135
|
+
wireShape: "openai-responses",
|
|
136
|
+
provenance: "builtin"
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
if (/^gpt-5\.(?:4|5)(?:-\d{4}-\d{2}-\d{2})?$/.test(native)) {
|
|
140
|
+
return {
|
|
141
|
+
status: "supported",
|
|
142
|
+
efforts: ["none", "low", "medium", "high", "xhigh"].map((id) => ({ id })),
|
|
143
|
+
wireShape: "openai-responses",
|
|
144
|
+
provenance: "builtin"
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
status: "supported",
|
|
149
|
+
wireShape: "openai-responses",
|
|
150
|
+
provenance: "builtin"
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
function bedrockOpenAiDiscoveredModel(id) {
|
|
154
|
+
return {
|
|
155
|
+
id,
|
|
156
|
+
metadata: {
|
|
157
|
+
architecture: {
|
|
158
|
+
modality: "text+image->text",
|
|
159
|
+
inputModalities: ["text", "image"],
|
|
160
|
+
outputModalities: ["text"]
|
|
161
|
+
},
|
|
162
|
+
supportedParameters: ["tools", "tool_choice"],
|
|
163
|
+
provenance: "route"
|
|
164
|
+
},
|
|
165
|
+
reasoning: bedrockOpenAiReasoning(id),
|
|
166
|
+
capabilities: { streaming: "supported" }
|
|
167
|
+
};
|
|
168
|
+
}
|
|
18
169
|
function anthropicFoundationModel(model) {
|
|
19
170
|
return (model.providerName?.toLowerCase() === "anthropic" &&
|
|
20
171
|
model.modelLifecycle?.status === "ACTIVE" &&
|
|
@@ -96,20 +247,30 @@ export class BedrockProviderSource {
|
|
|
96
247
|
sourceId = "bedrock";
|
|
97
248
|
discovery;
|
|
98
249
|
requests;
|
|
99
|
-
responses
|
|
250
|
+
responses;
|
|
100
251
|
capabilities;
|
|
101
252
|
resource;
|
|
102
253
|
#control;
|
|
103
254
|
#runtime;
|
|
255
|
+
#env;
|
|
256
|
+
#injectedMantle;
|
|
257
|
+
#mantle;
|
|
104
258
|
#inferenceProfilesByFoundation = new Map();
|
|
105
259
|
constructor(options = {}) {
|
|
106
260
|
this.#control = options.controlClient ?? new BedrockClient({});
|
|
107
261
|
this.#runtime = options.runtimeClient ?? new BedrockRuntimeClient({});
|
|
262
|
+
this.#env = options.env ?? process.env;
|
|
263
|
+
this.#injectedMantle = options.mantleBackend;
|
|
108
264
|
this.discovery = { discoverModels: (signal) => this.#discoverModels(signal) };
|
|
109
265
|
this.requests = {
|
|
110
266
|
chat: (body, signal, requestOptions) => this.#chat(body, signal, requestOptions),
|
|
111
267
|
embeddings: () => Effect.succeed(Response.json({ error: { type: "not_implemented", message: "Bedrock embeddings are not supported" } }, { status: 501 }))
|
|
112
268
|
};
|
|
269
|
+
this.responses = {
|
|
270
|
+
kind: "responses",
|
|
271
|
+
supports: (model) => isBedrockOpenAiModel(model),
|
|
272
|
+
execute: (body, signal, requestOptions) => this.#responses(body, signal, requestOptions)
|
|
273
|
+
};
|
|
113
274
|
this.capabilities = {
|
|
114
275
|
forModel: () => ({}),
|
|
115
276
|
reasoningForModel: (model) => this.#reasoningCapabilities(model)
|
|
@@ -122,32 +283,73 @@ export class BedrockProviderSource {
|
|
|
122
283
|
})
|
|
123
284
|
};
|
|
124
285
|
}
|
|
286
|
+
#mantleBackend() {
|
|
287
|
+
if (this.#injectedMantle !== undefined)
|
|
288
|
+
return this.#injectedMantle;
|
|
289
|
+
if (this.#mantle !== undefined)
|
|
290
|
+
return this.#mantle;
|
|
291
|
+
const apiKey = mantleApiKey(this.#env);
|
|
292
|
+
const region = mantleRegion(this.#env);
|
|
293
|
+
if (apiKey === undefined || region === undefined)
|
|
294
|
+
return undefined;
|
|
295
|
+
this.#mantle = new OpenAiBackend({
|
|
296
|
+
baseUrl: mantleBaseUrl(region),
|
|
297
|
+
apiKey
|
|
298
|
+
});
|
|
299
|
+
return this.#mantle;
|
|
300
|
+
}
|
|
301
|
+
#missingMantleResponse() {
|
|
302
|
+
return Response.json({
|
|
303
|
+
error: {
|
|
304
|
+
type: "invalid_request_error",
|
|
305
|
+
message: "Bedrock OpenAI models require AWS_BEARER_TOKEN_BEDROCK and AWS_REGION"
|
|
306
|
+
}
|
|
307
|
+
}, { status: 400 });
|
|
308
|
+
}
|
|
125
309
|
#discoverModels(signal) {
|
|
126
310
|
const self = this;
|
|
127
311
|
return Effect.gen(function* () {
|
|
128
312
|
self.#inferenceProfilesByFoundation.clear();
|
|
129
313
|
const abort = signal === undefined ? undefined : { abortSignal: signal };
|
|
130
|
-
const
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
const
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
314
|
+
const discovered = yield* Effect.gen(function* () {
|
|
315
|
+
const foundation = yield* gatewayTryPromise(() => self.#control.send(new ListFoundationModelsCommand({ byProvider: "Anthropic" }), abort));
|
|
316
|
+
const foundations = (foundation.modelSummaries ?? []).filter(anthropicFoundationModel);
|
|
317
|
+
const byId = new Map(foundations.map((model) => [model.modelId, model]));
|
|
318
|
+
const ids = new Set(byId.keys());
|
|
319
|
+
const anthropic = new Map(foundations.map((model) => [
|
|
320
|
+
model.modelId,
|
|
321
|
+
bedrockDiscoveredModel(model.modelId, model)
|
|
322
|
+
]));
|
|
323
|
+
let nextToken;
|
|
324
|
+
do {
|
|
325
|
+
const profiles = yield* gatewayTryPromise(() => self.#control.send(new ListInferenceProfilesCommand({
|
|
326
|
+
...(nextToken !== undefined ? { nextToken } : {})
|
|
327
|
+
}), abort));
|
|
328
|
+
for (const profile of profiles.inferenceProfileSummaries ?? []) {
|
|
329
|
+
if (!activeAnthropicProfile(profile, ids))
|
|
330
|
+
continue;
|
|
331
|
+
const backingId = (profile.models ?? [])
|
|
332
|
+
.map((model) => foundationIdFromArn(model.modelArn))
|
|
333
|
+
.find((id) => id !== undefined && ids.has(id));
|
|
334
|
+
if (backingId !== undefined) {
|
|
335
|
+
self.#inferenceProfilesByFoundation.set(backingId, preferredInferenceProfile(self.#inferenceProfilesByFoundation.get(backingId), profile.inferenceProfileId));
|
|
336
|
+
anthropic.set(profile.inferenceProfileId, bedrockDiscoveredModel(profile.inferenceProfileId, byId.get(backingId)));
|
|
337
|
+
}
|
|
147
338
|
}
|
|
339
|
+
nextToken = profiles.nextToken;
|
|
340
|
+
} while (nextToken !== undefined && nextToken.length > 0);
|
|
341
|
+
return anthropic;
|
|
342
|
+
}).pipe(Effect.catch((error) => {
|
|
343
|
+
if (mantleApiKey(self.#env) === undefined)
|
|
344
|
+
return Effect.fail(error);
|
|
345
|
+
self.#inferenceProfilesByFoundation.clear();
|
|
346
|
+
return Effect.succeed(new Map());
|
|
347
|
+
}));
|
|
348
|
+
if (mantleApiKey(self.#env) !== undefined) {
|
|
349
|
+
for (const id of BEDROCK_OPENAI_ALLOWLIST) {
|
|
350
|
+
discovered.set(id, bedrockOpenAiDiscoveredModel(id));
|
|
148
351
|
}
|
|
149
|
-
|
|
150
|
-
} while (nextToken !== undefined && nextToken.length > 0);
|
|
352
|
+
}
|
|
151
353
|
if (discovered.size === 0) {
|
|
152
354
|
return yield* new RouteKitFailure({
|
|
153
355
|
message: "model discovery returned no active Anthropic Bedrock models"
|
|
@@ -156,9 +358,27 @@ export class BedrockProviderSource {
|
|
|
156
358
|
return [...discovered.values()];
|
|
157
359
|
});
|
|
158
360
|
}
|
|
159
|
-
#
|
|
361
|
+
#responses(body, signal, options) {
|
|
362
|
+
const requestedModel = record(body)?.model;
|
|
363
|
+
const model = typeof requestedModel === "string" ? requestedModel : "";
|
|
364
|
+
if (!isBedrockOpenAiModel(model)) {
|
|
365
|
+
return Effect.succeed(Response.json({ error: { type: "not_supported", message: "native Responses egress is not supported" } }, { status: 501 }));
|
|
366
|
+
}
|
|
367
|
+
const backend = this.#mantleBackend();
|
|
368
|
+
if (backend === undefined)
|
|
369
|
+
return Effect.succeed(this.#missingMantleResponse());
|
|
370
|
+
return backend.responses(sanitizeBedrockMantleRequestBody(body), signal, options);
|
|
371
|
+
}
|
|
372
|
+
#chat(body, signal, options) {
|
|
160
373
|
const self = this;
|
|
161
374
|
return Effect.gen(function* () {
|
|
375
|
+
const requestedModel = record(body)?.model;
|
|
376
|
+
if (typeof requestedModel === "string" && isBedrockOpenAiModel(requestedModel)) {
|
|
377
|
+
const backend = self.#mantleBackend();
|
|
378
|
+
if (backend === undefined)
|
|
379
|
+
return self.#missingMantleResponse();
|
|
380
|
+
return yield* backend.chat(sanitizeBedrockMantleRequestBody(body), signal, options);
|
|
381
|
+
}
|
|
162
382
|
const input = yield* gatewayTry(() => toBedrockConverseInput(body)).pipe(Effect.catch((error) => Effect.succeed(Response.json({
|
|
163
383
|
error: {
|
|
164
384
|
type: "invalid_request_error",
|
|
@@ -170,7 +390,7 @@ export class BedrockProviderSource {
|
|
|
170
390
|
const stream = record(body)?.stream === true;
|
|
171
391
|
const modelId = input.modelId;
|
|
172
392
|
if (modelId === undefined)
|
|
173
|
-
return errorResponse(
|
|
393
|
+
return errorResponse("Bedrock chat requires a model");
|
|
174
394
|
const runtimeModelId = modelId === "anthropic.claude-opus-5"
|
|
175
395
|
? (self.#inferenceProfilesByFoundation.get(modelId) ?? modelId)
|
|
176
396
|
: modelId;
|
|
@@ -181,7 +401,7 @@ export class BedrockProviderSource {
|
|
|
181
401
|
if (output instanceof Response)
|
|
182
402
|
return output;
|
|
183
403
|
if (output.stream === undefined)
|
|
184
|
-
return errorResponse(
|
|
404
|
+
return errorResponse("Bedrock returned no response stream");
|
|
185
405
|
return streamResponse(output.stream, modelId, signal);
|
|
186
406
|
}
|
|
187
407
|
const output = yield* gatewayTryPromise(() => self.#runtime.send(new ConverseCommand(runtimeInput), abort)).pipe(Effect.catch((error) => Effect.succeed(errorResponse(error))));
|
|
@@ -191,6 +411,9 @@ export class BedrockProviderSource {
|
|
|
191
411
|
});
|
|
192
412
|
}
|
|
193
413
|
#reasoningCapabilities(model) {
|
|
414
|
+
if (model !== undefined && isBedrockOpenAiModel(model)) {
|
|
415
|
+
return bedrockOpenAiReasoning(model);
|
|
416
|
+
}
|
|
194
417
|
const known = bedrockReasoningCapabilities(model);
|
|
195
418
|
if (known !== undefined)
|
|
196
419
|
return known;
|
package/dist/routing/router.js
CHANGED
|
@@ -128,7 +128,7 @@ export class RoutingBackend {
|
|
|
128
128
|
if (injected !== undefined)
|
|
129
129
|
return injected;
|
|
130
130
|
if (provider === "bedrock")
|
|
131
|
-
return new BedrockProviderSource();
|
|
131
|
+
return new BedrockProviderSource({ env });
|
|
132
132
|
if (isApiProvider(provider)) {
|
|
133
133
|
return (options.createApiSource?.(provider, env) ??
|
|
134
134
|
new ApiProviderSource({ provider, env }));
|
|
@@ -3,7 +3,8 @@ import { test } from "node:test";
|
|
|
3
3
|
import { ListFoundationModelsCommand } from "@aws-sdk/client-bedrock";
|
|
4
4
|
import { ConverseCommand, ConverseStreamCommand } from "@aws-sdk/client-bedrock-runtime";
|
|
5
5
|
import { runRouteKitEffect } from "@velum-labs/routekit-runtime/effect";
|
|
6
|
-
import {
|
|
6
|
+
import { Effect } from "effect";
|
|
7
|
+
import { BEDROCK_OPENAI_ALLOWLIST, BedrockProviderSource, isBedrockOpenAiModel, sanitizeBedrockMantleRequestBody, toBedrockConverseInput } from "../providers/bedrock-source.js";
|
|
7
8
|
test("Bedrock discovery includes active Anthropic foundations and paginated backed profiles", async () => {
|
|
8
9
|
const commands = [];
|
|
9
10
|
const source = new BedrockProviderSource({
|
|
@@ -124,6 +125,197 @@ test("Bedrock discovery includes active Anthropic foundations and paginated back
|
|
|
124
125
|
assert.deepEqual(discovered[3]?.reasoning, discovered[1]?.reasoning);
|
|
125
126
|
assert.equal(commands.length, 3);
|
|
126
127
|
});
|
|
128
|
+
test("Bedrock OpenAI model ids stay on mantle and never match Anthropic Converse", () => {
|
|
129
|
+
assert.equal(isBedrockOpenAiModel("openai.gpt-5.7"), true);
|
|
130
|
+
assert.equal(isBedrockOpenAiModel("openai.gpt-5.6-sol"), true);
|
|
131
|
+
assert.equal(isBedrockOpenAiModel("us.openai.gpt-5.6-terra"), true);
|
|
132
|
+
assert.equal(isBedrockOpenAiModel("anthropic.claude-3"), false);
|
|
133
|
+
assert.equal(isBedrockOpenAiModel("us.anthropic.claude-3"), false);
|
|
134
|
+
});
|
|
135
|
+
test("Bedrock discovery advertises only verified OpenAI mantle models", async () => {
|
|
136
|
+
assert.deepEqual(BEDROCK_OPENAI_ALLOWLIST, [
|
|
137
|
+
"openai.gpt-5.6-sol",
|
|
138
|
+
"openai.gpt-5.6-terra",
|
|
139
|
+
"openai.gpt-5.6-luna"
|
|
140
|
+
]);
|
|
141
|
+
const source = new BedrockProviderSource({
|
|
142
|
+
env: { AWS_BEARER_TOKEN_BEDROCK: "bedrock-key", AWS_REGION: "us-east-1" },
|
|
143
|
+
controlClient: {
|
|
144
|
+
send: async (command) => {
|
|
145
|
+
if (command instanceof ListFoundationModelsCommand) {
|
|
146
|
+
return {
|
|
147
|
+
modelSummaries: [
|
|
148
|
+
{
|
|
149
|
+
modelId: "anthropic.claude-3",
|
|
150
|
+
providerName: "Anthropic",
|
|
151
|
+
modelLifecycle: { status: "ACTIVE" }
|
|
152
|
+
}
|
|
153
|
+
]
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
return { inferenceProfileSummaries: [] };
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
runtimeClient: { send: async () => ({}) }
|
|
160
|
+
});
|
|
161
|
+
const discovered = await runRouteKitEffect(source.discovery.discoverModels());
|
|
162
|
+
assert.deepEqual(discovered.map((model) => model.id), ["anthropic.claude-3", ...BEDROCK_OPENAI_ALLOWLIST]);
|
|
163
|
+
const sol = discovered.find((model) => model.id === "openai.gpt-5.6-sol");
|
|
164
|
+
assert.equal(sol?.reasoning?.wireShape, "openai-responses");
|
|
165
|
+
assert.equal(sol?.reasoning?.defaultEffort, "medium");
|
|
166
|
+
});
|
|
167
|
+
test("Bedrock discovery keeps OpenAI models when Anthropic listing fails", async () => {
|
|
168
|
+
const source = new BedrockProviderSource({
|
|
169
|
+
env: { AWS_BEARER_TOKEN_BEDROCK: "bedrock-key", AWS_REGION: "us-east-1" },
|
|
170
|
+
controlClient: {
|
|
171
|
+
send: async () => {
|
|
172
|
+
throw new Error("not authorized");
|
|
173
|
+
}
|
|
174
|
+
},
|
|
175
|
+
runtimeClient: { send: async () => ({}) }
|
|
176
|
+
});
|
|
177
|
+
const discovered = await runRouteKitEffect(source.discovery.discoverModels());
|
|
178
|
+
assert.deepEqual(discovered.map((model) => model.id), [...BEDROCK_OPENAI_ALLOWLIST]);
|
|
179
|
+
});
|
|
180
|
+
test("Bedrock routes OpenAI models through mantle and leaves Anthropic on Converse", async () => {
|
|
181
|
+
const runtimeCalls = [];
|
|
182
|
+
const mantleCalls = [];
|
|
183
|
+
const source = new BedrockProviderSource({
|
|
184
|
+
env: { AWS_BEARER_TOKEN_BEDROCK: "bedrock-key", AWS_REGION: "us-east-1" },
|
|
185
|
+
controlClient: { send: async () => ({}) },
|
|
186
|
+
runtimeClient: {
|
|
187
|
+
send: async (value) => {
|
|
188
|
+
runtimeCalls.push(value);
|
|
189
|
+
return {
|
|
190
|
+
$metadata: { requestId: "req-1" },
|
|
191
|
+
output: { message: { role: "assistant", content: [{ text: "claude" }] } },
|
|
192
|
+
stopReason: "end_turn"
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
},
|
|
196
|
+
mantleBackend: {
|
|
197
|
+
chat: (body) => {
|
|
198
|
+
const model = typeof body.model === "string"
|
|
199
|
+
? body.model
|
|
200
|
+
: undefined;
|
|
201
|
+
mantleCalls.push({ kind: "chat", ...(model !== undefined ? { model } : {}) });
|
|
202
|
+
return Effect.succeed(Response.json({ id: "chat", model }));
|
|
203
|
+
},
|
|
204
|
+
responses: (body) => {
|
|
205
|
+
const model = typeof body.model === "string"
|
|
206
|
+
? body.model
|
|
207
|
+
: undefined;
|
|
208
|
+
mantleCalls.push({ kind: "responses", ...(model !== undefined ? { model } : {}) });
|
|
209
|
+
return Effect.succeed(Response.json({ id: "resp", model }));
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
assert.equal(source.responses.kind === "responses" && source.responses.supports("openai.gpt-5.6-sol"), true);
|
|
214
|
+
assert.equal(source.responses.kind === "responses" && source.responses.supports("anthropic.claude-3"), false);
|
|
215
|
+
assert.equal(source.capabilities.reasoningForModel("openai.gpt-5.6-sol")?.wireShape, "openai-responses");
|
|
216
|
+
const openaiChat = await runRouteKitEffect(source.requests.chat({
|
|
217
|
+
model: "openai.gpt-5.6-sol",
|
|
218
|
+
messages: [{ role: "user", content: "hi" }]
|
|
219
|
+
}));
|
|
220
|
+
assert.deepEqual(await openaiChat.json(), { id: "chat", model: "openai.gpt-5.6-sol" });
|
|
221
|
+
assert.equal(source.responses.kind, "responses");
|
|
222
|
+
if (source.responses.kind !== "responses")
|
|
223
|
+
throw new Error("expected Responses support");
|
|
224
|
+
const openaiResponses = await runRouteKitEffect(source.responses.execute({
|
|
225
|
+
model: "openai.gpt-5.6-terra",
|
|
226
|
+
input: "hi"
|
|
227
|
+
}));
|
|
228
|
+
assert.deepEqual(await openaiResponses.json(), {
|
|
229
|
+
id: "resp",
|
|
230
|
+
model: "openai.gpt-5.6-terra"
|
|
231
|
+
});
|
|
232
|
+
const anthropic = await runRouteKitEffect(source.requests.chat({
|
|
233
|
+
model: "anthropic.claude-3",
|
|
234
|
+
messages: [{ role: "user", content: "hi" }]
|
|
235
|
+
}));
|
|
236
|
+
assert.equal(anthropic.status, 200);
|
|
237
|
+
assert.equal(runtimeCalls.length, 1);
|
|
238
|
+
assert.equal(runtimeCalls[0] instanceof ConverseCommand, true);
|
|
239
|
+
assert.deepEqual(mantleCalls, [
|
|
240
|
+
{ kind: "chat", model: "openai.gpt-5.6-sol" },
|
|
241
|
+
{ kind: "responses", model: "openai.gpt-5.6-terra" }
|
|
242
|
+
]);
|
|
243
|
+
const rejected = await runRouteKitEffect(source.responses.execute({
|
|
244
|
+
model: "anthropic.claude-3",
|
|
245
|
+
input: "hi"
|
|
246
|
+
}));
|
|
247
|
+
assert.equal(rejected.status, 501);
|
|
248
|
+
});
|
|
249
|
+
test("Bedrock mantle sanitizes Codex-only request fields before egress", async () => {
|
|
250
|
+
const forwarded = [];
|
|
251
|
+
const source = new BedrockProviderSource({
|
|
252
|
+
env: { AWS_BEARER_TOKEN_BEDROCK: "bedrock-key", AWS_REGION: "us-east-1" },
|
|
253
|
+
controlClient: { send: async () => ({}) },
|
|
254
|
+
runtimeClient: { send: async () => ({}) },
|
|
255
|
+
mantleBackend: {
|
|
256
|
+
chat: () => Effect.succeed(Response.json({})),
|
|
257
|
+
responses: (body) => {
|
|
258
|
+
forwarded.push(body);
|
|
259
|
+
return Effect.succeed(Response.json({ id: "resp" }));
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
const body = {
|
|
264
|
+
model: "openai.gpt-5.6-sol",
|
|
265
|
+
tools: [{ type: "web_search", search_content_types: ["text", "image"] }],
|
|
266
|
+
input: [
|
|
267
|
+
{
|
|
268
|
+
type: "agent_message",
|
|
269
|
+
author: "/root",
|
|
270
|
+
recipient: "/root/worker",
|
|
271
|
+
content: [
|
|
272
|
+
{ type: "input_text", text: "hi" },
|
|
273
|
+
{ type: "encrypted_content", encrypted_content: " " }
|
|
274
|
+
]
|
|
275
|
+
},
|
|
276
|
+
{ type: "compaction", encrypted_content: "not-a-prefix" },
|
|
277
|
+
{ type: "reasoning", encrypted_content: "rsn_ok", summary: [] }
|
|
278
|
+
]
|
|
279
|
+
};
|
|
280
|
+
assert.deepEqual(sanitizeBedrockMantleRequestBody(body), {
|
|
281
|
+
model: "openai.gpt-5.6-sol",
|
|
282
|
+
tools: [{ type: "web_search" }],
|
|
283
|
+
input: [
|
|
284
|
+
{
|
|
285
|
+
type: "message",
|
|
286
|
+
role: "user",
|
|
287
|
+
content: [{ type: "input_text", text: "hi" }]
|
|
288
|
+
},
|
|
289
|
+
{ type: "reasoning", encrypted_content: "rsn_ok", summary: [] }
|
|
290
|
+
]
|
|
291
|
+
});
|
|
292
|
+
assert.equal(source.responses.kind, "responses");
|
|
293
|
+
if (source.responses.kind !== "responses")
|
|
294
|
+
throw new Error("expected Responses support");
|
|
295
|
+
const response = await runRouteKitEffect(source.responses.execute(body));
|
|
296
|
+
assert.equal(response.status, 200);
|
|
297
|
+
assert.deepEqual(forwarded, [sanitizeBedrockMantleRequestBody(body)]);
|
|
298
|
+
});
|
|
299
|
+
test("Bedrock OpenAI chat without a mantle key returns 400 and skips Converse", async () => {
|
|
300
|
+
const runtimeCalls = [];
|
|
301
|
+
const source = new BedrockProviderSource({
|
|
302
|
+
env: { AWS_REGION: "us-east-1" },
|
|
303
|
+
controlClient: { send: async () => ({}) },
|
|
304
|
+
runtimeClient: {
|
|
305
|
+
send: async (value) => {
|
|
306
|
+
runtimeCalls.push(value);
|
|
307
|
+
return {};
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
});
|
|
311
|
+
const response = await runRouteKitEffect(source.requests.chat({
|
|
312
|
+
model: "openai.gpt-5.6-luna",
|
|
313
|
+
messages: [{ role: "user", content: "hi" }]
|
|
314
|
+
}));
|
|
315
|
+
assert.equal(response.status, 400);
|
|
316
|
+
assert.match(await response.text(), /AWS_BEARER_TOKEN_BEDROCK/);
|
|
317
|
+
assert.equal(runtimeCalls.length, 0);
|
|
318
|
+
});
|
|
127
319
|
test("Bedrock request translation covers system, image, tools, results, and inference config", () => {
|
|
128
320
|
const input = toBedrockConverseInput({
|
|
129
321
|
model: "us.anthropic.claude-3",
|
package/dist/test/chat.test.js
CHANGED
|
@@ -134,7 +134,7 @@ test("GPT-5 Chat Completions translate max_tokens to max_completion_tokens", asy
|
|
|
134
134
|
for (const model of [
|
|
135
135
|
"gpt-5.6-sol",
|
|
136
136
|
"openai.gpt-5.6-luna",
|
|
137
|
-
"bedrock/openai.gpt-5.
|
|
137
|
+
"bedrock/openai.gpt-5.6-sol",
|
|
138
138
|
"routekit/bedrock/us.openai.gpt-5.7"
|
|
139
139
|
]) {
|
|
140
140
|
const response = await fetch(`${gateway.url()}/v1/chat/completions`, {
|
package/dist/test/router.test.js
CHANGED
|
@@ -936,3 +936,53 @@ test("Bedrock Opus 5 exposes reasoning controls and accepts routed effort select
|
|
|
936
936
|
const error = (await rejected.json());
|
|
937
937
|
assert.equal(error.error.code, "unsupported_reasoning_control");
|
|
938
938
|
});
|
|
939
|
+
test("Bedrock OpenAI models keep native ids and route through Responses", async () => {
|
|
940
|
+
const calls = [];
|
|
941
|
+
const bedrock = fakeSource("bedrock", [
|
|
942
|
+
{
|
|
943
|
+
id: "openai.gpt-5.6-sol",
|
|
944
|
+
reasoning: {
|
|
945
|
+
status: "supported",
|
|
946
|
+
efforts: ["none", "low", "medium", "high", "xhigh", "max"].map((id) => ({ id })),
|
|
947
|
+
defaultEffort: "medium",
|
|
948
|
+
wireShape: "openai-responses",
|
|
949
|
+
provenance: "builtin"
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
], calls);
|
|
953
|
+
const backend = await runRouteKitEffect(RoutingBackend.create({
|
|
954
|
+
config: { providers: { bedrock: {} }, defaultModel: "bedrock/openai.gpt-5.6-sol" },
|
|
955
|
+
sources: {
|
|
956
|
+
bedrock: {
|
|
957
|
+
...bedrock,
|
|
958
|
+
responses: {
|
|
959
|
+
kind: "responses",
|
|
960
|
+
supports: (model) => model.startsWith("openai."),
|
|
961
|
+
execute: (body) => {
|
|
962
|
+
const model = typeof body === "object" &&
|
|
963
|
+
body !== null &&
|
|
964
|
+
"model" in body &&
|
|
965
|
+
typeof body.model === "string"
|
|
966
|
+
? body.model
|
|
967
|
+
: undefined;
|
|
968
|
+
calls.push({
|
|
969
|
+
source: "bedrock-responses",
|
|
970
|
+
...(model !== undefined ? { model } : {})
|
|
971
|
+
});
|
|
972
|
+
return Effect.succeed(Response.json({ ok: true, model }));
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
}));
|
|
978
|
+
assert.deepEqual(backend.listModelIds(), ["bedrock/openai.gpt-5.6-sol"]);
|
|
979
|
+
assert.equal(backend.supportsResponses("bedrock/openai.gpt-5.6-sol"), true);
|
|
980
|
+
assert.equal(backend.modelInfo("bedrock/openai.gpt-5.6-sol")?.reasoning?.wireShape, "openai-responses");
|
|
981
|
+
const response = await runRouteKitEffect(backend.responses({
|
|
982
|
+
model: "bedrock/openai.gpt-5.6-sol",
|
|
983
|
+
input: "hi"
|
|
984
|
+
}));
|
|
985
|
+
assert.equal(response.status, 200);
|
|
986
|
+
assert.deepEqual(await response.json(), { ok: true, model: "openai.gpt-5.6-sol" });
|
|
987
|
+
assert.deepEqual(calls, [{ source: "bedrock-responses", model: "openai.gpt-5.6-sol" }]);
|
|
988
|
+
});
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@velum-labs/routekit-gateway",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "1.0.
|
|
4
|
+
"version": "1.0.7",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "git+https://github.com/velum-labs/routekit.git",
|
|
@@ -45,12 +45,12 @@
|
|
|
45
45
|
"@aws-sdk/client-bedrock": "3.1095.0",
|
|
46
46
|
"@aws-sdk/client-bedrock-runtime": "3.1095.0",
|
|
47
47
|
"effect": "4.0.0-rc.108",
|
|
48
|
-
"@velum-labs/routekit-config-core": "1.0.
|
|
49
|
-
"@velum-labs/routekit-contracts": "1.0.
|
|
50
|
-
"@velum-labs/routekit-eval-contracts": "1.0.
|
|
51
|
-
"@velum-labs/routekit-eval-core": "1.0.
|
|
52
|
-
"@velum-labs/routekit-registry": "1.0.
|
|
53
|
-
"@velum-labs/routekit-runtime": "1.0.
|
|
48
|
+
"@velum-labs/routekit-config-core": "1.0.7",
|
|
49
|
+
"@velum-labs/routekit-contracts": "1.0.7",
|
|
50
|
+
"@velum-labs/routekit-eval-contracts": "1.0.7",
|
|
51
|
+
"@velum-labs/routekit-eval-core": "1.0.7",
|
|
52
|
+
"@velum-labs/routekit-registry": "1.0.7",
|
|
53
|
+
"@velum-labs/routekit-runtime": "1.0.7"
|
|
54
54
|
},
|
|
55
55
|
"keywords": [
|
|
56
56
|
"routekit",
|