@velum-labs/routekit-gateway 1.3.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/responses-codec.js +7 -1
- package/dist/http/app.js +3 -3
- package/dist/http/health.d.ts +12 -0
- package/dist/http/health.js +24 -0
- package/dist/observability/cost.js +14 -18
- package/dist/providers/backend-core.d.ts +2 -0
- package/dist/providers/codex-responses-backend.js +18 -5
- package/dist/providers/codex-responses-codec.d.ts +5 -0
- package/dist/providers/codex-responses-codec.js +100 -5
- package/dist/switching-proxy.d.ts +2 -0
- package/dist/switching-proxy.js +12 -8
- package/dist/test/cost.test.js +1 -0
- package/dist/test/drain.test.js +6 -1
- package/dist/test/health.test.d.ts +1 -0
- package/dist/test/health.test.js +14 -0
- package/dist/test/provider-google-codex.test.js +222 -3
- package/dist/test/responses-conversion.test.js +28 -0
- package/dist/test/responses-reasoning.test.js +63 -0
- package/dist/test/server-resilience.test.js +6 -0
- package/package.json +7 -7
|
@@ -34,6 +34,9 @@ function mapTextFormat(text) {
|
|
|
34
34
|
type: "json_schema",
|
|
35
35
|
json_schema: {
|
|
36
36
|
...(typeof format.name === "string" ? { name: format.name } : {}),
|
|
37
|
+
...(typeof format.description === "string"
|
|
38
|
+
? { description: format.description }
|
|
39
|
+
: {}),
|
|
37
40
|
...(format.schema !== undefined ? { schema: format.schema } : {}),
|
|
38
41
|
...(typeof format.strict === "boolean" ? { strict: format.strict } : {})
|
|
39
42
|
}
|
|
@@ -601,8 +604,11 @@ export function responsesToChat(body, backendModel, options = {}) {
|
|
|
601
604
|
if (choice !== undefined)
|
|
602
605
|
chat.tool_choice = choice;
|
|
603
606
|
}
|
|
604
|
-
|
|
607
|
+
// Do not synthesize a Chat-only field when the next adapter returns to
|
|
608
|
+
// Responses; Codex reports usage in its native terminal event.
|
|
609
|
+
if (body.stream === true && options.destinationWireShape !== "openai-responses") {
|
|
605
610
|
chat.stream_options = { include_usage: true };
|
|
611
|
+
}
|
|
606
612
|
return chat;
|
|
607
613
|
}
|
|
608
614
|
// ---- non-streaming response translation ----
|
package/dist/http/app.js
CHANGED
|
@@ -5,6 +5,7 @@ import * as HttpEffect from "effect/unstable/http/HttpEffect";
|
|
|
5
5
|
import { HttpServerError } from "effect/unstable/http/HttpServerError";
|
|
6
6
|
import { parsePrincipalHeader, ROUTEKIT_PRINCIPAL_HEADER } from "./auth.js";
|
|
7
7
|
import { gatewayErrorResponse } from "./errors.js";
|
|
8
|
+
import { gatewayHealth, RUNNING_GATEWAY_VERSION } from "./health.js";
|
|
8
9
|
import { NO_BODY, readJson } from "./request.js";
|
|
9
10
|
import { handleModelCall, streamFetchResponse } from "../model-call-service.js";
|
|
10
11
|
function jsonResponse(status, value, headers = {}) {
|
|
@@ -114,9 +115,8 @@ export function buildGatewayHttpEffect(state) {
|
|
|
114
115
|
const request = yield* HttpServerRequest.HttpServerRequest;
|
|
115
116
|
const url = new URL(request.url, "http://localhost");
|
|
116
117
|
if (url.pathname === "/health") {
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
});
|
|
118
|
+
const draining = state.draining();
|
|
119
|
+
return jsonResponse(draining ? 503 : 200, gatewayHealth(draining ? "draining" : "ok", RUNNING_GATEWAY_VERSION));
|
|
120
120
|
}
|
|
121
121
|
if (state.draining()) {
|
|
122
122
|
return jsonResponse(503, {
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export type GatewayHealth = {
|
|
2
|
+
status: "ok" | "draining";
|
|
3
|
+
version: string;
|
|
4
|
+
pid?: number;
|
|
5
|
+
};
|
|
6
|
+
type ProcessIdentity = {
|
|
7
|
+
readonly pid: number;
|
|
8
|
+
};
|
|
9
|
+
/** Captured when this module loads, so later PATH or on-disk installs cannot change it. */
|
|
10
|
+
export declare const RUNNING_GATEWAY_VERSION: string;
|
|
11
|
+
export declare function gatewayHealth(status: GatewayHealth["status"], version: string, runningProcess?: ProcessIdentity): GatewayHealth;
|
|
12
|
+
export {};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
function readRunningPackageVersion() {
|
|
4
|
+
try {
|
|
5
|
+
const manifest = JSON.parse(readFileSync(fileURLToPath(new URL("../../package.json", import.meta.url)), "utf8"));
|
|
6
|
+
return typeof manifest.version === "string" && manifest.version.length > 0
|
|
7
|
+
? manifest.version
|
|
8
|
+
: "0.0.0";
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return "0.0.0";
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
/** Captured when this module loads, so later PATH or on-disk installs cannot change it. */
|
|
15
|
+
export const RUNNING_GATEWAY_VERSION = readRunningPackageVersion();
|
|
16
|
+
export function gatewayHealth(status, version, runningProcess = process) {
|
|
17
|
+
try {
|
|
18
|
+
const pid = runningProcess.pid;
|
|
19
|
+
return Number.isSafeInteger(pid) && pid > 0 ? { status, version, pid } : { status, version };
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return { status, version };
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { DEFAULT_MODEL_PRICING as REGISTRY_MODEL_PRICING, attributeModelCosts, lookupModelPricing } from "@velum-labs/routekit-registry";
|
|
2
2
|
import { decodeBufferedSse } from "../sse/parse.js";
|
|
3
3
|
const DEFAULT_CURRENCY = "USD";
|
|
4
4
|
export const DEFAULT_MODEL_PRICING = REGISTRY_MODEL_PRICING;
|
|
@@ -44,21 +44,8 @@ export function parseUsageFromSse(text) {
|
|
|
44
44
|
}
|
|
45
45
|
return usage;
|
|
46
46
|
}
|
|
47
|
-
function canonicalPricingKey(model) {
|
|
48
|
-
const direct = Object.keys(DEFAULT_MODEL_PRICING).find((candidate) => candidate.toLowerCase() === model.toLowerCase());
|
|
49
|
-
if (direct !== undefined)
|
|
50
|
-
return direct;
|
|
51
|
-
const alias = Object.entries(PRICING_ALIASES).find(([candidate]) => candidate.toLowerCase() === model.toLowerCase());
|
|
52
|
-
return alias?.[1] ?? model;
|
|
53
|
-
}
|
|
54
47
|
export function lookupPricing(model, overrides = {}) {
|
|
55
|
-
|
|
56
|
-
...DEFAULT_MODEL_PRICING,
|
|
57
|
-
...overrides
|
|
58
|
-
};
|
|
59
|
-
const key = canonicalPricingKey(model);
|
|
60
|
-
const entry = Object.entries(combined).find(([candidate]) => candidate.toLowerCase() === key.toLowerCase());
|
|
61
|
-
return entry?.[1];
|
|
48
|
+
return lookupModelPricing(model, overrides);
|
|
62
49
|
}
|
|
63
50
|
export function estimateCost(usage, pricing) {
|
|
64
51
|
if (pricing === undefined)
|
|
@@ -67,10 +54,19 @@ export function estimateCost(usage, pricing) {
|
|
|
67
54
|
const hasCompletion = usage.completionTokens !== undefined;
|
|
68
55
|
if (!hasPrompt && !hasCompletion)
|
|
69
56
|
return undefined;
|
|
57
|
+
const actuals = attributeModelCosts([
|
|
58
|
+
{
|
|
59
|
+
model: "configured",
|
|
60
|
+
usage: {
|
|
61
|
+
...(usage.promptTokens === undefined ? {} : { inputTokens: usage.promptTokens }),
|
|
62
|
+
...(usage.completionTokens === undefined ? {} : { outputTokens: usage.completionTokens })
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
], { configured: pricing });
|
|
66
|
+
if (actuals.knownCostUsd === undefined)
|
|
67
|
+
return undefined;
|
|
70
68
|
return {
|
|
71
|
-
costUsd:
|
|
72
|
-
(usage.completionTokens ?? 0) * pricing.outputPer1mTokens) /
|
|
73
|
-
1_000_000,
|
|
69
|
+
costUsd: actuals.knownCostUsd,
|
|
74
70
|
partialUsage: !hasPrompt || !hasCompletion
|
|
75
71
|
};
|
|
76
72
|
}
|
|
@@ -21,6 +21,7 @@ export type ChatMessage = {
|
|
|
21
21
|
tool_call_id?: string;
|
|
22
22
|
};
|
|
23
23
|
export type ChatBody = {
|
|
24
|
+
[key: string]: unknown;
|
|
24
25
|
model?: string;
|
|
25
26
|
messages?: ChatMessage[];
|
|
26
27
|
tools?: Array<{
|
|
@@ -40,6 +41,7 @@ export type ChatBody = {
|
|
|
40
41
|
type?: string;
|
|
41
42
|
json_schema?: {
|
|
42
43
|
name?: string;
|
|
44
|
+
description?: string;
|
|
43
45
|
schema?: unknown;
|
|
44
46
|
strict?: boolean;
|
|
45
47
|
};
|
|
@@ -7,7 +7,7 @@ import { StreamPump } from "@velum-labs/routekit-runtime/sse";
|
|
|
7
7
|
import { Effect } from "effect";
|
|
8
8
|
import { routeKitRequestValidationErrorOf } from "../adapters/openai-chat-wire.js";
|
|
9
9
|
import { joinPath } from "./backend.js";
|
|
10
|
-
import { applyCodexForceStreamEvent, codexCompletionResponse, codexForceStreamResponse, codexReasoningModeError, codexSseToChatChunks, createCodexForceStreamState, createCodexStreamState, responsesRequest } from "./codex-responses-codec.js";
|
|
10
|
+
import { applyCodexForceStreamEvent, CodexResponsesTranslationError, codexCompletionResponse, codexForceStreamResponse, codexReasoningModeError, codexSseToChatChunks, createCodexForceStreamState, createCodexStreamState, responsesRequest } from "./codex-responses-codec.js";
|
|
11
11
|
import { gatewayTry, gatewayTryPromise } from "../effect/gateway.js";
|
|
12
12
|
import { copyFailure, jsonResponse } from "../http/response.js";
|
|
13
13
|
import { bodyRecord, HttpProviderBackend, invalidReasoningControlResponse, mapSse, providerTransport } from "./backend-core.js";
|
|
@@ -46,6 +46,22 @@ export class CodexResponsesBackend extends HttpProviderBackend {
|
|
|
46
46
|
}
|
|
47
47
|
}, 400);
|
|
48
48
|
}
|
|
49
|
+
const encoded = yield* gatewayTry(() => responsesRequest(body, model, {
|
|
50
|
+
forceStream: self.#forceStream,
|
|
51
|
+
omitSampling: self.#omitSampling
|
|
52
|
+
})).pipe(Effect.map((body) => ({ ok: true, body })), Effect.catch((failure) => failure.cause instanceof CodexResponsesTranslationError
|
|
53
|
+
? Effect.succeed({ ok: false, cause: failure.cause })
|
|
54
|
+
: Effect.fail(failure)));
|
|
55
|
+
if (!encoded.ok) {
|
|
56
|
+
return jsonResponse({
|
|
57
|
+
error: {
|
|
58
|
+
type: "invalid_request_error",
|
|
59
|
+
code: encoded.cause.code,
|
|
60
|
+
param: encoded.cause.field,
|
|
61
|
+
message: encoded.cause.message
|
|
62
|
+
}
|
|
63
|
+
}, 400);
|
|
64
|
+
}
|
|
49
65
|
const response = yield* providerTransport(self.transport, joinPath(self.baseUrl, "/responses"), {
|
|
50
66
|
method: "POST",
|
|
51
67
|
headers: {
|
|
@@ -54,10 +70,7 @@ export class CodexResponsesBackend extends HttpProviderBackend {
|
|
|
54
70
|
...(self.#accountId !== undefined ? { "chatgpt-account-id": self.#accountId } : {}),
|
|
55
71
|
...self.extraHeaders
|
|
56
72
|
},
|
|
57
|
-
body: JSON.stringify(
|
|
58
|
-
forceStream: self.#forceStream,
|
|
59
|
-
omitSampling: self.#omitSampling
|
|
60
|
-
})),
|
|
73
|
+
body: JSON.stringify(encoded.body),
|
|
61
74
|
...(signal !== undefined ? { signal } : {})
|
|
62
75
|
}, options);
|
|
63
76
|
if (!response.ok)
|
|
@@ -9,6 +9,11 @@
|
|
|
9
9
|
import { type ResponsesReasoningOwner } from "../adapters/openai-responses-wire.js";
|
|
10
10
|
import { type ChatBody } from "./backend-core.js";
|
|
11
11
|
import { type ProviderRecord } from "./protocol.js";
|
|
12
|
+
export declare class CodexResponsesTranslationError extends Error {
|
|
13
|
+
readonly field: string;
|
|
14
|
+
readonly code = "codex_responses_unmappable_chat_field";
|
|
15
|
+
constructor(field: string, message?: string);
|
|
16
|
+
}
|
|
12
17
|
export declare function responsesRequest(body: ChatBody, model: string, options: {
|
|
13
18
|
forceStream: boolean;
|
|
14
19
|
omitSampling: boolean;
|
|
@@ -13,8 +13,97 @@ import { normalizeOpenAiResponsesCallIds, prepareResponsesReasoningInput, wrapRe
|
|
|
13
13
|
import { jsonResponse } from "../http/response.js";
|
|
14
14
|
import { chatCompletion, normalizedOpenAiUsage, textContent } from "./backend-core.js";
|
|
15
15
|
import { isProviderRecord, ProviderProtocolError } from "./protocol.js";
|
|
16
|
+
export class CodexResponsesTranslationError extends Error {
|
|
17
|
+
field;
|
|
18
|
+
code = "codex_responses_unmappable_chat_field";
|
|
19
|
+
constructor(field, message = `Codex Responses cannot represent Chat field "${field}"`) {
|
|
20
|
+
super(message);
|
|
21
|
+
this.field = field;
|
|
22
|
+
this.name = "CodexResponsesTranslationError";
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
const supportedChatFields = new Set([
|
|
26
|
+
"model",
|
|
27
|
+
"messages",
|
|
28
|
+
"tools",
|
|
29
|
+
"tool_choice",
|
|
30
|
+
"parallel_tool_calls",
|
|
31
|
+
"stream",
|
|
32
|
+
"max_tokens",
|
|
33
|
+
"max_completion_tokens",
|
|
34
|
+
"temperature",
|
|
35
|
+
"top_p",
|
|
36
|
+
"reasoning_effort",
|
|
37
|
+
"response_format",
|
|
38
|
+
// RouteKit's private reasoning envelope is consumed by reasoningSelectionOf.
|
|
39
|
+
"x_routekit"
|
|
40
|
+
]);
|
|
41
|
+
function assertTotalChatRequest(body, options) {
|
|
42
|
+
for (const field of Object.keys(body)) {
|
|
43
|
+
if (!supportedChatFields.has(field)) {
|
|
44
|
+
throw new CodexResponsesTranslationError(field);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (body.max_tokens !== undefined && body.max_completion_tokens !== undefined) {
|
|
48
|
+
throw new CodexResponsesTranslationError("max_completion_tokens", 'Codex Responses cannot represent both Chat fields "max_tokens" and "max_completion_tokens"');
|
|
49
|
+
}
|
|
50
|
+
if (options.omitSampling) {
|
|
51
|
+
for (const field of ["temperature", "top_p"]) {
|
|
52
|
+
if (body[field] !== undefined) {
|
|
53
|
+
throw new CodexResponsesTranslationError(field, `Codex subscription Responses does not support Chat field "${field}"`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
function responsesTextFormat(responseFormat) {
|
|
59
|
+
if (responseFormat === undefined)
|
|
60
|
+
return undefined;
|
|
61
|
+
switch (responseFormat.type) {
|
|
62
|
+
case "text":
|
|
63
|
+
case "json_object":
|
|
64
|
+
return { type: responseFormat.type };
|
|
65
|
+
case "json_schema": {
|
|
66
|
+
const jsonSchema = responseFormat.json_schema;
|
|
67
|
+
if (jsonSchema === undefined ||
|
|
68
|
+
typeof jsonSchema.name !== "string" ||
|
|
69
|
+
jsonSchema.name.length === 0 ||
|
|
70
|
+
jsonSchema.schema === undefined) {
|
|
71
|
+
throw new CodexResponsesTranslationError("response_format.json_schema", 'Codex Responses requires Chat "response_format.json_schema" to include name and schema');
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
type: "json_schema",
|
|
75
|
+
name: jsonSchema.name,
|
|
76
|
+
...(jsonSchema.description !== undefined
|
|
77
|
+
? { description: jsonSchema.description }
|
|
78
|
+
: {}),
|
|
79
|
+
schema: jsonSchema.schema,
|
|
80
|
+
...(jsonSchema.strict !== undefined ? { strict: jsonSchema.strict } : {})
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
default:
|
|
84
|
+
throw new CodexResponsesTranslationError("response_format.type", `Codex Responses cannot represent Chat response format ${JSON.stringify(responseFormat.type)}`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function responsesToolChoice(choice) {
|
|
88
|
+
if (choice === undefined)
|
|
89
|
+
return undefined;
|
|
90
|
+
if (choice === "auto" || choice === "none" || choice === "required")
|
|
91
|
+
return choice;
|
|
92
|
+
if (!isProviderRecord(choice) || choice.type !== "function") {
|
|
93
|
+
throw new CodexResponsesTranslationError("tool_choice");
|
|
94
|
+
}
|
|
95
|
+
const fn = isProviderRecord(choice.function) ? choice.function : undefined;
|
|
96
|
+
if (typeof fn?.name !== "string" || fn.name.length === 0) {
|
|
97
|
+
throw new CodexResponsesTranslationError("tool_choice.function.name", 'Codex Responses requires a named function in Chat "tool_choice"');
|
|
98
|
+
}
|
|
99
|
+
return { type: "function", name: fn.name };
|
|
100
|
+
}
|
|
16
101
|
export function responsesRequest(body, model, options) {
|
|
102
|
+
assertTotalChatRequest(body, options);
|
|
17
103
|
const reasoning = reasoningSelectionOf(body);
|
|
104
|
+
const textFormat = responsesTextFormat(body.response_format);
|
|
105
|
+
const toolChoice = responsesToolChoice(body.tool_choice);
|
|
106
|
+
const maxOutputTokens = body.max_completion_tokens ?? body.max_tokens;
|
|
18
107
|
const input = (body.messages ?? []).flatMap((message) => {
|
|
19
108
|
if (message.role === "tool") {
|
|
20
109
|
return [
|
|
@@ -63,13 +152,16 @@ export function responsesRequest(body, model, options) {
|
|
|
63
152
|
store: false,
|
|
64
153
|
...(includeEncryptedContent ? { include: ["reasoning.encrypted_content"] } : {}),
|
|
65
154
|
...(reasoning.mode === "effort" ? { reasoning: { effort: reasoning.effort } } : {}),
|
|
66
|
-
...(
|
|
67
|
-
? { max_output_tokens: body.max_tokens }
|
|
68
|
-
: {}),
|
|
155
|
+
...(maxOutputTokens !== undefined ? { max_output_tokens: maxOutputTokens } : {}),
|
|
69
156
|
...(!options.omitSampling && body.temperature !== undefined
|
|
70
157
|
? { temperature: body.temperature }
|
|
71
158
|
: {}),
|
|
72
|
-
...(body.
|
|
159
|
+
...(!options.omitSampling && body.top_p !== undefined ? { top_p: body.top_p } : {}),
|
|
160
|
+
...(textFormat !== undefined ? { text: { format: textFormat } } : {}),
|
|
161
|
+
...(toolChoice !== undefined ? { tool_choice: toolChoice } : {}),
|
|
162
|
+
...(body.parallel_tool_calls !== undefined
|
|
163
|
+
? { parallel_tool_calls: body.parallel_tool_calls }
|
|
164
|
+
: {}),
|
|
73
165
|
...(body.tools !== undefined
|
|
74
166
|
? {
|
|
75
167
|
tools: body.tools.flatMap((tool) => tool.function === undefined
|
|
@@ -79,7 +171,10 @@ export function responsesRequest(body, model, options) {
|
|
|
79
171
|
type: "function",
|
|
80
172
|
name: tool.function.name,
|
|
81
173
|
description: tool.function.description,
|
|
82
|
-
parameters: tool.function.parameters ?? { type: "object" }
|
|
174
|
+
parameters: tool.function.parameters ?? { type: "object" },
|
|
175
|
+
...(typeof tool.function.strict === "boolean"
|
|
176
|
+
? { strict: tool.function.strict }
|
|
177
|
+
: {})
|
|
83
178
|
}
|
|
84
179
|
])
|
|
85
180
|
}
|
|
@@ -22,6 +22,8 @@ export type SwitchingGatewayProxy = {
|
|
|
22
22
|
};
|
|
23
23
|
export type SwitchingGatewayProxyOptions = {
|
|
24
24
|
target: string;
|
|
25
|
+
/** Running daemon package version captured by its loaded module. */
|
|
26
|
+
packageVersion?: string;
|
|
25
27
|
host?: string;
|
|
26
28
|
port?: number;
|
|
27
29
|
authToken?: string;
|
package/dist/switching-proxy.js
CHANGED
|
@@ -10,12 +10,13 @@
|
|
|
10
10
|
import { createServer } from "node:http";
|
|
11
11
|
import { EVAL_ATTRIBUTION_HEADER, EVAL_POLICY_BYPASS_HEADER, isForbiddenEvalModel } from "@velum-labs/routekit-eval-contracts";
|
|
12
12
|
import { assertAuthenticatedBind, trimTrailingSlashes } from "@velum-labs/routekit-runtime/network";
|
|
13
|
-
import { createNodeHttpHandlerEffect, executeWebRequest, RouteKitLive, toRouteKitFailure } from "@velum-labs/routekit-runtime/effect";
|
|
13
|
+
import { createNodeHttpHandlerEffect, executeWebRequest, RouteKitFailure, RouteKitLive, toRouteKitFailure } from "@velum-labs/routekit-runtime/effect";
|
|
14
14
|
import { Context, Deferred, Effect, Layer, ManagedRuntime, Stream } from "effect";
|
|
15
15
|
import { HttpClient, HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
|
|
16
16
|
import * as HttpEffect from "effect/unstable/http/HttpEffect";
|
|
17
17
|
import { authorizedRequest, ROUTEKIT_PRINCIPAL_HEADER, resolvePrincipal } from "./http/auth.js";
|
|
18
18
|
import { gatewayTryPromise } from "./effect/gateway.js";
|
|
19
|
+
import { gatewayHealth, RUNNING_GATEWAY_VERSION } from "./http/health.js";
|
|
19
20
|
const MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
20
21
|
const DEFAULT_UPSTREAM_HEADERS_TIMEOUT_MS = 10 * 60 * 1000;
|
|
21
22
|
const HOP_BY_HOP = new Set([
|
|
@@ -182,6 +183,7 @@ const startSwitchingGatewayProxyOperation = Effect.fn("SwitchingGatewayProxy.sta
|
|
|
182
183
|
let draining = false;
|
|
183
184
|
let retiring = false;
|
|
184
185
|
let inflight = 0;
|
|
186
|
+
const healthVersion = input.packageVersion ?? RUNNING_GATEWAY_VERSION;
|
|
185
187
|
const httpEffect = yield* Effect.gen(function* () {
|
|
186
188
|
const router = yield* HttpRouter.make;
|
|
187
189
|
const httpClient = yield* HttpClient.HttpClient;
|
|
@@ -222,14 +224,18 @@ const startSwitchingGatewayProxyOperation = Effect.fn("SwitchingGatewayProxy.sta
|
|
|
222
224
|
error: { message: rejection.message, type: "invalid_request_error" }
|
|
223
225
|
});
|
|
224
226
|
}
|
|
225
|
-
const
|
|
226
|
-
const timeout = setTimeout(() => headersTimeout.abort(new Error(`gateway generation did not return headers within ${input.upstreamHeadersTimeoutMs ?? DEFAULT_UPSTREAM_HEADERS_TIMEOUT_MS}ms`)), input.upstreamHeadersTimeoutMs ?? DEFAULT_UPSTREAM_HEADERS_TIMEOUT_MS);
|
|
227
|
+
const timeoutMs = input.upstreamHeadersTimeoutMs ?? DEFAULT_UPSTREAM_HEADERS_TIMEOUT_MS;
|
|
227
228
|
const upstream = yield* executeWebRequest(`${selected.url}${path}`, {
|
|
228
229
|
method: nodeReq.method ?? "GET",
|
|
229
230
|
headers: requestHeaders(nodeReq.headers, principal),
|
|
230
231
|
...(body !== undefined ? { body } : {}),
|
|
231
|
-
signal:
|
|
232
|
-
}).pipe(Effect.
|
|
232
|
+
signal: aborter.signal
|
|
233
|
+
}).pipe(Effect.timeoutOrElse({
|
|
234
|
+
duration: timeoutMs,
|
|
235
|
+
orElse: () => Effect.fail(new RouteKitFailure({
|
|
236
|
+
message: `gateway generation did not return headers within ${timeoutMs}ms`
|
|
237
|
+
}))
|
|
238
|
+
}));
|
|
233
239
|
return HttpEffect.scopeTransferToStream(proxyResponse(upstream, retiring));
|
|
234
240
|
}).pipe(Effect.orElseSucceed(() => jsonResponse(502, {
|
|
235
241
|
error: { message: "router generation unavailable", type: "upstream_error" }
|
|
@@ -243,9 +249,7 @@ const startSwitchingGatewayProxyOperation = Effect.fn("SwitchingGatewayProxy.sta
|
|
|
243
249
|
const request = yield* HttpServerRequest.HttpServerRequest;
|
|
244
250
|
const path = new URL(request.url, "http://localhost").pathname;
|
|
245
251
|
if (path === "/health") {
|
|
246
|
-
return jsonResponse(draining ? 503 : 200,
|
|
247
|
-
status: draining ? "draining" : "ok"
|
|
248
|
-
});
|
|
252
|
+
return jsonResponse(draining ? 503 : 200, gatewayHealth(draining ? "draining" : "ok", healthVersion));
|
|
249
253
|
}
|
|
250
254
|
if (draining) {
|
|
251
255
|
return jsonResponse(503, {
|
package/dist/test/cost.test.js
CHANGED
|
@@ -12,6 +12,7 @@ test("single-call metering normalizes provider usage and registry pricing", () =
|
|
|
12
12
|
outputPer1mTokens: 10
|
|
13
13
|
});
|
|
14
14
|
assert.deepEqual(estimateCost({ promptTokens: 1_000_000, completionTokens: 1_000_000 }, lookupPricing("gpt-5.5")), { costUsd: 11.25, partialUsage: false });
|
|
15
|
+
assert.equal(estimateCost({ promptTokens: 1_000_000 }, lookupPricing("gpt-5.5")), undefined);
|
|
15
16
|
});
|
|
16
17
|
test("SSE usage extraction retains the last provider usage block", () => {
|
|
17
18
|
const text = `data: ${JSON.stringify({ choices: [{ delta: { content: "first" } }] })}\n\n` +
|
package/dist/test/drain.test.js
CHANGED
|
@@ -4,6 +4,7 @@ import { RouteKitFailure, runRouteKitEffect } from "@velum-labs/routekit-runtime
|
|
|
4
4
|
import { Effect } from "effect";
|
|
5
5
|
import { borrowedBackendPorts, staticBackendModelPort } from "../providers/backend.js";
|
|
6
6
|
import { startGateway } from "../gateway-service.js";
|
|
7
|
+
import { RUNNING_GATEWAY_VERSION } from "../http/health.js";
|
|
7
8
|
/**
|
|
8
9
|
* Graceful drain: a draining gateway must report unhealthy and reject new
|
|
9
10
|
* model calls while letting in-flight requests (long-lived LLM streams)
|
|
@@ -50,7 +51,11 @@ test("drain finishes in-flight streams, rejects new work, and flips /health to 5
|
|
|
50
51
|
// Health flips to 503 so probes stop routing new work here.
|
|
51
52
|
const health = await fetch(`${gateway.url()}/health`);
|
|
52
53
|
assert.equal(health.status, 503);
|
|
53
|
-
assert.deepEqual(await health.json(), {
|
|
54
|
+
assert.deepEqual(await health.json(), {
|
|
55
|
+
status: "draining",
|
|
56
|
+
version: RUNNING_GATEWAY_VERSION,
|
|
57
|
+
pid: process.pid
|
|
58
|
+
});
|
|
54
59
|
// New model calls are rejected while draining.
|
|
55
60
|
const rejected = await fetch(`${gateway.url()}/v1/chat/completions`, {
|
|
56
61
|
method: "POST",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import { gatewayHealth } from "../http/health.js";
|
|
4
|
+
test("health keeps version and omits pid when process identity lookup fails", () => {
|
|
5
|
+
const unavailablePid = {
|
|
6
|
+
get pid() {
|
|
7
|
+
throw new Error("pid unavailable");
|
|
8
|
+
}
|
|
9
|
+
};
|
|
10
|
+
assert.deepEqual(gatewayHealth("ok", "1.2.3", unavailablePid), {
|
|
11
|
+
status: "ok",
|
|
12
|
+
version: "1.2.3"
|
|
13
|
+
});
|
|
14
|
+
});
|
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
2
3
|
import { test } from "node:test";
|
|
3
|
-
import { runRouteKitEffect } from "@velum-labs/routekit-runtime/effect";
|
|
4
|
+
import { RouteKitLive, runRouteKitEffect } from "@velum-labs/routekit-runtime/effect";
|
|
5
|
+
import { Effect } from "effect";
|
|
4
6
|
import { responsesReasoningMetadataOf } from "../adapters/openai-chat-wire.js";
|
|
5
7
|
import { parseResponsesEncryptedContent, wrapResponsesEncryptedContent } from "../adapters/openai-responses-wire.js";
|
|
6
8
|
import { responsesToChat } from "../adapters/responses.js";
|
|
7
9
|
import { OpenAiBackend } from "../providers/openai-backend.js";
|
|
8
10
|
import { AnthropicBackend, CodexResponsesBackend, GoogleGenAiBackend } from "../providers/backends.js";
|
|
11
|
+
import { languageModelDimensionRequestDecomposerLayer } from "../routing/dimension-request-decomposer.js";
|
|
12
|
+
import { classifierResponseSchemaV3 } from "../routing/classifier-v3-protocol.js";
|
|
13
|
+
import { lunaDirectRequestDecomposerLayer } from "../routing/luna-direct-classifier.js";
|
|
14
|
+
import { buildTaskContextV3 } from "../routing/task-context.js";
|
|
15
|
+
import { RequestDecomposer } from "../services/request-decomposer/service.js";
|
|
9
16
|
import { ChatStreamAssembler } from "../sse/chat-assembler.js";
|
|
10
17
|
import { SseDecoder } from "../sse/parse.js";
|
|
11
18
|
import { asTransport, sse } from "./provider-backends-fixtures.js";
|
|
@@ -439,6 +446,220 @@ test("Codex Responses egress preserves subscription auth and tool output", async
|
|
|
439
446
|
globalThis.fetch = original;
|
|
440
447
|
}
|
|
441
448
|
});
|
|
449
|
+
test("dimension classifier reaches Codex Responses with its exact schema and output budget", async () => {
|
|
450
|
+
const original = globalThis.fetch;
|
|
451
|
+
let request;
|
|
452
|
+
const dimensions = [
|
|
453
|
+
["provider-adapters", "Provider-specific protocol adapters"],
|
|
454
|
+
["eval-routing", "Evaluation-driven model routing"],
|
|
455
|
+
["gateway-protocol", "OpenAI-compatible gateway protocol behavior"],
|
|
456
|
+
["daemon-lifecycle", "Daemon startup and lifecycle"],
|
|
457
|
+
["remote-enrollment", "Remote enrollment and control relays"]
|
|
458
|
+
].map(([id, description]) => ({
|
|
459
|
+
id: id,
|
|
460
|
+
description: description,
|
|
461
|
+
includes: [`${description} work`],
|
|
462
|
+
excludes: [`work outside ${description.toLowerCase()}`]
|
|
463
|
+
}));
|
|
464
|
+
globalThis.fetch = async (input, init) => {
|
|
465
|
+
request = new Request(input, init);
|
|
466
|
+
return sse([
|
|
467
|
+
{
|
|
468
|
+
event: "response.completed",
|
|
469
|
+
data: {
|
|
470
|
+
response: {
|
|
471
|
+
output: [
|
|
472
|
+
{
|
|
473
|
+
type: "message",
|
|
474
|
+
content: [
|
|
475
|
+
{
|
|
476
|
+
type: "output_text",
|
|
477
|
+
text: '{"weights":{"provider-adapters":1,"eval-routing":0,"gateway-protocol":0,"daemon-lifecycle":0,"remote-enrollment":0},"unknownWeight":0}'
|
|
478
|
+
}
|
|
479
|
+
]
|
|
480
|
+
}
|
|
481
|
+
]
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
]);
|
|
486
|
+
};
|
|
487
|
+
try {
|
|
488
|
+
const backend = new CodexResponsesBackend({
|
|
489
|
+
baseUrl: "https://chatgpt.test/backend-api/codex",
|
|
490
|
+
apiKey: "oauth",
|
|
491
|
+
defaultModel: "codex-test",
|
|
492
|
+
forceStream: true,
|
|
493
|
+
omitSampling: true
|
|
494
|
+
});
|
|
495
|
+
const classifier = await runRouteKitEffect(RequestDecomposer.pipe(Effect.provide(languageModelDimensionRequestDecomposerLayer({
|
|
496
|
+
model: "codex/gpt-5.6-luna",
|
|
497
|
+
complete: (body, signal) => backend.chat(body, signal).pipe(Effect.provide(RouteKitLive))
|
|
498
|
+
}))));
|
|
499
|
+
const result = await runRouteKitEffect(classifier.decompose({
|
|
500
|
+
request: "Repair the Codex Responses adapter",
|
|
501
|
+
dimensions
|
|
502
|
+
}));
|
|
503
|
+
assert.deepEqual(result, {
|
|
504
|
+
weights: dimensions.map((dimension) => ({
|
|
505
|
+
dimensionId: dimension.id,
|
|
506
|
+
weight: dimension.id === "provider-adapters" ? 1 : 0
|
|
507
|
+
})),
|
|
508
|
+
unknownWeight: 0
|
|
509
|
+
});
|
|
510
|
+
const outbound = (await request?.json());
|
|
511
|
+
assert.equal(outbound.max_output_tokens, 256);
|
|
512
|
+
assert.equal(outbound.text?.format?.type, "json_schema");
|
|
513
|
+
assert.equal(outbound.text?.format?.name, "routekit_request_decomposition");
|
|
514
|
+
assert.equal(outbound.text?.format?.strict, true);
|
|
515
|
+
assert.equal(outbound.text?.format?.schema?.additionalProperties, false);
|
|
516
|
+
assert.deepEqual(outbound.text?.format?.schema?.properties?.weights?.required, [
|
|
517
|
+
"provider-adapters",
|
|
518
|
+
"eval-routing",
|
|
519
|
+
"gateway-protocol",
|
|
520
|
+
"daemon-lifecycle",
|
|
521
|
+
"remote-enrollment"
|
|
522
|
+
]);
|
|
523
|
+
}
|
|
524
|
+
finally {
|
|
525
|
+
globalThis.fetch = original;
|
|
526
|
+
}
|
|
527
|
+
});
|
|
528
|
+
test("V3 classifier reaches Codex Responses with its exact schema and output budget", async () => {
|
|
529
|
+
const original = globalThis.fetch;
|
|
530
|
+
let request;
|
|
531
|
+
const basis = JSON.parse(readFileSync(new URL("../../../../test/fixtures/routing-v3/examples/routing-basis-v3.example.json", import.meta.url), "utf8"));
|
|
532
|
+
const activation = JSON.parse(readFileSync(new URL("../../../../test/fixtures/routing-v3/examples/published-routing-activation-v3.example.json", import.meta.url), "utf8"));
|
|
533
|
+
const scores = Object.fromEntries(basis.dimensions.map((dimension) => [dimension.id, 0]));
|
|
534
|
+
globalThis.fetch = async (input, init) => {
|
|
535
|
+
request = new Request(input, init);
|
|
536
|
+
return sse([
|
|
537
|
+
{
|
|
538
|
+
event: "response.completed",
|
|
539
|
+
data: {
|
|
540
|
+
response: {
|
|
541
|
+
output: [
|
|
542
|
+
{
|
|
543
|
+
type: "message",
|
|
544
|
+
content: [
|
|
545
|
+
{
|
|
546
|
+
type: "output_text",
|
|
547
|
+
text: JSON.stringify({
|
|
548
|
+
dimension_scores: scores,
|
|
549
|
+
unknown_probability: 0
|
|
550
|
+
})
|
|
551
|
+
}
|
|
552
|
+
]
|
|
553
|
+
}
|
|
554
|
+
]
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
]);
|
|
559
|
+
};
|
|
560
|
+
try {
|
|
561
|
+
const backend = new CodexResponsesBackend({
|
|
562
|
+
baseUrl: "https://chatgpt.test/backend-api/codex",
|
|
563
|
+
apiKey: "oauth",
|
|
564
|
+
defaultModel: "codex-test",
|
|
565
|
+
forceStream: true,
|
|
566
|
+
omitSampling: true
|
|
567
|
+
});
|
|
568
|
+
const classifier = await runRouteKitEffect(RequestDecomposer.pipe(Effect.provide(lunaDirectRequestDecomposerLayer({
|
|
569
|
+
model: "codex/gpt-5.6-luna",
|
|
570
|
+
modelAvailable: () => true,
|
|
571
|
+
complete: (body, signal) => backend.chat(body, signal).pipe(Effect.provide(RouteKitLive))
|
|
572
|
+
}))));
|
|
573
|
+
const context = buildTaskContextV3({
|
|
574
|
+
currentRequest: "Repair the Codex Responses adapter",
|
|
575
|
+
earlierUserContext: [],
|
|
576
|
+
relevantDiagnostics: []
|
|
577
|
+
}, basis.facetSnapshot, activation.classifier.context);
|
|
578
|
+
const result = await runRouteKitEffect(classifier.classify({
|
|
579
|
+
basis,
|
|
580
|
+
context,
|
|
581
|
+
config: activation.classifier
|
|
582
|
+
}));
|
|
583
|
+
assert.deepEqual(result.scores, basis.dimensions.map((dimension) => ({ dimensionId: dimension.id, score: 0 })));
|
|
584
|
+
const outbound = (await request?.json());
|
|
585
|
+
assert.equal(outbound.max_output_tokens, activation.classifier.maxCompletionTokens);
|
|
586
|
+
assert.deepEqual(outbound.text?.format, {
|
|
587
|
+
type: "json_schema",
|
|
588
|
+
name: "routekit_independent_dimension_scores_v3",
|
|
589
|
+
strict: true,
|
|
590
|
+
schema: classifierResponseSchemaV3(basis)
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
finally {
|
|
594
|
+
globalThis.fetch = original;
|
|
595
|
+
}
|
|
596
|
+
});
|
|
597
|
+
test("Codex subscription egress named-rejects an unmappable Chat field before I/O", async () => {
|
|
598
|
+
const original = globalThis.fetch;
|
|
599
|
+
let requests = 0;
|
|
600
|
+
globalThis.fetch = async () => {
|
|
601
|
+
requests += 1;
|
|
602
|
+
return Response.json({});
|
|
603
|
+
};
|
|
604
|
+
try {
|
|
605
|
+
const backend = new CodexResponsesBackend({
|
|
606
|
+
baseUrl: "https://chatgpt.test/backend-api/codex",
|
|
607
|
+
apiKey: "oauth",
|
|
608
|
+
defaultModel: "codex-test",
|
|
609
|
+
forceStream: true,
|
|
610
|
+
omitSampling: true
|
|
611
|
+
});
|
|
612
|
+
const response = await runRouteKitEffect(backend.chat({
|
|
613
|
+
messages: [{ role: "user", content: "reply" }],
|
|
614
|
+
top_k: 40
|
|
615
|
+
}));
|
|
616
|
+
const payload = (await response.json());
|
|
617
|
+
assert.equal(response.status, 400);
|
|
618
|
+
assert.equal(requests, 0);
|
|
619
|
+
assert.deepEqual(payload.error, {
|
|
620
|
+
type: "invalid_request_error",
|
|
621
|
+
code: "codex_responses_unmappable_chat_field",
|
|
622
|
+
param: "top_k",
|
|
623
|
+
message: 'Codex Responses cannot represent Chat field "top_k"'
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
finally {
|
|
627
|
+
globalThis.fetch = original;
|
|
628
|
+
}
|
|
629
|
+
});
|
|
630
|
+
test("Codex subscription egress named-rejects an unknown Chat field before I/O", async () => {
|
|
631
|
+
const original = globalThis.fetch;
|
|
632
|
+
let requests = 0;
|
|
633
|
+
globalThis.fetch = async () => {
|
|
634
|
+
requests += 1;
|
|
635
|
+
return Response.json({});
|
|
636
|
+
};
|
|
637
|
+
try {
|
|
638
|
+
const backend = new CodexResponsesBackend({
|
|
639
|
+
baseUrl: "https://chatgpt.test/backend-api/codex",
|
|
640
|
+
apiKey: "oauth",
|
|
641
|
+
defaultModel: "codex-test",
|
|
642
|
+
forceStream: true,
|
|
643
|
+
omitSampling: true
|
|
644
|
+
});
|
|
645
|
+
const response = await runRouteKitEffect(backend.chat({
|
|
646
|
+
messages: [{ role: "user", content: "reply" }],
|
|
647
|
+
best_of: 2
|
|
648
|
+
}));
|
|
649
|
+
const payload = (await response.json());
|
|
650
|
+
assert.equal(response.status, 400);
|
|
651
|
+
assert.equal(requests, 0);
|
|
652
|
+
assert.deepEqual(payload.error, {
|
|
653
|
+
type: "invalid_request_error",
|
|
654
|
+
code: "codex_responses_unmappable_chat_field",
|
|
655
|
+
param: "best_of",
|
|
656
|
+
message: 'Codex Responses cannot represent Chat field "best_of"'
|
|
657
|
+
});
|
|
658
|
+
}
|
|
659
|
+
finally {
|
|
660
|
+
globalThis.fetch = original;
|
|
661
|
+
}
|
|
662
|
+
});
|
|
442
663
|
test("Codex subscription egress forces SSE and omits unsupported sampling", async () => {
|
|
443
664
|
const original = globalThis.fetch;
|
|
444
665
|
let request;
|
|
@@ -471,8 +692,6 @@ test("Codex subscription egress forces SSE and omits unsupported sampling", asyn
|
|
|
471
692
|
});
|
|
472
693
|
const response = await runRouteKitEffect(backend.chat({
|
|
473
694
|
stream: false,
|
|
474
|
-
max_tokens: 16,
|
|
475
|
-
temperature: 0,
|
|
476
695
|
messages: [{ role: "user", content: "reply" }]
|
|
477
696
|
}));
|
|
478
697
|
const outbound = (await request?.json());
|
|
@@ -132,6 +132,34 @@ test("responsesToChat tolerates reasoning: null and text: null (Codex custom-pro
|
|
|
132
132
|
assert.equal(chat.model, "grok-4");
|
|
133
133
|
assert.equal(chat.reasoning_effort, undefined);
|
|
134
134
|
assert.equal(chat.response_format, undefined);
|
|
135
|
+
assert.deepEqual(chat.stream_options, { include_usage: true });
|
|
136
|
+
});
|
|
137
|
+
test("responsesToChat does not add Chat stream options before Codex Responses egress", () => {
|
|
138
|
+
const chat = responsesToChat({
|
|
139
|
+
model: "codex/matrix-codex",
|
|
140
|
+
input: [
|
|
141
|
+
{
|
|
142
|
+
type: "message",
|
|
143
|
+
role: "user",
|
|
144
|
+
content: [{ type: "input_text", text: "Use the declared tool." }]
|
|
145
|
+
}
|
|
146
|
+
],
|
|
147
|
+
stream: true,
|
|
148
|
+
tools: [
|
|
149
|
+
{
|
|
150
|
+
type: "function",
|
|
151
|
+
name: "read_file",
|
|
152
|
+
description: "read a file",
|
|
153
|
+
parameters: {
|
|
154
|
+
type: "object",
|
|
155
|
+
properties: { path: { type: "string" } }
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
]
|
|
159
|
+
}, "matrix-codex", { destinationWireShape: "openai-responses" });
|
|
160
|
+
assert.equal(chat.stream, true);
|
|
161
|
+
assert.equal(chat.stream_options, undefined);
|
|
162
|
+
assert.equal((chat.tools[0]?.function?.name), "read_file");
|
|
135
163
|
});
|
|
136
164
|
test("responsesToChat treats Codex reasoning effort null as absent", () => {
|
|
137
165
|
const chat = responsesToChat({ model: "gpt-5.5", input: "say OK", reasoning: { effort: null } }, "gpt-5.5");
|
|
@@ -839,6 +839,69 @@ test("Responses follows ModelRoutedBackend reasoning wire capability", async ()
|
|
|
839
839
|
await Effect.runPromise(gateway.close);
|
|
840
840
|
}
|
|
841
841
|
});
|
|
842
|
+
test("streaming Responses requests reach Codex without Chat-only stream options", async () => {
|
|
843
|
+
let codexBody;
|
|
844
|
+
const codex = new CodexResponsesBackend({
|
|
845
|
+
baseUrl: "https://codex.test",
|
|
846
|
+
apiKey: "x",
|
|
847
|
+
defaultModel: "codex-native",
|
|
848
|
+
transport: asTransport(async (_url, init) => {
|
|
849
|
+
codexBody = JSON.parse(String(init.body));
|
|
850
|
+
return new Response([
|
|
851
|
+
'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"ok"}\n\n',
|
|
852
|
+
'event: response.completed\ndata: {"type":"response.completed","response":{"output":[{"type":"message","content":[{"type":"output_text","text":"ok"}]}]}}\n\n'
|
|
853
|
+
].join(""), { headers: { "content-type": "text/event-stream" } });
|
|
854
|
+
})
|
|
855
|
+
});
|
|
856
|
+
const primary = {
|
|
857
|
+
defaultModel: "primary-model",
|
|
858
|
+
ports: borrowedBackendPorts("primary-model"),
|
|
859
|
+
chat: () => Effect.succeed(Response.json({ choices: [] })),
|
|
860
|
+
models: () => Effect.succeed(Response.json({ data: [] })),
|
|
861
|
+
embeddings: () => Effect.succeed(Response.json({}))
|
|
862
|
+
};
|
|
863
|
+
const backend = new ModelRoutedBackend({
|
|
864
|
+
routedModelIds: ["codex-model"],
|
|
865
|
+
routed: codex,
|
|
866
|
+
primary
|
|
867
|
+
});
|
|
868
|
+
const gateway = await startGateway({ backend });
|
|
869
|
+
try {
|
|
870
|
+
const response = await fetch(`${gateway.url()}/v1/responses`, {
|
|
871
|
+
method: "POST",
|
|
872
|
+
headers: { "content-type": "application/json" },
|
|
873
|
+
body: JSON.stringify({
|
|
874
|
+
model: "codex-model",
|
|
875
|
+
input: [
|
|
876
|
+
{
|
|
877
|
+
type: "message",
|
|
878
|
+
role: "user",
|
|
879
|
+
content: [{ type: "input_text", text: "Use the declared tool." }]
|
|
880
|
+
}
|
|
881
|
+
],
|
|
882
|
+
stream: true,
|
|
883
|
+
tools: [
|
|
884
|
+
{
|
|
885
|
+
type: "function",
|
|
886
|
+
name: "read_file",
|
|
887
|
+
description: "read a file",
|
|
888
|
+
parameters: {
|
|
889
|
+
type: "object",
|
|
890
|
+
properties: { path: { type: "string" } }
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
]
|
|
894
|
+
})
|
|
895
|
+
});
|
|
896
|
+
assert.equal(response.status, 200, await response.clone().text());
|
|
897
|
+
assert.equal(codexBody?.stream, true);
|
|
898
|
+
assert.equal("stream_options" in (codexBody ?? {}), false);
|
|
899
|
+
assert.equal(codexBody?.tools?.[0]?.name, "read_file");
|
|
900
|
+
}
|
|
901
|
+
finally {
|
|
902
|
+
await Effect.runPromise(gateway.close);
|
|
903
|
+
}
|
|
904
|
+
});
|
|
842
905
|
test("native Responses swaps isolate provider reasoning and restore it on A to B to A", async () => {
|
|
843
906
|
const requests = [];
|
|
844
907
|
const routes = {
|
|
@@ -9,6 +9,7 @@ import { borrowedBackendPorts } from "../providers/backend.js";
|
|
|
9
9
|
import { OpenAiBackend } from "../providers/openai-backend.js";
|
|
10
10
|
import { startGateway } from "../gateway-service.js";
|
|
11
11
|
import { startSwitchingGatewayProxy } from "../switching-proxy.js";
|
|
12
|
+
import { RUNNING_GATEWAY_VERSION } from "../http/health.js";
|
|
12
13
|
/**
|
|
13
14
|
* Crash resilience: an upstream stream that dies mid-response (the shape of a
|
|
14
15
|
* local model server being OOM-killed during a turn) must fail only that one
|
|
@@ -90,6 +91,11 @@ test("a mid-stream upstream failure does not kill the gateway process", async ()
|
|
|
90
91
|
// The gateway (and its hosting process) is still alive and serving.
|
|
91
92
|
const health = await fetch(`${gateway.url()}/health`);
|
|
92
93
|
assert.equal(health.status, 200);
|
|
94
|
+
assert.deepEqual(await health.json(), {
|
|
95
|
+
status: "ok",
|
|
96
|
+
version: RUNNING_GATEWAY_VERSION,
|
|
97
|
+
pid: process.pid
|
|
98
|
+
});
|
|
93
99
|
const models = await fetch(`${gateway.url()}/v1/models`);
|
|
94
100
|
assert.equal(models.status, 200);
|
|
95
101
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@velum-labs/routekit-gateway",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.4.0",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "git+https://github.com/velum-labs/routekit.git",
|
|
@@ -45,12 +45,12 @@
|
|
|
45
45
|
"@aws-sdk/client-bedrock": "3.1095.0",
|
|
46
46
|
"@aws-sdk/client-bedrock-runtime": "3.1095.0",
|
|
47
47
|
"effect": "4.0.0-rc.108",
|
|
48
|
-
"@velum-labs/routekit-config-core": "1.
|
|
49
|
-
"@velum-labs/routekit-contracts": "1.
|
|
50
|
-
"@velum-labs/routekit-eval-contracts": "1.
|
|
51
|
-
"@velum-labs/routekit-eval-core": "1.
|
|
52
|
-
"@velum-labs/routekit-registry": "1.
|
|
53
|
-
"@velum-labs/routekit-runtime": "1.
|
|
48
|
+
"@velum-labs/routekit-config-core": "1.4.0",
|
|
49
|
+
"@velum-labs/routekit-contracts": "1.4.0",
|
|
50
|
+
"@velum-labs/routekit-eval-contracts": "1.4.0",
|
|
51
|
+
"@velum-labs/routekit-eval-core": "1.4.0",
|
|
52
|
+
"@velum-labs/routekit-registry": "1.4.0",
|
|
53
|
+
"@velum-labs/routekit-runtime": "1.4.0"
|
|
54
54
|
},
|
|
55
55
|
"keywords": [
|
|
56
56
|
"routekit",
|