@velum-labs/routekit-gateway 1.5.0 → 1.5.2

package/dist/adapters/responses-usage.d.ts

@@ -1,7 +1,10 @@
 import { type Usage } from "@velum-labs/routekit-contracts/protocol-ir";
 /**
  * Translate Chat Completions token accounting to the Responses vocabulary.
- * Detail objects are copied so compatible current and future counters survive;
- * notably cached_tokens and reasoning_tokens retain their native values.
+ * Codex requires cached_tokens and reasoning_tokens when their optional detail
+ * objects are present. Emit details only when those counters are known, so an
+ * empty provider object cannot break completion parsing and unknown accounting
+ * is not replaced by a measured zero. Complete detail objects retain compatible
+ * current and future counters.
  */
 export declare function chatUsageToResponses(usage: Usage | null | undefined): Record<string, unknown> | null;

package/dist/adapters/responses-usage.js

@@ -1,8 +1,11 @@
 import { extensionValue } from "@velum-labs/routekit-contracts/protocol-ir";
 /**
  * Translate Chat Completions token accounting to the Responses vocabulary.
- * Detail objects are copied so compatible current and future counters survive;
- * notably cached_tokens and reasoning_tokens retain their native values.
+ * Codex requires cached_tokens and reasoning_tokens when their optional detail
+ * objects are present. Emit details only when those counters are known, so an
+ * empty provider object cannot break completion parsing and unknown accounting
+ * is not replaced by a measured zero. Complete detail objects retain compatible
+ * current and future counters.
  */
 export function chatUsageToResponses(usage) {
     if (usage == null)
@@ -16,18 +19,24 @@ export function chatUsageToResponses(usage) {
     const details = extensionValue(usage.extensions, "openai.chat.usage-details");
     const inputDetails = details?.promptTokens;
     const outputDetails = details?.completionTokens;
+    const cachedTokens = inputDetails?.cached_tokens ?? usage.cachedInputTokens;
+    const reasoningTokens = outputDetails?.reasoning_tokens;
     if (inputTokens === undefined &&
         outputTokens === undefined &&
         totalTokens === undefined &&
-        inputDetails == null &&
-        outputDetails == null) {
+        cachedTokens == null &&
+        reasoningTokens == null) {
         return null;
     }
     return {
         ...(inputTokens !== undefined ? { input_tokens: inputTokens } : {}),
         ...(outputTokens !== undefined ? { output_tokens: outputTokens } : {}),
         ...(totalTokens !== undefined ? { total_tokens: totalTokens } : {}),
-        ...(inputDetails != null ? { input_tokens_details: { ...inputDetails } } : {}),
-        ...(outputDetails != null ? { output_tokens_details: { ...outputDetails } } : {})
+        ...(cachedTokens != null
+            ? { input_tokens_details: { ...inputDetails, cached_tokens: cachedTokens } }
+            : {}),
+        ...(reasoningTokens != null
+            ? { output_tokens_details: { ...outputDetails, reasoning_tokens: reasoningTokens } }
+            : {})
     };
 }

package/dist/test/anthropic-stream-usage.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/anthropic-stream-usage.test.js

@@ -0,0 +1,172 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { AnthropicBackend, decodeBufferedSse } from "@velum-labs/routekit-provider-core";
+import { Effect } from "effect";
+import { startGateway } from "../gateway-service.js";
+import { parseUsageFromSse } from "../observability/cost.js";
+import { asTransport, sse } from "./provider-backends-fixtures.js";
+const MODEL = "claude-stream-usage-test";
+const INITIAL_USAGE = {
+    input_tokens: 64,
+    output_tokens: 0,
+    cache_read_input_tokens: 8,
+    cache_creation_input_tokens: 2
+};
+const cases = [
+    { name: "split counters", initial: INITIAL_USAGE, terminal: { output_tokens: 26 }, total: 100 },
+    {
+        name: "full terminal counters",
+        initial: INITIAL_USAGE,
+        terminal: { ...INITIAL_USAGE, output_tokens: 26 },
+        total: 100
+    },
+    {
+        name: "terminal null input and cache counters",
+        initial: INITIAL_USAGE,
+        terminal: {
+            input_tokens: null,
+            output_tokens: 26,
+            cache_read_input_tokens: null,
+            cache_creation_input_tokens: null
+        },
+        total: 100
+    },
+    {
+        name: "provider-reported initial total",
+        initial: { ...INITIAL_USAGE, total_tokens: 123 },
+        terminal: { output_tokens: 26 },
+        total: 123
+    },
+    {
+        name: "provider-reported terminal total replaces the initial total",
+        initial: { ...INITIAL_USAGE, total_tokens: 123 },
+        terminal: { output_tokens: 26, total_tokens: 124 },
+        total: 124
+    }
+];
+function nativeStream(initial, terminal) {
+    return sse([
+        {
+            event: "message_start",
+            data: {
+                type: "message_start",
+                message: {
+                    id: "msg_stream_usage",
+                    type: "message",
+                    role: "assistant",
+                    model: MODEL,
+                    content: [],
+                    usage: initial
+                }
+            }
+        },
+        {
+            event: "content_block_start",
+            data: {
+                type: "content_block_start",
+                index: 0,
+                content_block: { type: "text", text: "" }
+            }
+        },
+        {
+            event: "content_block_delta",
+            data: {
+                type: "content_block_delta",
+                index: 0,
+                delta: { type: "text_delta", text: "STREAM_USAGE_OK" }
+            }
+        },
+        { event: "content_block_stop", data: { type: "content_block_stop", index: 0 } },
+        {
+            event: "message_delta",
+            data: { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: terminal }
+        },
+        { event: "message_stop", data: { type: "message_stop" } }
+    ]);
+}
+for (const endpoint of ["chat/completions", "responses"]) {
+    for (const fixture of cases) {
+        test(`Anthropic ${endpoint} streaming usage keeps ${fixture.name} complete and request-local`, async () => {
+            let submissions = 0;
+            const backend = new AnthropicBackend({
+                baseUrl: "https://api.anthropic.test/v1",
+                apiKey: "test-only",
+                defaultModel: MODEL,
+                transport: asTransport(async (_url, init) => {
+                    const body = JSON.parse(String(init.body));
+                    assert.equal(body.stream, true);
+                    submissions += 1;
+                    assert.ok(submissions <= 2, "each gateway request must submit upstream exactly once");
+                    return submissions === 1
+                        ? nativeStream(fixture.initial, fixture.terminal)
+                        : nativeStream({ input_tokens: 5, output_tokens: 0 }, { output_tokens: 3 });
+                })
+            });
+            const gateway = await startGateway({ backend });
+            try {
+                for (const requestNumber of [1, 2]) {
+                    const response = await fetch(`${gateway.url()}/v1/${endpoint}`, {
+                        method: "POST",
+                        headers: { "content-type": "application/json" },
+                        body: JSON.stringify({
+                            model: MODEL,
+                            stream: true,
+                            ...(endpoint === "responses"
+                                ? { input: "hello" }
+                                : {
+                                    messages: [{ role: "user", content: "hello" }],
+                                    stream_options: { include_usage: true }
+                                })
+                        })
+                    });
+                    assert.equal(response.status, 200);
+                    const wire = await response.text();
+                    assert.match(wire, /STREAM_USAGE_OK/);
+                    const frames = decodeBufferedSse(wire)
+                        .filter((event) => event.data.trim() !== "[DONE]")
+                        .map((event) => JSON.parse(event.data));
+                    let finalUsage;
+                    if (endpoint === "responses") {
+                        const completed = frames.filter((frame) => frame.type === "response.completed");
+                        assert.equal(completed.length, 1);
+                        assert.doesNotMatch(wire, /event: response\.(incomplete|failed)/);
+                        const finalResponse = completed[0]?.response;
+                        assert.equal(finalResponse.status, "completed");
+                        finalUsage = finalResponse.usage;
+                    }
+                    else {
+                        const terminal = frames.filter((frame) => frame.choices.some((choice) => choice.finish_reason != null));
+                        assert.equal(terminal.length, 1);
+                        finalUsage = terminal[0]?.usage;
+                        const initialUsage = frames.find((frame) => frame.usage != null)?.usage;
+                        assert.equal(initialUsage?.total_tokens, requestNumber === 1
+                            ? "total_tokens" in fixture.initial
+                                ? fixture.initial.total_tokens
+                                : 74
+                            : 5, "each usage snapshot reports the current cumulative total");
+                    }
+                    const firstRequest = requestNumber === 1;
+                    const input = firstRequest ? 74 : 5;
+                    const output = firstRequest ? 26 : 3;
+                    const total = firstRequest ? fixture.total : 8;
+                    assert.equal(finalUsage[endpoint === "responses" ? "input_tokens" : "prompt_tokens"], input);
+                    assert.equal(finalUsage[endpoint === "responses" ? "output_tokens" : "completion_tokens"], output);
+                    assert.equal(finalUsage.total_tokens, total);
+                    const observed = parseUsageFromSse(wire);
+                    assert.equal(observed?.promptTokens, input);
+                    assert.equal(observed?.completionTokens, output);
+                    assert.equal(observed?.totalTokens, total);
+                    assert.equal(observed?.cachedInputTokens, firstRequest ? 8 : undefined);
+                    assert.equal(observed?.reasoningTokens, undefined);
+                    if (endpoint === "chat/completions") {
+                        assert.equal(observed?.cacheCreationInputTokens, firstRequest ? 2 : undefined);
+                    }
+                    assert.equal(submissions, requestNumber);
+                }
+            }
+            finally {
+                await Effect.runPromise(gateway.close);
+            }
+        });
+    }
+}

package/dist/test/claude-responses-refusal.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/claude-responses-refusal.test.js

@@ -0,0 +1,107 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { AnthropicBackend } from "@velum-labs/routekit-provider-core";
+import { runRouteKitEffect } from "@velum-labs/routekit-runtime/effect";
+import { Effect } from "effect";
+import { handleResponses } from "../adapters/responses.js";
+const MODEL = "claude-fable-5-1";
+const TOOL = "staging_subscription_echo";
+const PARTIAL_ARGUMENTS = { marker: "RO" };
+const COMPLETE_MARKER = "ROUTEKIT_CLAUDE_TOOL_OK";
+function refusalResponse(stream) {
+    const tool = { type: "tool_use", id: "toolu_refused", name: TOOL };
+    const usage = { input_tokens: 467, output_tokens: 36 };
+    if (!stream) {
+        return Response.json({
+            id: "msg_refused",
+            type: "message",
+            role: "assistant",
+            model: MODEL,
+            content: [{ ...tool, input: PARTIAL_ARGUMENTS }],
+            stop_reason: "refusal",
+            usage
+        });
+    }
+    const events = [
+        {
+            type: "message_start",
+            message: { id: "msg_refused", content: [], usage: { input_tokens: 467, output_tokens: 0 } }
+        },
+        { type: "content_block_start", index: 0, content_block: { ...tool, input: {} } },
+        {
+            type: "content_block_delta",
+            index: 0,
+            delta: { type: "input_json_delta", partial_json: '{"marker":"' }
+        },
+        {
+            type: "content_block_delta",
+            index: 0,
+            delta: { type: "input_json_delta", partial_json: 'RO"}' }
+        },
+        { type: "content_block_stop", index: 0 },
+        { type: "message_delta", delta: { stop_reason: "refusal" }, usage },
+        { type: "message_stop" }
+    ];
+    return new Response(events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""), { headers: { "content-type": "text/event-stream" } });
+}
+for (const stream of [false, true]) {
+    test(`Claude ${stream ? "streamed" : "buffered"} refusal remains an incomplete Responses tool turn`, async () => {
+        const submitted = [];
+        const backend = new AnthropicBackend({
+            baseUrl: "https://anthropic.test/v1",
+            apiKey: "test-only",
+            defaultModel: MODEL,
+            transport: (_url, init) => {
+                submitted.push(JSON.parse(String(init.body)));
+                return Effect.succeed(refusalResponse(stream));
+            }
+        });
+        const response = await runRouteKitEffect(handleResponses(backend, {
+            model: MODEL,
+            stream,
+            input: `Call ${TOOL} with marker ${COMPLETE_MARKER}.`,
+            tool_choice: "auto",
+            tools: [
+                {
+                    type: "function",
+                    name: TOOL,
+                    parameters: {
+                        type: "object",
+                        properties: { marker: { type: "string" } },
+                        required: ["marker"]
+                    }
+                }
+            ]
+        }));
+        assert.equal(response.status, 200, "provider refusal is a terminal model outcome");
+        assert.equal(submitted.length, 1, "the gateway must not issue a follow-up call after refusal");
+        assert.equal(submitted[0]?.stream, stream);
+        assert.deepEqual(submitted[0]?.tool_choice, { type: "auto" });
+        let terminal;
+        if (stream) {
+            const wire = await response.text();
+            const events = wire
+                .split("\n\n")
+                .flatMap((frame) => {
+                const data = frame.split("\n").find((line) => line.startsWith("data: "));
+                return data === undefined ? [] : [JSON.parse(data.slice(6))];
+            });
+            assert.equal(events.filter((event) => event.type === "response.completed").length, 0);
+            const incomplete = events.filter((event) => event.type === "response.incomplete");
+            assert.equal(incomplete.length, 1);
+            terminal = incomplete[0]?.response;
+        }
+        else {
+            terminal = (await response.json());
+        }
+        assert.equal(terminal.status, "incomplete");
+        assert.deepEqual(terminal.incomplete_details, { reason: "content_filter" });
+        const output = terminal.output;
+        assert.equal(output.length, 1);
+        assert.equal(output[0]?.type, "function_call");
+        assert.equal(output[0]?.call_id, "toolu_refused");
+        assert.equal(output[0]?.name, TOOL);
+        assert.deepEqual(JSON.parse(String(output[0]?.arguments)), PARTIAL_ARGUMENTS);
+        assert.notEqual(JSON.parse(String(output[0]?.arguments)).marker, COMPLETE_MARKER);
+    });
+}

package/dist/test/responses-usage.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/responses-usage.test.js

@@ -0,0 +1,213 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { AnthropicBackend } from "@velum-labs/routekit-provider-core";
+import { Effect } from "effect";
+import { chatUsageToResponses } from "../adapters/responses-usage.js";
+import { startGateway } from "../gateway-service.js";
+import { parseUsage, parseUsageFromSse } from "../observability/cost.js";
+import { asTransport, sse } from "./provider-backends-fixtures.js";
+test("generated Responses usage omits unknown details without changing provider accounting", () => {
+    for (const details of [
+        undefined,
+        { promptTokens: {}, completionTokens: {} },
+        { promptTokens: { audio_tokens: 1 }, completionTokens: { accepted_prediction_tokens: 2 } }
+    ]) {
+        const usage = Object.freeze({
+            inputTokens: 64,
+            outputTokens: 26,
+            ...(details === undefined
+                ? {}
+                : {
+                    extensions: Object.freeze([
+                        {
+                            namespace: "openai.chat.usage-details",
+                            value: Object.freeze({
+                                promptTokens: Object.freeze(details.promptTokens),
+                                completionTokens: Object.freeze(details.completionTokens)
+                            })
+                        }
+                    ])
+                })
+        });
+        const before = structuredClone(usage);
+        const wire = chatUsageToResponses(usage);
+        assert.equal(wire?.input_tokens, 64);
+        assert.equal(wire?.output_tokens, 26);
+        assert.equal(wire?.total_tokens, 90);
+        assert.equal(wire?.input_tokens_details, undefined);
+        assert.equal(wire?.output_tokens_details, undefined);
+        assert.deepEqual(parseUsage(wire), { promptTokens: 64, completionTokens: 26, totalTokens: 90 });
+        assert.deepEqual(usage, before);
+    }
+    assert.equal(chatUsageToResponses(undefined), null);
+    assert.equal(chatUsageToResponses(null), null);
+    assert.equal(chatUsageToResponses({}), null);
+    assert.equal(chatUsageToResponses({
+        extensions: [
+            {
+                namespace: "openai.chat.usage-details",
+                value: { promptTokens: {}, completionTokens: { accepted_prediction_tokens: 2 } }
+            }
+        ]
+    }), null);
+});
+test("generated Responses usage preserves measured details and cached-input accounting", () => {
+    const usage = {
+        inputTokens: 12,
+        outputTokens: 7,
+        cachedInputTokens: 6,
+        extensions: [
+            {
+                namespace: "openai.chat.usage-details",
+                value: {
+                    promptTokens: { cached_tokens: 8, cache_write_tokens: 2, audio_tokens: 1 },
+                    completionTokens: { reasoning_tokens: 5, accepted_prediction_tokens: 2 }
+                }
+            }
+        ]
+    };
+    assert.deepEqual(chatUsageToResponses(usage), {
+        input_tokens: 12,
+        output_tokens: 7,
+        total_tokens: 19,
+        input_tokens_details: { cached_tokens: 8, cache_write_tokens: 2, audio_tokens: 1 },
+        output_tokens_details: { reasoning_tokens: 5, accepted_prediction_tokens: 2 }
+    });
+    assert.deepEqual(parseUsage(chatUsageToResponses(usage)), {
+        promptTokens: 12,
+        cachedInputTokens: 8,
+        completionTokens: 7,
+        reasoningTokens: 5,
+        totalTokens: 19
+    });
+    assert.deepEqual(chatUsageToResponses({
+        inputTokens: 12,
+        outputTokens: 7,
+        extensions: [
+            {
+                namespace: "openai.chat.usage-details",
+                value: { promptTokens: { cached_tokens: 0 }, completionTokens: { reasoning_tokens: 0 } }
+            }
+        ]
+    })?.output_tokens_details, { reasoning_tokens: 0 });
+    assert.deepEqual(chatUsageToResponses({ inputTokens: 12, outputTokens: 7, cachedInputTokens: 6 })
+        ?.input_tokens_details, {
+        cached_tokens: 6
+    });
+});
+for (const { stream, providerTotal } of [
+    { stream: false, providerTotal: undefined },
+    { stream: true, providerTotal: undefined },
+    { stream: true, providerTotal: 123 }
+]) {
+    test(`Claude Messages ${stream ? "SSE" : "JSON"} completes with Codex-compatible Responses usage${providerTotal === undefined ? "" : " and a provider-reported total"}`, async () => {
+        let submissions = 0;
+        const backend = new AnthropicBackend({
+            baseUrl: "https://api.anthropic.test/v1",
+            apiKey: "unused",
+            defaultModel: "claude-fable-5.1",
+            transport: asTransport(async (_url, init) => {
+                submissions += 1;
+                const body = JSON.parse(String(init.body));
+                assert.equal(body.stream === true, stream);
+                return stream
+                    ? sse([
+                        {
+                            event: "message_start",
+                            data: {
+                                type: "message_start",
+                                message: {
+                                    id: "msg_claude_usage",
+                                    type: "message",
+                                    role: "assistant",
+                                    model: "claude-fable-5.1",
+                                    content: [],
+                                    usage: {
+                                        input_tokens: 64,
+                                        output_tokens: 0,
+                                        cache_read_input_tokens: 0,
+                                        cache_creation_input_tokens: 0,
+                                        ...(providerTotal === undefined ? {} : { total_tokens: providerTotal })
+                                    }
+                                }
+                            }
+                        },
+                        {
+                            event: "content_block_start",
+                            data: {
+                                type: "content_block_start",
+                                index: 0,
+                                content_block: { type: "text", text: "" }
+                            }
+                        },
+                        {
+                            event: "content_block_delta",
+                            data: {
+                                type: "content_block_delta",
+                                index: 0,
+                                delta: { type: "text_delta", text: "CLAUDE_USAGE_OK" }
+                            }
+                        },
+                        { event: "content_block_stop", data: { type: "content_block_stop", index: 0 } },
+                        {
+                            event: "message_delta",
+                            data: {
+                                type: "message_delta",
+                                delta: { stop_reason: "end_turn" },
+                                usage: { output_tokens: 26 }
+                            }
+                        },
+                        { event: "message_stop", data: { type: "message_stop" } }
+                    ])
+                    : Response.json({
+                        id: "msg_claude_usage",
+                        type: "message",
+                        role: "assistant",
+                        model: "claude-fable-5.1",
+                        content: [{ type: "text", text: "CLAUDE_USAGE_OK" }],
+                        stop_reason: "end_turn",
+                        usage: { input_tokens: 64, output_tokens: 26, cache_read_input_tokens: 0 }
+                    });
+            })
+        });
+        const gateway = await startGateway({ backend });
+        try {
+            const response = await fetch(`${gateway.url()}/v1/responses`, {
+                method: "POST",
+                headers: { "content-type": "application/json" },
+                body: JSON.stringify({ input: "hello", stream })
+            });
+            assert.equal(response.status, 200);
+            const text = await response.text();
+            assert.match(text, /CLAUDE_USAGE_OK/);
+            const terminal = stream
+                ? text.split("\n\n").filter((event) => event.startsWith("event: response.completed"))
+                : [];
+            if (stream) {
+                assert.equal(terminal.length, 1);
+                assert.doesNotMatch(text, /event: response\.(incomplete|failed)/);
+            }
+            const completed = stream
+                ? JSON.parse(terminal[0].slice(terminal[0].indexOf("data:") + 5)).response
+                : JSON.parse(text);
+            assert.equal(completed.status, "completed");
+            const usage = completed.usage;
+            assert.equal(usage.input_tokens, 64);
+            assert.equal(usage.output_tokens, 26);
+            assert.equal(usage.total_tokens, providerTotal ?? 90);
+            assert.deepEqual(usage.input_tokens_details, { cached_tokens: 0 });
+            assert.equal(usage.output_tokens_details, undefined);
+            const observed = stream ? parseUsageFromSse(text) : parseUsage(usage);
+            assert.deepEqual(observed, {
+                promptTokens: 64,
+                cachedInputTokens: 0,
+                completionTokens: 26,
+                totalTokens: providerTotal ?? 90
+            });
+            assert.equal(submissions, 1);
+        }
+        finally {
+            await Effect.runPromise(gateway.close);
+        }
+    });
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-gateway",
   "private": false,
-  "version": "1.5.0",
+  "version": "1.5.2",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -43,13 +43,13 @@
   },
   "dependencies": {
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-config-core": "1.5.0",
-    "@velum-labs/routekit-contracts": "1.5.0",
-    "@velum-labs/routekit-eval-contracts": "1.5.0",
-    "@velum-labs/routekit-eval-core": "1.5.0",
-    "@velum-labs/routekit-registry": "1.5.0",
-    "@velum-labs/routekit-runtime": "1.5.0",
-    "@velum-labs/routekit-provider-core": "1.5.0"
+    "@velum-labs/routekit-config-core": "1.5.2",
+    "@velum-labs/routekit-contracts": "1.5.2",
+    "@velum-labs/routekit-eval-contracts": "1.5.2",
+    "@velum-labs/routekit-eval-core": "1.5.2",
+    "@velum-labs/routekit-registry": "1.5.2",
+    "@velum-labs/routekit-runtime": "1.5.2",
+    "@velum-labs/routekit-provider-core": "1.5.2"
   },
   "keywords": [
     "routekit",