@velum-labs/routekit-gateway 0.18.1 → 0.18.2

package/dist/backend.d.ts

@@ -44,8 +44,8 @@ export type Backend = {
     resolveModelRoute?(requested: string | undefined, nativeProvider?: string): BackendModelRoute | undefined;
     /**
      * Whether the backend serves this exact model id itself. Unlike
-     * {@link resolveModel} — which folds unknown
-     * ids into the default — this distinguishes "mine" from "unknown", so the
+     * The defaulting resolver folds unknown ids into the default; this
+     * distinguishes "mine" from "unknown", so the
      * gateway can hand unknown ids to a relay (e.g. the Codex backend relay)
      * instead of silently routing them to the default.
      */

package/dist/switching-proxy.d.ts

@@ -5,6 +5,7 @@ export type SwitchingGatewayProxy = {
     target(): string;
     swapTarget(target: string): string;
     waitForTargetIdle(target: string, graceMs: number): Promise<boolean>;
+    retire(graceMs?: number): Promise<void>;
     drain(graceMs?: number): Promise<void>;
     close(): Promise<void>;
 };

package/dist/switching-proxy.js

@@ -68,13 +68,17 @@ function writeJson(res, status, value) {
     res.setHeader("content-length", String(payload.length));
     res.end(payload);
 }
-async function pipe(res, upstream) {
+async function pipe(res, upstream, closeConnection) {
     res.statusCode = upstream.status;
     for (const [name, value] of upstream.headers) {
         if (!HOP_BY_HOP.has(name.toLowerCase()) && name.toLowerCase() !== "content-length") {
             res.setHeader(name, value);
         }
     }
+    if (closeConnection()) {
+        res.shouldKeepAlive = false;
+        res.setHeader("connection", "close");
+    }
     if (upstream.body === null) {
         res.end();
         return;
@@ -108,8 +112,13 @@ export async function startSwitchingGatewayProxy(input) {
     };
     const generations = new Map([[active.url, active]]);
     let draining = false;
+    let retiring = false;
     let inflight = 0;
     const server = createServer((req, res) => {
+        if (retiring) {
+            res.shouldKeepAlive = false;
+            res.setHeader("connection", "close");
+        }
         inflight += 1;
         res.once("close", () => {
             inflight -= 1;
@@ -170,7 +179,7 @@ export async function startSwitchingGatewayProxy(input) {
                         AbortSignal.timeout(10 * 60 * 1000)
                     ])
                 });
-                await pipe(res, upstream);
+                await pipe(res, upstream, () => retiring);
             }
             catch {
                 if (!res.destroyed && !res.headersSent) {
@@ -205,14 +214,33 @@ export async function startSwitchingGatewayProxy(input) {
     const address = server.address();
     const port = typeof address === "object" && address !== null ? address.port : input.port ?? 0;
     let drainRun;
+    let retireRun;
+    const waitForInflight = async (graceMs) => {
+        const deadline = Date.now() + graceMs;
+        while (inflight > 0 && Date.now() < deadline) {
+            await new Promise((resolve) => setTimeout(resolve, 50));
+        }
+    };
+    const closeRetiringListener = async (graceMs) => {
+        server.closeIdleConnections();
+        const closed = new Promise((resolve) => server.close(() => resolve()));
+        await waitForInflight(graceMs);
+        if (inflight > 0)
+            server.closeAllConnections();
+        await closed;
+    };
+    const retire = (graceMs = 0) => {
+        retireRun ??= (async () => {
+            retiring = true;
+            await closeRetiringListener(graceMs);
+        })();
+        return retireRun;
+    };
     const drain = (graceMs = 0) => {
         drainRun ??= (async () => {
             draining = true;
             server.closeIdleConnections();
-            const deadline = Date.now() + graceMs;
-            while (inflight > 0 && Date.now() < deadline) {
-                await new Promise((resolve) => setTimeout(resolve, 50));
-            }
+            await waitForInflight(graceMs);
             const closed = new Promise((resolve) => server.close(() => resolve()));
             server.closeAllConnections();
             await closed;
@@ -252,6 +280,7 @@ export async function startSwitchingGatewayProxy(input) {
                 generations.delete(generation.url);
             return result;
         },
+        retire,
         drain,
         close: async () => await drain(0)
     };

package/dist/test/switching-proxy-retire.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/switching-proxy-retire.test.js

@@ -0,0 +1,88 @@
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import test from "node:test";
+import { startSwitchingGatewayProxy } from "../switching-proxy.js";
+async function heldTarget() {
+    let markStarted;
+    const started = new Promise((resolve) => {
+        markStarted = resolve;
+    });
+    let release;
+    const released = new Promise((resolve) => {
+        release = resolve;
+    });
+    const server = createServer((request, response) => {
+        if (request.url === "/stream") {
+            response.setHeader("content-type", "text/plain");
+            response.write("first\n");
+            markStarted();
+            void released.then(() => response.end("last\n"));
+            return;
+        }
+        response.end("ok");
+    });
+    await new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(0, "127.0.0.1", () => resolve());
+    });
+    const port = server.address().port;
+    return {
+        url: `http://127.0.0.1:${port}`,
+        started,
+        release,
+        close: async () => await new Promise((resolve, reject) => server.close((error) => (error === undefined ? resolve() : reject(error))))
+    };
+}
+test("switching proxy retirement preserves an admitted streaming response", async () => {
+    const target = await heldTarget();
+    const proxy = await startSwitchingGatewayProxy({ target: target.url });
+    try {
+        const response = await fetch(`${proxy.url()}/stream`);
+        const reader = response.body?.getReader();
+        assert.ok(reader !== undefined);
+        const first = await reader.read();
+        assert.match(Buffer.from(first.value ?? []).toString("utf8"), /first/);
+        await target.started;
+        const retiring = proxy.retire(2_000);
+        target.release();
+        let remainder = "";
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            remainder += Buffer.from(value ?? []).toString("utf8");
+        }
+        assert.match(remainder, /last/);
+        await retiring;
+    }
+    finally {
+        await proxy.close();
+        await target.close();
+    }
+});
+test("switching proxy retirement deadline terminates a stream that never finishes", async () => {
+    const target = await heldTarget();
+    const proxy = await startSwitchingGatewayProxy({ target: target.url });
+    try {
+        const response = await fetch(`${proxy.url()}/stream`);
+        const reader = response.body?.getReader();
+        assert.ok(reader !== undefined);
+        await reader.read();
+        await target.started;
+        const startedAt = Date.now();
+        await proxy.retire(250);
+        assert.ok(Date.now() - startedAt >= 200);
+        await assert.rejects(async () => {
+            for (;;) {
+                const { done } = await reader.read();
+                if (done)
+                    break;
+            }
+        });
+    }
+    finally {
+        target.release();
+        await proxy.close();
+        await target.close();
+    }
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-gateway",
   "private": false,
-  "version": "0.18.1",
+  "version": "0.18.2",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -29,10 +29,10 @@
     "@aws-sdk/client-bedrock": "3.1095.0",
     "@aws-sdk/client-bedrock-runtime": "3.1095.0",
     "zod": "4.4.3",
-    "@velum-labs/routekit-contracts": "0.18.1",
-    "@velum-labs/routekit-registry": "0.18.1",
-    "@velum-labs/routekit-runtime": "0.18.1",
-    "@velum-labs/routekit-tracing": "0.18.1"
+    "@velum-labs/routekit-contracts": "0.18.2",
+    "@velum-labs/routekit-registry": "0.18.2",
+    "@velum-labs/routekit-runtime": "0.18.2",
+    "@velum-labs/routekit-tracing": "0.18.2"
   },
   "keywords": [
     "routekit",