@velum-labs/routekit-gateway 1.3.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/http/app.js +3 -3
- package/dist/http/health.d.ts +12 -0
- package/dist/http/health.js +24 -0
- package/dist/switching-proxy.d.ts +2 -0
- package/dist/switching-proxy.js +3 -3
- package/dist/test/drain.test.js +6 -1
- package/dist/test/health.test.d.ts +1 -0
- package/dist/test/health.test.js +14 -0
- package/dist/test/server-resilience.test.js +6 -0
- package/package.json +7 -7
package/dist/http/app.js
CHANGED
|
@@ -5,6 +5,7 @@ import * as HttpEffect from "effect/unstable/http/HttpEffect";
|
|
|
5
5
|
import { HttpServerError } from "effect/unstable/http/HttpServerError";
|
|
6
6
|
import { parsePrincipalHeader, ROUTEKIT_PRINCIPAL_HEADER } from "./auth.js";
|
|
7
7
|
import { gatewayErrorResponse } from "./errors.js";
|
|
8
|
+
import { gatewayHealth, RUNNING_GATEWAY_VERSION } from "./health.js";
|
|
8
9
|
import { NO_BODY, readJson } from "./request.js";
|
|
9
10
|
import { handleModelCall, streamFetchResponse } from "../model-call-service.js";
|
|
10
11
|
function jsonResponse(status, value, headers = {}) {
|
|
@@ -114,9 +115,8 @@ export function buildGatewayHttpEffect(state) {
|
|
|
114
115
|
const request = yield* HttpServerRequest.HttpServerRequest;
|
|
115
116
|
const url = new URL(request.url, "http://localhost");
|
|
116
117
|
if (url.pathname === "/health") {
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
});
|
|
118
|
+
const draining = state.draining();
|
|
119
|
+
return jsonResponse(draining ? 503 : 200, gatewayHealth(draining ? "draining" : "ok", RUNNING_GATEWAY_VERSION));
|
|
120
120
|
}
|
|
121
121
|
if (state.draining()) {
|
|
122
122
|
return jsonResponse(503, {
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export type GatewayHealth = {
|
|
2
|
+
status: "ok" | "draining";
|
|
3
|
+
version: string;
|
|
4
|
+
pid?: number;
|
|
5
|
+
};
|
|
6
|
+
type ProcessIdentity = {
|
|
7
|
+
readonly pid: number;
|
|
8
|
+
};
|
|
9
|
+
/** Captured when this module loads, so later PATH or on-disk installs cannot change it. */
|
|
10
|
+
export declare const RUNNING_GATEWAY_VERSION: string;
|
|
11
|
+
export declare function gatewayHealth(status: GatewayHealth["status"], version: string, runningProcess?: ProcessIdentity): GatewayHealth;
|
|
12
|
+
export {};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
function readRunningPackageVersion() {
|
|
4
|
+
try {
|
|
5
|
+
const manifest = JSON.parse(readFileSync(fileURLToPath(new URL("../../package.json", import.meta.url)), "utf8"));
|
|
6
|
+
return typeof manifest.version === "string" && manifest.version.length > 0
|
|
7
|
+
? manifest.version
|
|
8
|
+
: "0.0.0";
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return "0.0.0";
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
/** Captured when this module loads, so later PATH or on-disk installs cannot change it. */
|
|
15
|
+
export const RUNNING_GATEWAY_VERSION = readRunningPackageVersion();
|
|
16
|
+
export function gatewayHealth(status, version, runningProcess = process) {
|
|
17
|
+
try {
|
|
18
|
+
const pid = runningProcess.pid;
|
|
19
|
+
return Number.isSafeInteger(pid) && pid > 0 ? { status, version, pid } : { status, version };
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return { status, version };
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -22,6 +22,8 @@ export type SwitchingGatewayProxy = {
|
|
|
22
22
|
};
|
|
23
23
|
export type SwitchingGatewayProxyOptions = {
|
|
24
24
|
target: string;
|
|
25
|
+
/** Running daemon package version captured by its loaded module. */
|
|
26
|
+
packageVersion?: string;
|
|
25
27
|
host?: string;
|
|
26
28
|
port?: number;
|
|
27
29
|
authToken?: string;
|
package/dist/switching-proxy.js
CHANGED
|
@@ -16,6 +16,7 @@ import { HttpClient, HttpRouter, HttpServerRequest, HttpServerResponse } from "e
|
|
|
16
16
|
import * as HttpEffect from "effect/unstable/http/HttpEffect";
|
|
17
17
|
import { authorizedRequest, ROUTEKIT_PRINCIPAL_HEADER, resolvePrincipal } from "./http/auth.js";
|
|
18
18
|
import { gatewayTryPromise } from "./effect/gateway.js";
|
|
19
|
+
import { gatewayHealth, RUNNING_GATEWAY_VERSION } from "./http/health.js";
|
|
19
20
|
const MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
20
21
|
const DEFAULT_UPSTREAM_HEADERS_TIMEOUT_MS = 10 * 60 * 1000;
|
|
21
22
|
const HOP_BY_HOP = new Set([
|
|
@@ -182,6 +183,7 @@ const startSwitchingGatewayProxyOperation = Effect.fn("SwitchingGatewayProxy.sta
|
|
|
182
183
|
let draining = false;
|
|
183
184
|
let retiring = false;
|
|
184
185
|
let inflight = 0;
|
|
186
|
+
const healthVersion = input.packageVersion ?? RUNNING_GATEWAY_VERSION;
|
|
185
187
|
const httpEffect = yield* Effect.gen(function* () {
|
|
186
188
|
const router = yield* HttpRouter.make;
|
|
187
189
|
const httpClient = yield* HttpClient.HttpClient;
|
|
@@ -243,9 +245,7 @@ const startSwitchingGatewayProxyOperation = Effect.fn("SwitchingGatewayProxy.sta
|
|
|
243
245
|
const request = yield* HttpServerRequest.HttpServerRequest;
|
|
244
246
|
const path = new URL(request.url, "http://localhost").pathname;
|
|
245
247
|
if (path === "/health") {
|
|
246
|
-
return jsonResponse(draining ? 503 : 200,
|
|
247
|
-
status: draining ? "draining" : "ok"
|
|
248
|
-
});
|
|
248
|
+
return jsonResponse(draining ? 503 : 200, gatewayHealth(draining ? "draining" : "ok", healthVersion));
|
|
249
249
|
}
|
|
250
250
|
if (draining) {
|
|
251
251
|
return jsonResponse(503, {
|
package/dist/test/drain.test.js
CHANGED
|
@@ -4,6 +4,7 @@ import { RouteKitFailure, runRouteKitEffect } from "@velum-labs/routekit-runtime
|
|
|
4
4
|
import { Effect } from "effect";
|
|
5
5
|
import { borrowedBackendPorts, staticBackendModelPort } from "../providers/backend.js";
|
|
6
6
|
import { startGateway } from "../gateway-service.js";
|
|
7
|
+
import { RUNNING_GATEWAY_VERSION } from "../http/health.js";
|
|
7
8
|
/**
|
|
8
9
|
* Graceful drain: a draining gateway must report unhealthy and reject new
|
|
9
10
|
* model calls while letting in-flight requests (long-lived LLM streams)
|
|
@@ -50,7 +51,11 @@ test("drain finishes in-flight streams, rejects new work, and flips /health to 5
|
|
|
50
51
|
// Health flips to 503 so probes stop routing new work here.
|
|
51
52
|
const health = await fetch(`${gateway.url()}/health`);
|
|
52
53
|
assert.equal(health.status, 503);
|
|
53
|
-
assert.deepEqual(await health.json(), {
|
|
54
|
+
assert.deepEqual(await health.json(), {
|
|
55
|
+
status: "draining",
|
|
56
|
+
version: RUNNING_GATEWAY_VERSION,
|
|
57
|
+
pid: process.pid
|
|
58
|
+
});
|
|
54
59
|
// New model calls are rejected while draining.
|
|
55
60
|
const rejected = await fetch(`${gateway.url()}/v1/chat/completions`, {
|
|
56
61
|
method: "POST",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import { gatewayHealth } from "../http/health.js";
|
|
4
|
+
test("health keeps version and omits pid when process identity lookup fails", () => {
|
|
5
|
+
const unavailablePid = {
|
|
6
|
+
get pid() {
|
|
7
|
+
throw new Error("pid unavailable");
|
|
8
|
+
}
|
|
9
|
+
};
|
|
10
|
+
assert.deepEqual(gatewayHealth("ok", "1.2.3", unavailablePid), {
|
|
11
|
+
status: "ok",
|
|
12
|
+
version: "1.2.3"
|
|
13
|
+
});
|
|
14
|
+
});
|
|
@@ -9,6 +9,7 @@ import { borrowedBackendPorts } from "../providers/backend.js";
|
|
|
9
9
|
import { OpenAiBackend } from "../providers/openai-backend.js";
|
|
10
10
|
import { startGateway } from "../gateway-service.js";
|
|
11
11
|
import { startSwitchingGatewayProxy } from "../switching-proxy.js";
|
|
12
|
+
import { RUNNING_GATEWAY_VERSION } from "../http/health.js";
|
|
12
13
|
/**
|
|
13
14
|
* Crash resilience: an upstream stream that dies mid-response (the shape of a
|
|
14
15
|
* local model server being OOM-killed during a turn) must fail only that one
|
|
@@ -90,6 +91,11 @@ test("a mid-stream upstream failure does not kill the gateway process", async ()
|
|
|
90
91
|
// The gateway (and its hosting process) is still alive and serving.
|
|
91
92
|
const health = await fetch(`${gateway.url()}/health`);
|
|
92
93
|
assert.equal(health.status, 200);
|
|
94
|
+
assert.deepEqual(await health.json(), {
|
|
95
|
+
status: "ok",
|
|
96
|
+
version: RUNNING_GATEWAY_VERSION,
|
|
97
|
+
pid: process.pid
|
|
98
|
+
});
|
|
93
99
|
const models = await fetch(`${gateway.url()}/v1/models`);
|
|
94
100
|
assert.equal(models.status, 200);
|
|
95
101
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@velum-labs/routekit-gateway",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "1.3.
|
|
4
|
+
"version": "1.3.2",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "git+https://github.com/velum-labs/routekit.git",
|
|
@@ -45,12 +45,12 @@
|
|
|
45
45
|
"@aws-sdk/client-bedrock": "3.1095.0",
|
|
46
46
|
"@aws-sdk/client-bedrock-runtime": "3.1095.0",
|
|
47
47
|
"effect": "4.0.0-rc.108",
|
|
48
|
-
"@velum-labs/routekit-config-core": "1.3.
|
|
49
|
-
"@velum-labs/routekit-contracts": "1.3.
|
|
50
|
-
"@velum-labs/routekit-eval-contracts": "1.3.
|
|
51
|
-
"@velum-labs/routekit-eval-core": "1.3.
|
|
52
|
-
"@velum-labs/routekit-registry": "1.3.
|
|
53
|
-
"@velum-labs/routekit-runtime": "1.3.
|
|
48
|
+
"@velum-labs/routekit-config-core": "1.3.2",
|
|
49
|
+
"@velum-labs/routekit-contracts": "1.3.2",
|
|
50
|
+
"@velum-labs/routekit-eval-contracts": "1.3.2",
|
|
51
|
+
"@velum-labs/routekit-eval-core": "1.3.2",
|
|
52
|
+
"@velum-labs/routekit-registry": "1.3.2",
|
|
53
|
+
"@velum-labs/routekit-runtime": "1.3.2"
|
|
54
54
|
},
|
|
55
55
|
"keywords": [
|
|
56
56
|
"routekit",
|