@neutrome/open-ai-router 0.4.2 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dev.vars.example +2 -0
- package/README.md +5 -1
- package/package.json +5 -4
- package/src/admission/access.ts +0 -1
- package/src/admission/types.ts +0 -1
- package/src/app/factory.test.ts +40 -34
- package/src/app/factory.ts +120 -173
- package/src/app/google-genai-route.ts +48 -0
- package/src/app/http-utils.ts +14 -0
- package/src/app/model-listing.ts +29 -0
- package/src/app/response-lifecycle.ts +66 -0
- package/src/app/types.ts +38 -0
- package/src/index.ts +11 -43
- package/src/otlp.ts +155 -0
- package/src/router/audit.ts +5 -12
- package/src/router/config.ts +4 -4
- package/src/router/error-codec.ts +170 -0
- package/src/router/errors.ts +38 -1
- package/src/router/execute.test.ts +177 -107
- package/src/router/execute.ts +68 -924
- package/src/router/execution-events.ts +39 -0
- package/src/router/execution-invocation.ts +144 -0
- package/src/router/execution-observation.ts +161 -0
- package/src/router/execution-resolution.ts +120 -0
- package/src/router/execution-runtime.test.ts +156 -83
- package/src/router/execution-runtime.ts +144 -542
- package/src/router/execution-transforms.ts +82 -0
- package/src/router/execution-types.ts +8 -19
- package/src/router/index.ts +6 -5
- package/src/router/mcp.ts +12 -6
- package/src/router/provider-invoker.ts +134 -0
- package/src/router/provider-stream.ts +192 -0
- package/src/router/provider-telemetry.ts +59 -0
- package/src/router/remote-mcp-execution.ts +61 -0
- package/src/router/request-program.ts +16 -0
- package/src/router/request-target.ts +56 -0
- package/src/router/resolve.test.ts +43 -10
- package/src/router/resolve.ts +50 -28
- package/src/router/response-delivery.ts +141 -0
- package/src/router/runtime.test.ts +56 -0
- package/src/router/runtime.ts +79 -9
- package/src/router/targets.ts +3 -1
- package/src/router/upstream-client.ts +137 -0
- package/src/telemetry.test.ts +78 -0
- package/src/telemetry.ts +316 -0
- package/src/trace-labels.ts +29 -0
- package/src/upstream-auth/env.ts +15 -3
- package/src/upstream-auth/proxy.ts +6 -1
- package/src/upstream-auth/registry.ts +3 -1
- package/src/util/glob.ts +1 -1
- package/src/util/model-syntax.ts +3 -3
- package/src/worker.ts +12 -6
- package/worker-configuration.d.ts +195 -164
- package/pnpm-workspace.yaml +0 -4
- package/src/admission/jwt.test.ts +0 -96
- package/src/admission/jwt.ts +0 -221
- package/src/app/handlers.ts +0 -198
- package/src/example.test.ts +0 -377
- package/src/example.ts +0 -73
- package/src/observer.test.ts +0 -54
- package/src/observer.ts +0 -315
- package/src/timing.ts +0 -36
package/README.md
CHANGED
|
@@ -5,7 +5,11 @@ Deployable router app and reusable router/runtime library built on
|
|
|
5
5
|
executor/tool contracts.
|
|
6
6
|
|
|
7
7
|
```ts
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
createRouterApp,
|
|
10
|
+
createRouterRuntime,
|
|
11
|
+
type RouterConfig,
|
|
12
|
+
} from "@neutrome/open-ai-router";
|
|
9
13
|
```
|
|
10
14
|
|
|
11
15
|
Primary runtime model:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@neutrome/open-ai-router",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./src/index.ts"
|
|
@@ -8,16 +8,17 @@
|
|
|
8
8
|
"dependencies": {
|
|
9
9
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
10
10
|
"hono": "^4.12.18",
|
|
11
|
-
"@neutrome/
|
|
12
|
-
"@neutrome/
|
|
11
|
+
"@neutrome/lilsdk-ts": "0.3.0",
|
|
12
|
+
"@neutrome/lil-engine": "0.3.0"
|
|
13
13
|
},
|
|
14
14
|
"devDependencies": {
|
|
15
15
|
"@types/node": "^25.9.3",
|
|
16
16
|
"typescript": "^6.0.3",
|
|
17
17
|
"vitest": "^4.1.6",
|
|
18
|
-
"wrangler": "
|
|
18
|
+
"wrangler": "4.103.0"
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
21
|
+
"generate": "pnpm run cf-typegen",
|
|
21
22
|
"cf-typegen": "wrangler types",
|
|
22
23
|
"test": "vitest run",
|
|
23
24
|
"typecheck": "tsc -p tsconfig.json --noEmit",
|
package/src/admission/access.ts
CHANGED
package/src/admission/types.ts
CHANGED
package/src/app/factory.test.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it, vi } from "vitest";
|
|
2
|
-
import type {
|
|
2
|
+
import type { ExecutionSummary } from "../telemetry.ts";
|
|
3
3
|
import type { RouterConfig } from "../router/index.ts";
|
|
4
4
|
import { createRouterApp } from "./factory.ts";
|
|
5
5
|
|
|
@@ -8,7 +8,7 @@ const encoder = new TextEncoder();
|
|
|
8
8
|
const config: RouterConfig = {
|
|
9
9
|
providers: {
|
|
10
10
|
openrouter: {
|
|
11
|
-
|
|
11
|
+
apiBaseUrl: "https://openrouter.ai/api/v1",
|
|
12
12
|
style: "chat-completions",
|
|
13
13
|
exports: ["*"],
|
|
14
14
|
},
|
|
@@ -26,7 +26,7 @@ const config: RouterConfig = {
|
|
|
26
26
|
describe("createRouterApp trace finalization", () => {
|
|
27
27
|
it("keeps non-stream trace hooks alive with waitUntil", async () => {
|
|
28
28
|
const waited: Promise<unknown>[] = [];
|
|
29
|
-
const
|
|
29
|
+
const onExecutionSummary = vi.fn(async (_summary: ExecutionSummary) => {});
|
|
30
30
|
const app = createRouterApp({
|
|
31
31
|
config,
|
|
32
32
|
executorImplementations: {},
|
|
@@ -35,22 +35,24 @@ describe("createRouterApp trace finalization", () => {
|
|
|
35
35
|
return principal();
|
|
36
36
|
},
|
|
37
37
|
},
|
|
38
|
-
hooks: {
|
|
39
|
-
fetchImpl: vi.fn(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
38
|
+
hooks: { onExecutionSummary },
|
|
39
|
+
fetchImpl: vi.fn(
|
|
40
|
+
async () =>
|
|
41
|
+
new Response(
|
|
42
|
+
JSON.stringify({
|
|
43
|
+
id: "resp_1",
|
|
44
|
+
model: "gpt-4o",
|
|
45
|
+
choices: [
|
|
46
|
+
{
|
|
47
|
+
index: 0,
|
|
48
|
+
message: { role: "assistant", content: "hello" },
|
|
49
|
+
finish_reason: "stop",
|
|
50
|
+
},
|
|
51
|
+
],
|
|
52
|
+
}),
|
|
53
|
+
{ headers: { "content-type": "application/json; charset=utf-8" } },
|
|
54
|
+
),
|
|
55
|
+
),
|
|
54
56
|
});
|
|
55
57
|
|
|
56
58
|
const response = await app.fetch(
|
|
@@ -69,12 +71,12 @@ describe("createRouterApp trace finalization", () => {
|
|
|
69
71
|
expect(response.status).toBe(200);
|
|
70
72
|
expect(waited).toHaveLength(1);
|
|
71
73
|
await waited[0];
|
|
72
|
-
expect(
|
|
74
|
+
expect(onExecutionSummary).toHaveBeenCalledTimes(1);
|
|
73
75
|
});
|
|
74
76
|
|
|
75
77
|
it("keeps stream trace hooks alive until the stream closes", async () => {
|
|
76
78
|
const waited: Promise<unknown>[] = [];
|
|
77
|
-
const
|
|
79
|
+
const onExecutionSummary = vi.fn(async (_summary: ExecutionSummary) => {});
|
|
78
80
|
const app = createRouterApp({
|
|
79
81
|
config,
|
|
80
82
|
executorImplementations: {},
|
|
@@ -83,7 +85,7 @@ describe("createRouterApp trace finalization", () => {
|
|
|
83
85
|
return principal();
|
|
84
86
|
},
|
|
85
87
|
},
|
|
86
|
-
hooks: {
|
|
88
|
+
hooks: { onExecutionSummary },
|
|
87
89
|
fetchImpl: vi.fn(async () => {
|
|
88
90
|
const stream = new ReadableStream<Uint8Array>({
|
|
89
91
|
start(controller) {
|
|
@@ -92,7 +94,9 @@ describe("createRouterApp trace finalization", () => {
|
|
|
92
94
|
`data: ${JSON.stringify({
|
|
93
95
|
id: "chunk_1",
|
|
94
96
|
model: "gpt-4o",
|
|
95
|
-
choices: [
|
|
97
|
+
choices: [
|
|
98
|
+
{ index: 0, delta: { role: "assistant", content: "Hi" } },
|
|
99
|
+
],
|
|
96
100
|
})}\n\n`,
|
|
97
101
|
),
|
|
98
102
|
);
|
|
@@ -130,23 +134,25 @@ describe("createRouterApp trace finalization", () => {
|
|
|
130
134
|
createExecutionContext(waited) as never,
|
|
131
135
|
);
|
|
132
136
|
|
|
133
|
-
expect(response.headers.get("content-type")).toBe(
|
|
137
|
+
expect(response.headers.get("content-type")).toBe(
|
|
138
|
+
"text/event-stream; charset=utf-8",
|
|
139
|
+
);
|
|
134
140
|
expect(waited).toHaveLength(0);
|
|
135
141
|
|
|
136
142
|
await response.text();
|
|
137
143
|
|
|
138
144
|
expect(waited).toHaveLength(1);
|
|
139
145
|
await waited[0];
|
|
140
|
-
expect(
|
|
141
|
-
const
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
expect(
|
|
145
|
-
expect(
|
|
146
|
-
expect(
|
|
147
|
-
expect(
|
|
148
|
-
expect(
|
|
149
|
-
expect(
|
|
146
|
+
expect(onExecutionSummary).toHaveBeenCalledTimes(1);
|
|
147
|
+
const events = onExecutionSummary.mock.calls[0]?.[0].spans.map(
|
|
148
|
+
(span) => span.event,
|
|
149
|
+
);
|
|
150
|
+
expect(events).toContain("admission.authenticate");
|
|
151
|
+
expect(events).toContain("upstream_auth.collect_incoming");
|
|
152
|
+
expect(events).toContain("upstream_auth.resolve_target");
|
|
153
|
+
expect(events).toContain("provider.stream_first_byte");
|
|
154
|
+
expect(events).toContain("response.stream_first_chunk");
|
|
155
|
+
expect(events).toContain("provider.stream_started");
|
|
150
156
|
});
|
|
151
157
|
});
|
|
152
158
|
|
package/src/app/factory.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { Hono, type Context } from "hono";
|
|
2
|
-
import
|
|
3
|
-
|
|
2
|
+
import {
|
|
3
|
+
getModel,
|
|
4
|
+
setStreaming,
|
|
5
|
+
type Program,
|
|
6
|
+
type ProviderStyle,
|
|
7
|
+
} from "@neutrome/lil-engine";
|
|
4
8
|
import { cors } from "./cors.ts";
|
|
5
9
|
import { healthHandler } from "./health.ts";
|
|
6
|
-
import {
|
|
10
|
+
import { createRouterTelemetry } from "../telemetry.ts";
|
|
7
11
|
import { canAccessPrincipalModel } from "../admission/access.ts";
|
|
8
12
|
import {
|
|
9
13
|
createRouterRuntime,
|
|
@@ -12,47 +16,32 @@ import {
|
|
|
12
16
|
handleGoogleGenAI,
|
|
13
17
|
parseRequestProgram,
|
|
14
18
|
handleResponses,
|
|
15
|
-
listConfiguredModels,
|
|
16
19
|
} from "../router/index.ts";
|
|
17
|
-
import type { AuthPrincipal, RuntimeAuthenticator } from "../admission/types.ts";
|
|
18
20
|
import type { RequestContext } from "../upstream-auth/types.ts";
|
|
19
|
-
import type { RouterConfig } from "../router/config.ts";
|
|
20
|
-
import type { RemoteMcpClientFactory } from "../router/mcp.ts";
|
|
21
21
|
import type { ExecutionRuntimeOptions } from "../router/execution-types.ts";
|
|
22
|
-
import {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
};
|
|
22
|
+
import { observeTimedExecution } from "../router/execution-events.ts";
|
|
23
|
+
import { createOpenAiErrorResponse } from "../router/errors.ts";
|
|
24
|
+
import {
|
|
25
|
+
parseGoogleGenAIRoute,
|
|
26
|
+
rewriteGoogleGenAIRequest,
|
|
27
|
+
} from "./google-genai-route.ts";
|
|
28
|
+
import {
|
|
29
|
+
finalizeWhenStreamCloses,
|
|
30
|
+
isEventStreamResponse,
|
|
31
|
+
runBackgroundTask,
|
|
32
|
+
} from "./response-lifecycle.ts";
|
|
33
|
+
import { createModelListingHandler } from "./model-listing.ts";
|
|
34
|
+
import { honoEnv, unauthorizedResponse } from "./http-utils.ts";
|
|
35
|
+
import type { RouterAppOptions } from "./types.ts";
|
|
35
36
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
transforms?: Readonly<Record<string, ProgramTransform>>;
|
|
40
|
-
auth: RouterAppAuth;
|
|
41
|
-
hooks?: RouterAppHooks;
|
|
42
|
-
fetchImpl?: typeof fetch;
|
|
43
|
-
knownSuffixes?: string[];
|
|
44
|
-
observe?: ExecutionRuntimeOptions["observe"];
|
|
45
|
-
requestIdFactory?: ExecutionRuntimeOptions["requestIdFactory"];
|
|
46
|
-
executionIdFactory?: ExecutionRuntimeOptions["executionIdFactory"];
|
|
47
|
-
upstreamTimeoutMs?: number;
|
|
48
|
-
remoteMcpClientFactory?: RemoteMcpClientFactory;
|
|
49
|
-
};
|
|
37
|
+
type ProtocolHandler = (
|
|
38
|
+
options: Parameters<typeof handleChatCompletions>[0],
|
|
39
|
+
) => Promise<Response>;
|
|
50
40
|
|
|
51
41
|
export function createRouterApp(options: RouterAppOptions) {
|
|
52
42
|
const runtime = createRouterRuntime({
|
|
53
43
|
config: options.config,
|
|
54
44
|
...(options.fetchImpl ? { fetchImpl: options.fetchImpl } : {}),
|
|
55
|
-
...(options.knownSuffixes ? { knownSuffixes: options.knownSuffixes } : {}),
|
|
56
45
|
});
|
|
57
46
|
|
|
58
47
|
const app = new Hono();
|
|
@@ -61,30 +50,34 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
61
50
|
app.get("/health", healthHandler());
|
|
62
51
|
|
|
63
52
|
app.get("/v1/models", async (c) => {
|
|
64
|
-
const
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
.filter((id) => canAccessPrincipalModel(principal, id))
|
|
70
|
-
.map((id) => ({
|
|
71
|
-
id,
|
|
72
|
-
object: "model" as const,
|
|
73
|
-
created: 0,
|
|
74
|
-
owned_by: id.split("/")[0] || "unknown",
|
|
75
|
-
}));
|
|
76
|
-
return c.json({ object: "list", data: models });
|
|
53
|
+
const response = await createModelListingHandler({
|
|
54
|
+
runtime,
|
|
55
|
+
auth: options.auth,
|
|
56
|
+
})(c.req.raw, honoEnv(c));
|
|
57
|
+
return response instanceof Response ? response : c.json(response);
|
|
77
58
|
});
|
|
78
59
|
|
|
79
|
-
app.post("/v1/chat/completions", (c) =>
|
|
80
|
-
|
|
81
|
-
|
|
60
|
+
app.post("/v1/chat/completions", (c) =>
|
|
61
|
+
runAuthorized(c, c.req.raw, "chat-completions", handleChatCompletions),
|
|
62
|
+
);
|
|
63
|
+
app.post("/v1/responses", (c) =>
|
|
64
|
+
runAuthorized(c, c.req.raw, "responses", handleResponses),
|
|
65
|
+
);
|
|
66
|
+
app.post("/v1/messages", (c) =>
|
|
67
|
+
runAuthorized(c, c.req.raw, "anthropic-messages", handleAnthropicMessages),
|
|
68
|
+
);
|
|
82
69
|
|
|
83
70
|
app.post("/v1/models/*", async (c) => {
|
|
84
71
|
const route = parseGoogleGenAIRoute(c.req.raw);
|
|
85
72
|
if (!route) return c.text("Not Found", 404);
|
|
86
73
|
const request = await rewriteGoogleGenAIRequest(c.req.raw, route);
|
|
87
|
-
return runAuthorized(
|
|
74
|
+
return runAuthorized(
|
|
75
|
+
c,
|
|
76
|
+
request,
|
|
77
|
+
"google-genai",
|
|
78
|
+
handleGoogleGenAI,
|
|
79
|
+
route.stream,
|
|
80
|
+
);
|
|
88
81
|
});
|
|
89
82
|
|
|
90
83
|
app.all("*", (c) => c.text("Not Found", 404));
|
|
@@ -93,31 +86,35 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
93
86
|
c: Context,
|
|
94
87
|
request: Request,
|
|
95
88
|
style: ProviderStyle,
|
|
96
|
-
handler:
|
|
89
|
+
handler: ProtocolHandler,
|
|
97
90
|
forceStreaming?: boolean,
|
|
98
91
|
): Promise<Response> {
|
|
99
92
|
const env = honoEnv(c);
|
|
100
93
|
const incomingExecutionId = `incoming_${crypto.randomUUID()}`;
|
|
101
|
-
const
|
|
102
|
-
const
|
|
103
|
-
const observe = (
|
|
104
|
-
|
|
94
|
+
const telemetry = createRouterTelemetry(env);
|
|
95
|
+
const onExecutionSummary = options.hooks?.onExecutionSummary;
|
|
96
|
+
const observe = (
|
|
97
|
+
event: Parameters<NonNullable<ExecutionRuntimeOptions["observe"]>>[0],
|
|
98
|
+
) => {
|
|
99
|
+
telemetry.observe(event);
|
|
105
100
|
options.observe?.(event);
|
|
106
101
|
};
|
|
107
102
|
let traceFinalized = false;
|
|
108
103
|
const finalizeTrace = () => {
|
|
109
104
|
if (traceFinalized) return;
|
|
110
105
|
traceFinalized = true;
|
|
111
|
-
if (!onTrace) return;
|
|
112
106
|
runBackgroundTask(
|
|
113
107
|
c,
|
|
114
|
-
() =>
|
|
115
|
-
|
|
108
|
+
async () => {
|
|
109
|
+
await telemetry.flush();
|
|
110
|
+
await onExecutionSummary?.(telemetry.summary(), env);
|
|
111
|
+
},
|
|
112
|
+
"onExecutionSummary hook",
|
|
116
113
|
);
|
|
117
114
|
};
|
|
118
115
|
|
|
119
116
|
const requestStartedMs = Date.now();
|
|
120
|
-
|
|
117
|
+
observeTimedExecution({
|
|
121
118
|
observe,
|
|
122
119
|
kind: "request.started",
|
|
123
120
|
requestId: incomingExecutionId,
|
|
@@ -128,7 +125,7 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
128
125
|
|
|
129
126
|
const authStartedMs = Date.now();
|
|
130
127
|
const principal = await options.auth.authenticate(request, env);
|
|
131
|
-
|
|
128
|
+
observeTimedExecution({
|
|
132
129
|
observe,
|
|
133
130
|
kind: "admission.authenticate",
|
|
134
131
|
requestId: incomingExecutionId,
|
|
@@ -137,14 +134,22 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
137
134
|
finishedAtMs: Date.now(),
|
|
138
135
|
data: { authenticated: !!principal },
|
|
139
136
|
});
|
|
140
|
-
if (!principal)
|
|
137
|
+
if (!principal) {
|
|
138
|
+
telemetry.recordError({
|
|
139
|
+
message: "Authentication failed",
|
|
140
|
+
requestId: incomingExecutionId,
|
|
141
|
+
executionId: incomingExecutionId,
|
|
142
|
+
});
|
|
143
|
+
finalizeTrace();
|
|
144
|
+
return unauthorizedResponse();
|
|
145
|
+
}
|
|
141
146
|
|
|
142
147
|
let program: Program;
|
|
143
148
|
try {
|
|
144
149
|
const parseStartedMs = Date.now();
|
|
145
150
|
program = await parseRequestProgram(style, request.clone());
|
|
146
151
|
if (forceStreaming) program = setStreaming(program, true);
|
|
147
|
-
|
|
152
|
+
observeTimedExecution({
|
|
148
153
|
observe,
|
|
149
154
|
kind: "request.parsed",
|
|
150
155
|
requestId: incomingExecutionId,
|
|
@@ -154,7 +159,12 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
154
159
|
data: { style, streaming: forceStreaming === true },
|
|
155
160
|
});
|
|
156
161
|
} catch {
|
|
157
|
-
|
|
162
|
+
telemetry.recordError({
|
|
163
|
+
message: "Invalid request body",
|
|
164
|
+
requestId: incomingExecutionId,
|
|
165
|
+
executionId: incomingExecutionId,
|
|
166
|
+
});
|
|
167
|
+
observeTimedExecution({
|
|
158
168
|
observe,
|
|
159
169
|
kind: "request.parse_failed",
|
|
160
170
|
requestId: incomingExecutionId,
|
|
@@ -164,13 +174,17 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
164
174
|
data: { style },
|
|
165
175
|
});
|
|
166
176
|
finalizeTrace();
|
|
167
|
-
return
|
|
177
|
+
return createOpenAiErrorResponse(
|
|
178
|
+
"Invalid request body",
|
|
179
|
+
400,
|
|
180
|
+
"invalid_request_body",
|
|
181
|
+
);
|
|
168
182
|
}
|
|
169
183
|
|
|
170
184
|
const model = getModel(program) || null;
|
|
171
185
|
const accessStartedMs = Date.now();
|
|
172
186
|
const hasModelAccess = !model || canAccessPrincipalModel(principal, model);
|
|
173
|
-
|
|
187
|
+
observeTimedExecution({
|
|
174
188
|
observe,
|
|
175
189
|
kind: "admission.model_access",
|
|
176
190
|
requestId: incomingExecutionId,
|
|
@@ -180,14 +194,28 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
180
194
|
data: { model: model ?? undefined, allowed: hasModelAccess },
|
|
181
195
|
});
|
|
182
196
|
if (!hasModelAccess) {
|
|
197
|
+
telemetry.recordError({
|
|
198
|
+
message: "Model access denied",
|
|
199
|
+
requestId: incomingExecutionId,
|
|
200
|
+
executionId: incomingExecutionId,
|
|
201
|
+
});
|
|
183
202
|
finalizeTrace();
|
|
184
|
-
return
|
|
203
|
+
return createOpenAiErrorResponse(
|
|
204
|
+
`Model \`${model}\` does not exist or you do not have access.`,
|
|
205
|
+
404,
|
|
206
|
+
"model_not_found",
|
|
207
|
+
);
|
|
185
208
|
}
|
|
186
209
|
|
|
187
210
|
if (options.hooks?.beforeRequest) {
|
|
188
211
|
const hookStartedMs = Date.now();
|
|
189
|
-
const hookResponse = await options.hooks.beforeRequest({
|
|
190
|
-
|
|
212
|
+
const hookResponse = await options.hooks.beforeRequest({
|
|
213
|
+
request,
|
|
214
|
+
model,
|
|
215
|
+
principal,
|
|
216
|
+
env,
|
|
217
|
+
});
|
|
218
|
+
observeTimedExecution({
|
|
191
219
|
observe,
|
|
192
220
|
kind: "request.before_hook",
|
|
193
221
|
requestId: incomingExecutionId,
|
|
@@ -197,6 +225,13 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
197
225
|
data: { returnedResponse: !!hookResponse },
|
|
198
226
|
});
|
|
199
227
|
if (hookResponse) {
|
|
228
|
+
if (hookResponse.status >= 400) {
|
|
229
|
+
telemetry.recordError({
|
|
230
|
+
message: `Request hook returned ${hookResponse.status}`,
|
|
231
|
+
requestId: incomingExecutionId,
|
|
232
|
+
executionId: incomingExecutionId,
|
|
233
|
+
});
|
|
234
|
+
}
|
|
200
235
|
finalizeTrace();
|
|
201
236
|
return hookResponse;
|
|
202
237
|
}
|
|
@@ -211,7 +246,7 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
211
246
|
for (const driver of runtime.upstreamAuthChain) {
|
|
212
247
|
const collectStartedMs = Date.now();
|
|
213
248
|
driver.collectIncoming?.(ctx);
|
|
214
|
-
|
|
249
|
+
observeTimedExecution({
|
|
215
250
|
observe,
|
|
216
251
|
kind: "upstream_auth.collect_incoming",
|
|
217
252
|
requestId: incomingExecutionId,
|
|
@@ -232,10 +267,14 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
232
267
|
...(options.transforms ? { transforms: options.transforms } : {}),
|
|
233
268
|
observe,
|
|
234
269
|
incomingExecutionId,
|
|
235
|
-
...(options.upstreamTimeoutMs !== undefined
|
|
236
|
-
|
|
270
|
+
...(options.upstreamTimeoutMs !== undefined
|
|
271
|
+
? { upstreamTimeoutMs: options.upstreamTimeoutMs }
|
|
272
|
+
: {}),
|
|
273
|
+
...(options.remoteMcpClientFactory
|
|
274
|
+
? { remoteMcpClientFactory: options.remoteMcpClientFactory }
|
|
275
|
+
: {}),
|
|
237
276
|
});
|
|
238
|
-
|
|
277
|
+
observeTimedExecution({
|
|
239
278
|
observe,
|
|
240
279
|
kind: "response.ready",
|
|
241
280
|
requestId: incomingExecutionId,
|
|
@@ -252,8 +291,14 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
252
291
|
finalizeTrace();
|
|
253
292
|
return response;
|
|
254
293
|
}
|
|
255
|
-
return
|
|
294
|
+
return finalizeWhenStreamCloses(response, finalizeTrace);
|
|
256
295
|
} catch (error) {
|
|
296
|
+
telemetry.recordError({
|
|
297
|
+
message:
|
|
298
|
+
error instanceof Error ? error.message : "Request execution failed",
|
|
299
|
+
requestId: incomingExecutionId,
|
|
300
|
+
executionId: incomingExecutionId,
|
|
301
|
+
});
|
|
257
302
|
finalizeTrace();
|
|
258
303
|
throw error;
|
|
259
304
|
}
|
|
@@ -261,101 +306,3 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
261
306
|
|
|
262
307
|
return app;
|
|
263
308
|
}
|
|
264
|
-
|
|
265
|
-
function honoEnv(c: Context): Record<string, unknown> {
|
|
266
|
-
return (c.env ?? {}) as Record<string, unknown>;
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
function parseGoogleGenAIRoute(request: Request): { model: string; stream: boolean } | null {
|
|
270
|
-
const pathname = new URL(request.url).pathname;
|
|
271
|
-
const match = pathname.match(/^\/v1\/models\/(.+):(generateContent|streamGenerateContent)$/);
|
|
272
|
-
if (!match?.[1] || !match[2]) return null;
|
|
273
|
-
try {
|
|
274
|
-
return { model: decodeURIComponent(match[1]), stream: match[2] === "streamGenerateContent" };
|
|
275
|
-
} catch {
|
|
276
|
-
return null;
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
async function rewriteGoogleGenAIRequest(
|
|
281
|
-
request: Request,
|
|
282
|
-
route: { model: string; stream: boolean },
|
|
283
|
-
): Promise<Request> {
|
|
284
|
-
const bodyText = await request.text();
|
|
285
|
-
const headers = new Headers(request.headers);
|
|
286
|
-
try {
|
|
287
|
-
const raw = JSON.parse(bodyText) as Record<string, unknown>;
|
|
288
|
-
if (raw && typeof raw === "object" && !Array.isArray(raw)) {
|
|
289
|
-
return new Request(request.url, {
|
|
290
|
-
method: request.method,
|
|
291
|
-
headers,
|
|
292
|
-
body: JSON.stringify({ ...raw, model: route.model, ...(route.stream ? { stream: true } : {}) }),
|
|
293
|
-
});
|
|
294
|
-
}
|
|
295
|
-
} catch {}
|
|
296
|
-
return new Request(request.url, { method: request.method, headers, body: bodyText });
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
function unauthorized(): Response {
|
|
300
|
-
return errorResponse("Authentication required", 401, "auth_required");
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
function errorResponse(message: string, status: number, code: string): Response {
|
|
304
|
-
return new Response(
|
|
305
|
-
JSON.stringify({ error: { message, type: "invalid_request_error", param: null, code } }),
|
|
306
|
-
{ status, headers: { "content-type": "application/json; charset=utf-8" } },
|
|
307
|
-
);
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
function isEventStreamResponse(response: Response): boolean {
|
|
311
|
-
const contentType = response.headers.get("content-type") ?? "";
|
|
312
|
-
return contentType.startsWith("text/event-stream");
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
function runBackgroundTask(c: Context, task: () => unknown, label: string): void {
|
|
316
|
-
const tracked = Promise.resolve()
|
|
317
|
-
.then(task)
|
|
318
|
-
.catch((error) => {
|
|
319
|
-
console.error(`[open-ai-router] ${label} failed:`, error);
|
|
320
|
-
});
|
|
321
|
-
let executionCtx: { waitUntil?(promise: Promise<unknown>): void } | undefined;
|
|
322
|
-
try {
|
|
323
|
-
executionCtx = c.executionCtx as { waitUntil?(promise: Promise<unknown>): void } | undefined;
|
|
324
|
-
} catch {
|
|
325
|
-
executionCtx = undefined;
|
|
326
|
-
}
|
|
327
|
-
if (executionCtx?.waitUntil) {
|
|
328
|
-
executionCtx.waitUntil(tracked);
|
|
329
|
-
return;
|
|
330
|
-
}
|
|
331
|
-
void tracked;
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
function wrapResponseWithFinalize(response: Response, finalize: () => void): Response {
|
|
335
|
-
const reader = response.body!.getReader();
|
|
336
|
-
const body = new ReadableStream<Uint8Array>({
|
|
337
|
-
async pull(controller) {
|
|
338
|
-
try {
|
|
339
|
-
const { done, value } = await reader.read();
|
|
340
|
-
if (done) {
|
|
341
|
-
finalize();
|
|
342
|
-
controller.close();
|
|
343
|
-
return;
|
|
344
|
-
}
|
|
345
|
-
controller.enqueue(value);
|
|
346
|
-
} catch (error) {
|
|
347
|
-
finalize();
|
|
348
|
-
controller.error(error);
|
|
349
|
-
}
|
|
350
|
-
},
|
|
351
|
-
async cancel(reason) {
|
|
352
|
-
finalize();
|
|
353
|
-
await reader.cancel(reason).catch(() => {});
|
|
354
|
-
},
|
|
355
|
-
});
|
|
356
|
-
return new Response(body, {
|
|
357
|
-
status: response.status,
|
|
358
|
-
statusText: response.statusText,
|
|
359
|
-
headers: new Headers(response.headers),
|
|
360
|
-
});
|
|
361
|
-
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
type GoogleGenAIRoute = { model: string; stream: boolean };
|
|
2
|
+
|
|
3
|
+
export function parseGoogleGenAIRoute(
|
|
4
|
+
request: Request,
|
|
5
|
+
): GoogleGenAIRoute | null {
|
|
6
|
+
const pathname = new URL(request.url).pathname;
|
|
7
|
+
const match = pathname.match(
|
|
8
|
+
/^\/v1\/models\/(.+):(generateContent|streamGenerateContent)$/,
|
|
9
|
+
);
|
|
10
|
+
if (!match?.[1] || !match[2]) return null;
|
|
11
|
+
try {
|
|
12
|
+
return {
|
|
13
|
+
model: decodeURIComponent(match[1]),
|
|
14
|
+
stream: match[2] === "streamGenerateContent",
|
|
15
|
+
};
|
|
16
|
+
} catch {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export async function rewriteGoogleGenAIRequest(
|
|
22
|
+
request: Request,
|
|
23
|
+
route: GoogleGenAIRoute,
|
|
24
|
+
): Promise<Request> {
|
|
25
|
+
const bodyText = await request.text();
|
|
26
|
+
const headers = new Headers(request.headers);
|
|
27
|
+
try {
|
|
28
|
+
const raw: unknown = JSON.parse(bodyText);
|
|
29
|
+
if (raw && typeof raw === "object" && !Array.isArray(raw)) {
|
|
30
|
+
return new Request(request.url, {
|
|
31
|
+
method: request.method,
|
|
32
|
+
headers,
|
|
33
|
+
body: JSON.stringify({
|
|
34
|
+
...raw,
|
|
35
|
+
model: route.model,
|
|
36
|
+
...(route.stream ? { stream: true } : {}),
|
|
37
|
+
}),
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
} catch {
|
|
41
|
+
// The protocol parser returns the canonical invalid-body response.
|
|
42
|
+
}
|
|
43
|
+
return new Request(request.url, {
|
|
44
|
+
method: request.method,
|
|
45
|
+
headers,
|
|
46
|
+
body: bodyText,
|
|
47
|
+
});
|
|
48
|
+
}
|