@neutrome/open-ai-router 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -2
- package/src/app/factory.ts +3 -0
- package/src/app/handlers.ts +3 -0
- package/src/index.ts +13 -0
- package/src/router/config.ts +16 -0
- package/src/router/execute.test.ts +133 -0
- package/src/router/execute.ts +105 -40
- package/src/router/index.ts +14 -0
- package/src/router/mcp.ts +157 -0
- package/src/router/resolve.ts +4 -0
- package/src/router/runtime.ts +17 -3
package/package.json
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@neutrome/open-ai-router",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.15",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./src/index.ts"
|
|
7
7
|
},
|
|
8
8
|
"dependencies": {
|
|
9
|
+
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
9
10
|
"hono": "^4.12.18",
|
|
10
|
-
"@neutrome/lil-engine": "0.1.
|
|
11
|
+
"@neutrome/lil-engine": "0.1.5"
|
|
11
12
|
},
|
|
12
13
|
"devDependencies": {
|
|
13
14
|
"@types/node": "^25.9.3",
|
package/src/app/factory.ts
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
} from "../router/index.ts";
|
|
16
16
|
import type { RequestContext } from "../auth/types.ts";
|
|
17
17
|
import type { RouterConfig } from "../router/config.ts";
|
|
18
|
+
import type { RemoteMcpClientFactory } from "../router/mcp.ts";
|
|
18
19
|
|
|
19
20
|
export type RouterAppAuth = {
|
|
20
21
|
authenticate(request: Request, env: Record<string, unknown>): Promise<object | null>;
|
|
@@ -38,6 +39,7 @@ export type RouterAppOptions = {
|
|
|
38
39
|
requestIdFactory?: ExecutionRuntimeOptions["requestIdFactory"];
|
|
39
40
|
executionIdFactory?: ExecutionRuntimeOptions["executionIdFactory"];
|
|
40
41
|
upstreamTimeoutMs?: number;
|
|
42
|
+
remoteMcpClientFactory?: RemoteMcpClientFactory;
|
|
41
43
|
};
|
|
42
44
|
|
|
43
45
|
export function createRouterApp(options: RouterAppOptions) {
|
|
@@ -137,6 +139,7 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
137
139
|
...(options.transforms ? { transforms: options.transforms } : {}),
|
|
138
140
|
observe,
|
|
139
141
|
...(options.upstreamTimeoutMs !== undefined ? { upstreamTimeoutMs: options.upstreamTimeoutMs } : {}),
|
|
142
|
+
...(options.remoteMcpClientFactory ? { remoteMcpClientFactory: options.remoteMcpClientFactory } : {}),
|
|
140
143
|
...(forceStreaming ? { forceStreaming: true } : {}),
|
|
141
144
|
});
|
|
142
145
|
if (!isEventStreamResponse(response) || !response.body) {
|
package/src/app/handlers.ts
CHANGED
|
@@ -132,6 +132,9 @@ function createExecutionHandlerOptions(
|
|
|
132
132
|
if (options.upstreamTimeoutMs !== undefined) {
|
|
133
133
|
handlerOptions.upstreamTimeoutMs = options.upstreamTimeoutMs;
|
|
134
134
|
}
|
|
135
|
+
if (options.remoteMcpClientFactory) {
|
|
136
|
+
handlerOptions.remoteMcpClientFactory = options.remoteMcpClientFactory;
|
|
137
|
+
}
|
|
135
138
|
|
|
136
139
|
return handlerOptions;
|
|
137
140
|
}
|
package/src/index.ts
CHANGED
|
@@ -43,11 +43,24 @@ export type {
|
|
|
43
43
|
ProviderApiStyle as ProviderStyle,
|
|
44
44
|
ProviderRuntime,
|
|
45
45
|
ProviderTargetConfig,
|
|
46
|
+
RemoteMcpServerConfig,
|
|
47
|
+
RemoteMcpToolConfig,
|
|
46
48
|
ResolvedTarget,
|
|
47
49
|
RouterConfig,
|
|
48
50
|
RouterExecutionOptions,
|
|
49
51
|
RouterRuntime,
|
|
50
52
|
} from "./router/index.ts";
|
|
53
|
+
export type {
|
|
54
|
+
RemoteMcpClientFactory,
|
|
55
|
+
RemoteMcpToolCallClient,
|
|
56
|
+
RemoteMcpToolSet,
|
|
57
|
+
} from "./router/index.ts";
|
|
58
|
+
export {
|
|
59
|
+
createRemoteMcpToolSet,
|
|
60
|
+
createSseMcpClient,
|
|
61
|
+
stringifyMcpToolResult,
|
|
62
|
+
wireToolName,
|
|
63
|
+
} from "./router/index.ts";
|
|
51
64
|
|
|
52
65
|
export type {
|
|
53
66
|
AuthDriver,
|
package/src/router/config.ts
CHANGED
|
@@ -17,11 +17,13 @@ export type ExecutorRouteConfig =
|
|
|
17
17
|
executor: string;
|
|
18
18
|
alias?: string;
|
|
19
19
|
transforms?: string[];
|
|
20
|
+
mcps?: string[];
|
|
20
21
|
}
|
|
21
22
|
| {
|
|
22
23
|
provider: string;
|
|
23
24
|
model: string;
|
|
24
25
|
transforms?: string[];
|
|
26
|
+
mcps?: string[];
|
|
25
27
|
};
|
|
26
28
|
|
|
27
29
|
export type ExecutorNamespaceConfig = {
|
|
@@ -29,9 +31,23 @@ export type ExecutorNamespaceConfig = {
|
|
|
29
31
|
models: Record<string, ExecutorRouteConfig>;
|
|
30
32
|
};
|
|
31
33
|
|
|
34
|
+
export type RemoteMcpToolConfig = {
|
|
35
|
+
name: string;
|
|
36
|
+
description?: string;
|
|
37
|
+
inputSchema: Record<string, unknown>;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export type RemoteMcpServerConfig = {
|
|
41
|
+
type: "mcp_sse";
|
|
42
|
+
url: string;
|
|
43
|
+
headers?: Record<string, string>;
|
|
44
|
+
tools: RemoteMcpToolConfig[];
|
|
45
|
+
};
|
|
46
|
+
|
|
32
47
|
export type RouterConfig = {
|
|
33
48
|
trace?: boolean;
|
|
34
49
|
auth?: AuthConfig;
|
|
35
50
|
providers: Record<string, ProviderTargetConfig>;
|
|
36
51
|
executors: Record<string, ExecutorNamespaceConfig>;
|
|
52
|
+
mcps?: Record<string, RemoteMcpServerConfig>;
|
|
37
53
|
};
|
|
@@ -2,6 +2,7 @@ import { appendAssistantMessage } from "@neutrome/lil-engine";
|
|
|
2
2
|
import { describe, expect, it, vi } from "vitest";
|
|
3
3
|
import type { RequestContext } from "../auth/types.ts";
|
|
4
4
|
import { handleChatCompletions } from "./execute.ts";
|
|
5
|
+
import type { RemoteMcpClientFactory } from "./mcp.ts";
|
|
5
6
|
import { createRouterRuntime } from "./runtime.ts";
|
|
6
7
|
|
|
7
8
|
const encoder = new TextEncoder();
|
|
@@ -93,6 +94,138 @@ describe("router execution", () => {
|
|
|
93
94
|
});
|
|
94
95
|
});
|
|
95
96
|
|
|
97
|
+
it("injects and executes attached remote mcp tools", async () => {
|
|
98
|
+
const capturedBodies: Record<string, unknown>[] = [];
|
|
99
|
+
const close = vi.fn(async () => {});
|
|
100
|
+
const callTool = vi.fn(async (name: string, args: Record<string, unknown>) => {
|
|
101
|
+
expect(name).toBe("lookup");
|
|
102
|
+
expect(args).toEqual({ id: "42" });
|
|
103
|
+
return { content: [{ type: "text", text: "customer is active" }] };
|
|
104
|
+
});
|
|
105
|
+
const remoteMcpClientFactory: RemoteMcpClientFactory = vi.fn(async () => ({
|
|
106
|
+
callTool,
|
|
107
|
+
close,
|
|
108
|
+
}));
|
|
109
|
+
|
|
110
|
+
const runtime = createRouterRuntime({
|
|
111
|
+
config: {
|
|
112
|
+
providers: {
|
|
113
|
+
openrouter: {
|
|
114
|
+
api_base_url: "https://openrouter.ai/api/v1",
|
|
115
|
+
style: "chat-completions",
|
|
116
|
+
exports: ["*"],
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
mcps: {
|
|
120
|
+
crm: {
|
|
121
|
+
type: "mcp_sse",
|
|
122
|
+
url: "https://mcp.example.test/sse",
|
|
123
|
+
headers: { authorization: "Bearer static" },
|
|
124
|
+
tools: [{
|
|
125
|
+
name: "lookup",
|
|
126
|
+
description: "Look up a customer",
|
|
127
|
+
inputSchema: {
|
|
128
|
+
type: "object",
|
|
129
|
+
properties: { id: { type: "string" } },
|
|
130
|
+
required: ["id"],
|
|
131
|
+
},
|
|
132
|
+
}],
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
executors: {
|
|
136
|
+
default: {
|
|
137
|
+
exports: ["*"],
|
|
138
|
+
models: {
|
|
139
|
+
"gpt-4o": {
|
|
140
|
+
provider: "openrouter",
|
|
141
|
+
model: "gpt-4o",
|
|
142
|
+
mcps: ["crm"],
|
|
143
|
+
},
|
|
144
|
+
},
|
|
145
|
+
},
|
|
146
|
+
},
|
|
147
|
+
},
|
|
148
|
+
fetchImpl: vi.fn(async (_input, init) => {
|
|
149
|
+
const body = JSON.parse(decoder.decode(new Uint8Array(init?.body as ArrayBuffer)));
|
|
150
|
+
capturedBodies.push(body);
|
|
151
|
+
if (capturedBodies.length === 1) {
|
|
152
|
+
expect(body.tools).toEqual([{
|
|
153
|
+
type: "function",
|
|
154
|
+
function: {
|
|
155
|
+
name: "crm__lookup",
|
|
156
|
+
description: "Look up a customer",
|
|
157
|
+
parameters: {
|
|
158
|
+
type: "object",
|
|
159
|
+
properties: { id: { type: "string" } },
|
|
160
|
+
required: ["id"],
|
|
161
|
+
},
|
|
162
|
+
},
|
|
163
|
+
}]);
|
|
164
|
+
return new Response(
|
|
165
|
+
JSON.stringify({
|
|
166
|
+
id: "resp_1",
|
|
167
|
+
model: "gpt-4o",
|
|
168
|
+
choices: [{
|
|
169
|
+
index: 0,
|
|
170
|
+
message: {
|
|
171
|
+
role: "assistant",
|
|
172
|
+
tool_calls: [{
|
|
173
|
+
id: "call_1",
|
|
174
|
+
type: "function",
|
|
175
|
+
function: { name: "crm__lookup", arguments: "{\"id\":\"42\"}" },
|
|
176
|
+
}],
|
|
177
|
+
},
|
|
178
|
+
finish_reason: "tool_calls",
|
|
179
|
+
}],
|
|
180
|
+
}),
|
|
181
|
+
{ headers: { "content-type": "application/json; charset=utf-8" } },
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
expect(body.messages).toContainEqual({
|
|
185
|
+
role: "tool",
|
|
186
|
+
tool_call_id: "call_1",
|
|
187
|
+
content: "customer is active",
|
|
188
|
+
});
|
|
189
|
+
return new Response(
|
|
190
|
+
JSON.stringify({
|
|
191
|
+
id: "resp_2",
|
|
192
|
+
model: "gpt-4o",
|
|
193
|
+
choices: [{
|
|
194
|
+
index: 0,
|
|
195
|
+
message: { role: "assistant", content: "done" },
|
|
196
|
+
finish_reason: "stop",
|
|
197
|
+
}],
|
|
198
|
+
}),
|
|
199
|
+
{ headers: { "content-type": "application/json; charset=utf-8" } },
|
|
200
|
+
);
|
|
201
|
+
}),
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
const ctx: RequestContext = {
|
|
205
|
+
request: new Request("http://local/v1/chat/completions", {
|
|
206
|
+
method: "POST",
|
|
207
|
+
headers: { "content-type": "application/json" },
|
|
208
|
+
body: JSON.stringify({
|
|
209
|
+
model: "gpt-4o",
|
|
210
|
+
messages: [{ role: "user", content: "check customer 42" }],
|
|
211
|
+
}),
|
|
212
|
+
}),
|
|
213
|
+
incomingAuth: new Map(),
|
|
214
|
+
env: {},
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
const response = await handleChatCompletions({ runtime, ctx, remoteMcpClientFactory });
|
|
218
|
+
|
|
219
|
+
expect(response.status).toBe(200);
|
|
220
|
+
expect(remoteMcpClientFactory).toHaveBeenCalledOnce();
|
|
221
|
+
expect(callTool).toHaveBeenCalledOnce();
|
|
222
|
+
expect(close).toHaveBeenCalledOnce();
|
|
223
|
+
expect(capturedBodies).toHaveLength(2);
|
|
224
|
+
expect(await response.json()).toMatchObject({
|
|
225
|
+
choices: [{ message: { content: "done" } }],
|
|
226
|
+
});
|
|
227
|
+
});
|
|
228
|
+
|
|
96
229
|
it("routes responses-style providers through provider transport", async () => {
|
|
97
230
|
let capturedUrl = "";
|
|
98
231
|
let capturedBody: Record<string, unknown> | null = null;
|
package/src/router/execute.ts
CHANGED
|
@@ -14,7 +14,9 @@ import {
|
|
|
14
14
|
setStreaming,
|
|
15
15
|
type Program,
|
|
16
16
|
type ProviderStyle,
|
|
17
|
+
withTools,
|
|
17
18
|
usageObject,
|
|
19
|
+
type ExecutionTarget,
|
|
18
20
|
} from "@neutrome/lil-engine";
|
|
19
21
|
import {
|
|
20
22
|
createExecutionRuntime,
|
|
@@ -30,6 +32,10 @@ import { cloneForwardHeaders, isSse } from "../util/headers.ts";
|
|
|
30
32
|
import { eventDataLines, splitSseEvents } from "../util/sse.ts";
|
|
31
33
|
import { renderProgramAsLil } from "../observer.ts";
|
|
32
34
|
import { resolveInvocationTarget, type ResolvedTarget } from "./resolve.ts";
|
|
35
|
+
import {
|
|
36
|
+
createRemoteMcpToolSet,
|
|
37
|
+
type RemoteMcpClientFactory,
|
|
38
|
+
} from "./mcp.ts";
|
|
33
39
|
import type { ProviderRuntime, RouterRuntime } from "./runtime.ts";
|
|
34
40
|
|
|
35
41
|
const encoder = new TextEncoder();
|
|
@@ -43,6 +49,7 @@ export type RouterExecutionOptions = Pick<
|
|
|
43
49
|
executors?: Readonly<Record<string, Executor>>;
|
|
44
50
|
transforms?: Readonly<Record<string, ProgramTransform>>;
|
|
45
51
|
upstreamTimeoutMs?: number;
|
|
52
|
+
remoteMcpClientFactory?: RemoteMcpClientFactory;
|
|
46
53
|
};
|
|
47
54
|
|
|
48
55
|
export type HandleChatCompletionsOptions = RouterExecutionOptions & {
|
|
@@ -168,32 +175,39 @@ export async function handleChatCompletions(
|
|
|
168
175
|
executionId: incomingExecutionId,
|
|
169
176
|
});
|
|
170
177
|
const resolved = resolveRequestTarget(options.runtime, request);
|
|
171
|
-
const
|
|
172
|
-
|
|
173
|
-
options.ctx,
|
|
174
|
-
options,
|
|
175
|
-
);
|
|
178
|
+
const remoteMcp = prepareRemoteMcpExecution(options.runtime, resolved, options);
|
|
179
|
+
const execution = createRouterExecutionRuntime(options.runtime, options.ctx, remoteMcp.options);
|
|
176
180
|
|
|
177
181
|
const startTime = Date.now();
|
|
178
182
|
if (isStreaming(request)) {
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
+
try {
|
|
184
|
+
const chunks = execution.stream(request, { target: remoteMcp.target });
|
|
185
|
+
return await streamExecutionResponse(
|
|
186
|
+
"chat-completions",
|
|
187
|
+
chunks,
|
|
188
|
+
resolved,
|
|
189
|
+
startTime,
|
|
190
|
+
remoteMcp.close,
|
|
191
|
+
);
|
|
192
|
+
} catch (error) {
|
|
193
|
+
await remoteMcp.close();
|
|
194
|
+
throw error;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
try {
|
|
199
|
+
const response = await execution.execute(request, {
|
|
200
|
+
target: remoteMcp.target,
|
|
201
|
+
});
|
|
202
|
+
const userTime = Date.now() - startTime;
|
|
203
|
+
return jsonExecutionResponse(
|
|
204
|
+
emitChatCompletionsResponse(response),
|
|
183
205
|
resolved,
|
|
184
|
-
|
|
206
|
+
userTime,
|
|
185
207
|
);
|
|
208
|
+
} finally {
|
|
209
|
+
await remoteMcp.close();
|
|
186
210
|
}
|
|
187
|
-
|
|
188
|
-
const response = await execution.execute(request, {
|
|
189
|
-
target: resolved.target,
|
|
190
|
-
});
|
|
191
|
-
const userTime = Date.now() - startTime;
|
|
192
|
-
return jsonExecutionResponse(
|
|
193
|
-
emitChatCompletionsResponse(response),
|
|
194
|
-
resolved,
|
|
195
|
-
userTime,
|
|
196
|
-
);
|
|
197
211
|
} catch (error) {
|
|
198
212
|
return errorResponse(error);
|
|
199
213
|
}
|
|
@@ -240,32 +254,39 @@ async function handleProviderStyle(
|
|
|
240
254
|
executionId: incomingExecutionId,
|
|
241
255
|
});
|
|
242
256
|
const resolved = resolveRequestTarget(options.runtime, request);
|
|
243
|
-
const
|
|
244
|
-
|
|
245
|
-
options.ctx,
|
|
246
|
-
options,
|
|
247
|
-
);
|
|
257
|
+
const remoteMcp = prepareRemoteMcpExecution(options.runtime, resolved, options);
|
|
258
|
+
const execution = createRouterExecutionRuntime(options.runtime, options.ctx, remoteMcp.options);
|
|
248
259
|
|
|
249
260
|
const startTime = Date.now();
|
|
250
261
|
if (isStreaming(request)) {
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
262
|
+
try {
|
|
263
|
+
const chunks = execution.stream(request, { target: remoteMcp.target });
|
|
264
|
+
return await streamExecutionResponse(
|
|
265
|
+
style,
|
|
266
|
+
chunks,
|
|
267
|
+
resolved,
|
|
268
|
+
startTime,
|
|
269
|
+
remoteMcp.close,
|
|
270
|
+
);
|
|
271
|
+
} catch (error) {
|
|
272
|
+
await remoteMcp.close();
|
|
273
|
+
throw error;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
try {
|
|
278
|
+
const response = await execution.execute(request, {
|
|
279
|
+
target: remoteMcp.target,
|
|
280
|
+
});
|
|
281
|
+
const userTime = Date.now() - startTime;
|
|
282
|
+
return jsonExecutionResponse(
|
|
283
|
+
emitProviderResponse(style, response),
|
|
255
284
|
resolved,
|
|
256
|
-
|
|
285
|
+
userTime,
|
|
257
286
|
);
|
|
287
|
+
} finally {
|
|
288
|
+
await remoteMcp.close();
|
|
258
289
|
}
|
|
259
|
-
|
|
260
|
-
const response = await execution.execute(request, {
|
|
261
|
-
target: resolved.target,
|
|
262
|
-
});
|
|
263
|
-
const userTime = Date.now() - startTime;
|
|
264
|
-
return jsonExecutionResponse(
|
|
265
|
-
emitProviderResponse(style, response),
|
|
266
|
-
resolved,
|
|
267
|
-
userTime,
|
|
268
|
-
);
|
|
269
290
|
} catch (error) {
|
|
270
291
|
return errorResponse(error);
|
|
271
292
|
}
|
|
@@ -451,6 +472,7 @@ async function streamExecutionResponse(
|
|
|
451
472
|
chunks: AsyncIterable<Program>,
|
|
452
473
|
resolved: ResolvedTarget,
|
|
453
474
|
startTime: number,
|
|
475
|
+
cleanup?: () => Promise<void>,
|
|
454
476
|
): Promise<Response> {
|
|
455
477
|
const iterator = chunks[Symbol.asyncIterator]();
|
|
456
478
|
|
|
@@ -481,8 +503,10 @@ async function streamExecutionResponse(
|
|
|
481
503
|
if (responseStyle === "chat-completions") {
|
|
482
504
|
controller.enqueue(encoder.encode("data: [DONE]\n\n"));
|
|
483
505
|
}
|
|
506
|
+
await cleanup?.();
|
|
484
507
|
controller.close();
|
|
485
508
|
} catch (error) {
|
|
509
|
+
await cleanup?.();
|
|
486
510
|
controller.enqueue(
|
|
487
511
|
encoder.encode(
|
|
488
512
|
`event: error\ndata: ${JSON.stringify({ message: error instanceof Error ? error.message : String(error) })}\n\n`,
|
|
@@ -492,6 +516,7 @@ async function streamExecutionResponse(
|
|
|
492
516
|
}
|
|
493
517
|
},
|
|
494
518
|
async cancel() {
|
|
519
|
+
await cleanup?.();
|
|
495
520
|
await iterator.return?.();
|
|
496
521
|
},
|
|
497
522
|
});
|
|
@@ -505,6 +530,46 @@ async function streamExecutionResponse(
|
|
|
505
530
|
});
|
|
506
531
|
}
|
|
507
532
|
|
|
533
|
+
function prepareRemoteMcpExecution(
|
|
534
|
+
runtime: RouterRuntime,
|
|
535
|
+
resolved: ResolvedTarget,
|
|
536
|
+
options: RouterExecutionOptions,
|
|
537
|
+
): {
|
|
538
|
+
target: ExecutionTarget;
|
|
539
|
+
options: RouterExecutionOptions;
|
|
540
|
+
close: () => Promise<void>;
|
|
541
|
+
} {
|
|
542
|
+
if (resolved.mcps.length === 0) {
|
|
543
|
+
return { target: resolved.target, options, close: async () => {} };
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
const remoteMcp = createRemoteMcpToolSet(
|
|
547
|
+
runtime,
|
|
548
|
+
resolved.mcps,
|
|
549
|
+
options.remoteMcpClientFactory,
|
|
550
|
+
);
|
|
551
|
+
if (remoteMcp.tools.length === 0) {
|
|
552
|
+
return { target: resolved.target, options, close: remoteMcp.close };
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
const executorName = `__remote_mcp_${crypto.randomUUID()}`;
|
|
556
|
+
return {
|
|
557
|
+
target: {
|
|
558
|
+
kind: "executor",
|
|
559
|
+
executor: executorName,
|
|
560
|
+
alias: resolved.requestedModel,
|
|
561
|
+
},
|
|
562
|
+
options: {
|
|
563
|
+
...options,
|
|
564
|
+
executors: {
|
|
565
|
+
...(options.executors ?? {}),
|
|
566
|
+
[executorName]: withTools(remoteMcp.tools, resolved.target),
|
|
567
|
+
},
|
|
568
|
+
},
|
|
569
|
+
close: remoteMcp.close,
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
|
|
508
573
|
function jsonExecutionResponse(
|
|
509
574
|
body: Uint8Array,
|
|
510
575
|
resolved: ResolvedTarget,
|
package/src/router/index.ts
CHANGED
|
@@ -11,12 +11,20 @@ export {
|
|
|
11
11
|
ProviderHttpError,
|
|
12
12
|
} from "./execute.ts";
|
|
13
13
|
export { listConfiguredModels, resolveInvocationTarget, resolveInvocationTargets } from "./resolve.ts";
|
|
14
|
+
export {
|
|
15
|
+
createRemoteMcpToolSet,
|
|
16
|
+
createSseMcpClient,
|
|
17
|
+
stringifyMcpToolResult,
|
|
18
|
+
wireToolName,
|
|
19
|
+
} from "./mcp.ts";
|
|
14
20
|
export type {
|
|
15
21
|
AuthConfig,
|
|
16
22
|
ExecutorNamespaceConfig,
|
|
17
23
|
ExecutorRouteConfig,
|
|
18
24
|
ProviderApiStyle,
|
|
19
25
|
ProviderTargetConfig,
|
|
26
|
+
RemoteMcpServerConfig,
|
|
27
|
+
RemoteMcpToolConfig,
|
|
20
28
|
RouterConfig,
|
|
21
29
|
} from "./config.ts";
|
|
22
30
|
export type {
|
|
@@ -24,6 +32,7 @@ export type {
|
|
|
24
32
|
ExecutorRouteRuntime,
|
|
25
33
|
ExecutorRuntime,
|
|
26
34
|
ProviderRuntime,
|
|
35
|
+
RemoteMcpServerRuntime,
|
|
27
36
|
RouterRuntime,
|
|
28
37
|
} from "./runtime.ts";
|
|
29
38
|
export type {
|
|
@@ -33,4 +42,9 @@ export type {
|
|
|
33
42
|
HandleResponsesOptions,
|
|
34
43
|
RouterExecutionOptions,
|
|
35
44
|
} from "./execute.ts";
|
|
45
|
+
export type {
|
|
46
|
+
RemoteMcpClientFactory,
|
|
47
|
+
RemoteMcpToolCallClient,
|
|
48
|
+
RemoteMcpToolSet,
|
|
49
|
+
} from "./mcp.ts";
|
|
36
50
|
export type { ResolvedTarget } from "./resolve.ts";
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
2
|
+
import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
|
|
3
|
+
import type { ExecutionContext, Tool } from "@neutrome/lil-engine";
|
|
4
|
+
import type { RemoteMcpServerRuntime, RouterRuntime } from "./runtime.ts";
|
|
5
|
+
|
|
6
|
+
export type RemoteMcpToolCallClient = {
|
|
7
|
+
callTool(name: string, args: Record<string, unknown>): Promise<unknown>;
|
|
8
|
+
close(): Promise<void>;
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
export type RemoteMcpClientFactory = (
|
|
12
|
+
server: RemoteMcpServerRuntime,
|
|
13
|
+
signal: AbortSignal,
|
|
14
|
+
) => Promise<RemoteMcpToolCallClient>;
|
|
15
|
+
|
|
16
|
+
export type RemoteMcpToolSet = {
|
|
17
|
+
tools: Tool[];
|
|
18
|
+
close(): Promise<void>;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export function createRemoteMcpToolSet(
|
|
22
|
+
runtime: RouterRuntime,
|
|
23
|
+
serverNames: readonly string[],
|
|
24
|
+
factory?: RemoteMcpClientFactory,
|
|
25
|
+
): RemoteMcpToolSet {
|
|
26
|
+
const pool = new RemoteMcpSessionPool(
|
|
27
|
+
factory ?? ((server, signal) => createSseMcpClient(server, signal, runtime.fetchImpl)),
|
|
28
|
+
);
|
|
29
|
+
const tools: Tool[] = [];
|
|
30
|
+
const usedNames = new Map<string, number>();
|
|
31
|
+
|
|
32
|
+
for (const serverName of unique(serverNames)) {
|
|
33
|
+
const server = runtime.mcps.get(serverName);
|
|
34
|
+
if (!server) continue;
|
|
35
|
+
|
|
36
|
+
for (const snapshot of server.tools) {
|
|
37
|
+
const baseName = wireToolName(server.name, snapshot.name);
|
|
38
|
+
const count = usedNames.get(baseName) ?? 0;
|
|
39
|
+
usedNames.set(baseName, count + 1);
|
|
40
|
+
const name = count === 0 ? baseName : `${baseName}_${count + 1}`;
|
|
41
|
+
|
|
42
|
+
tools.push({
|
|
43
|
+
name,
|
|
44
|
+
description: snapshot.description || `Remote MCP tool ${server.name}/${snapshot.name}`,
|
|
45
|
+
schema: normalizeSchema(snapshot.inputSchema),
|
|
46
|
+
async execute(args: Record<string, unknown>, ctx: ExecutionContext) {
|
|
47
|
+
const client = await pool.get(server, ctx.signal);
|
|
48
|
+
const result = await client.callTool(snapshot.name, args);
|
|
49
|
+
return stringifyMcpToolResult(result);
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return { tools, close: () => pool.closeAll() };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function createSseMcpClient(
|
|
59
|
+
server: RemoteMcpServerRuntime,
|
|
60
|
+
signal?: AbortSignal,
|
|
61
|
+
fetchImpl?: typeof fetch,
|
|
62
|
+
): Promise<RemoteMcpToolCallClient> {
|
|
63
|
+
const client = new Client({ name: "open-ai-router", version: "0.1.0" });
|
|
64
|
+
const transport = new SSEClientTransport(new URL(server.url), {
|
|
65
|
+
requestInit: { headers: server.headers ?? {} },
|
|
66
|
+
...(fetchImpl ? { fetch: fetchImpl } : {}),
|
|
67
|
+
});
|
|
68
|
+
signal?.addEventListener("abort", () => void client.close(), { once: true });
|
|
69
|
+
await client.connect(transport);
|
|
70
|
+
return {
|
|
71
|
+
callTool(name, args) {
|
|
72
|
+
return client.callTool({ name, arguments: args });
|
|
73
|
+
},
|
|
74
|
+
close() {
|
|
75
|
+
return client.close();
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function wireToolName(serverName: string, toolName: string): string {
|
|
81
|
+
return `${safeToolNamePart(serverName)}__${safeToolNamePart(toolName)}`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function stringifyMcpToolResult(result: unknown): string {
|
|
85
|
+
if (!isRecord(result)) return JSON.stringify(result);
|
|
86
|
+
if ("toolResult" in result) return JSON.stringify(result.toolResult);
|
|
87
|
+
|
|
88
|
+
const chunks: string[] = [];
|
|
89
|
+
const content = result.content;
|
|
90
|
+
if (Array.isArray(content)) {
|
|
91
|
+
for (const item of content) chunks.push(stringifyContent(item));
|
|
92
|
+
}
|
|
93
|
+
if (isRecord(result.structuredContent)) {
|
|
94
|
+
chunks.push(JSON.stringify(result.structuredContent));
|
|
95
|
+
}
|
|
96
|
+
const body = chunks.filter(Boolean).join("\n");
|
|
97
|
+
if (result.isError === true) {
|
|
98
|
+
return body ? `MCP tool returned an error:\n${body}` : "MCP tool returned an error.";
|
|
99
|
+
}
|
|
100
|
+
return body || JSON.stringify(result);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
class RemoteMcpSessionPool {
|
|
104
|
+
readonly #factory: RemoteMcpClientFactory;
|
|
105
|
+
readonly #clients = new Map<string, Promise<RemoteMcpToolCallClient>>();
|
|
106
|
+
|
|
107
|
+
constructor(factory: RemoteMcpClientFactory) {
|
|
108
|
+
this.#factory = factory;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
get(server: RemoteMcpServerRuntime, signal: AbortSignal) {
|
|
112
|
+
let client = this.#clients.get(server.name);
|
|
113
|
+
if (!client) {
|
|
114
|
+
client = this.#factory(server, signal);
|
|
115
|
+
this.#clients.set(server.name, client);
|
|
116
|
+
}
|
|
117
|
+
return client;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async closeAll(): Promise<void> {
|
|
121
|
+
const clients = await Promise.allSettled(this.#clients.values());
|
|
122
|
+
this.#clients.clear();
|
|
123
|
+
await Promise.allSettled(
|
|
124
|
+
clients.flatMap((result) =>
|
|
125
|
+
result.status === "fulfilled" ? [result.value.close()] : [],
|
|
126
|
+
),
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function stringifyContent(item: unknown): string {
|
|
132
|
+
if (!isRecord(item)) return JSON.stringify(item);
|
|
133
|
+
if (item.type === "text" && typeof item.text === "string") return item.text;
|
|
134
|
+
if (item.type === "resource" && isRecord(item.resource)) {
|
|
135
|
+
if (typeof item.resource.text === "string") return item.resource.text;
|
|
136
|
+
return JSON.stringify(item.resource);
|
|
137
|
+
}
|
|
138
|
+
return JSON.stringify(item);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function normalizeSchema(value: unknown): Record<string, unknown> {
|
|
142
|
+
if (!isRecord(value)) return { type: "object" };
|
|
143
|
+
return { ...value, type: "object" };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function safeToolNamePart(value: string): string {
|
|
147
|
+
const cleaned = value.trim().replace(/[^A-Za-z0-9_-]+/g, "_");
|
|
148
|
+
return cleaned.replace(/^_+|_+$/g, "") || "tool";
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function unique(values: readonly string[]): string[] {
|
|
152
|
+
return [...new Set(values)];
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
156
|
+
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
157
|
+
}
|
package/src/router/resolve.ts
CHANGED
|
@@ -8,6 +8,7 @@ export type ResolvedTarget = {
|
|
|
8
8
|
suffixes: ModelSuffixSpec[];
|
|
9
9
|
namespace: string;
|
|
10
10
|
source: "provider" | "executor";
|
|
11
|
+
mcps: readonly string[];
|
|
11
12
|
};
|
|
12
13
|
|
|
13
14
|
export function resolveInvocationTarget(
|
|
@@ -43,6 +44,7 @@ export function resolveInvocationTargets(
|
|
|
43
44
|
namespace,
|
|
44
45
|
source: route.target.kind,
|
|
45
46
|
suffixes: parsed.suffixes,
|
|
47
|
+
mcps: route.mcps,
|
|
46
48
|
target: withSuffixTransforms(route.target, parsed.suffixes),
|
|
47
49
|
});
|
|
48
50
|
}
|
|
@@ -90,6 +92,7 @@ function resolveQualified(
|
|
|
90
92
|
namespace,
|
|
91
93
|
source: route.target.kind,
|
|
92
94
|
suffixes,
|
|
95
|
+
mcps: route.mcps,
|
|
93
96
|
target: withSuffixTransforms(route.target, suffixes),
|
|
94
97
|
}];
|
|
95
98
|
}
|
|
@@ -103,6 +106,7 @@ function resolveQualified(
|
|
|
103
106
|
namespace,
|
|
104
107
|
source: "provider",
|
|
105
108
|
suffixes,
|
|
109
|
+
mcps: [],
|
|
106
110
|
target: {
|
|
107
111
|
kind: "provider",
|
|
108
112
|
provider: namespace,
|
package/src/router/runtime.ts
CHANGED
|
@@ -6,6 +6,7 @@ import type {
|
|
|
6
6
|
ExecutorNamespaceConfig,
|
|
7
7
|
ExecutorRouteConfig,
|
|
8
8
|
ProviderTargetConfig,
|
|
9
|
+
RemoteMcpServerConfig,
|
|
9
10
|
RouterConfig,
|
|
10
11
|
} from "./config.ts";
|
|
11
12
|
|
|
@@ -22,6 +23,7 @@ export type ExecutorRouteRuntime = {
|
|
|
22
23
|
requestedAlias: string;
|
|
23
24
|
target: ExecutionTarget;
|
|
24
25
|
transforms: readonly string[];
|
|
26
|
+
mcps: readonly string[];
|
|
25
27
|
};
|
|
26
28
|
|
|
27
29
|
export type ExecutorRuntime = {
|
|
@@ -36,11 +38,16 @@ export type RouterRuntime = {
|
|
|
36
38
|
providerOrder: readonly string[];
|
|
37
39
|
executors: ReadonlyMap<string, ExecutorRuntime>;
|
|
38
40
|
executorOrder: readonly string[];
|
|
41
|
+
mcps: ReadonlyMap<string, RemoteMcpServerRuntime>;
|
|
39
42
|
authChain: readonly AuthDriver[];
|
|
40
43
|
knownSuffixes: ReadonlySet<string>;
|
|
41
44
|
fetchImpl: typeof fetch;
|
|
42
45
|
};
|
|
43
46
|
|
|
47
|
+
export type RemoteMcpServerRuntime = RemoteMcpServerConfig & {
|
|
48
|
+
name: string;
|
|
49
|
+
};
|
|
50
|
+
|
|
44
51
|
export type CreateRouterOptions = {
|
|
45
52
|
config: RouterConfig;
|
|
46
53
|
fetchImpl?: typeof fetch;
|
|
@@ -51,6 +58,7 @@ export type CreateRouterOptions = {
|
|
|
51
58
|
export function createRouterRuntime(options: CreateRouterOptions): RouterRuntime {
|
|
52
59
|
const providers = new Map<string, ProviderRuntime>();
|
|
53
60
|
const executors = new Map<string, ExecutorRuntime>();
|
|
61
|
+
const mcps = new Map<string, RemoteMcpServerRuntime>();
|
|
54
62
|
const providerOrder: string[] = [];
|
|
55
63
|
const executorOrder: string[] = [];
|
|
56
64
|
|
|
@@ -63,6 +71,9 @@ export function createRouterRuntime(options: CreateRouterOptions): RouterRuntime
|
|
|
63
71
|
executorOrder.push(name);
|
|
64
72
|
executors.set(name, buildExecutorRuntime(name, config));
|
|
65
73
|
}
|
|
74
|
+
for (const [name, config] of Object.entries(options.config.mcps ?? {})) {
|
|
75
|
+
mcps.set(name, { name, ...config });
|
|
76
|
+
}
|
|
66
77
|
|
|
67
78
|
return {
|
|
68
79
|
config: options.config,
|
|
@@ -70,6 +81,7 @@ export function createRouterRuntime(options: CreateRouterOptions): RouterRuntime
|
|
|
70
81
|
providerOrder,
|
|
71
82
|
executors,
|
|
72
83
|
executorOrder,
|
|
84
|
+
mcps,
|
|
73
85
|
authChain: options.authDrivers ?? buildAuthChain(options.config.auth),
|
|
74
86
|
knownSuffixes: new Set(options.knownSuffixes ?? ["slwin", "kvtools"]),
|
|
75
87
|
fetchImpl: options.fetchImpl ?? globalThis.fetch.bind(globalThis),
|
|
@@ -105,6 +117,7 @@ function buildExecutorRoute(alias: string, route: ExecutorRouteConfig): Executor
|
|
|
105
117
|
return {
|
|
106
118
|
requestedAlias: alias,
|
|
107
119
|
transforms: route.transforms ?? [],
|
|
120
|
+
mcps: route.mcps ?? [],
|
|
108
121
|
target: {
|
|
109
122
|
kind: "provider",
|
|
110
123
|
provider: route.provider,
|
|
@@ -115,9 +128,10 @@ function buildExecutorRoute(alias: string, route: ExecutorRouteConfig): Executor
|
|
|
115
128
|
}
|
|
116
129
|
|
|
117
130
|
return {
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
131
|
+
requestedAlias: alias,
|
|
132
|
+
transforms: route.transforms ?? [],
|
|
133
|
+
mcps: route.mcps ?? [],
|
|
134
|
+
target: {
|
|
121
135
|
kind: "executor",
|
|
122
136
|
executor: route.executor,
|
|
123
137
|
alias: route.alias ?? alias,
|