@neutrome/open-ai-router 0.5.8 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +3 -3
- package/src/app/factory.test.ts +1 -1
- package/src/app/factory.ts +2 -0
- package/src/app/types.ts +2 -1
- package/src/index.ts +1 -0
- package/src/router/audit.ts +1 -1
- package/src/router/config.ts +8 -7
- package/src/router/execute.test.ts +10 -14
- package/src/router/execute.ts +8 -1
- package/src/router/execution-events.ts +2 -2
- package/src/router/execution-invocation.ts +1 -1
- package/src/router/execution-observation.ts +2 -2
- package/src/router/execution-runtime.test.ts +108 -62
- package/src/router/execution-runtime.ts +45 -16
- package/src/router/execution-transforms.ts +2 -2
- package/src/router/execution-types.ts +24 -5
- package/src/router/index.ts +1 -0
- package/src/router/mcp.ts +1 -1
- package/src/router/provider-telemetry.ts +1 -1
- package/src/router/remote-mcp-execution.ts +2 -6
- package/src/router/request-target.ts +2 -6
- package/src/router/resolve.test.ts +11 -7
- package/src/router/resolve.ts +1 -1
- package/src/router/runtime.test.ts +3 -1
- package/src/router/runtime.ts +11 -8
- package/src/telemetry/router.ts +1 -1
- package/src/telemetry.test.ts +1 -1
- package/src/trace-labels.ts +1 -1
package/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# `@neutrome/open-ai-router`
|
|
2
2
|
|
|
3
3
|
Deployable router app and reusable router/runtime library built on
|
|
4
|
-
`@neutrome/lil-engine` for protocol conversion and `@neutrome/lilsdk
|
|
4
|
+
`@neutrome/lil-engine` for protocol conversion and `@neutrome/lilsdk` for
|
|
5
5
|
executor/tool contracts.
|
|
6
6
|
|
|
7
7
|
```ts
|
|
@@ -30,4 +30,4 @@ Notes:
|
|
|
30
30
|
|
|
31
31
|
- `RouterConfig` is the canonical router config surface.
|
|
32
32
|
- `/v1/models` lists configured executor aliases.
|
|
33
|
-
- Executor contracts and `connectTools` come from `@neutrome/lilsdk
|
|
33
|
+
- Executor contracts and `connectTools` come from `@neutrome/lilsdk`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@neutrome/open-ai-router",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./src/index.ts"
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
"hono": "^4.12.18",
|
|
12
12
|
"posthog-node": "^5.41.0",
|
|
13
13
|
"zod": "^4.2.1",
|
|
14
|
-
"@neutrome/lil-engine": "0.
|
|
15
|
-
"@neutrome/lilsdk
|
|
14
|
+
"@neutrome/lil-engine": "0.4.0",
|
|
15
|
+
"@neutrome/lilsdk": "0.4.0"
|
|
16
16
|
},
|
|
17
17
|
"devDependencies": {
|
|
18
18
|
"@types/node": "^25.9.3",
|
package/src/app/factory.test.ts
CHANGED
package/src/app/factory.ts
CHANGED
|
@@ -257,6 +257,7 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
257
257
|
}
|
|
258
258
|
|
|
259
259
|
const ctx: RequestContext = { request, env };
|
|
260
|
+
const cache = options.cache?.(env);
|
|
260
261
|
|
|
261
262
|
try {
|
|
262
263
|
const handlerStartedMs = Date.now();
|
|
@@ -264,6 +265,7 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
264
265
|
runtime,
|
|
265
266
|
ctx,
|
|
266
267
|
program,
|
|
268
|
+
...(cache ? { cache } : {}),
|
|
267
269
|
executorImplementations: options.executorImplementations,
|
|
268
270
|
...(options.transforms ? { transforms: options.transforms } : {}),
|
|
269
271
|
observe,
|
package/src/app/types.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Executor, ProgramTransform } from "@neutrome/lilsdk
|
|
1
|
+
import type { Cache, Executor, ProgramTransform } from "@neutrome/lilsdk";
|
|
2
2
|
import type {
|
|
3
3
|
AuthPrincipal,
|
|
4
4
|
RuntimeAuthenticator,
|
|
@@ -27,6 +27,7 @@ export type RouterAppOptions = {
|
|
|
27
27
|
config: RouterConfig;
|
|
28
28
|
executorImplementations: Readonly<Record<string, Executor>>;
|
|
29
29
|
transforms?: Readonly<Record<string, ProgramTransform>>;
|
|
30
|
+
cache?(env: Record<string, unknown>): Cache | undefined;
|
|
30
31
|
auth: RouterAppAuth;
|
|
31
32
|
hooks?: RouterAppHooks;
|
|
32
33
|
fetchImpl?: typeof fetch;
|
package/src/index.ts
CHANGED
package/src/router/audit.ts
CHANGED
package/src/router/config.ts
CHANGED
|
@@ -28,20 +28,21 @@ export type ProviderTargetConfig = {
|
|
|
28
28
|
auth?: ProviderAuthDriverConfig[];
|
|
29
29
|
};
|
|
30
30
|
|
|
31
|
-
export type
|
|
31
|
+
export type ModelRouteTarget =
|
|
32
32
|
| {
|
|
33
|
-
|
|
34
|
-
alias?: string;
|
|
35
|
-
transforms?: string[];
|
|
36
|
-
mcps?: string[];
|
|
33
|
+
executor: string;
|
|
37
34
|
}
|
|
38
35
|
| {
|
|
39
36
|
provider: string;
|
|
40
37
|
model: string;
|
|
41
|
-
transforms?: string[];
|
|
42
|
-
mcps?: string[];
|
|
43
38
|
};
|
|
44
39
|
|
|
40
|
+
export type ModelRouteConfig = {
|
|
41
|
+
target: ModelRouteTarget;
|
|
42
|
+
transforms?: string[];
|
|
43
|
+
mcps?: string[];
|
|
44
|
+
};
|
|
45
|
+
|
|
45
46
|
export type ModelNamespaceConfig = {
|
|
46
47
|
exports?: string[];
|
|
47
48
|
models: Record<string, ModelRouteConfig>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { appendAssistantMessage } from "@neutrome/lilsdk
|
|
1
|
+
import { appendAssistantMessage } from "@neutrome/lilsdk/synthetic";
|
|
2
2
|
import { describe, expect, it, vi } from "vitest";
|
|
3
3
|
import type { RequestContext } from "../upstream-auth/types.ts";
|
|
4
4
|
import { handleChatCompletions } from "./execute.ts";
|
|
@@ -27,7 +27,7 @@ describe("router execution", () => {
|
|
|
27
27
|
default: {
|
|
28
28
|
exports: ["*"],
|
|
29
29
|
models: {
|
|
30
|
-
"gpt-4o": { provider: "openrouter", model: "gpt-4o" },
|
|
30
|
+
"gpt-4o": { target: { provider: "openrouter", model: "gpt-4o" } },
|
|
31
31
|
},
|
|
32
32
|
},
|
|
33
33
|
},
|
|
@@ -146,8 +146,7 @@ describe("router execution", () => {
|
|
|
146
146
|
exports: ["*"],
|
|
147
147
|
models: {
|
|
148
148
|
"gpt-4o": {
|
|
149
|
-
provider: "openrouter",
|
|
150
|
-
model: "gpt-4o",
|
|
149
|
+
target: { provider: "openrouter", model: "gpt-4o" },
|
|
151
150
|
mcps: ["crm"],
|
|
152
151
|
},
|
|
153
152
|
},
|
|
@@ -283,7 +282,7 @@ describe("router execution", () => {
|
|
|
283
282
|
default: {
|
|
284
283
|
exports: ["*"],
|
|
285
284
|
models: {
|
|
286
|
-
"gpt-5": { provider: "openai", model: "gpt-5" },
|
|
285
|
+
"gpt-5": { target: { provider: "openai", model: "gpt-5" } },
|
|
287
286
|
},
|
|
288
287
|
},
|
|
289
288
|
},
|
|
@@ -363,8 +362,7 @@ describe("router execution", () => {
|
|
|
363
362
|
exports: ["*"],
|
|
364
363
|
models: {
|
|
365
364
|
"claude-sonnet-4": {
|
|
366
|
-
provider: "anthropic",
|
|
367
|
-
model: "claude-sonnet-4",
|
|
365
|
+
target: { provider: "anthropic", model: "claude-sonnet-4" },
|
|
368
366
|
},
|
|
369
367
|
},
|
|
370
368
|
},
|
|
@@ -445,8 +443,7 @@ describe("router execution", () => {
|
|
|
445
443
|
exports: ["*"],
|
|
446
444
|
models: {
|
|
447
445
|
"gemini-2.5-flash": {
|
|
448
|
-
provider: "google",
|
|
449
|
-
model: "gemini-2.5-flash",
|
|
446
|
+
target: { provider: "google", model: "gemini-2.5-flash" },
|
|
450
447
|
},
|
|
451
448
|
},
|
|
452
449
|
},
|
|
@@ -519,7 +516,7 @@ describe("router execution", () => {
|
|
|
519
516
|
enei: {
|
|
520
517
|
exports: ["enei-*"],
|
|
521
518
|
models: {
|
|
522
|
-
"enei-1": {
|
|
519
|
+
"enei-1": { target: { executor: "enei-1" } },
|
|
523
520
|
},
|
|
524
521
|
},
|
|
525
522
|
},
|
|
@@ -583,7 +580,7 @@ describe("router execution", () => {
|
|
|
583
580
|
default: {
|
|
584
581
|
exports: ["*"],
|
|
585
582
|
models: {
|
|
586
|
-
"gpt-4o": { provider: "openrouter", model: "gpt-4o" },
|
|
583
|
+
"gpt-4o": { target: { provider: "openrouter", model: "gpt-4o" } },
|
|
587
584
|
},
|
|
588
585
|
},
|
|
589
586
|
},
|
|
@@ -661,7 +658,7 @@ describe("router execution", () => {
|
|
|
661
658
|
default: {
|
|
662
659
|
exports: ["*"],
|
|
663
660
|
models: {
|
|
664
|
-
"gpt-4o": { provider: "openrouter", model: "gpt-4o" },
|
|
661
|
+
"gpt-4o": { target: { provider: "openrouter", model: "gpt-4o" } },
|
|
665
662
|
},
|
|
666
663
|
},
|
|
667
664
|
},
|
|
@@ -723,8 +720,7 @@ describe("router execution", () => {
|
|
|
723
720
|
exports: ["*"],
|
|
724
721
|
models: {
|
|
725
722
|
"claude-sonnet-4": {
|
|
726
|
-
provider: "anthropic",
|
|
727
|
-
model: "claude-sonnet-4",
|
|
723
|
+
target: { provider: "anthropic", model: "claude-sonnet-4" },
|
|
728
724
|
},
|
|
729
725
|
},
|
|
730
726
|
},
|
package/src/router/execute.ts
CHANGED
|
@@ -66,7 +66,11 @@ const DEFAULT_UPSTREAM_TIMEOUT_MS = 120_000;
|
|
|
66
66
|
|
|
67
67
|
export type RouterExecutionOptions = Pick<
|
|
68
68
|
ExecutionRuntimeOptions,
|
|
69
|
-
|
|
69
|
+
| "cache"
|
|
70
|
+
| "observe"
|
|
71
|
+
| "requestIdFactory"
|
|
72
|
+
| "executionIdFactory"
|
|
73
|
+
| "resolveExecutor"
|
|
70
74
|
> & {
|
|
71
75
|
executorImplementations?: Readonly<Record<string, Executor>>;
|
|
72
76
|
transforms?: Readonly<Record<string, ProgramTransform>>;
|
|
@@ -122,6 +126,9 @@ export function createRouterExecutionRuntime(
|
|
|
122
126
|
if (options.executorImplementations) {
|
|
123
127
|
runtimeOptions.executorImplementations = options.executorImplementations;
|
|
124
128
|
}
|
|
129
|
+
if (options.cache) {
|
|
130
|
+
runtimeOptions.cache = options.cache;
|
|
131
|
+
}
|
|
125
132
|
if (options.transforms) {
|
|
126
133
|
runtimeOptions.transforms = options.transforms;
|
|
127
134
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createExecutionEvent } from "@neutrome/lilsdk
|
|
2
|
-
import type { ExecutionEvent } from "@neutrome/lilsdk
|
|
1
|
+
import { createExecutionEvent } from "@neutrome/lilsdk";
|
|
2
|
+
import type { ExecutionEvent } from "@neutrome/lilsdk";
|
|
3
3
|
|
|
4
4
|
type TimedExecutionInput = {
|
|
5
5
|
observe?: ((event: ExecutionEvent) => void) | undefined;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { validateProgram } from "@neutrome/lil-engine";
|
|
2
2
|
import type { Program, ValidationMode } from "@neutrome/lil-engine";
|
|
3
|
-
import { createExecutionEvent } from "@neutrome/lilsdk
|
|
4
|
-
import type { ExecutionEvent } from "@neutrome/lilsdk
|
|
3
|
+
import { createExecutionEvent } from "@neutrome/lilsdk";
|
|
4
|
+
import type { ExecutionEvent } from "@neutrome/lilsdk";
|
|
5
5
|
import { ExecutionError } from "./errors.ts";
|
|
6
6
|
import { executionTargetAuditRef, executionTargetId } from "./targets.ts";
|
|
7
7
|
import type { ExecutionTarget } from "./execution-types.ts";
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
cloneProgram,
|
|
4
|
+
createProgram,
|
|
5
|
+
getModel,
|
|
6
|
+
Opcode,
|
|
7
|
+
type Program,
|
|
8
|
+
} from "@neutrome/lil-engine";
|
|
3
9
|
import {
|
|
4
10
|
createExecutionRuntime,
|
|
5
11
|
createInMemoryAuditSink,
|
|
@@ -7,10 +13,62 @@ import {
|
|
|
7
13
|
executionTargetAuditRef,
|
|
8
14
|
executionTargetId,
|
|
9
15
|
} from "./index.ts";
|
|
10
|
-
import {
|
|
11
|
-
|
|
16
|
+
import {
|
|
17
|
+
createExecutionEvent as createAuditEvent,
|
|
18
|
+
type ExecutorContext,
|
|
19
|
+
} from "@neutrome/lilsdk";
|
|
20
|
+
import { observeExecutionStream } from "@neutrome/lilsdk/stream";
|
|
12
21
|
|
|
13
22
|
describe("router execution runtime", () => {
|
|
23
|
+
it("shares one cache with transforms and executors", async () => {
|
|
24
|
+
const values = new Map<string, string>();
|
|
25
|
+
const cache = {
|
|
26
|
+
async get(key: string) {
|
|
27
|
+
return values.get(key) ?? null;
|
|
28
|
+
},
|
|
29
|
+
async put(key: string, value: string) {
|
|
30
|
+
values.set(key, value);
|
|
31
|
+
},
|
|
32
|
+
async delete(key: string) {
|
|
33
|
+
values.delete(key);
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
const runtime = createExecutionRuntime({
|
|
37
|
+
cache,
|
|
38
|
+
transforms: {
|
|
39
|
+
cached: {
|
|
40
|
+
name: "cached",
|
|
41
|
+
capabilities: [],
|
|
42
|
+
async apply(program, ctx) {
|
|
43
|
+
await ctx.cache.put("shared", "from-transform");
|
|
44
|
+
return program;
|
|
45
|
+
},
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
executorImplementations: {
|
|
49
|
+
cached: {
|
|
50
|
+
async execute(program, ctx) {
|
|
51
|
+
expect(ctx.cache).toBe(cache);
|
|
52
|
+
expect(await ctx.cache.get("shared")).toBe("from-transform");
|
|
53
|
+
return program;
|
|
54
|
+
},
|
|
55
|
+
async *stream() {
|
|
56
|
+
return;
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
await runtime.execute(createProgram(), {
|
|
63
|
+
target: {
|
|
64
|
+
kind: "executor",
|
|
65
|
+
executorId: "cached",
|
|
66
|
+
alias: "cached",
|
|
67
|
+
transforms: ["cached"],
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
|
|
14
72
|
it("creates audit events with a stable envelope", () => {
|
|
15
73
|
const event = createAuditEvent({
|
|
16
74
|
kind: "executor.started",
|
|
@@ -66,33 +124,31 @@ describe("router execution runtime", () => {
|
|
|
66
124
|
|
|
67
125
|
it("invokes nested executors without router re-entry and preserves request identity", async () => {
|
|
68
126
|
const sink = createInMemoryAuditSink();
|
|
127
|
+
const childExecutor = {
|
|
128
|
+
async execute(request: Program) {
|
|
129
|
+
const result = cloneProgram(request);
|
|
130
|
+
result.code.push({
|
|
131
|
+
opcode: Opcode.RESP_DONE,
|
|
132
|
+
value: { kind: "string", value: "stop" },
|
|
133
|
+
});
|
|
134
|
+
return result;
|
|
135
|
+
},
|
|
136
|
+
async *stream() {
|
|
137
|
+
return;
|
|
138
|
+
},
|
|
139
|
+
};
|
|
69
140
|
const runtime = createExecutionRuntime({
|
|
70
141
|
observe: sink.observe,
|
|
71
142
|
executorImplementations: {
|
|
72
143
|
parent: {
|
|
73
144
|
async execute(request, ctx) {
|
|
74
|
-
const childResult = await ctx.invoke(
|
|
75
|
-
target: { kind: "executor", executorId: "child", alias: "child" },
|
|
76
|
-
});
|
|
145
|
+
const childResult = await ctx.invoke(childExecutor, request);
|
|
77
146
|
return childResult;
|
|
78
147
|
},
|
|
79
148
|
async *stream() {
|
|
80
149
|
return;
|
|
81
150
|
},
|
|
82
151
|
},
|
|
83
|
-
child: {
|
|
84
|
-
async execute(request) {
|
|
85
|
-
const result = cloneProgram(request);
|
|
86
|
-
result.code.push({
|
|
87
|
-
opcode: Opcode.RESP_DONE,
|
|
88
|
-
value: { kind: "string", value: "stop" },
|
|
89
|
-
});
|
|
90
|
-
return result;
|
|
91
|
-
},
|
|
92
|
-
async *stream() {
|
|
93
|
-
return;
|
|
94
|
-
},
|
|
95
|
-
},
|
|
96
152
|
},
|
|
97
153
|
});
|
|
98
154
|
|
|
@@ -119,12 +175,9 @@ describe("router execution runtime", () => {
|
|
|
119
175
|
const started = sink.events.filter(
|
|
120
176
|
(event) => event.kind === "executor.started",
|
|
121
177
|
);
|
|
122
|
-
expect(started).toHaveLength(
|
|
178
|
+
expect(started).toHaveLength(1);
|
|
123
179
|
expect(started[0]?.requestId).toBe("req_nested");
|
|
124
|
-
expect(started[1]?.requestId).toBe("req_nested");
|
|
125
|
-
expect(started[1]?.parentExecutionId).toBe("exec_root");
|
|
126
180
|
expect(started[0]?.data?.depth).toBe(0);
|
|
127
|
-
expect(started[1]?.data?.depth).toBe(1);
|
|
128
181
|
});
|
|
129
182
|
|
|
130
183
|
it("applies target transforms before provider invocation", async () => {
|
|
@@ -181,6 +234,9 @@ describe("router execution runtime", () => {
|
|
|
181
234
|
const providerCalls: string[] = [];
|
|
182
235
|
const finalMarkers: Record<string, string> = {};
|
|
183
236
|
const runtime = createExecutionRuntime({
|
|
237
|
+
resolveTarget(request) {
|
|
238
|
+
return { kind: "provider", model: getModel(request) ?? "" };
|
|
239
|
+
},
|
|
184
240
|
providerInvoker: {
|
|
185
241
|
async execute(request, ctx) {
|
|
186
242
|
providerCalls.push(`execute:${ctx.target.model}`);
|
|
@@ -223,13 +279,10 @@ describe("router execution runtime", () => {
|
|
|
223
279
|
name: "enrich",
|
|
224
280
|
capabilities: ["write_messages"],
|
|
225
281
|
async apply(program, ctx) {
|
|
226
|
-
const invoked = await ctx.invoke(
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
model: "nested-exec",
|
|
231
|
-
},
|
|
232
|
-
});
|
|
282
|
+
const invoked = await ctx.invoke(
|
|
283
|
+
"openrouter/nested-exec",
|
|
284
|
+
createProgram(),
|
|
285
|
+
);
|
|
233
286
|
let invokeModel = "";
|
|
234
287
|
for (const instruction of invoked.code) {
|
|
235
288
|
if (
|
|
@@ -242,13 +295,10 @@ describe("router execution runtime", () => {
|
|
|
242
295
|
}
|
|
243
296
|
|
|
244
297
|
let streamModel = "";
|
|
245
|
-
for await (const chunk of ctx.invokeStream(
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
model: "nested-stream",
|
|
250
|
-
},
|
|
251
|
-
})) {
|
|
298
|
+
for await (const chunk of ctx.invokeStream(
|
|
299
|
+
"openrouter/nested-stream",
|
|
300
|
+
createProgram(),
|
|
301
|
+
)) {
|
|
252
302
|
const instruction = chunk.code[0];
|
|
253
303
|
if (
|
|
254
304
|
instruction?.opcode === Opcode.STREAM_DELTA &&
|
|
@@ -295,13 +345,13 @@ describe("router execution runtime", () => {
|
|
|
295
345
|
});
|
|
296
346
|
|
|
297
347
|
expect(providerCalls).toEqual([
|
|
298
|
-
"execute:nested-exec",
|
|
299
|
-
"stream:nested-stream",
|
|
348
|
+
"execute:openrouter/nested-exec",
|
|
349
|
+
"stream:openrouter/nested-stream",
|
|
300
350
|
"execute:final",
|
|
301
351
|
]);
|
|
302
352
|
expect(finalMarkers).toMatchObject({
|
|
303
|
-
"from-invoke": "nested-exec",
|
|
304
|
-
"from-stream": "nested-stream",
|
|
353
|
+
"from-invoke": "openrouter/nested-exec",
|
|
354
|
+
"from-stream": "openrouter/nested-stream",
|
|
305
355
|
});
|
|
306
356
|
});
|
|
307
357
|
|
|
@@ -341,33 +391,29 @@ describe("router execution runtime", () => {
|
|
|
341
391
|
|
|
342
392
|
it("enforces recursion limits with typed errors", async () => {
|
|
343
393
|
const sink = createInMemoryAuditSink();
|
|
394
|
+
const leaf = {
|
|
395
|
+
async execute(request: Program) {
|
|
396
|
+
return request;
|
|
397
|
+
},
|
|
398
|
+
async *stream() {
|
|
399
|
+
return;
|
|
400
|
+
},
|
|
401
|
+
};
|
|
402
|
+
const child = {
|
|
403
|
+
async execute(request: Program, ctx: ExecutorContext) {
|
|
404
|
+
return ctx.invoke(leaf, request);
|
|
405
|
+
},
|
|
406
|
+
async *stream() {
|
|
407
|
+
return;
|
|
408
|
+
},
|
|
409
|
+
};
|
|
344
410
|
const runtime = createExecutionRuntime({
|
|
345
411
|
observe: sink.observe,
|
|
346
412
|
maxDepth: 1,
|
|
347
413
|
executorImplementations: {
|
|
348
414
|
parent: {
|
|
349
415
|
async execute(request, ctx) {
|
|
350
|
-
return ctx.invoke(
|
|
351
|
-
target: { kind: "executor", executorId: "child", alias: "child" },
|
|
352
|
-
});
|
|
353
|
-
},
|
|
354
|
-
async *stream() {
|
|
355
|
-
return;
|
|
356
|
-
},
|
|
357
|
-
},
|
|
358
|
-
child: {
|
|
359
|
-
async execute(request, ctx) {
|
|
360
|
-
return ctx.invoke(request, {
|
|
361
|
-
target: { kind: "executor", executorId: "leaf", alias: "leaf" },
|
|
362
|
-
});
|
|
363
|
-
},
|
|
364
|
-
async *stream() {
|
|
365
|
-
return;
|
|
366
|
-
},
|
|
367
|
-
},
|
|
368
|
-
leaf: {
|
|
369
|
-
async execute(request) {
|
|
370
|
-
return request;
|
|
416
|
+
return ctx.invoke(child, request);
|
|
371
417
|
},
|
|
372
418
|
async *stream() {
|
|
373
419
|
return;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { cloneProgram } from "@neutrome/lil-engine";
|
|
1
|
+
import { cloneProgram, setModel } from "@neutrome/lil-engine";
|
|
2
2
|
import type { Program, ValidationMode } from "@neutrome/lil-engine";
|
|
3
|
-
import type { ExecutorContext } from "@neutrome/lilsdk
|
|
3
|
+
import type { ExecutorContext } from "@neutrome/lilsdk";
|
|
4
4
|
import { validateProgramOrThrow as validateObservedProgram } from "./execution-observation.ts";
|
|
5
5
|
import {
|
|
6
6
|
ensureActiveSignal as assertActiveSignal,
|
|
@@ -24,6 +24,11 @@ import type {
|
|
|
24
24
|
} from "./execution-types.ts";
|
|
25
25
|
|
|
26
26
|
const DEFAULT_MAX_DEPTH = 8;
|
|
27
|
+
const unavailableCache: ExecutorContext["cache"] = {
|
|
28
|
+
get: () => Promise.reject(cacheUnavailable()),
|
|
29
|
+
put: () => Promise.reject(cacheUnavailable()),
|
|
30
|
+
delete: () => Promise.reject(cacheUnavailable()),
|
|
31
|
+
};
|
|
27
32
|
|
|
28
33
|
export function createExecutionRuntime(
|
|
29
34
|
options: ExecutionRuntimeOptions = {},
|
|
@@ -198,28 +203,48 @@ export function createExecutionRuntime(
|
|
|
198
203
|
depth: number,
|
|
199
204
|
): ExecutorContext {
|
|
200
205
|
return {
|
|
206
|
+
cache: options.cache ?? unavailableCache,
|
|
201
207
|
requestId,
|
|
202
208
|
executionId,
|
|
203
209
|
...(parentExecutionId ? { parentExecutionId } : {}),
|
|
204
|
-
async invoke(
|
|
210
|
+
async invoke(executor, request) {
|
|
211
|
+
if (typeof executor !== "string") {
|
|
212
|
+
throwIfDepthExceeded(depth + 1, maxDepth);
|
|
213
|
+
return executor.execute(
|
|
214
|
+
cloneProgram(request),
|
|
215
|
+
createExecutionContext(
|
|
216
|
+
requestId,
|
|
217
|
+
executionIdFactory(),
|
|
218
|
+
executionId,
|
|
219
|
+
signal,
|
|
220
|
+
depth + 1,
|
|
221
|
+
),
|
|
222
|
+
);
|
|
223
|
+
}
|
|
205
224
|
return executeTarget(
|
|
206
|
-
cloneProgram(request),
|
|
207
|
-
{
|
|
208
|
-
...invokeOptions,
|
|
209
|
-
requestId,
|
|
210
|
-
parentExecutionId: executionId,
|
|
211
|
-
},
|
|
225
|
+
setModel(cloneProgram(request), executor),
|
|
226
|
+
{ requestId, parentExecutionId: executionId },
|
|
212
227
|
depth + 1,
|
|
213
228
|
);
|
|
214
229
|
},
|
|
215
|
-
async *invokeStream(
|
|
230
|
+
async *invokeStream(executor, request) {
|
|
231
|
+
if (typeof executor !== "string") {
|
|
232
|
+
throwIfDepthExceeded(depth + 1, maxDepth);
|
|
233
|
+
yield* executor.stream(
|
|
234
|
+
cloneProgram(request),
|
|
235
|
+
createExecutionContext(
|
|
236
|
+
requestId,
|
|
237
|
+
executionIdFactory(),
|
|
238
|
+
executionId,
|
|
239
|
+
signal,
|
|
240
|
+
depth + 1,
|
|
241
|
+
),
|
|
242
|
+
);
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
216
245
|
yield* streamTarget(
|
|
217
|
-
cloneProgram(request),
|
|
218
|
-
{
|
|
219
|
-
...invokeOptions,
|
|
220
|
-
requestId,
|
|
221
|
-
parentExecutionId: executionId,
|
|
222
|
-
},
|
|
246
|
+
setModel(cloneProgram(request), executor),
|
|
247
|
+
{ requestId, parentExecutionId: executionId },
|
|
223
248
|
depth + 1,
|
|
224
249
|
);
|
|
225
250
|
},
|
|
@@ -279,3 +304,7 @@ export function createExecutionRuntime(
|
|
|
279
304
|
});
|
|
280
305
|
}
|
|
281
306
|
}
|
|
307
|
+
|
|
308
|
+
function cacheUnavailable(): Error {
|
|
309
|
+
return new Error("Runtime cache is not configured");
|
|
310
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { cloneProgram } from "@neutrome/lil-engine";
|
|
2
2
|
import type { Program } from "@neutrome/lil-engine";
|
|
3
|
-
import { createExecutionEvent } from "@neutrome/lilsdk
|
|
4
|
-
import type { ExecutorContext } from "@neutrome/lilsdk
|
|
3
|
+
import { createExecutionEvent } from "@neutrome/lilsdk";
|
|
4
|
+
import type { ExecutorContext } from "@neutrome/lilsdk";
|
|
5
5
|
import { ExecutionError } from "./errors.ts";
|
|
6
6
|
import { executionTargetAuditRef, executionTargetId } from "./targets.ts";
|
|
7
7
|
import type { ExecutionTarget, ProgramTransform } from "./execution-types.ts";
|
|
@@ -1,17 +1,37 @@
|
|
|
1
1
|
import type { Program, ValidationIssue } from "@neutrome/lil-engine";
|
|
2
2
|
import type {
|
|
3
|
-
|
|
3
|
+
Cache,
|
|
4
4
|
ExecutionEvent,
|
|
5
5
|
Executor,
|
|
6
|
-
InvokeOptions,
|
|
7
6
|
ProgramTransform,
|
|
8
|
-
} from "@neutrome/lilsdk
|
|
7
|
+
} from "@neutrome/lilsdk";
|
|
9
8
|
import type {
|
|
10
9
|
ExecutionErrorDetails,
|
|
11
10
|
ExecutionErrorKind,
|
|
12
11
|
ExecutionErrorStage,
|
|
13
12
|
} from "./errors.ts";
|
|
14
13
|
|
|
14
|
+
export type ExecutionTarget =
|
|
15
|
+
| {
|
|
16
|
+
kind: "provider";
|
|
17
|
+
provider?: string;
|
|
18
|
+
model: string;
|
|
19
|
+
transforms?: string[];
|
|
20
|
+
}
|
|
21
|
+
| {
|
|
22
|
+
kind: "executor";
|
|
23
|
+
executorId: string;
|
|
24
|
+
alias: string;
|
|
25
|
+
transforms?: string[];
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export type InvokeOptions = {
|
|
29
|
+
target?: ExecutionTarget;
|
|
30
|
+
requestId?: string;
|
|
31
|
+
executionId?: string;
|
|
32
|
+
parentExecutionId?: string;
|
|
33
|
+
};
|
|
34
|
+
|
|
15
35
|
export type ProviderInvocationContext = {
|
|
16
36
|
requestId: string;
|
|
17
37
|
executionId: string;
|
|
@@ -39,6 +59,7 @@ export type ResolveExecutor = (
|
|
|
39
59
|
) => Executor | Promise<Executor | null> | null;
|
|
40
60
|
|
|
41
61
|
export type ExecutionRuntimeOptions = {
|
|
62
|
+
cache?: Cache;
|
|
42
63
|
executorImplementations?: Readonly<Record<string, Executor>>;
|
|
43
64
|
resolveExecutor?: ResolveExecutor;
|
|
44
65
|
transforms?: Readonly<Record<string, ProgramTransform>>;
|
|
@@ -61,9 +82,7 @@ export type {
|
|
|
61
82
|
ExecutionErrorDetails,
|
|
62
83
|
ExecutionErrorKind,
|
|
63
84
|
ExecutionErrorStage,
|
|
64
|
-
ExecutionTarget,
|
|
65
85
|
ExecutionEvent,
|
|
66
86
|
Executor,
|
|
67
|
-
InvokeOptions,
|
|
68
87
|
ProgramTransform,
|
|
69
88
|
};
|
package/src/router/index.ts
CHANGED
package/src/router/mcp.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
2
2
|
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
3
3
|
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
|
|
4
|
-
import type { ExecutorContext, Tool } from "@neutrome/lilsdk
|
|
4
|
+
import type { ExecutorContext, Tool } from "@neutrome/lilsdk";
|
|
5
5
|
import type { RemoteMcpServerRuntime, RouterRuntime } from "./runtime.ts";
|
|
6
6
|
import { observeTimedExecution } from "./execution-events.ts";
|
|
7
7
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { usageObject, type Program } from "@neutrome/lil-engine";
|
|
2
|
-
import { createExecutionEvent } from "@neutrome/lilsdk
|
|
2
|
+
import { createExecutionEvent } from "@neutrome/lilsdk";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import type { ProviderInvocationContext } from "./execution-types.ts";
|
|
5
5
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { connectTools } from "@neutrome/lilsdk-ts/tools";
|
|
1
|
+
import { connectTools } from "@neutrome/lilsdk/tools";
|
|
3
2
|
import type {
|
|
4
3
|
ExecutionTarget,
|
|
5
4
|
Executor,
|
|
@@ -50,10 +49,7 @@ export function prepareRemoteMcpExecution(
|
|
|
50
49
|
...options,
|
|
51
50
|
executorImplementations: {
|
|
52
51
|
...(options.executorImplementations ?? {}),
|
|
53
|
-
[executorName]: connectTools(
|
|
54
|
-
remoteMcp.tools,
|
|
55
|
-
createTargetExecutor(resolved.target),
|
|
56
|
-
),
|
|
52
|
+
[executorName]: connectTools(remoteMcp.tools, resolved.requestedModel),
|
|
57
53
|
},
|
|
58
54
|
},
|
|
59
55
|
close: remoteMcp.close,
|
|
@@ -18,9 +18,7 @@ export function resolveModelNamespaceExecutor(
|
|
|
18
18
|
"Request is missing model for namespace executor resolution",
|
|
19
19
|
{ stage: "target_resolution" },
|
|
20
20
|
);
|
|
21
|
-
return ctx.invoke(request
|
|
22
|
-
target: resolveInvocationTarget(runtime, `${name}/${model}`).target,
|
|
23
|
-
});
|
|
21
|
+
return ctx.invoke(`${name}/${model}`, request);
|
|
24
22
|
},
|
|
25
23
|
async *stream(request, ctx) {
|
|
26
24
|
const model = getModel(request);
|
|
@@ -30,9 +28,7 @@ export function resolveModelNamespaceExecutor(
|
|
|
30
28
|
"Request is missing model for namespace executor resolution",
|
|
31
29
|
{ stage: "target_resolution" },
|
|
32
30
|
);
|
|
33
|
-
yield* ctx.invokeStream(request
|
|
34
|
-
target: resolveInvocationTarget(runtime, `${name}/${model}`).target,
|
|
35
|
-
});
|
|
31
|
+
yield* ctx.invokeStream(`${name}/${model}`, request);
|
|
36
32
|
},
|
|
37
33
|
};
|
|
38
34
|
}
|
|
@@ -22,9 +22,9 @@ describe("router resolution", () => {
|
|
|
22
22
|
enei: {
|
|
23
23
|
exports: ["enei-*"],
|
|
24
24
|
models: {
|
|
25
|
-
"enei-1": {
|
|
25
|
+
"enei-1": { target: { executor: "enei-1" } },
|
|
26
26
|
"enei-1-pro": {
|
|
27
|
-
|
|
27
|
+
target: { executor: "enei-1-pro" },
|
|
28
28
|
transforms: ["reasoning:high"],
|
|
29
29
|
},
|
|
30
30
|
},
|
|
@@ -106,8 +106,10 @@ describe("router resolution", () => {
|
|
|
106
106
|
exports: ["enei-*"],
|
|
107
107
|
models: {
|
|
108
108
|
"enei-1": {
|
|
109
|
-
|
|
110
|
-
|
|
109
|
+
target: {
|
|
110
|
+
provider: "openrouter",
|
|
111
|
+
model: "openai/gpt-4.1-mini",
|
|
112
|
+
},
|
|
111
113
|
},
|
|
112
114
|
},
|
|
113
115
|
},
|
|
@@ -140,8 +142,8 @@ describe("router resolution", () => {
|
|
|
140
142
|
enei: {
|
|
141
143
|
exports: ["enei-*"],
|
|
142
144
|
models: {
|
|
143
|
-
"enei-1": {
|
|
144
|
-
"enei-1-pro": {
|
|
145
|
+
"enei-1": { target: { executor: "enei-1" } },
|
|
146
|
+
"enei-1-pro": { target: { executor: "enei-1-pro" } },
|
|
145
147
|
},
|
|
146
148
|
},
|
|
147
149
|
},
|
|
@@ -168,7 +170,9 @@ describe("router resolution", () => {
|
|
|
168
170
|
default: {
|
|
169
171
|
exports: ["*"],
|
|
170
172
|
models: {
|
|
171
|
-
"gpt-4o": {
|
|
173
|
+
"gpt-4o": {
|
|
174
|
+
target: { provider: "openrouter", model: "openai/gpt-4o" },
|
|
175
|
+
},
|
|
172
176
|
},
|
|
173
177
|
},
|
|
174
178
|
},
|
package/src/router/resolve.ts
CHANGED
|
@@ -25,7 +25,9 @@ describe("router runtime configuration", () => {
|
|
|
25
25
|
providers: { upstream: provider },
|
|
26
26
|
modelNamespaces: {
|
|
27
27
|
default: {
|
|
28
|
-
models: {
|
|
28
|
+
models: {
|
|
29
|
+
chat: { target: { provider: "missing", model: "chat" } },
|
|
30
|
+
},
|
|
29
31
|
},
|
|
30
32
|
},
|
|
31
33
|
},
|
package/src/router/runtime.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ExecutionTarget } from "
|
|
1
|
+
import type { ExecutionTarget } from "./execution-types.ts";
|
|
2
2
|
import { createExportsMatcher } from "../util/glob.ts";
|
|
3
3
|
import type {
|
|
4
4
|
ModelNamespaceConfig,
|
|
@@ -101,9 +101,12 @@ function validateRouterConfig(config: RouterConfig): void {
|
|
|
101
101
|
throw new Error(`Model namespace ${namespace} has no routes`);
|
|
102
102
|
}
|
|
103
103
|
for (const [alias, route] of Object.entries(namespaceConfig.models)) {
|
|
104
|
-
if (
|
|
104
|
+
if (
|
|
105
|
+
"provider" in route.target &&
|
|
106
|
+
!config.providers[route.target.provider]
|
|
107
|
+
) {
|
|
105
108
|
throw new Error(
|
|
106
|
-
`Model route ${namespace}/${alias} references unknown provider ${route.provider}`,
|
|
109
|
+
`Model route ${namespace}/${alias} references unknown provider ${route.target.provider}`,
|
|
107
110
|
);
|
|
108
111
|
}
|
|
109
112
|
for (const mcp of route.mcps ?? []) {
|
|
@@ -178,15 +181,15 @@ function buildModelRoute(
|
|
|
178
181
|
alias: string,
|
|
179
182
|
route: ModelRouteConfig,
|
|
180
183
|
): ModelRouteRuntime {
|
|
181
|
-
if ("provider" in route) {
|
|
184
|
+
if ("provider" in route.target) {
|
|
182
185
|
return {
|
|
183
186
|
requestedAlias: alias,
|
|
184
187
|
transforms: route.transforms ?? [],
|
|
185
188
|
mcps: route.mcps ?? [],
|
|
186
189
|
target: {
|
|
187
190
|
kind: "provider",
|
|
188
|
-
provider: route.provider,
|
|
189
|
-
model: route.model,
|
|
191
|
+
provider: route.target.provider,
|
|
192
|
+
model: route.target.model,
|
|
190
193
|
transforms: route.transforms ?? [],
|
|
191
194
|
},
|
|
192
195
|
};
|
|
@@ -198,8 +201,8 @@ function buildModelRoute(
|
|
|
198
201
|
mcps: route.mcps ?? [],
|
|
199
202
|
target: {
|
|
200
203
|
kind: "executor",
|
|
201
|
-
executorId: route.
|
|
202
|
-
alias
|
|
204
|
+
executorId: route.target.executor,
|
|
205
|
+
alias,
|
|
203
206
|
transforms: route.transforms ?? [],
|
|
204
207
|
},
|
|
205
208
|
};
|
package/src/telemetry/router.ts
CHANGED
package/src/telemetry.test.ts
CHANGED
package/src/trace-labels.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ExecutionEvent as AuditEvent } from "@neutrome/lilsdk
|
|
1
|
+
import type { ExecutionEvent as AuditEvent } from "@neutrome/lilsdk";
|
|
2
2
|
|
|
3
3
|
export function traceTimingKind(event: AuditEvent): string {
|
|
4
4
|
if (event.kind.startsWith("upstream_auth.")) return "upstream_auth";
|