@neutrome/open-ai-router 0.4.2 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dev.vars.example +2 -0
- package/README.md +5 -1
- package/package.json +5 -4
- package/src/admission/access.ts +0 -1
- package/src/admission/types.ts +0 -1
- package/src/app/factory.test.ts +40 -34
- package/src/app/factory.ts +120 -173
- package/src/app/google-genai-route.ts +48 -0
- package/src/app/http-utils.ts +14 -0
- package/src/app/model-listing.ts +29 -0
- package/src/app/response-lifecycle.ts +66 -0
- package/src/app/types.ts +38 -0
- package/src/index.ts +11 -43
- package/src/otlp.ts +155 -0
- package/src/router/audit.ts +5 -12
- package/src/router/config.ts +4 -4
- package/src/router/error-codec.ts +170 -0
- package/src/router/errors.ts +38 -1
- package/src/router/execute.test.ts +177 -107
- package/src/router/execute.ts +68 -924
- package/src/router/execution-events.ts +39 -0
- package/src/router/execution-invocation.ts +144 -0
- package/src/router/execution-observation.ts +161 -0
- package/src/router/execution-resolution.ts +120 -0
- package/src/router/execution-runtime.test.ts +156 -83
- package/src/router/execution-runtime.ts +144 -542
- package/src/router/execution-transforms.ts +82 -0
- package/src/router/execution-types.ts +8 -19
- package/src/router/index.ts +6 -5
- package/src/router/mcp.ts +12 -6
- package/src/router/provider-invoker.ts +134 -0
- package/src/router/provider-stream.ts +192 -0
- package/src/router/provider-telemetry.ts +59 -0
- package/src/router/remote-mcp-execution.ts +61 -0
- package/src/router/request-program.ts +16 -0
- package/src/router/request-target.ts +56 -0
- package/src/router/resolve.test.ts +43 -10
- package/src/router/resolve.ts +50 -28
- package/src/router/response-delivery.ts +141 -0
- package/src/router/runtime.test.ts +56 -0
- package/src/router/runtime.ts +79 -9
- package/src/router/targets.ts +3 -1
- package/src/router/upstream-client.ts +137 -0
- package/src/telemetry.test.ts +78 -0
- package/src/telemetry.ts +316 -0
- package/src/trace-labels.ts +29 -0
- package/src/upstream-auth/env.ts +15 -3
- package/src/upstream-auth/proxy.ts +6 -1
- package/src/upstream-auth/registry.ts +3 -1
- package/src/util/glob.ts +1 -1
- package/src/util/model-syntax.ts +3 -3
- package/src/worker.ts +12 -6
- package/worker-configuration.d.ts +195 -164
- package/pnpm-workspace.yaml +0 -4
- package/src/admission/jwt.test.ts +0 -96
- package/src/admission/jwt.ts +0 -221
- package/src/app/handlers.ts +0 -198
- package/src/example.test.ts +0 -377
- package/src/example.ts +0 -73
- package/src/observer.test.ts +0 -54
- package/src/observer.ts +0 -315
- package/src/timing.ts +0 -36
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { cloneProgram, createProgram, Opcode } from "@neutrome/lil-engine";
|
|
2
3
|
import {
|
|
3
|
-
cloneProgram,
|
|
4
|
-
createProgram,
|
|
5
|
-
Opcode,
|
|
6
|
-
} from "@neutrome/lil-engine";
|
|
7
|
-
import {
|
|
8
|
-
createAuditEvent,
|
|
9
4
|
createExecutionRuntime,
|
|
10
5
|
createInMemoryAuditSink,
|
|
11
6
|
ExecutionError,
|
|
12
7
|
executionTargetAuditRef,
|
|
13
8
|
executionTargetId,
|
|
14
9
|
} from "./index.ts";
|
|
10
|
+
import { createExecutionEvent as createAuditEvent } from "@neutrome/lilsdk-ts";
|
|
15
11
|
import { observeExecutionStream } from "@neutrome/lilsdk-ts/stream";
|
|
16
12
|
|
|
17
13
|
describe("router execution runtime", () => {
|
|
@@ -33,12 +29,14 @@ describe("router execution runtime", () => {
|
|
|
33
29
|
|
|
34
30
|
it("collects audit events in memory", () => {
|
|
35
31
|
const sink = createInMemoryAuditSink();
|
|
36
|
-
sink.observe(
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
32
|
+
sink.observe(
|
|
33
|
+
createAuditEvent({
|
|
34
|
+
kind: "program.validated",
|
|
35
|
+
requestId: "req_2",
|
|
36
|
+
executionId: "exec_2",
|
|
37
|
+
data: { issues: 0 },
|
|
38
|
+
}),
|
|
39
|
+
);
|
|
42
40
|
|
|
43
41
|
expect(sink.events).toHaveLength(1);
|
|
44
42
|
expect(sink.events[0]?.kind).toBe("program.validated");
|
|
@@ -46,17 +44,21 @@ describe("router execution runtime", () => {
|
|
|
46
44
|
});
|
|
47
45
|
|
|
48
46
|
it("normalizes execution target identifiers", () => {
|
|
49
|
-
expect(
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
47
|
+
expect(
|
|
48
|
+
executionTargetId({
|
|
49
|
+
kind: "provider",
|
|
50
|
+
provider: "openrouter",
|
|
51
|
+
model: "google/gemma-4-31b-it",
|
|
52
|
+
}),
|
|
53
|
+
).toBe("openrouter/google/gemma-4-31b-it");
|
|
54
|
+
|
|
55
|
+
expect(
|
|
56
|
+
executionTargetAuditRef({
|
|
57
|
+
kind: "executor",
|
|
58
|
+
executorId: "enei",
|
|
59
|
+
alias: "enei-1",
|
|
60
|
+
}),
|
|
61
|
+
).toEqual({
|
|
60
62
|
kind: "executor",
|
|
61
63
|
id: "enei/enei-1",
|
|
62
64
|
});
|
|
@@ -96,7 +98,10 @@ describe("router execution runtime", () => {
|
|
|
96
98
|
|
|
97
99
|
const request = createProgram({
|
|
98
100
|
code: [
|
|
99
|
-
{
|
|
101
|
+
{
|
|
102
|
+
opcode: Opcode.SET_MODEL,
|
|
103
|
+
value: { kind: "string", value: "enei-1" },
|
|
104
|
+
},
|
|
100
105
|
],
|
|
101
106
|
});
|
|
102
107
|
|
|
@@ -111,7 +116,9 @@ describe("router execution runtime", () => {
|
|
|
111
116
|
value: { kind: "string", value: "stop" },
|
|
112
117
|
});
|
|
113
118
|
|
|
114
|
-
const started = sink.events.filter(
|
|
119
|
+
const started = sink.events.filter(
|
|
120
|
+
(event) => event.kind === "executor.started",
|
|
121
|
+
);
|
|
115
122
|
expect(started).toHaveLength(2);
|
|
116
123
|
expect(started[0]?.requestId).toBe("req_nested");
|
|
117
124
|
expect(started[1]?.requestId).toBe("req_nested");
|
|
@@ -125,13 +132,16 @@ describe("router execution runtime", () => {
|
|
|
125
132
|
const runtime = createExecutionRuntime({
|
|
126
133
|
providerInvoker: {
|
|
127
134
|
async execute(request) {
|
|
128
|
-
const marker = request.code.find(
|
|
129
|
-
instruction
|
|
130
|
-
|
|
131
|
-
|
|
135
|
+
const marker = request.code.find(
|
|
136
|
+
(instruction) =>
|
|
137
|
+
instruction.opcode === Opcode.SET_META &&
|
|
138
|
+
instruction.value.kind === "key_string" &&
|
|
139
|
+
instruction.value.key === "transform",
|
|
132
140
|
);
|
|
133
141
|
providerCalls.push(
|
|
134
|
-
marker?.value.kind === "key_string"
|
|
142
|
+
marker?.value.kind === "key_string"
|
|
143
|
+
? marker.value.value
|
|
144
|
+
: "missing",
|
|
135
145
|
);
|
|
136
146
|
return cloneProgram(request);
|
|
137
147
|
},
|
|
@@ -176,7 +186,10 @@ describe("router execution runtime", () => {
|
|
|
176
186
|
providerCalls.push(`execute:${ctx.target.model}`);
|
|
177
187
|
if (ctx.target.model === "final") {
|
|
178
188
|
for (const instruction of request.code) {
|
|
179
|
-
if (
|
|
189
|
+
if (
|
|
190
|
+
instruction.opcode === Opcode.SET_META &&
|
|
191
|
+
instruction.value.kind === "key_string"
|
|
192
|
+
) {
|
|
180
193
|
finalMarkers[instruction.value.key] = instruction.value.value;
|
|
181
194
|
}
|
|
182
195
|
}
|
|
@@ -185,17 +198,23 @@ describe("router execution runtime", () => {
|
|
|
185
198
|
const result = cloneProgram(request);
|
|
186
199
|
result.code.push({
|
|
187
200
|
opcode: Opcode.SET_META,
|
|
188
|
-
value: {
|
|
201
|
+
value: {
|
|
202
|
+
kind: "key_string",
|
|
203
|
+
key: "nested-invoke",
|
|
204
|
+
value: ctx.target.model,
|
|
205
|
+
},
|
|
189
206
|
});
|
|
190
207
|
return result;
|
|
191
208
|
},
|
|
192
209
|
async *stream(_request, ctx) {
|
|
193
210
|
providerCalls.push(`stream:${ctx.target.model}`);
|
|
194
211
|
yield createProgram({
|
|
195
|
-
code: [
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
212
|
+
code: [
|
|
213
|
+
{
|
|
214
|
+
opcode: Opcode.STREAM_DELTA,
|
|
215
|
+
value: { kind: "string", value: ctx.target.model },
|
|
216
|
+
},
|
|
217
|
+
],
|
|
199
218
|
});
|
|
200
219
|
},
|
|
201
220
|
},
|
|
@@ -205,7 +224,11 @@ describe("router execution runtime", () => {
|
|
|
205
224
|
capabilities: ["write_messages"],
|
|
206
225
|
async apply(program, ctx) {
|
|
207
226
|
const invoked = await ctx.invoke(createProgram(), {
|
|
208
|
-
target: {
|
|
227
|
+
target: {
|
|
228
|
+
kind: "provider",
|
|
229
|
+
provider: "openrouter",
|
|
230
|
+
model: "nested-exec",
|
|
231
|
+
},
|
|
209
232
|
});
|
|
210
233
|
let invokeModel = "";
|
|
211
234
|
for (const instruction of invoked.code) {
|
|
@@ -220,7 +243,11 @@ describe("router execution runtime", () => {
|
|
|
220
243
|
|
|
221
244
|
let streamModel = "";
|
|
222
245
|
for await (const chunk of ctx.invokeStream(createProgram(), {
|
|
223
|
-
target: {
|
|
246
|
+
target: {
|
|
247
|
+
kind: "provider",
|
|
248
|
+
provider: "openrouter",
|
|
249
|
+
model: "nested-stream",
|
|
250
|
+
},
|
|
224
251
|
})) {
|
|
225
252
|
const instruction = chunk.code[0];
|
|
226
253
|
if (
|
|
@@ -235,11 +262,19 @@ describe("router execution runtime", () => {
|
|
|
235
262
|
result.code.push(
|
|
236
263
|
{
|
|
237
264
|
opcode: Opcode.SET_META,
|
|
238
|
-
value: {
|
|
265
|
+
value: {
|
|
266
|
+
kind: "key_string",
|
|
267
|
+
key: "from-invoke",
|
|
268
|
+
value: invokeModel,
|
|
269
|
+
},
|
|
239
270
|
},
|
|
240
271
|
{
|
|
241
272
|
opcode: Opcode.SET_META,
|
|
242
|
-
value: {
|
|
273
|
+
value: {
|
|
274
|
+
kind: "key_string",
|
|
275
|
+
key: "from-stream",
|
|
276
|
+
value: streamModel,
|
|
277
|
+
},
|
|
243
278
|
},
|
|
244
279
|
);
|
|
245
280
|
return result;
|
|
@@ -280,7 +315,10 @@ describe("router execution runtime", () => {
|
|
|
280
315
|
yield cloneProgram(request);
|
|
281
316
|
yield createProgram({
|
|
282
317
|
code: [
|
|
283
|
-
{
|
|
318
|
+
{
|
|
319
|
+
opcode: Opcode.STREAM_DELTA,
|
|
320
|
+
value: { kind: "string", value: "hello" },
|
|
321
|
+
},
|
|
284
322
|
],
|
|
285
323
|
});
|
|
286
324
|
},
|
|
@@ -338,14 +376,18 @@ describe("router execution runtime", () => {
|
|
|
338
376
|
},
|
|
339
377
|
});
|
|
340
378
|
|
|
341
|
-
await expect(
|
|
342
|
-
|
|
343
|
-
|
|
379
|
+
await expect(
|
|
380
|
+
runtime.execute(createProgram(), {
|
|
381
|
+
target: { kind: "executor", executorId: "parent", alias: "parent" },
|
|
382
|
+
}),
|
|
383
|
+
).rejects.toMatchObject({
|
|
344
384
|
kind: "recursion_limit",
|
|
345
385
|
stage: "target_resolution",
|
|
346
386
|
} satisfies Partial<ExecutionError>);
|
|
347
387
|
|
|
348
|
-
const failed = sink.events.find(
|
|
388
|
+
const failed = sink.events.find(
|
|
389
|
+
(event) => event.kind === "execution.failed",
|
|
390
|
+
);
|
|
349
391
|
expect(failed?.errorKind).toBe("recursion_limit");
|
|
350
392
|
expect(failed?.data?.depth).toBe(2);
|
|
351
393
|
});
|
|
@@ -358,28 +400,40 @@ describe("router execution runtime", () => {
|
|
|
358
400
|
},
|
|
359
401
|
async *stream() {
|
|
360
402
|
yield createProgram({
|
|
361
|
-
code: [
|
|
362
|
-
{ opcode: Opcode.MSG_START, value: { kind: "none" } },
|
|
363
|
-
],
|
|
403
|
+
code: [{ opcode: Opcode.MSG_START, value: { kind: "none" } }],
|
|
364
404
|
});
|
|
365
405
|
},
|
|
366
406
|
},
|
|
367
407
|
});
|
|
368
408
|
|
|
369
|
-
await expect(
|
|
370
|
-
|
|
371
|
-
{
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
409
|
+
await expect(
|
|
410
|
+
runtime.execute(
|
|
411
|
+
createProgram({
|
|
412
|
+
code: [
|
|
413
|
+
{
|
|
414
|
+
opcode: Opcode.EXT_DATA,
|
|
415
|
+
value: {
|
|
416
|
+
kind: "key_json",
|
|
417
|
+
key: "provider",
|
|
418
|
+
value: new Uint8Array([123, 125]),
|
|
419
|
+
},
|
|
420
|
+
},
|
|
421
|
+
],
|
|
422
|
+
}),
|
|
423
|
+
{
|
|
424
|
+
target: { kind: "provider", provider: "openrouter", model: "gpt-4o" },
|
|
425
|
+
},
|
|
426
|
+
),
|
|
427
|
+
).rejects.toMatchObject({
|
|
376
428
|
kind: "validation",
|
|
377
429
|
stage: "request",
|
|
378
430
|
} satisfies Partial<ExecutionError>);
|
|
379
431
|
|
|
380
|
-
const iterator = runtime
|
|
381
|
-
|
|
382
|
-
|
|
432
|
+
const iterator = runtime
|
|
433
|
+
.stream(createProgram(), {
|
|
434
|
+
target: { kind: "provider", provider: "openrouter", model: "gpt-4o" },
|
|
435
|
+
})
|
|
436
|
+
[Symbol.asyncIterator]();
|
|
383
437
|
await expect(iterator.next()).rejects.toMatchObject({
|
|
384
438
|
kind: "validation",
|
|
385
439
|
stage: "stream_chunk",
|
|
@@ -387,19 +441,30 @@ describe("router execution runtime", () => {
|
|
|
387
441
|
});
|
|
388
442
|
|
|
389
443
|
it("observes text-first streams and tool-call streams", async () => {
|
|
390
|
-
const textObserved = await observeExecutionStream(
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
444
|
+
const textObserved = await observeExecutionStream(
|
|
445
|
+
(async function* () {
|
|
446
|
+
yield createProgram({
|
|
447
|
+
code: [
|
|
448
|
+
{
|
|
449
|
+
opcode: Opcode.STREAM_DELTA,
|
|
450
|
+
value: { kind: "string", value: "hello" },
|
|
451
|
+
},
|
|
452
|
+
],
|
|
453
|
+
});
|
|
454
|
+
yield createProgram({
|
|
455
|
+
code: [
|
|
456
|
+
{
|
|
457
|
+
opcode: Opcode.STREAM_DELTA,
|
|
458
|
+
value: { kind: "string", value: " world" },
|
|
459
|
+
},
|
|
460
|
+
{
|
|
461
|
+
opcode: Opcode.RESP_DONE,
|
|
462
|
+
value: { kind: "string", value: "stop" },
|
|
463
|
+
},
|
|
464
|
+
],
|
|
465
|
+
});
|
|
466
|
+
})(),
|
|
467
|
+
);
|
|
403
468
|
|
|
404
469
|
expect(textObserved).toEqual({
|
|
405
470
|
mode: "text",
|
|
@@ -407,18 +472,26 @@ describe("router execution runtime", () => {
|
|
|
407
472
|
emittedText: true,
|
|
408
473
|
});
|
|
409
474
|
|
|
410
|
-
const toolObserved = await observeExecutionStream(
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
475
|
+
const toolObserved = await observeExecutionStream(
|
|
476
|
+
(async function* () {
|
|
477
|
+
yield createProgram({
|
|
478
|
+
code: [
|
|
479
|
+
{
|
|
480
|
+
opcode: Opcode.STREAM_TOOL_DELTA,
|
|
481
|
+
value: { kind: "json", value: new Uint8Array([123, 125]) },
|
|
482
|
+
},
|
|
483
|
+
],
|
|
484
|
+
});
|
|
485
|
+
yield createProgram({
|
|
486
|
+
code: [
|
|
487
|
+
{
|
|
488
|
+
opcode: Opcode.RESP_DONE,
|
|
489
|
+
value: { kind: "string", value: "tool_calls" },
|
|
490
|
+
},
|
|
491
|
+
],
|
|
492
|
+
});
|
|
493
|
+
})(),
|
|
494
|
+
);
|
|
422
495
|
|
|
423
496
|
expect(toolObserved.mode).toBe("tool");
|
|
424
497
|
if (toolObserved.mode === "tool") {
|