@neutrome/open-ai-router 0.4.2 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/.dev.vars.example +2 -0
  2. package/README.md +5 -1
  3. package/package.json +5 -4
  4. package/src/admission/access.ts +0 -1
  5. package/src/admission/types.ts +0 -1
  6. package/src/app/factory.test.ts +40 -34
  7. package/src/app/factory.ts +120 -173
  8. package/src/app/google-genai-route.ts +48 -0
  9. package/src/app/http-utils.ts +14 -0
  10. package/src/app/model-listing.ts +29 -0
  11. package/src/app/response-lifecycle.ts +66 -0
  12. package/src/app/types.ts +38 -0
  13. package/src/index.ts +11 -43
  14. package/src/otlp.ts +148 -0
  15. package/src/router/audit.ts +5 -12
  16. package/src/router/config.ts +4 -4
  17. package/src/router/error-codec.ts +170 -0
  18. package/src/router/errors.ts +38 -1
  19. package/src/router/execute.test.ts +177 -107
  20. package/src/router/execute.ts +68 -924
  21. package/src/router/execution-events.ts +39 -0
  22. package/src/router/execution-invocation.ts +144 -0
  23. package/src/router/execution-observation.ts +161 -0
  24. package/src/router/execution-resolution.ts +120 -0
  25. package/src/router/execution-runtime.test.ts +156 -83
  26. package/src/router/execution-runtime.ts +144 -542
  27. package/src/router/execution-transforms.ts +82 -0
  28. package/src/router/execution-types.ts +8 -19
  29. package/src/router/index.ts +6 -5
  30. package/src/router/mcp.ts +12 -6
  31. package/src/router/provider-invoker.ts +134 -0
  32. package/src/router/provider-stream.ts +192 -0
  33. package/src/router/provider-telemetry.ts +59 -0
  34. package/src/router/remote-mcp-execution.ts +61 -0
  35. package/src/router/request-program.ts +16 -0
  36. package/src/router/request-target.ts +56 -0
  37. package/src/router/resolve.test.ts +43 -10
  38. package/src/router/resolve.ts +50 -28
  39. package/src/router/response-delivery.ts +141 -0
  40. package/src/router/runtime.test.ts +56 -0
  41. package/src/router/runtime.ts +79 -9
  42. package/src/router/targets.ts +3 -1
  43. package/src/router/upstream-client.ts +137 -0
  44. package/src/telemetry.test.ts +50 -0
  45. package/src/telemetry.ts +316 -0
  46. package/src/trace-labels.ts +29 -0
  47. package/src/upstream-auth/env.ts +15 -3
  48. package/src/upstream-auth/proxy.ts +6 -1
  49. package/src/upstream-auth/registry.ts +3 -1
  50. package/src/util/glob.ts +1 -1
  51. package/src/util/model-syntax.ts +3 -3
  52. package/src/worker.ts +12 -6
  53. package/worker-configuration.d.ts +195 -164
  54. package/pnpm-workspace.yaml +0 -4
  55. package/src/admission/jwt.test.ts +0 -96
  56. package/src/admission/jwt.ts +0 -221
  57. package/src/app/handlers.ts +0 -198
  58. package/src/example.test.ts +0 -377
  59. package/src/example.ts +0 -73
  60. package/src/observer.test.ts +0 -54
  61. package/src/observer.ts +0 -315
  62. package/src/timing.ts +0 -36
@@ -1,17 +1,13 @@
1
1
  import { describe, expect, it } from "vitest";
2
+ import { cloneProgram, createProgram, Opcode } from "@neutrome/lil-engine";
2
3
  import {
3
- cloneProgram,
4
- createProgram,
5
- Opcode,
6
- } from "@neutrome/lil-engine";
7
- import {
8
- createAuditEvent,
9
4
  createExecutionRuntime,
10
5
  createInMemoryAuditSink,
11
6
  ExecutionError,
12
7
  executionTargetAuditRef,
13
8
  executionTargetId,
14
9
  } from "./index.ts";
10
+ import { createExecutionEvent as createAuditEvent } from "@neutrome/lilsdk-ts";
15
11
  import { observeExecutionStream } from "@neutrome/lilsdk-ts/stream";
16
12
 
17
13
  describe("router execution runtime", () => {
@@ -33,12 +29,14 @@ describe("router execution runtime", () => {
33
29
 
34
30
  it("collects audit events in memory", () => {
35
31
  const sink = createInMemoryAuditSink();
36
- sink.observe(createAuditEvent({
37
- kind: "program.validated",
38
- requestId: "req_2",
39
- executionId: "exec_2",
40
- data: { issues: 0 },
41
- }));
32
+ sink.observe(
33
+ createAuditEvent({
34
+ kind: "program.validated",
35
+ requestId: "req_2",
36
+ executionId: "exec_2",
37
+ data: { issues: 0 },
38
+ }),
39
+ );
42
40
 
43
41
  expect(sink.events).toHaveLength(1);
44
42
  expect(sink.events[0]?.kind).toBe("program.validated");
@@ -46,17 +44,21 @@ describe("router execution runtime", () => {
46
44
  });
47
45
 
48
46
  it("normalizes execution target identifiers", () => {
49
- expect(executionTargetId({
50
- kind: "provider",
51
- provider: "openrouter",
52
- model: "google/gemma-4-31b-it",
53
- })).toBe("openrouter/google/gemma-4-31b-it");
54
-
55
- expect(executionTargetAuditRef({
56
- kind: "executor",
57
- executorId: "enei",
58
- alias: "enei-1",
59
- })).toEqual({
47
+ expect(
48
+ executionTargetId({
49
+ kind: "provider",
50
+ provider: "openrouter",
51
+ model: "google/gemma-4-31b-it",
52
+ }),
53
+ ).toBe("openrouter/google/gemma-4-31b-it");
54
+
55
+ expect(
56
+ executionTargetAuditRef({
57
+ kind: "executor",
58
+ executorId: "enei",
59
+ alias: "enei-1",
60
+ }),
61
+ ).toEqual({
60
62
  kind: "executor",
61
63
  id: "enei/enei-1",
62
64
  });
@@ -96,7 +98,10 @@ describe("router execution runtime", () => {
96
98
 
97
99
  const request = createProgram({
98
100
  code: [
99
- { opcode: Opcode.SET_MODEL, value: { kind: "string", value: "enei-1" } },
101
+ {
102
+ opcode: Opcode.SET_MODEL,
103
+ value: { kind: "string", value: "enei-1" },
104
+ },
100
105
  ],
101
106
  });
102
107
 
@@ -111,7 +116,9 @@ describe("router execution runtime", () => {
111
116
  value: { kind: "string", value: "stop" },
112
117
  });
113
118
 
114
- const started = sink.events.filter((event) => event.kind === "executor.started");
119
+ const started = sink.events.filter(
120
+ (event) => event.kind === "executor.started",
121
+ );
115
122
  expect(started).toHaveLength(2);
116
123
  expect(started[0]?.requestId).toBe("req_nested");
117
124
  expect(started[1]?.requestId).toBe("req_nested");
@@ -125,13 +132,16 @@ describe("router execution runtime", () => {
125
132
  const runtime = createExecutionRuntime({
126
133
  providerInvoker: {
127
134
  async execute(request) {
128
- const marker = request.code.find((instruction) =>
129
- instruction.opcode === Opcode.SET_META &&
130
- instruction.value.kind === "key_string" &&
131
- instruction.value.key === "transform",
135
+ const marker = request.code.find(
136
+ (instruction) =>
137
+ instruction.opcode === Opcode.SET_META &&
138
+ instruction.value.kind === "key_string" &&
139
+ instruction.value.key === "transform",
132
140
  );
133
141
  providerCalls.push(
134
- marker?.value.kind === "key_string" ? marker.value.value : "missing",
142
+ marker?.value.kind === "key_string"
143
+ ? marker.value.value
144
+ : "missing",
135
145
  );
136
146
  return cloneProgram(request);
137
147
  },
@@ -176,7 +186,10 @@ describe("router execution runtime", () => {
176
186
  providerCalls.push(`execute:${ctx.target.model}`);
177
187
  if (ctx.target.model === "final") {
178
188
  for (const instruction of request.code) {
179
- if (instruction.opcode === Opcode.SET_META && instruction.value.kind === "key_string") {
189
+ if (
190
+ instruction.opcode === Opcode.SET_META &&
191
+ instruction.value.kind === "key_string"
192
+ ) {
180
193
  finalMarkers[instruction.value.key] = instruction.value.value;
181
194
  }
182
195
  }
@@ -185,17 +198,23 @@ describe("router execution runtime", () => {
185
198
  const result = cloneProgram(request);
186
199
  result.code.push({
187
200
  opcode: Opcode.SET_META,
188
- value: { kind: "key_string", key: "nested-invoke", value: ctx.target.model },
201
+ value: {
202
+ kind: "key_string",
203
+ key: "nested-invoke",
204
+ value: ctx.target.model,
205
+ },
189
206
  });
190
207
  return result;
191
208
  },
192
209
  async *stream(_request, ctx) {
193
210
  providerCalls.push(`stream:${ctx.target.model}`);
194
211
  yield createProgram({
195
- code: [{
196
- opcode: Opcode.STREAM_DELTA,
197
- value: { kind: "string", value: ctx.target.model },
198
- }],
212
+ code: [
213
+ {
214
+ opcode: Opcode.STREAM_DELTA,
215
+ value: { kind: "string", value: ctx.target.model },
216
+ },
217
+ ],
199
218
  });
200
219
  },
201
220
  },
@@ -205,7 +224,11 @@ describe("router execution runtime", () => {
205
224
  capabilities: ["write_messages"],
206
225
  async apply(program, ctx) {
207
226
  const invoked = await ctx.invoke(createProgram(), {
208
- target: { kind: "provider", provider: "openrouter", model: "nested-exec" },
227
+ target: {
228
+ kind: "provider",
229
+ provider: "openrouter",
230
+ model: "nested-exec",
231
+ },
209
232
  });
210
233
  let invokeModel = "";
211
234
  for (const instruction of invoked.code) {
@@ -220,7 +243,11 @@ describe("router execution runtime", () => {
220
243
 
221
244
  let streamModel = "";
222
245
  for await (const chunk of ctx.invokeStream(createProgram(), {
223
- target: { kind: "provider", provider: "openrouter", model: "nested-stream" },
246
+ target: {
247
+ kind: "provider",
248
+ provider: "openrouter",
249
+ model: "nested-stream",
250
+ },
224
251
  })) {
225
252
  const instruction = chunk.code[0];
226
253
  if (
@@ -235,11 +262,19 @@ describe("router execution runtime", () => {
235
262
  result.code.push(
236
263
  {
237
264
  opcode: Opcode.SET_META,
238
- value: { kind: "key_string", key: "from-invoke", value: invokeModel },
265
+ value: {
266
+ kind: "key_string",
267
+ key: "from-invoke",
268
+ value: invokeModel,
269
+ },
239
270
  },
240
271
  {
241
272
  opcode: Opcode.SET_META,
242
- value: { kind: "key_string", key: "from-stream", value: streamModel },
273
+ value: {
274
+ kind: "key_string",
275
+ key: "from-stream",
276
+ value: streamModel,
277
+ },
243
278
  },
244
279
  );
245
280
  return result;
@@ -280,7 +315,10 @@ describe("router execution runtime", () => {
280
315
  yield cloneProgram(request);
281
316
  yield createProgram({
282
317
  code: [
283
- { opcode: Opcode.STREAM_DELTA, value: { kind: "string", value: "hello" } },
318
+ {
319
+ opcode: Opcode.STREAM_DELTA,
320
+ value: { kind: "string", value: "hello" },
321
+ },
284
322
  ],
285
323
  });
286
324
  },
@@ -338,14 +376,18 @@ describe("router execution runtime", () => {
338
376
  },
339
377
  });
340
378
 
341
- await expect(runtime.execute(createProgram(), {
342
- target: { kind: "executor", executorId: "parent", alias: "parent" },
343
- })).rejects.toMatchObject({
379
+ await expect(
380
+ runtime.execute(createProgram(), {
381
+ target: { kind: "executor", executorId: "parent", alias: "parent" },
382
+ }),
383
+ ).rejects.toMatchObject({
344
384
  kind: "recursion_limit",
345
385
  stage: "target_resolution",
346
386
  } satisfies Partial<ExecutionError>);
347
387
 
348
- const failed = sink.events.find((event) => event.kind === "execution.failed");
388
+ const failed = sink.events.find(
389
+ (event) => event.kind === "execution.failed",
390
+ );
349
391
  expect(failed?.errorKind).toBe("recursion_limit");
350
392
  expect(failed?.data?.depth).toBe(2);
351
393
  });
@@ -358,28 +400,40 @@ describe("router execution runtime", () => {
358
400
  },
359
401
  async *stream() {
360
402
  yield createProgram({
361
- code: [
362
- { opcode: Opcode.MSG_START, value: { kind: "none" } },
363
- ],
403
+ code: [{ opcode: Opcode.MSG_START, value: { kind: "none" } }],
364
404
  });
365
405
  },
366
406
  },
367
407
  });
368
408
 
369
- await expect(runtime.execute(createProgram({
370
- code: [
371
- { opcode: Opcode.EXT_DATA, value: { kind: "key_json", key: "provider", value: new Uint8Array([123, 125]) } },
372
- ],
373
- }), {
374
- target: { kind: "provider", provider: "openrouter", model: "gpt-4o" },
375
- })).rejects.toMatchObject({
409
+ await expect(
410
+ runtime.execute(
411
+ createProgram({
412
+ code: [
413
+ {
414
+ opcode: Opcode.EXT_DATA,
415
+ value: {
416
+ kind: "key_json",
417
+ key: "provider",
418
+ value: new Uint8Array([123, 125]),
419
+ },
420
+ },
421
+ ],
422
+ }),
423
+ {
424
+ target: { kind: "provider", provider: "openrouter", model: "gpt-4o" },
425
+ },
426
+ ),
427
+ ).rejects.toMatchObject({
376
428
  kind: "validation",
377
429
  stage: "request",
378
430
  } satisfies Partial<ExecutionError>);
379
431
 
380
- const iterator = runtime.stream(createProgram(), {
381
- target: { kind: "provider", provider: "openrouter", model: "gpt-4o" },
382
- })[Symbol.asyncIterator]();
432
+ const iterator = runtime
433
+ .stream(createProgram(), {
434
+ target: { kind: "provider", provider: "openrouter", model: "gpt-4o" },
435
+ })
436
+ [Symbol.asyncIterator]();
383
437
  await expect(iterator.next()).rejects.toMatchObject({
384
438
  kind: "validation",
385
439
  stage: "stream_chunk",
@@ -387,19 +441,30 @@ describe("router execution runtime", () => {
387
441
  });
388
442
 
389
443
  it("observes text-first streams and tool-call streams", async () => {
390
- const textObserved = await observeExecutionStream((async function* () {
391
- yield createProgram({
392
- code: [
393
- { opcode: Opcode.STREAM_DELTA, value: { kind: "string", value: "hello" } },
394
- ],
395
- });
396
- yield createProgram({
397
- code: [
398
- { opcode: Opcode.STREAM_DELTA, value: { kind: "string", value: " world" } },
399
- { opcode: Opcode.RESP_DONE, value: { kind: "string", value: "stop" } },
400
- ],
401
- });
402
- })());
444
+ const textObserved = await observeExecutionStream(
445
+ (async function* () {
446
+ yield createProgram({
447
+ code: [
448
+ {
449
+ opcode: Opcode.STREAM_DELTA,
450
+ value: { kind: "string", value: "hello" },
451
+ },
452
+ ],
453
+ });
454
+ yield createProgram({
455
+ code: [
456
+ {
457
+ opcode: Opcode.STREAM_DELTA,
458
+ value: { kind: "string", value: " world" },
459
+ },
460
+ {
461
+ opcode: Opcode.RESP_DONE,
462
+ value: { kind: "string", value: "stop" },
463
+ },
464
+ ],
465
+ });
466
+ })(),
467
+ );
403
468
 
404
469
  expect(textObserved).toEqual({
405
470
  mode: "text",
@@ -407,18 +472,26 @@ describe("router execution runtime", () => {
407
472
  emittedText: true,
408
473
  });
409
474
 
410
- const toolObserved = await observeExecutionStream((async function* () {
411
- yield createProgram({
412
- code: [
413
- { opcode: Opcode.STREAM_TOOL_DELTA, value: { kind: "json", value: new Uint8Array([123, 125]) } },
414
- ],
415
- });
416
- yield createProgram({
417
- code: [
418
- { opcode: Opcode.RESP_DONE, value: { kind: "string", value: "tool_calls" } },
419
- ],
420
- });
421
- })());
475
+ const toolObserved = await observeExecutionStream(
476
+ (async function* () {
477
+ yield createProgram({
478
+ code: [
479
+ {
480
+ opcode: Opcode.STREAM_TOOL_DELTA,
481
+ value: { kind: "json", value: new Uint8Array([123, 125]) },
482
+ },
483
+ ],
484
+ });
485
+ yield createProgram({
486
+ code: [
487
+ {
488
+ opcode: Opcode.RESP_DONE,
489
+ value: { kind: "string", value: "tool_calls" },
490
+ },
491
+ ],
492
+ });
493
+ })(),
494
+ );
422
495
 
423
496
  expect(toolObserved.mode).toBe("tool");
424
497
  if (toolObserved.mode === "tool") {