@neutrome/open-ai-router 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/app/factory.test.ts +12 -2
- package/src/app/factory.ts +112 -20
- package/src/observer.test.ts +6 -2
- package/src/observer.ts +31 -6
- package/src/router/execute.ts +146 -2
- package/src/router/execution-runtime.ts +36 -4
- package/src/router/mcp.ts +9 -6
- package/src/timing.ts +36 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@neutrome/open-ai-router",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./src/index.ts"
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
10
10
|
"hono": "^4.12.18",
|
|
11
11
|
"@neutrome/lil-engine": "0.2.1",
|
|
12
|
-
"@neutrome/lilsdk-ts": "0.2.
|
|
12
|
+
"@neutrome/lilsdk-ts": "0.2.4"
|
|
13
13
|
},
|
|
14
14
|
"devDependencies": {
|
|
15
15
|
"@types/node": "^25.9.3",
|
package/src/app/factory.test.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it, vi } from "vitest";
|
|
2
|
+
import type { ExecutionTrace } from "../observer.ts";
|
|
2
3
|
import type { RouterConfig } from "../router/index.ts";
|
|
3
4
|
import { createRouterApp } from "./factory.ts";
|
|
4
5
|
|
|
@@ -25,7 +26,7 @@ const config: RouterConfig = {
|
|
|
25
26
|
describe("createRouterApp trace finalization", () => {
|
|
26
27
|
it("keeps non-stream trace hooks alive with waitUntil", async () => {
|
|
27
28
|
const waited: Promise<unknown>[] = [];
|
|
28
|
-
const onTrace = vi.fn(async () => {});
|
|
29
|
+
const onTrace = vi.fn(async (_trace: ExecutionTrace) => {});
|
|
29
30
|
const app = createRouterApp({
|
|
30
31
|
config,
|
|
31
32
|
executorImplementations: {},
|
|
@@ -73,7 +74,7 @@ describe("createRouterApp trace finalization", () => {
|
|
|
73
74
|
|
|
74
75
|
it("keeps stream trace hooks alive until the stream closes", async () => {
|
|
75
76
|
const waited: Promise<unknown>[] = [];
|
|
76
|
-
const onTrace = vi.fn(async () => {});
|
|
77
|
+
const onTrace = vi.fn(async (_trace: ExecutionTrace) => {});
|
|
77
78
|
const app = createRouterApp({
|
|
78
79
|
config,
|
|
79
80
|
executorImplementations: {},
|
|
@@ -137,6 +138,15 @@ describe("createRouterApp trace finalization", () => {
|
|
|
137
138
|
expect(waited).toHaveLength(1);
|
|
138
139
|
await waited[0];
|
|
139
140
|
expect(onTrace).toHaveBeenCalledTimes(1);
|
|
141
|
+
const rawTrace = onTrace.mock.calls[0]?.[0].rawTrace ?? "";
|
|
142
|
+
expect(rawTrace).toContain('"event":"admission.authenticate"');
|
|
143
|
+
expect(rawTrace).toContain('"event":"upstream_auth.collect_incoming"');
|
|
144
|
+
expect(rawTrace).toContain('"event":"upstream_auth.resolve_target"');
|
|
145
|
+
expect(rawTrace).toContain('"event":"provider.stream_first_byte"');
|
|
146
|
+
expect(rawTrace).toContain('"event":"response.stream_first_chunk"');
|
|
147
|
+
expect(rawTrace).toContain('"event":"provider.stream_finished"');
|
|
148
|
+
expect(rawTrace).toContain('"startedAt"');
|
|
149
|
+
expect(rawTrace).toContain('"finishedAt"');
|
|
140
150
|
});
|
|
141
151
|
});
|
|
142
152
|
|
package/src/app/factory.ts
CHANGED
|
@@ -19,6 +19,7 @@ import type { RequestContext } from "../upstream-auth/types.ts";
|
|
|
19
19
|
import type { RouterConfig } from "../router/config.ts";
|
|
20
20
|
import type { RemoteMcpClientFactory } from "../router/mcp.ts";
|
|
21
21
|
import type { ExecutionRuntimeOptions } from "../router/execution-types.ts";
|
|
22
|
+
import { observeTimingEvent } from "../timing.ts";
|
|
22
23
|
|
|
23
24
|
export type RouterAppAuth = RuntimeAuthenticator;
|
|
24
25
|
|
|
@@ -96,25 +97,109 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
96
97
|
forceStreaming?: boolean,
|
|
97
98
|
): Promise<Response> {
|
|
98
99
|
const env = honoEnv(c);
|
|
100
|
+
const incomingExecutionId = `incoming_${crypto.randomUUID()}`;
|
|
101
|
+
const collector = createTraceCollector();
|
|
102
|
+
const onTrace = options.hooks?.onTrace;
|
|
103
|
+
const observe = (event: Parameters<NonNullable<ExecutionRuntimeOptions["observe"]>>[0]) => {
|
|
104
|
+
collector.observe(event);
|
|
105
|
+
options.observe?.(event);
|
|
106
|
+
};
|
|
107
|
+
let traceFinalized = false;
|
|
108
|
+
const finalizeTrace = () => {
|
|
109
|
+
if (traceFinalized) return;
|
|
110
|
+
traceFinalized = true;
|
|
111
|
+
if (!onTrace) return;
|
|
112
|
+
runBackgroundTask(
|
|
113
|
+
c,
|
|
114
|
+
() => onTrace(collector.getTrace(), env),
|
|
115
|
+
"onTrace hook",
|
|
116
|
+
);
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
const requestStartedMs = Date.now();
|
|
120
|
+
observeTimingEvent({
|
|
121
|
+
observe,
|
|
122
|
+
kind: "request.started",
|
|
123
|
+
requestId: incomingExecutionId,
|
|
124
|
+
executionId: incomingExecutionId,
|
|
125
|
+
startedAtMs: requestStartedMs,
|
|
126
|
+
data: { style },
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
const authStartedMs = Date.now();
|
|
99
130
|
const principal = await options.auth.authenticate(request, env);
|
|
131
|
+
observeTimingEvent({
|
|
132
|
+
observe,
|
|
133
|
+
kind: "admission.authenticate",
|
|
134
|
+
requestId: incomingExecutionId,
|
|
135
|
+
executionId: incomingExecutionId,
|
|
136
|
+
startedAtMs: authStartedMs,
|
|
137
|
+
finishedAtMs: Date.now(),
|
|
138
|
+
data: { authenticated: !!principal },
|
|
139
|
+
});
|
|
100
140
|
if (!principal) return unauthorized();
|
|
101
141
|
|
|
102
142
|
let program: Program;
|
|
103
143
|
try {
|
|
144
|
+
const parseStartedMs = Date.now();
|
|
104
145
|
program = await parseRequestProgram(style, request.clone());
|
|
105
146
|
if (forceStreaming) program = setStreaming(program, true);
|
|
147
|
+
observeTimingEvent({
|
|
148
|
+
observe,
|
|
149
|
+
kind: "request.parsed",
|
|
150
|
+
requestId: incomingExecutionId,
|
|
151
|
+
executionId: incomingExecutionId,
|
|
152
|
+
startedAtMs: parseStartedMs,
|
|
153
|
+
finishedAtMs: Date.now(),
|
|
154
|
+
data: { style, streaming: forceStreaming === true },
|
|
155
|
+
});
|
|
106
156
|
} catch {
|
|
157
|
+
observeTimingEvent({
|
|
158
|
+
observe,
|
|
159
|
+
kind: "request.parse_failed",
|
|
160
|
+
requestId: incomingExecutionId,
|
|
161
|
+
executionId: incomingExecutionId,
|
|
162
|
+
startedAtMs: requestStartedMs,
|
|
163
|
+
finishedAtMs: Date.now(),
|
|
164
|
+
data: { style },
|
|
165
|
+
});
|
|
166
|
+
finalizeTrace();
|
|
107
167
|
return errorResponse("Invalid request body", 400, "invalid_request_body");
|
|
108
168
|
}
|
|
109
169
|
|
|
110
170
|
const model = getModel(program) || null;
|
|
111
|
-
|
|
171
|
+
const accessStartedMs = Date.now();
|
|
172
|
+
const hasModelAccess = !model || canAccessPrincipalModel(principal, model);
|
|
173
|
+
observeTimingEvent({
|
|
174
|
+
observe,
|
|
175
|
+
kind: "admission.model_access",
|
|
176
|
+
requestId: incomingExecutionId,
|
|
177
|
+
executionId: incomingExecutionId,
|
|
178
|
+
startedAtMs: accessStartedMs,
|
|
179
|
+
finishedAtMs: Date.now(),
|
|
180
|
+
data: { model: model ?? undefined, allowed: hasModelAccess },
|
|
181
|
+
});
|
|
182
|
+
if (!hasModelAccess) {
|
|
183
|
+
finalizeTrace();
|
|
112
184
|
return errorResponse(`Model \`${model}\` does not exist or you do not have access.`, 404, "model_not_found");
|
|
113
185
|
}
|
|
114
186
|
|
|
115
187
|
if (options.hooks?.beforeRequest) {
|
|
188
|
+
const hookStartedMs = Date.now();
|
|
116
189
|
const hookResponse = await options.hooks.beforeRequest({ request, model, principal, env });
|
|
117
|
-
|
|
190
|
+
observeTimingEvent({
|
|
191
|
+
observe,
|
|
192
|
+
kind: "request.before_hook",
|
|
193
|
+
requestId: incomingExecutionId,
|
|
194
|
+
executionId: incomingExecutionId,
|
|
195
|
+
startedAtMs: hookStartedMs,
|
|
196
|
+
finishedAtMs: Date.now(),
|
|
197
|
+
data: { returnedResponse: !!hookResponse },
|
|
198
|
+
});
|
|
199
|
+
if (hookResponse) {
|
|
200
|
+
finalizeTrace();
|
|
201
|
+
return hookResponse;
|
|
202
|
+
}
|
|
118
203
|
}
|
|
119
204
|
|
|
120
205
|
const ctx: RequestContext = {
|
|
@@ -124,28 +209,21 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
124
209
|
state: new Map(),
|
|
125
210
|
};
|
|
126
211
|
for (const driver of runtime.upstreamAuthChain) {
|
|
212
|
+
const collectStartedMs = Date.now();
|
|
127
213
|
driver.collectIncoming?.(ctx);
|
|
214
|
+
observeTimingEvent({
|
|
215
|
+
observe,
|
|
216
|
+
kind: "upstream_auth.collect_incoming",
|
|
217
|
+
requestId: incomingExecutionId,
|
|
218
|
+
executionId: incomingExecutionId,
|
|
219
|
+
startedAtMs: collectStartedMs,
|
|
220
|
+
finishedAtMs: Date.now(),
|
|
221
|
+
data: { driver: driver.name },
|
|
222
|
+
});
|
|
128
223
|
}
|
|
129
224
|
|
|
130
|
-
const collector = createTraceCollector();
|
|
131
|
-
const onTrace = options.hooks?.onTrace;
|
|
132
|
-
const observe = (event: Parameters<NonNullable<ExecutionRuntimeOptions["observe"]>>[0]) => {
|
|
133
|
-
collector.observe(event);
|
|
134
|
-
options.observe?.(event);
|
|
135
|
-
};
|
|
136
|
-
let traceFinalized = false;
|
|
137
|
-
const finalizeTrace = () => {
|
|
138
|
-
if (traceFinalized) return;
|
|
139
|
-
traceFinalized = true;
|
|
140
|
-
if (!onTrace) return;
|
|
141
|
-
runBackgroundTask(
|
|
142
|
-
c,
|
|
143
|
-
() => onTrace(collector.getTrace(), env),
|
|
144
|
-
"onTrace hook",
|
|
145
|
-
);
|
|
146
|
-
};
|
|
147
|
-
|
|
148
225
|
try {
|
|
226
|
+
const handlerStartedMs = Date.now();
|
|
149
227
|
const response = await handler({
|
|
150
228
|
runtime,
|
|
151
229
|
ctx,
|
|
@@ -153,9 +231,23 @@ export function createRouterApp(options: RouterAppOptions) {
|
|
|
153
231
|
executorImplementations: options.executorImplementations,
|
|
154
232
|
...(options.transforms ? { transforms: options.transforms } : {}),
|
|
155
233
|
observe,
|
|
234
|
+
incomingExecutionId,
|
|
156
235
|
...(options.upstreamTimeoutMs !== undefined ? { upstreamTimeoutMs: options.upstreamTimeoutMs } : {}),
|
|
157
236
|
...(options.remoteMcpClientFactory ? { remoteMcpClientFactory: options.remoteMcpClientFactory } : {}),
|
|
158
237
|
});
|
|
238
|
+
observeTimingEvent({
|
|
239
|
+
observe,
|
|
240
|
+
kind: "response.ready",
|
|
241
|
+
requestId: incomingExecutionId,
|
|
242
|
+
executionId: incomingExecutionId,
|
|
243
|
+
startedAtMs: handlerStartedMs,
|
|
244
|
+
finishedAtMs: Date.now(),
|
|
245
|
+
data: {
|
|
246
|
+
style,
|
|
247
|
+
streaming: isEventStreamResponse(response),
|
|
248
|
+
status: response.status,
|
|
249
|
+
},
|
|
250
|
+
});
|
|
159
251
|
if (!isEventStreamResponse(response) || !response.body) {
|
|
160
252
|
finalizeTrace();
|
|
161
253
|
return response;
|
package/src/observer.test.ts
CHANGED
|
@@ -40,11 +40,15 @@ describe("createTraceCollector", () => {
|
|
|
40
40
|
requestId: "request-1",
|
|
41
41
|
executionId: "provider-1",
|
|
42
42
|
timestamp: "2026-06-24T12:00:03.000Z",
|
|
43
|
-
data: {
|
|
43
|
+
data: {
|
|
44
|
+
startedAt: "2026-06-24T12:00:02.877Z",
|
|
45
|
+
finishedAt: "2026-06-24T12:00:03.000Z",
|
|
46
|
+
durationMs: 123,
|
|
47
|
+
},
|
|
44
48
|
});
|
|
45
49
|
|
|
46
50
|
expect(collector.getTrace().rawTrace).toBe(
|
|
47
|
-
';timing {"kind":"provider","event":"provider.finished","executionId":"provider-1","durationMs":123}',
|
|
51
|
+
';timing {"kind":"provider","event":"provider.finished","requestId":"request-1","executionId":"provider-1","startedAt":"2026-06-24T12:00:02.877Z","finishedAt":"2026-06-24T12:00:03.000Z","durationMs":123}',
|
|
48
52
|
);
|
|
49
53
|
});
|
|
50
54
|
});
|
package/src/observer.ts
CHANGED
|
@@ -52,8 +52,8 @@ export function createTraceCollector(spanId = crypto.randomUUID()): {
|
|
|
52
52
|
|
|
53
53
|
return {
|
|
54
54
|
observe(event) {
|
|
55
|
-
appendLilSegment(event);
|
|
56
55
|
appendTimingComment(event);
|
|
56
|
+
appendLilSegment(event);
|
|
57
57
|
|
|
58
58
|
switch (event.kind) {
|
|
59
59
|
case "request.received":
|
|
@@ -159,19 +159,40 @@ export function createTraceCollector(spanId = crypto.randomUUID()): {
|
|
|
159
159
|
}
|
|
160
160
|
|
|
161
161
|
function appendTimingComment(event: AuditEvent): void {
|
|
162
|
+
const startedAt = asString(event.data?.startedAt);
|
|
163
|
+
const finishedAt = asString(event.data?.finishedAt);
|
|
162
164
|
const rawDurationMs = event.data?.durationMs;
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
165
|
+
const durationMs = typeof rawDurationMs === "number" && Number.isFinite(rawDurationMs) && rawDurationMs >= 0
|
|
166
|
+
? Math.round(rawDurationMs)
|
|
167
|
+
: undefined;
|
|
168
|
+
if (!startedAt && !finishedAt && durationMs === undefined) return;
|
|
169
|
+
|
|
170
|
+
rawSegments.push(`;timing ${JSON.stringify(compactObject({
|
|
166
171
|
kind: traceTimingKind(event),
|
|
167
172
|
event: event.kind,
|
|
173
|
+
requestId: event.requestId,
|
|
168
174
|
executionId: event.executionId,
|
|
175
|
+
...(event.parentExecutionId ? { parentExecutionId: event.parentExecutionId } : {}),
|
|
176
|
+
startedAt,
|
|
177
|
+
finishedAt: finishedAt ?? (durationMs !== undefined ? event.timestamp : undefined),
|
|
169
178
|
durationMs,
|
|
170
|
-
|
|
179
|
+
provider: asString(event.data?.provider),
|
|
180
|
+
model: asString(event.data?.model),
|
|
181
|
+
toolName: asString(event.data?.toolName),
|
|
182
|
+
executor: asString(event.data?.executor) || asString(event.data?.executorId),
|
|
183
|
+
driver: asString(event.data?.driver),
|
|
184
|
+
source: asString(event.data?.source),
|
|
185
|
+
matched: typeof event.data?.matched === "boolean" ? event.data.matched : undefined,
|
|
186
|
+
status: typeof event.data?.status === "number" ? event.data.status : undefined,
|
|
187
|
+
bytes: typeof event.data?.bytes === "number" ? event.data.bytes : undefined,
|
|
188
|
+
}))}`);
|
|
171
189
|
}
|
|
172
190
|
}
|
|
173
191
|
|
|
174
|
-
function traceTimingKind(event: AuditEvent):
|
|
192
|
+
function traceTimingKind(event: AuditEvent): string {
|
|
193
|
+
if (event.kind.startsWith("upstream_auth.")) return "upstream_auth";
|
|
194
|
+
if (event.kind.startsWith("admission.")) return "admission";
|
|
195
|
+
if (event.kind.startsWith("response.")) return "response";
|
|
175
196
|
if (event.kind.startsWith("provider.")) return "provider";
|
|
176
197
|
if (event.kind.startsWith("executor.")) return "executor";
|
|
177
198
|
if (event.kind.startsWith("remote_mcp.")) return "remote_mcp";
|
|
@@ -288,3 +309,7 @@ function asNumber(value: unknown): number {
|
|
|
288
309
|
function withMaybeString<K extends string>(key: K, value: string | undefined): Partial<Record<K, string>> {
|
|
289
310
|
return value ? { [key]: value } as Record<K, string> : {};
|
|
290
311
|
}
|
|
312
|
+
|
|
313
|
+
function compactObject<T extends Record<string, unknown>>(value: T): Record<string, unknown> {
|
|
314
|
+
return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined));
|
|
315
|
+
}
|
package/src/router/execute.ts
CHANGED
|
@@ -45,6 +45,7 @@ import {
|
|
|
45
45
|
type RemoteMcpClientFactory,
|
|
46
46
|
} from "./mcp.ts";
|
|
47
47
|
import type { ProviderRuntime, RouterRuntime } from "./runtime.ts";
|
|
48
|
+
import { observeTimingEvent } from "../timing.ts";
|
|
48
49
|
|
|
49
50
|
const encoder = new TextEncoder();
|
|
50
51
|
const decoder = new TextDecoder();
|
|
@@ -58,6 +59,7 @@ export type RouterExecutionOptions = Pick<
|
|
|
58
59
|
transforms?: Readonly<Record<string, ProgramTransform>>;
|
|
59
60
|
upstreamTimeoutMs?: number;
|
|
60
61
|
remoteMcpClientFactory?: RemoteMcpClientFactory;
|
|
62
|
+
incomingExecutionId?: string;
|
|
61
63
|
};
|
|
62
64
|
|
|
63
65
|
export type HandleChatCompletionsOptions = RouterExecutionOptions & {
|
|
@@ -179,7 +181,7 @@ export async function handleChatCompletions(
|
|
|
179
181
|
): Promise<Response> {
|
|
180
182
|
try {
|
|
181
183
|
const request = options.program ?? await parseRequestProgram("chat-completions", options.ctx.request);
|
|
182
|
-
const incomingExecutionId = `incoming_${crypto.randomUUID()}`;
|
|
184
|
+
const incomingExecutionId = options.incomingExecutionId ?? `incoming_${crypto.randomUUID()}`;
|
|
183
185
|
emitObservedProgram(options.observe, "request.received", request, {
|
|
184
186
|
requestId: incomingExecutionId,
|
|
185
187
|
executionId: incomingExecutionId,
|
|
@@ -190,11 +192,18 @@ export async function handleChatCompletions(
|
|
|
190
192
|
|
|
191
193
|
if (isStreaming(request)) {
|
|
192
194
|
try {
|
|
195
|
+
const responseStartedMs = Date.now();
|
|
193
196
|
const chunks = execution.stream(request, { target: remoteMcp.target });
|
|
194
197
|
return await streamExecutionResponse(
|
|
195
198
|
"chat-completions",
|
|
196
199
|
chunks,
|
|
197
200
|
resolved,
|
|
201
|
+
options.observe,
|
|
202
|
+
{
|
|
203
|
+
requestId: incomingExecutionId,
|
|
204
|
+
executionId: incomingExecutionId,
|
|
205
|
+
startedAtMs: responseStartedMs,
|
|
206
|
+
},
|
|
198
207
|
remoteMcp.close,
|
|
199
208
|
);
|
|
200
209
|
} catch (error) {
|
|
@@ -256,7 +265,7 @@ async function handleProviderStyle(
|
|
|
256
265
|
try {
|
|
257
266
|
const baseRequest = options.program ?? await parseRequestProgram(style, options.ctx.request);
|
|
258
267
|
const request = mutateRequest ? mutateRequest(baseRequest) : baseRequest;
|
|
259
|
-
const incomingExecutionId = `incoming_${crypto.randomUUID()}`;
|
|
268
|
+
const incomingExecutionId = options.incomingExecutionId ?? `incoming_${crypto.randomUUID()}`;
|
|
260
269
|
emitObservedProgram(options.observe, "request.received", request, {
|
|
261
270
|
requestId: incomingExecutionId,
|
|
262
271
|
executionId: incomingExecutionId,
|
|
@@ -267,11 +276,18 @@ async function handleProviderStyle(
|
|
|
267
276
|
|
|
268
277
|
if (isStreaming(request)) {
|
|
269
278
|
try {
|
|
279
|
+
const responseStartedMs = Date.now();
|
|
270
280
|
const chunks = execution.stream(request, { target: remoteMcp.target });
|
|
271
281
|
return await streamExecutionResponse(
|
|
272
282
|
style,
|
|
273
283
|
chunks,
|
|
274
284
|
resolved,
|
|
285
|
+
options.observe,
|
|
286
|
+
{
|
|
287
|
+
requestId: incomingExecutionId,
|
|
288
|
+
executionId: incomingExecutionId,
|
|
289
|
+
startedAtMs: responseStartedMs,
|
|
290
|
+
},
|
|
275
291
|
remoteMcp.close,
|
|
276
292
|
);
|
|
277
293
|
} catch (error) {
|
|
@@ -348,18 +364,49 @@ export function createFetchProviderInvoker(
|
|
|
348
364
|
provider: provider.name,
|
|
349
365
|
model: providerCtx.target.model,
|
|
350
366
|
});
|
|
367
|
+
const encodeStartedMs = Date.now();
|
|
351
368
|
let reqBody = emitProviderRequest(provider.style, resolvedRequest);
|
|
352
369
|
if (provider.style === "chat-completions" && isStreaming(resolvedRequest)) {
|
|
353
370
|
reqBody = injectStreamUsage(reqBody);
|
|
354
371
|
}
|
|
372
|
+
observeTimingEvent({
|
|
373
|
+
observe: providerCtx.observe,
|
|
374
|
+
kind: "provider.request_encoded",
|
|
375
|
+
requestId: providerCtx.requestId,
|
|
376
|
+
executionId: providerCtx.executionId,
|
|
377
|
+
...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
|
|
378
|
+
startedAtMs: encodeStartedMs,
|
|
379
|
+
finishedAtMs: Date.now(),
|
|
380
|
+
data: {
|
|
381
|
+
provider: provider.name,
|
|
382
|
+
model: providerCtx.target.model,
|
|
383
|
+
bytes: reqBody.byteLength,
|
|
384
|
+
},
|
|
385
|
+
});
|
|
355
386
|
const reqBytes = reqBody.byteLength;
|
|
387
|
+
const fetchStartedMs = Date.now();
|
|
356
388
|
const upstream = await fetchProvider(
|
|
357
389
|
runtime,
|
|
358
390
|
ctx,
|
|
359
391
|
provider,
|
|
360
392
|
reqBody,
|
|
361
393
|
timed.signal,
|
|
394
|
+
providerCtx,
|
|
362
395
|
);
|
|
396
|
+
observeTimingEvent({
|
|
397
|
+
observe: providerCtx.observe,
|
|
398
|
+
kind: "provider.fetch_headers",
|
|
399
|
+
requestId: providerCtx.requestId,
|
|
400
|
+
executionId: providerCtx.executionId,
|
|
401
|
+
...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
|
|
402
|
+
startedAtMs: fetchStartedMs,
|
|
403
|
+
finishedAtMs: Date.now(),
|
|
404
|
+
data: {
|
|
405
|
+
provider: provider.name,
|
|
406
|
+
model: providerCtx.target.model,
|
|
407
|
+
status: upstream.status,
|
|
408
|
+
},
|
|
409
|
+
});
|
|
363
410
|
if (!upstream.ok) {
|
|
364
411
|
const providerError = await toProviderError(provider, upstream);
|
|
365
412
|
emitObservedProgram(providerCtx.observe, "provider.response", providerError.program, {
|
|
@@ -372,7 +419,22 @@ export function createFetchProviderInvoker(
|
|
|
372
419
|
throw providerError;
|
|
373
420
|
}
|
|
374
421
|
|
|
422
|
+
const readStartedMs = Date.now();
|
|
375
423
|
const bytes = new Uint8Array(await upstream.arrayBuffer());
|
|
424
|
+
observeTimingEvent({
|
|
425
|
+
observe: providerCtx.observe,
|
|
426
|
+
kind: "provider.response_body_read",
|
|
427
|
+
requestId: providerCtx.requestId,
|
|
428
|
+
executionId: providerCtx.executionId,
|
|
429
|
+
...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
|
|
430
|
+
startedAtMs: readStartedMs,
|
|
431
|
+
finishedAtMs: Date.now(),
|
|
432
|
+
data: {
|
|
433
|
+
provider: provider.name,
|
|
434
|
+
model: providerCtx.target.model,
|
|
435
|
+
bytes: bytes.byteLength,
|
|
436
|
+
},
|
|
437
|
+
});
|
|
376
438
|
const response = parseProviderResponse(provider.style, bytes);
|
|
377
439
|
emitObservedProgram(providerCtx.observe, "provider.response", response, {
|
|
378
440
|
requestId: providerCtx.requestId,
|
|
@@ -405,18 +467,49 @@ export function createFetchProviderInvoker(
|
|
|
405
467
|
provider: provider.name,
|
|
406
468
|
model: providerCtx.target.model,
|
|
407
469
|
});
|
|
470
|
+
const encodeStartedMs = Date.now();
|
|
408
471
|
let reqBody = emitProviderRequest(provider.style, resolvedRequest);
|
|
409
472
|
if (provider.style === "chat-completions" && isStreaming(resolvedRequest)) {
|
|
410
473
|
reqBody = injectStreamUsage(reqBody);
|
|
411
474
|
}
|
|
475
|
+
observeTimingEvent({
|
|
476
|
+
observe: providerCtx.observe,
|
|
477
|
+
kind: "provider.request_encoded",
|
|
478
|
+
requestId: providerCtx.requestId,
|
|
479
|
+
executionId: providerCtx.executionId,
|
|
480
|
+
...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
|
|
481
|
+
startedAtMs: encodeStartedMs,
|
|
482
|
+
finishedAtMs: Date.now(),
|
|
483
|
+
data: {
|
|
484
|
+
provider: provider.name,
|
|
485
|
+
model: providerCtx.target.model,
|
|
486
|
+
bytes: reqBody.byteLength,
|
|
487
|
+
},
|
|
488
|
+
});
|
|
412
489
|
const reqBytes = reqBody.byteLength;
|
|
490
|
+
const fetchStartedMs = Date.now();
|
|
413
491
|
const upstream = await fetchProvider(
|
|
414
492
|
runtime,
|
|
415
493
|
ctx,
|
|
416
494
|
provider,
|
|
417
495
|
reqBody,
|
|
418
496
|
timed.signal,
|
|
497
|
+
providerCtx,
|
|
419
498
|
);
|
|
499
|
+
observeTimingEvent({
|
|
500
|
+
observe: providerCtx.observe,
|
|
501
|
+
kind: "provider.fetch_headers",
|
|
502
|
+
requestId: providerCtx.requestId,
|
|
503
|
+
executionId: providerCtx.executionId,
|
|
504
|
+
...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
|
|
505
|
+
startedAtMs: fetchStartedMs,
|
|
506
|
+
finishedAtMs: Date.now(),
|
|
507
|
+
data: {
|
|
508
|
+
provider: provider.name,
|
|
509
|
+
model: providerCtx.target.model,
|
|
510
|
+
status: upstream.status,
|
|
511
|
+
},
|
|
512
|
+
});
|
|
420
513
|
if (!upstream.ok) {
|
|
421
514
|
const providerError = await toProviderError(provider, upstream);
|
|
422
515
|
emitObservedProgram(providerCtx.observe, "provider.response", providerError.program, {
|
|
@@ -437,6 +530,8 @@ export function createFetchProviderInvoker(
|
|
|
437
530
|
}
|
|
438
531
|
|
|
439
532
|
reader = upstream.body.getReader();
|
|
533
|
+
let firstByteObserved = false;
|
|
534
|
+
const streamReadStartedMs = Date.now();
|
|
440
535
|
let lastChunkWithUsage: Program | null = null;
|
|
441
536
|
let respBytes = 0;
|
|
442
537
|
while (true) {
|
|
@@ -444,6 +539,23 @@ export function createFetchProviderInvoker(
|
|
|
444
539
|
if (done) {
|
|
445
540
|
break;
|
|
446
541
|
}
|
|
542
|
+
if (!firstByteObserved) {
|
|
543
|
+
firstByteObserved = true;
|
|
544
|
+
observeTimingEvent({
|
|
545
|
+
observe: providerCtx.observe,
|
|
546
|
+
kind: "provider.stream_first_byte",
|
|
547
|
+
requestId: providerCtx.requestId,
|
|
548
|
+
executionId: providerCtx.executionId,
|
|
549
|
+
...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
|
|
550
|
+
startedAtMs: streamReadStartedMs,
|
|
551
|
+
finishedAtMs: Date.now(),
|
|
552
|
+
data: {
|
|
553
|
+
provider: provider.name,
|
|
554
|
+
model: providerCtx.target.model,
|
|
555
|
+
bytes: value.byteLength,
|
|
556
|
+
},
|
|
557
|
+
});
|
|
558
|
+
}
|
|
447
559
|
respBytes += value.byteLength;
|
|
448
560
|
buffer += decoder.decode(value, { stream: true });
|
|
449
561
|
const split = splitSseEvents(buffer);
|
|
@@ -510,11 +622,25 @@ async function streamExecutionResponse(
|
|
|
510
622
|
responseStyle: ProviderStyle,
|
|
511
623
|
chunks: AsyncIterable<Program>,
|
|
512
624
|
resolved: ResolvedTarget,
|
|
625
|
+
observe: ExecutionRuntimeOptions["observe"] | undefined,
|
|
626
|
+
input: { requestId: string; executionId: string; startedAtMs: number },
|
|
513
627
|
cleanup?: () => Promise<void>,
|
|
514
628
|
): Promise<Response> {
|
|
515
629
|
const iterator = chunks[Symbol.asyncIterator]();
|
|
516
630
|
|
|
517
631
|
const first = await iterator.next();
|
|
632
|
+
observeTimingEvent({
|
|
633
|
+
observe,
|
|
634
|
+
kind: "response.stream_first_chunk",
|
|
635
|
+
requestId: input.requestId,
|
|
636
|
+
executionId: input.executionId,
|
|
637
|
+
startedAtMs: input.startedAtMs,
|
|
638
|
+
finishedAtMs: Date.now(),
|
|
639
|
+
data: {
|
|
640
|
+
streaming: true,
|
|
641
|
+
source: first.done ? "empty" : "chunk",
|
|
642
|
+
},
|
|
643
|
+
});
|
|
518
644
|
if (first.done) {
|
|
519
645
|
await cleanup?.();
|
|
520
646
|
const stream = new ReadableStream<Uint8Array>({
|
|
@@ -662,6 +788,7 @@ async function fetchProvider(
|
|
|
662
788
|
provider: ProviderRuntime,
|
|
663
789
|
body: Uint8Array,
|
|
664
790
|
signal: AbortSignal,
|
|
791
|
+
providerCtx: ProviderInvocationContext,
|
|
665
792
|
): Promise<Response> {
|
|
666
793
|
const headers = cloneForwardHeaders(ctx.request.headers);
|
|
667
794
|
headers.set("content-type", "application/json");
|
|
@@ -671,7 +798,24 @@ async function fetchProvider(
|
|
|
671
798
|
|
|
672
799
|
for (const driver of runtime.upstreamAuthChain) {
|
|
673
800
|
const targetCtx: TargetAuthContext = { ...ctx, providerName: provider.name };
|
|
801
|
+
const authStartedMs = Date.now();
|
|
674
802
|
const auth = await driver.collectTarget(targetCtx);
|
|
803
|
+
observeTimingEvent({
|
|
804
|
+
observe: providerCtx.observe,
|
|
805
|
+
kind: "upstream_auth.resolve_target",
|
|
806
|
+
requestId: providerCtx.requestId,
|
|
807
|
+
executionId: providerCtx.executionId,
|
|
808
|
+
...(providerCtx.parentExecutionId ? { parentExecutionId: providerCtx.parentExecutionId } : {}),
|
|
809
|
+
startedAtMs: authStartedMs,
|
|
810
|
+
finishedAtMs: Date.now(),
|
|
811
|
+
data: {
|
|
812
|
+
provider: provider.name,
|
|
813
|
+
model: providerCtx.target.model,
|
|
814
|
+
driver: driver.name,
|
|
815
|
+
source: auth?.source,
|
|
816
|
+
matched: !!auth,
|
|
817
|
+
},
|
|
818
|
+
});
|
|
675
819
|
if (!auth) {
|
|
676
820
|
continue;
|
|
677
821
|
}
|
|
@@ -77,6 +77,7 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
77
77
|
if (target.kind === "executor") {
|
|
78
78
|
const executor = await getExecutorOrThrow(target.executorId);
|
|
79
79
|
const startedMs = Date.now();
|
|
80
|
+
const startedAt = new Date(startedMs).toISOString();
|
|
80
81
|
observe(createAuditEvent({
|
|
81
82
|
kind: "executor.started",
|
|
82
83
|
requestId,
|
|
@@ -88,6 +89,7 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
88
89
|
transforms: target.transforms ?? [],
|
|
89
90
|
depth,
|
|
90
91
|
maxDepth,
|
|
92
|
+
startedAt,
|
|
91
93
|
},
|
|
92
94
|
...withParentExecutionId(parentExecutionId),
|
|
93
95
|
}));
|
|
@@ -107,12 +109,17 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
107
109
|
depth,
|
|
108
110
|
emitSuccess: true,
|
|
109
111
|
});
|
|
112
|
+
const finishedMs = Date.now();
|
|
110
113
|
observe(createAuditEvent({
|
|
111
114
|
kind: "executor.finished",
|
|
112
115
|
requestId,
|
|
113
116
|
executionId,
|
|
114
117
|
target: executionTargetAuditRef(target),
|
|
115
|
-
data: {
|
|
118
|
+
data: {
|
|
119
|
+
startedAt,
|
|
120
|
+
finishedAt: new Date(finishedMs).toISOString(),
|
|
121
|
+
durationMs: finishedMs - startedMs,
|
|
122
|
+
},
|
|
116
123
|
...withParentExecutionId(parentExecutionId),
|
|
117
124
|
}));
|
|
118
125
|
return result;
|
|
@@ -125,6 +132,7 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
125
132
|
|
|
126
133
|
const providerInvoker = getProviderInvokerOrThrow();
|
|
127
134
|
const startedMs = Date.now();
|
|
135
|
+
const startedAt = new Date(startedMs).toISOString();
|
|
128
136
|
const providerCtx = createProviderInvocationContext(
|
|
129
137
|
requestId,
|
|
130
138
|
executionId,
|
|
@@ -144,6 +152,7 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
144
152
|
transforms: target.transforms ?? [],
|
|
145
153
|
depth,
|
|
146
154
|
maxDepth,
|
|
155
|
+
startedAt,
|
|
147
156
|
},
|
|
148
157
|
...withParentExecutionId(parentExecutionId),
|
|
149
158
|
}));
|
|
@@ -160,12 +169,19 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
160
169
|
depth,
|
|
161
170
|
emitSuccess: true,
|
|
162
171
|
});
|
|
172
|
+
const finishedMs = Date.now();
|
|
163
173
|
observe(createAuditEvent({
|
|
164
174
|
kind: "provider.finished",
|
|
165
175
|
requestId,
|
|
166
176
|
executionId,
|
|
167
177
|
target: executionTargetAuditRef(target),
|
|
168
|
-
data: {
|
|
178
|
+
data: {
|
|
179
|
+
provider: target.provider ?? "",
|
|
180
|
+
model: target.model,
|
|
181
|
+
startedAt,
|
|
182
|
+
finishedAt: new Date(finishedMs).toISOString(),
|
|
183
|
+
durationMs: finishedMs - startedMs,
|
|
184
|
+
},
|
|
169
185
|
...withParentExecutionId(parentExecutionId),
|
|
170
186
|
}));
|
|
171
187
|
return result;
|
|
@@ -213,6 +229,7 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
213
229
|
if (target.kind === "executor") {
|
|
214
230
|
const executor = await getExecutorOrThrow(target.executorId);
|
|
215
231
|
const startedMs = Date.now();
|
|
232
|
+
const startedAt = new Date(startedMs).toISOString();
|
|
216
233
|
observe(createAuditEvent({
|
|
217
234
|
kind: "executor.stream_started",
|
|
218
235
|
requestId,
|
|
@@ -224,6 +241,7 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
224
241
|
transforms: target.transforms ?? [],
|
|
225
242
|
depth,
|
|
226
243
|
maxDepth,
|
|
244
|
+
startedAt,
|
|
227
245
|
},
|
|
228
246
|
...withParentExecutionId(parentExecutionId),
|
|
229
247
|
}));
|
|
@@ -245,12 +263,17 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
245
263
|
});
|
|
246
264
|
yield chunk;
|
|
247
265
|
}
|
|
266
|
+
const finishedMs = Date.now();
|
|
248
267
|
observe(createAuditEvent({
|
|
249
268
|
kind: "executor.stream_finished",
|
|
250
269
|
requestId,
|
|
251
270
|
executionId,
|
|
252
271
|
target: executionTargetAuditRef(target),
|
|
253
|
-
data: {
|
|
272
|
+
data: {
|
|
273
|
+
startedAt,
|
|
274
|
+
finishedAt: new Date(finishedMs).toISOString(),
|
|
275
|
+
durationMs: finishedMs - startedMs,
|
|
276
|
+
},
|
|
254
277
|
...withParentExecutionId(parentExecutionId),
|
|
255
278
|
}));
|
|
256
279
|
return;
|
|
@@ -263,6 +286,7 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
263
286
|
|
|
264
287
|
const providerInvoker = getProviderInvokerOrThrow();
|
|
265
288
|
const startedMs = Date.now();
|
|
289
|
+
const startedAt = new Date(startedMs).toISOString();
|
|
266
290
|
const providerCtx = createProviderInvocationContext(
|
|
267
291
|
requestId,
|
|
268
292
|
executionId,
|
|
@@ -282,6 +306,7 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
282
306
|
transforms: target.transforms ?? [],
|
|
283
307
|
depth,
|
|
284
308
|
maxDepth,
|
|
309
|
+
startedAt,
|
|
285
310
|
},
|
|
286
311
|
...withParentExecutionId(parentExecutionId),
|
|
287
312
|
}));
|
|
@@ -300,12 +325,19 @@ export function createExecutionRuntime(options: ExecutionRuntimeOptions = {}): E
|
|
|
300
325
|
});
|
|
301
326
|
yield chunk;
|
|
302
327
|
}
|
|
328
|
+
const finishedMs = Date.now();
|
|
303
329
|
observe(createAuditEvent({
|
|
304
330
|
kind: "provider.stream_finished",
|
|
305
331
|
requestId,
|
|
306
332
|
executionId,
|
|
307
333
|
target: executionTargetAuditRef(target),
|
|
308
|
-
data: {
|
|
334
|
+
data: {
|
|
335
|
+
provider: target.provider ?? "",
|
|
336
|
+
model: target.model,
|
|
337
|
+
startedAt,
|
|
338
|
+
finishedAt: new Date(finishedMs).toISOString(),
|
|
339
|
+
durationMs: finishedMs - startedMs,
|
|
340
|
+
},
|
|
309
341
|
...withParentExecutionId(parentExecutionId),
|
|
310
342
|
}));
|
|
311
343
|
} catch (error) {
|
package/src/router/mcp.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/
|
|
|
3
3
|
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
|
|
4
4
|
import type { ExecutorContext, Tool } from "@neutrome/lilsdk-ts";
|
|
5
5
|
import type { RemoteMcpServerRuntime, RouterRuntime } from "./runtime.ts";
|
|
6
|
+
import { observeTimingEvent } from "../timing.ts";
|
|
6
7
|
|
|
7
8
|
export type RemoteMcpToolCallClient = {
|
|
8
9
|
callTool(name: string, args: Record<string, unknown>): Promise<unknown>;
|
|
@@ -47,31 +48,33 @@ export function createRemoteMcpToolSet(
|
|
|
47
48
|
async execute(args: Record<string, unknown>, ctx: ExecutorContext) {
|
|
48
49
|
const connectStartedMs = Date.now();
|
|
49
50
|
const client = await pool.get(server, ctx.signal);
|
|
50
|
-
|
|
51
|
+
observeTimingEvent({
|
|
52
|
+
observe: ctx.observe,
|
|
51
53
|
kind: "remote_mcp.connect",
|
|
52
54
|
requestId: ctx.requestId,
|
|
53
55
|
executionId: `remote_mcp_connect_${server.name}_${crypto.randomUUID()}`,
|
|
54
56
|
parentExecutionId: ctx.executionId,
|
|
55
|
-
|
|
57
|
+
startedAtMs: connectStartedMs,
|
|
58
|
+
finishedAtMs: Date.now(),
|
|
56
59
|
data: {
|
|
57
60
|
server: server.name,
|
|
58
61
|
toolName: name,
|
|
59
|
-
durationMs: Date.now() - connectStartedMs,
|
|
60
62
|
},
|
|
61
63
|
});
|
|
62
64
|
const callStartedMs = Date.now();
|
|
63
65
|
const result = await client.callTool(snapshot.name, args);
|
|
64
|
-
|
|
66
|
+
observeTimingEvent({
|
|
67
|
+
observe: ctx.observe,
|
|
65
68
|
kind: "remote_mcp.tool_call",
|
|
66
69
|
requestId: ctx.requestId,
|
|
67
70
|
executionId: `remote_mcp_tool_${server.name}_${snapshot.name}_${crypto.randomUUID()}`,
|
|
68
71
|
parentExecutionId: ctx.executionId,
|
|
69
|
-
|
|
72
|
+
startedAtMs: callStartedMs,
|
|
73
|
+
finishedAtMs: Date.now(),
|
|
70
74
|
data: {
|
|
71
75
|
server: server.name,
|
|
72
76
|
toolName: name,
|
|
73
77
|
remoteToolName: snapshot.name,
|
|
74
|
-
durationMs: Date.now() - callStartedMs,
|
|
75
78
|
},
|
|
76
79
|
});
|
|
77
80
|
return stringifyMcpToolResult(result);
|
package/src/timing.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { createAuditEvent } from "./router/audit.ts";
|
|
2
|
+
import type { AuditEvent } from "./router/execution-types.ts";
|
|
3
|
+
|
|
4
|
+
export type TimingEventInput = {
|
|
5
|
+
observe?: ((event: AuditEvent) => void) | undefined;
|
|
6
|
+
kind: string;
|
|
7
|
+
requestId: string;
|
|
8
|
+
executionId: string;
|
|
9
|
+
parentExecutionId?: string | undefined;
|
|
10
|
+
startedAtMs: number;
|
|
11
|
+
finishedAtMs?: number | undefined;
|
|
12
|
+
data?: Record<string, unknown> | undefined;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export function observeTimingEvent(input: TimingEventInput): void {
|
|
16
|
+
if (!input.observe) return;
|
|
17
|
+
|
|
18
|
+
const finishedAtMs = input.finishedAtMs;
|
|
19
|
+
const data: Record<string, unknown> = {
|
|
20
|
+
...(input.data ?? {}),
|
|
21
|
+
startedAt: new Date(input.startedAtMs).toISOString(),
|
|
22
|
+
};
|
|
23
|
+
if (finishedAtMs !== undefined) {
|
|
24
|
+
data.finishedAt = new Date(finishedAtMs).toISOString();
|
|
25
|
+
data.durationMs = Math.max(0, finishedAtMs - input.startedAtMs);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
input.observe(createAuditEvent({
|
|
29
|
+
kind: input.kind,
|
|
30
|
+
requestId: input.requestId,
|
|
31
|
+
executionId: input.executionId,
|
|
32
|
+
...(input.parentExecutionId ? { parentExecutionId: input.parentExecutionId } : {}),
|
|
33
|
+
timestamp: new Date(finishedAtMs ?? input.startedAtMs).toISOString(),
|
|
34
|
+
data,
|
|
35
|
+
}));
|
|
36
|
+
}
|