@hebo-ai/gateway 0.4.0-beta.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -5
- package/dist/config.js +21 -7
- package/dist/endpoints/chat-completions/converters.d.ts +3 -3
- package/dist/endpoints/chat-completions/converters.js +16 -8
- package/dist/endpoints/chat-completions/handler.js +34 -27
- package/dist/endpoints/chat-completions/otel.d.ts +6 -0
- package/dist/endpoints/chat-completions/otel.js +127 -0
- package/dist/endpoints/embeddings/handler.js +19 -10
- package/dist/endpoints/embeddings/otel.d.ts +6 -0
- package/dist/endpoints/embeddings/otel.js +35 -0
- package/dist/endpoints/models/handler.js +3 -4
- package/dist/errors/gateway.d.ts +1 -1
- package/dist/errors/gateway.js +3 -4
- package/dist/errors/openai.js +11 -12
- package/dist/errors/utils.d.ts +3 -4
- package/dist/errors/utils.js +6 -6
- package/dist/gateway.js +1 -1
- package/dist/lifecycle.js +71 -29
- package/dist/middleware/matcher.js +1 -1
- package/dist/models/amazon/presets.d.ts +37 -37
- package/dist/models/amazon/presets.js +1 -1
- package/dist/models/anthropic/presets.d.ts +56 -56
- package/dist/models/cohere/presets.d.ts +54 -54
- package/dist/models/cohere/presets.js +2 -2
- package/dist/models/google/presets.d.ts +31 -31
- package/dist/models/google/presets.js +1 -1
- package/dist/models/meta/presets.d.ts +42 -42
- package/dist/models/openai/presets.d.ts +96 -96
- package/dist/models/openai/presets.js +1 -1
- package/dist/models/types.d.ts +1 -1
- package/dist/models/voyage/presets.d.ts +92 -92
- package/dist/models/voyage/presets.js +1 -1
- package/dist/providers/registry.js +2 -2
- package/dist/telemetry/baggage.d.ts +1 -0
- package/dist/telemetry/baggage.js +24 -0
- package/dist/telemetry/fetch.d.ts +2 -1
- package/dist/telemetry/fetch.js +13 -3
- package/dist/telemetry/gen-ai.d.ts +5 -0
- package/dist/telemetry/gen-ai.js +60 -0
- package/dist/telemetry/http.d.ts +3 -0
- package/dist/telemetry/http.js +57 -0
- package/dist/telemetry/memory.d.ts +2 -0
- package/dist/telemetry/memory.js +27 -0
- package/dist/telemetry/span.d.ts +6 -3
- package/dist/telemetry/span.js +24 -36
- package/dist/telemetry/stream.d.ts +3 -7
- package/dist/telemetry/stream.js +26 -29
- package/dist/types.d.ts +16 -15
- package/dist/utils/headers.d.ts +1 -1
- package/dist/utils/headers.js +7 -9
- package/dist/utils/request.d.ts +0 -4
- package/dist/utils/request.js +0 -9
- package/dist/utils/response.js +1 -1
- package/package.json +5 -2
- package/src/config.ts +28 -7
- package/src/endpoints/chat-completions/converters.ts +18 -11
- package/src/endpoints/chat-completions/handler.ts +46 -28
- package/src/endpoints/chat-completions/otel.ts +161 -0
- package/src/endpoints/embeddings/handler.test.ts +2 -2
- package/src/endpoints/embeddings/handler.ts +28 -10
- package/src/endpoints/embeddings/otel.ts +56 -0
- package/src/endpoints/models/handler.ts +3 -5
- package/src/errors/gateway.ts +5 -5
- package/src/errors/openai.ts +25 -17
- package/src/errors/utils.ts +6 -7
- package/src/gateway.ts +1 -1
- package/src/lifecycle.ts +85 -32
- package/src/middleware/matcher.ts +1 -1
- package/src/models/amazon/presets.ts +1 -1
- package/src/models/cohere/presets.ts +2 -2
- package/src/models/google/presets.ts +1 -1
- package/src/models/openai/presets.ts +1 -1
- package/src/models/types.ts +1 -1
- package/src/models/voyage/presets.ts +1 -1
- package/src/providers/registry.ts +2 -2
- package/src/telemetry/baggage.ts +27 -0
- package/src/telemetry/fetch.ts +15 -3
- package/src/telemetry/gen-ai.ts +88 -0
- package/src/telemetry/http.ts +65 -0
- package/src/telemetry/memory.ts +36 -0
- package/src/telemetry/span.ts +28 -40
- package/src/telemetry/stream.ts +36 -40
- package/src/types.ts +18 -18
- package/src/utils/headers.ts +8 -19
- package/src/utils/request.ts +0 -11
- package/src/utils/response.ts +1 -1
- package/dist/telemetry/otel.d.ts +0 -2
- package/dist/telemetry/otel.js +0 -50
- package/dist/telemetry/utils.d.ts +0 -4
- package/dist/telemetry/utils.js +0 -223
- package/src/telemetry/otel.ts +0 -91
- package/src/telemetry/utils.ts +0 -273
package/src/telemetry/span.ts
CHANGED
|
@@ -1,28 +1,13 @@
|
|
|
1
|
-
import type { Attributes,
|
|
1
|
+
import type { Attributes, SpanOptions, Tracer } from "@opentelemetry/api";
|
|
2
2
|
|
|
3
3
|
import { INVALID_SPAN_CONTEXT, SpanKind, SpanStatusCode, context, trace } from "@opentelemetry/api";
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
const mem = () => process?.memoryUsage?.();
|
|
5
|
+
import type { TelemetrySignalLevel } from "../types";
|
|
7
6
|
|
|
8
|
-
const
|
|
7
|
+
const DEFAULT_TRACER_NAME = "@hebo/gateway";
|
|
9
8
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
if (Object.keys(attrs).length === 0) return;
|
|
13
|
-
span.setAttributes(attrs);
|
|
14
|
-
};
|
|
15
|
-
|
|
16
|
-
const getMemoryAttributes = (): Attributes => {
|
|
17
|
-
const memory = mem();
|
|
18
|
-
if (!memory) return {};
|
|
19
|
-
|
|
20
|
-
return {
|
|
21
|
-
"process.memory.usage": memory.rss,
|
|
22
|
-
"process.memory.heap.used": memory.heapUsed,
|
|
23
|
-
"process.memory.heap.total": memory.heapTotal,
|
|
24
|
-
};
|
|
25
|
-
};
|
|
9
|
+
let spanTracer: Tracer | undefined;
|
|
10
|
+
let spanEventsEnabled = false;
|
|
26
11
|
|
|
27
12
|
const NOOP_SPAN = {
|
|
28
13
|
runWithContext: <T>(fn: () => Promise<T> | T) => fn(),
|
|
@@ -31,35 +16,38 @@ const NOOP_SPAN = {
|
|
|
31
16
|
isExisting: true,
|
|
32
17
|
};
|
|
33
18
|
|
|
34
|
-
export const
|
|
35
|
-
|
|
19
|
+
export const setSpanTracer = (tracer?: Tracer) => {
|
|
20
|
+
spanTracer = tracer ?? trace.getTracer(DEFAULT_TRACER_NAME);
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export const setSpanEventsEnabled = (level?: TelemetrySignalLevel) => {
|
|
24
|
+
spanEventsEnabled = level === "recommended" || level === "full";
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export const startSpan = (name: string, options?: SpanOptions) => {
|
|
28
|
+
if (!spanTracer) {
|
|
29
|
+
return Object.assign(trace.wrapSpanContext(INVALID_SPAN_CONTEXT), NOOP_SPAN);
|
|
30
|
+
}
|
|
36
31
|
|
|
37
32
|
const parentContext = context.active();
|
|
38
33
|
const activeSpan = trace.getActiveSpan();
|
|
39
34
|
|
|
40
|
-
const span =
|
|
35
|
+
const span = spanTracer.startSpan(
|
|
41
36
|
name,
|
|
42
37
|
{ kind: activeSpan ? SpanKind.INTERNAL : SpanKind.SERVER, ...options },
|
|
43
38
|
parentContext,
|
|
44
39
|
);
|
|
45
40
|
|
|
46
|
-
if (!span.isRecording()) {
|
|
47
|
-
return Object.assign(trace.wrapSpanContext(INVALID_SPAN_CONTEXT), NOOP_SPAN);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
maybeSetDynamicAttributes(span, getMemoryAttributes);
|
|
51
|
-
|
|
52
41
|
const runWithContext = <T>(fn: () => Promise<T> | T) =>
|
|
53
42
|
context.with(trace.setSpan(parentContext, span), fn);
|
|
54
43
|
|
|
55
44
|
const recordError = (error: unknown) => {
|
|
56
|
-
const err =
|
|
45
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
57
46
|
span.recordException(err);
|
|
58
47
|
span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
|
|
59
48
|
};
|
|
60
49
|
|
|
61
50
|
const finish = () => {
|
|
62
|
-
maybeSetDynamicAttributes(span, getMemoryAttributes);
|
|
63
51
|
span.end();
|
|
64
52
|
};
|
|
65
53
|
|
|
@@ -71,6 +59,10 @@ export const withSpan = async <T>(
|
|
|
71
59
|
run: () => Promise<T> | T,
|
|
72
60
|
options?: SpanOptions,
|
|
73
61
|
): Promise<T> => {
|
|
62
|
+
if (!spanTracer) {
|
|
63
|
+
return await run();
|
|
64
|
+
}
|
|
65
|
+
|
|
74
66
|
const started = startSpan(name, options);
|
|
75
67
|
try {
|
|
76
68
|
return await started.runWithContext(run);
|
|
@@ -83,15 +75,11 @@ export const withSpan = async <T>(
|
|
|
83
75
|
};
|
|
84
76
|
|
|
85
77
|
export const addSpanEvent = (name: string, attributes?: Attributes) => {
|
|
86
|
-
|
|
87
|
-
trace.getActiveSpan()?.addEvent(name,
|
|
78
|
+
if (!spanEventsEnabled) return;
|
|
79
|
+
trace.getActiveSpan()?.addEvent(name, attributes);
|
|
88
80
|
};
|
|
89
81
|
|
|
90
|
-
export const
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
const err = toError(error);
|
|
95
|
-
span.recordException(err);
|
|
96
|
-
span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
|
|
82
|
+
export const setSpanAttributes = (attributes?: Attributes) => {
|
|
83
|
+
if (!attributes) return;
|
|
84
|
+
trace.getActiveSpan()?.setAttributes(attributes);
|
|
97
85
|
};
|
package/src/telemetry/stream.ts
CHANGED
|
@@ -1,58 +1,51 @@
|
|
|
1
|
-
|
|
2
|
-
onComplete?: (status: number, stats: { bytes: number }) => void;
|
|
3
|
-
onError?: (error: unknown, status: number) => void;
|
|
4
|
-
};
|
|
5
|
-
|
|
6
|
-
export const instrumentStream = (
|
|
7
|
-
src: ReadableStream<Uint8Array>,
|
|
8
|
-
hooks: InstrumentStreamHooks,
|
|
9
|
-
signal?: AbortSignal,
|
|
10
|
-
): ReadableStream<Uint8Array> => {
|
|
11
|
-
const stats = { bytes: 0 };
|
|
12
|
-
let done = false;
|
|
1
|
+
import { toOpenAIError } from "#/errors/openai";
|
|
13
2
|
|
|
14
|
-
|
|
15
|
-
if (done) return;
|
|
16
|
-
done = true;
|
|
3
|
+
const isErrorChunk = (v: unknown) => v instanceof Error || !!(v as any)?.error;
|
|
17
4
|
|
|
18
|
-
|
|
5
|
+
export const wrapStream = (
|
|
6
|
+
src: ReadableStream,
|
|
7
|
+
hooks: { onDone?: (status: number, reason: unknown) => void },
|
|
8
|
+
): ReadableStream => {
|
|
9
|
+
let finished = false;
|
|
19
10
|
|
|
20
|
-
|
|
21
|
-
|
|
11
|
+
const done = (
|
|
12
|
+
reader: ReadableStreamDefaultReader,
|
|
13
|
+
controller: ReadableStreamDefaultController,
|
|
14
|
+
status: number,
|
|
15
|
+
reason?: unknown,
|
|
16
|
+
) => {
|
|
17
|
+
if (!finished) {
|
|
18
|
+
finished = true;
|
|
19
|
+
hooks.onDone?.(status, reason);
|
|
22
20
|
}
|
|
23
|
-
|
|
24
|
-
|
|
21
|
+
reader.cancel(reason).catch(() => {});
|
|
22
|
+
controller.close();
|
|
25
23
|
};
|
|
26
24
|
|
|
27
|
-
return new ReadableStream
|
|
25
|
+
return new ReadableStream({
|
|
28
26
|
async start(controller) {
|
|
29
27
|
const reader = src.getReader();
|
|
30
28
|
|
|
31
29
|
try {
|
|
32
30
|
for (;;) {
|
|
33
|
-
if (signal?.aborted) {
|
|
34
|
-
finish(499, signal.reason);
|
|
35
|
-
reader.cancel(signal.reason).catch(() => {});
|
|
36
|
-
controller.close();
|
|
37
|
-
return;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
31
|
// eslint-disable-next-line no-await-in-loop
|
|
41
|
-
const { value, done } = await reader.read();
|
|
42
|
-
if (
|
|
32
|
+
const { value, done: eof } = await reader.read();
|
|
33
|
+
if (eof) break;
|
|
43
34
|
|
|
44
|
-
|
|
45
|
-
controller.enqueue(
|
|
35
|
+
const out = isErrorChunk(value) ? toOpenAIError(value) : value;
|
|
36
|
+
controller.enqueue(out);
|
|
37
|
+
|
|
38
|
+
if (out !== value) {
|
|
39
|
+
const status = out.error?.type === "invalid_request_error" ? 422 : 502;
|
|
40
|
+
done(reader, controller, status, value);
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
46
43
|
}
|
|
47
44
|
|
|
48
|
-
|
|
49
|
-
controller.close();
|
|
45
|
+
done(reader, controller, 200);
|
|
50
46
|
} catch (err) {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
finish(status, err);
|
|
54
|
-
reader.cancel(err).catch(() => {});
|
|
55
|
-
controller.close();
|
|
47
|
+
controller.enqueue(toOpenAIError(err));
|
|
48
|
+
done(reader, controller, 502, err);
|
|
56
49
|
} finally {
|
|
57
50
|
try {
|
|
58
51
|
reader.releaseLock();
|
|
@@ -61,7 +54,10 @@ export const instrumentStream = (
|
|
|
61
54
|
},
|
|
62
55
|
|
|
63
56
|
cancel(reason) {
|
|
64
|
-
|
|
57
|
+
if (!finished) {
|
|
58
|
+
finished = true;
|
|
59
|
+
hooks.onDone?.(499, reason);
|
|
60
|
+
}
|
|
65
61
|
src.cancel(reason).catch(() => {});
|
|
66
62
|
},
|
|
67
63
|
});
|
package/src/types.ts
CHANGED
|
@@ -8,7 +8,6 @@ import type {
|
|
|
8
8
|
} from "./endpoints/chat-completions/schema";
|
|
9
9
|
import type { Embeddings, EmbeddingsBody } from "./endpoints/embeddings/schema";
|
|
10
10
|
import type { Model, ModelList } from "./endpoints/models";
|
|
11
|
-
import type { OpenAIError } from "./errors/openai";
|
|
12
11
|
import type { Logger, LoggerConfig } from "./logger";
|
|
13
12
|
import type { ModelCatalog, ModelId } from "./models/types";
|
|
14
13
|
import type { ProviderId, ProviderRegistry } from "./providers/types";
|
|
@@ -76,7 +75,7 @@ export type GatewayContext = {
|
|
|
76
75
|
*/
|
|
77
76
|
result?:
|
|
78
77
|
| ChatCompletions
|
|
79
|
-
| ReadableStream<ChatCompletionsChunk |
|
|
78
|
+
| ReadableStream<ChatCompletionsChunk | Error>
|
|
80
79
|
| Embeddings
|
|
81
80
|
| Model
|
|
82
81
|
| ModelList;
|
|
@@ -84,10 +83,6 @@ export type GatewayContext = {
|
|
|
84
83
|
* Response object returned by the handler.
|
|
85
84
|
*/
|
|
86
85
|
response?: Response;
|
|
87
|
-
/**
|
|
88
|
-
* Structured object result for streaming requests. Only available at the end of the stream.
|
|
89
|
-
*/
|
|
90
|
-
streamResult?: ChatCompletions;
|
|
91
86
|
};
|
|
92
87
|
|
|
93
88
|
/**
|
|
@@ -154,11 +149,9 @@ export type GatewayHooks = {
|
|
|
154
149
|
) =>
|
|
155
150
|
| void
|
|
156
151
|
| ChatCompletions
|
|
157
|
-
| ReadableStream<ChatCompletionsChunk |
|
|
152
|
+
| ReadableStream<ChatCompletionsChunk | Error>
|
|
158
153
|
| Embeddings
|
|
159
|
-
| Promise<
|
|
160
|
-
void | ChatCompletions | ReadableStream<ChatCompletionsChunk | OpenAIError> | Embeddings
|
|
161
|
-
>;
|
|
154
|
+
| Promise<void | ChatCompletions | ReadableStream<ChatCompletionsChunk | Error> | Embeddings>;
|
|
162
155
|
/**
|
|
163
156
|
* Runs after the lifecycle has produced the final Response.
|
|
164
157
|
* @returns Replacement Response, or undefined to keep original.
|
|
@@ -166,6 +159,8 @@ export type GatewayHooks = {
|
|
|
166
159
|
onResponse?: (ctx: OnResponseHookContext) => void | Response | Promise<void | Response>;
|
|
167
160
|
};
|
|
168
161
|
|
|
162
|
+
export type TelemetrySignalLevel = "off" | "required" | "recommended" | "full";
|
|
163
|
+
|
|
169
164
|
/**
|
|
170
165
|
* Main configuration object for the gateway.
|
|
171
166
|
*/
|
|
@@ -186,6 +181,10 @@ export type GatewayConfig = {
|
|
|
186
181
|
* Optional lifecycle hooks for routing, auth, and response shaping.
|
|
187
182
|
*/
|
|
188
183
|
hooks?: GatewayHooks;
|
|
184
|
+
/**
|
|
185
|
+
* Preferred logger configuration: custom logger or default logger settings.
|
|
186
|
+
*/
|
|
187
|
+
logger?: Logger | LoggerConfig | null;
|
|
189
188
|
/**
|
|
190
189
|
* Optional AI SDK telemetry configuration.
|
|
191
190
|
*/
|
|
@@ -200,17 +199,18 @@ export type GatewayConfig = {
|
|
|
200
199
|
*/
|
|
201
200
|
tracer?: Tracer;
|
|
202
201
|
/**
|
|
203
|
-
*
|
|
204
|
-
* -
|
|
202
|
+
* Telemetry signal levels by namespace.
|
|
203
|
+
* - off: disable the namespace
|
|
204
|
+
* - required: minimal baseline
|
|
205
205
|
* - recommended: practical defaults
|
|
206
|
-
* - full: include all available
|
|
206
|
+
* - full: include all available details
|
|
207
207
|
*/
|
|
208
|
-
|
|
208
|
+
signals?: {
|
|
209
|
+
gen_ai?: TelemetrySignalLevel;
|
|
210
|
+
http?: TelemetrySignalLevel;
|
|
211
|
+
hebo?: TelemetrySignalLevel;
|
|
212
|
+
};
|
|
209
213
|
};
|
|
210
|
-
/**
|
|
211
|
-
* Preferred logger configuration: custom logger or default logger settings.
|
|
212
|
-
*/
|
|
213
|
-
logger?: Logger | LoggerConfig | null;
|
|
214
214
|
};
|
|
215
215
|
|
|
216
216
|
export const kParsed = Symbol("hebo.gateway.parsed");
|
package/src/utils/headers.ts
CHANGED
|
@@ -1,32 +1,21 @@
|
|
|
1
1
|
export const REQUEST_ID_HEADER = "x-request-id";
|
|
2
2
|
|
|
3
|
-
type HeaderSource =
|
|
4
|
-
| string
|
|
5
|
-
| URL
|
|
6
|
-
| Headers
|
|
7
|
-
| Request
|
|
8
|
-
| Response
|
|
9
|
-
| RequestInit
|
|
10
|
-
| ResponseInit
|
|
11
|
-
| HeadersInit
|
|
12
|
-
| undefined;
|
|
3
|
+
type HeaderSource = Request | ResponseInit | undefined;
|
|
13
4
|
|
|
14
5
|
export const resolveRequestId = (source: HeaderSource): string | undefined => {
|
|
15
|
-
if (!source
|
|
6
|
+
if (!source) return undefined;
|
|
16
7
|
|
|
17
|
-
if (source instanceof Request
|
|
8
|
+
if (source instanceof Request) {
|
|
18
9
|
return source.headers.get(REQUEST_ID_HEADER) ?? undefined;
|
|
19
10
|
}
|
|
20
11
|
|
|
21
|
-
const headers =
|
|
22
|
-
if (!headers
|
|
12
|
+
const headers = source.headers;
|
|
13
|
+
if (!headers) return undefined;
|
|
23
14
|
|
|
24
|
-
if (
|
|
25
|
-
return (
|
|
15
|
+
if (headers instanceof Headers) {
|
|
16
|
+
return headers.get(REQUEST_ID_HEADER) ?? undefined;
|
|
26
17
|
}
|
|
27
18
|
|
|
28
|
-
if (headers instanceof Headers) return headers.get(REQUEST_ID_HEADER) ?? undefined;
|
|
29
|
-
|
|
30
19
|
if (Array.isArray(headers)) {
|
|
31
20
|
for (const [key, value] of headers) {
|
|
32
21
|
if (key.toLowerCase() === REQUEST_ID_HEADER) return value;
|
|
@@ -34,5 +23,5 @@ export const resolveRequestId = (source: HeaderSource): string | undefined => {
|
|
|
34
23
|
return undefined;
|
|
35
24
|
}
|
|
36
25
|
|
|
37
|
-
return
|
|
26
|
+
return headers[REQUEST_ID_HEADER];
|
|
38
27
|
};
|
package/src/utils/request.ts
CHANGED
|
@@ -18,17 +18,6 @@ export const prepareRequestHeaders = (request: Request) => {
|
|
|
18
18
|
return headers;
|
|
19
19
|
};
|
|
20
20
|
|
|
21
|
-
export const prepareRequestBody = async (request: Request) => {
|
|
22
|
-
let requestBytes = 0;
|
|
23
|
-
let body: ArrayBuffer | undefined;
|
|
24
|
-
if (request.body) {
|
|
25
|
-
body = await request.arrayBuffer();
|
|
26
|
-
requestBytes = body.byteLength;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
return { body, requestBytes };
|
|
30
|
-
};
|
|
31
|
-
|
|
32
21
|
export const prepareForwardHeaders = (request: Request): Record<string, string> => {
|
|
33
22
|
const userAgent = request.headers.get("user-agent");
|
|
34
23
|
const appendedUserAgent = userAgent
|
package/src/utils/response.ts
CHANGED
|
@@ -16,7 +16,7 @@ class JsonToSseTransformStream extends TransformStream<unknown, string> {
|
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
export const prepareResponseInit = (request: Request): ResponseInit => ({
|
|
19
|
-
headers: { [REQUEST_ID_HEADER]: resolveRequestId(request
|
|
19
|
+
headers: { [REQUEST_ID_HEADER]: resolveRequestId(request)! },
|
|
20
20
|
});
|
|
21
21
|
|
|
22
22
|
export const mergeResponseInit = (
|
package/dist/telemetry/otel.d.ts
DELETED
package/dist/telemetry/otel.js
DELETED
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
import { SpanStatusCode } from "@opentelemetry/api";
|
|
2
|
-
import { initFetch } from "./fetch";
|
|
3
|
-
import { startSpan } from "./span";
|
|
4
|
-
import { instrumentStream } from "./stream";
|
|
5
|
-
import { getAIAttributes, getBaggageAttributes, getRequestAttributes, getResponseAttributes, } from "./utils";
|
|
6
|
-
export const withOtel = (run, config) => async (ctx) => {
|
|
7
|
-
const requestStart = performance.now();
|
|
8
|
-
const aiSpan = startSpan(ctx.request.url, undefined, config.telemetry?.tracer);
|
|
9
|
-
initFetch();
|
|
10
|
-
const endAiSpan = (status, stats) => {
|
|
11
|
-
const attrs = getAIAttributes(ctx.body, ctx.streamResult ?? ctx.result, config.telemetry?.attributes, ctx.resolvedProviderId);
|
|
12
|
-
attrs["gen_ai.server.request.duration"] = Number(((performance.now() - requestStart) / 1000).toFixed(4));
|
|
13
|
-
if (!aiSpan.isExisting) {
|
|
14
|
-
Object.assign(attrs, getRequestAttributes(ctx.request, config.telemetry?.attributes), getResponseAttributes(ctx.response, config.telemetry?.attributes));
|
|
15
|
-
}
|
|
16
|
-
Object.assign(attrs, getBaggageAttributes(ctx.request));
|
|
17
|
-
if (config.telemetry?.attributes !== "required") {
|
|
18
|
-
attrs["http.request.body.size"] = Number(ctx.request.headers.get("content-length") || 0);
|
|
19
|
-
attrs["http.response.body.size"] =
|
|
20
|
-
stats?.bytes ?? Number(attrs["http.response.header.content-length"] || 0);
|
|
21
|
-
}
|
|
22
|
-
if (config.telemetry?.attributes === "full") {
|
|
23
|
-
attrs["http.request.body"] = JSON.stringify(ctx.body);
|
|
24
|
-
}
|
|
25
|
-
const realStatus = status === 200 ? (ctx.response?.status ?? status) : status;
|
|
26
|
-
attrs["http.response.status_code_effective"] = realStatus;
|
|
27
|
-
aiSpan.setStatus({ code: realStatus >= 500 ? SpanStatusCode.ERROR : SpanStatusCode.OK });
|
|
28
|
-
if (ctx.operation && ctx.modelId) {
|
|
29
|
-
aiSpan.updateName(`${ctx.operation} ${ctx.modelId}`);
|
|
30
|
-
}
|
|
31
|
-
else if (ctx.operation) {
|
|
32
|
-
aiSpan.updateName(`${ctx.operation}`);
|
|
33
|
-
}
|
|
34
|
-
aiSpan.setAttributes(attrs);
|
|
35
|
-
aiSpan.finish();
|
|
36
|
-
};
|
|
37
|
-
await aiSpan.runWithContext(() => run(ctx));
|
|
38
|
-
if (ctx.response.body instanceof ReadableStream) {
|
|
39
|
-
const instrumented = instrumentStream(ctx.response.body, {
|
|
40
|
-
onComplete: (status, params) => endAiSpan(status, params),
|
|
41
|
-
}, ctx.request.signal);
|
|
42
|
-
ctx.response = new Response(instrumented, {
|
|
43
|
-
status: ctx.response.status,
|
|
44
|
-
statusText: ctx.response.statusText,
|
|
45
|
-
headers: ctx.response.headers,
|
|
46
|
-
});
|
|
47
|
-
return;
|
|
48
|
-
}
|
|
49
|
-
endAiSpan(ctx.response.status);
|
|
50
|
-
};
|
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
export declare const getRequestAttributes: (request?: Request, attributesLevel?: string) => {};
|
|
2
|
-
export declare const getAIAttributes: (body?: object, result?: object, attributesLevel?: string, providerName?: string) => {};
|
|
3
|
-
export declare const getResponseAttributes: (response?: Response, attributesLevel?: string) => {};
|
|
4
|
-
export declare const getBaggageAttributes: (request?: Request) => Record<string, string>;
|
package/dist/telemetry/utils.js
DELETED
|
@@ -1,223 +0,0 @@
|
|
|
1
|
-
import { resolveRequestId } from "../utils/headers";
|
|
2
|
-
const DEFAULT_ATTRIBUTES_LEVEL = "recommended";
|
|
3
|
-
const HEBO_BAGGAGE_PREFIX = "hebo.";
|
|
4
|
-
const toTextPart = (content) => ({ type: "text", content });
|
|
5
|
-
const toMessageParts = (message) => {
|
|
6
|
-
if (message.role === "assistant") {
|
|
7
|
-
const parts = [];
|
|
8
|
-
if (typeof message.content === "string")
|
|
9
|
-
parts.push(toTextPart(message.content));
|
|
10
|
-
if (Array.isArray(message.tool_calls)) {
|
|
11
|
-
for (const call of message.tool_calls) {
|
|
12
|
-
parts.push({
|
|
13
|
-
type: "tool_call",
|
|
14
|
-
id: call.id,
|
|
15
|
-
name: call.function.name,
|
|
16
|
-
arguments: call.function.arguments,
|
|
17
|
-
});
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
return parts;
|
|
21
|
-
}
|
|
22
|
-
if (message.role === "tool") {
|
|
23
|
-
return [{ type: "tool_call_response", id: message.tool_call_id, content: message.content }];
|
|
24
|
-
}
|
|
25
|
-
if (message.role === "user") {
|
|
26
|
-
const parts = [];
|
|
27
|
-
if (typeof message.content === "string")
|
|
28
|
-
parts.push(toTextPart(message.content));
|
|
29
|
-
if (Array.isArray(message.content)) {
|
|
30
|
-
for (const part of message.content) {
|
|
31
|
-
if (part.type === "text") {
|
|
32
|
-
parts.push(toTextPart(part.text));
|
|
33
|
-
}
|
|
34
|
-
else if (part.type === "image_url") {
|
|
35
|
-
parts.push({ type: "image", content: part.image_url.url });
|
|
36
|
-
}
|
|
37
|
-
else {
|
|
38
|
-
parts.push({
|
|
39
|
-
type: "file",
|
|
40
|
-
// FUTURE: optionally expose safe metadata without raw binary payloads.
|
|
41
|
-
content: part.file.filename ?? "[REDACTED_BINARY_DATA]",
|
|
42
|
-
media_type: part.file.media_type,
|
|
43
|
-
});
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
return parts;
|
|
48
|
-
}
|
|
49
|
-
return [];
|
|
50
|
-
};
|
|
51
|
-
export const getRequestAttributes = (request, attributesLevel = DEFAULT_ATTRIBUTES_LEVEL) => {
|
|
52
|
-
if (!request)
|
|
53
|
-
return {};
|
|
54
|
-
let url;
|
|
55
|
-
try {
|
|
56
|
-
// FUTURE: use URL from lifecycle
|
|
57
|
-
url = new URL(request.url);
|
|
58
|
-
}
|
|
59
|
-
catch { }
|
|
60
|
-
const attrs = {
|
|
61
|
-
"http.request.method": request.method,
|
|
62
|
-
"url.full": request.url,
|
|
63
|
-
"url.path": url?.pathname,
|
|
64
|
-
"url.scheme": url?.protocol.replace(":", ""),
|
|
65
|
-
"server.address": url?.hostname,
|
|
66
|
-
"server.port": url
|
|
67
|
-
? url.port
|
|
68
|
-
? Number(url.port)
|
|
69
|
-
: url.protocol === "https:"
|
|
70
|
-
? 443
|
|
71
|
-
: 80
|
|
72
|
-
: undefined,
|
|
73
|
-
};
|
|
74
|
-
if (attributesLevel !== "required") {
|
|
75
|
-
Object.assign(attrs, {
|
|
76
|
-
"http.request.id": resolveRequestId(request),
|
|
77
|
-
"user_agent.original": request.headers.get("user-agent") ?? undefined,
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
|
-
if (attributesLevel === "full") {
|
|
81
|
-
Object.assign(attrs, {
|
|
82
|
-
// FUTURE: "url.query"
|
|
83
|
-
"http.request.header.content-type": [request.headers.get("content-type") ?? undefined],
|
|
84
|
-
"http.request.header.content-length": [request.headers.get("content-length") ?? undefined],
|
|
85
|
-
// FUTURE: "client.address"
|
|
86
|
-
});
|
|
87
|
-
}
|
|
88
|
-
return attrs;
|
|
89
|
-
};
|
|
90
|
-
export const getAIAttributes = (body, result, attributesLevel = DEFAULT_ATTRIBUTES_LEVEL, providerName) => {
|
|
91
|
-
if (!body && !result)
|
|
92
|
-
return {};
|
|
93
|
-
const isChat = !!body && "messages" in body;
|
|
94
|
-
const isEmbeddings = !!body && "input" in body;
|
|
95
|
-
const attrs = {
|
|
96
|
-
"gen_ai.operation.name": isEmbeddings ? "embeddings" : isChat ? "chat" : undefined,
|
|
97
|
-
"gen_ai.output.type": isEmbeddings ? "embedding" : isChat ? "text" : undefined,
|
|
98
|
-
"gen_ai.request.model": body && "model" in body ? body.model : undefined,
|
|
99
|
-
"gen_ai.provider.name": providerName,
|
|
100
|
-
};
|
|
101
|
-
if (isChat) {
|
|
102
|
-
if (body) {
|
|
103
|
-
const inputs = body;
|
|
104
|
-
if (inputs.seed !== undefined) {
|
|
105
|
-
Object.assign(attrs, { "gen_ai.request.seed": inputs.seed });
|
|
106
|
-
}
|
|
107
|
-
if (attributesLevel !== "required") {
|
|
108
|
-
Object.assign(attrs, {
|
|
109
|
-
"gen_ai.request.stream": inputs.stream,
|
|
110
|
-
"gen_ai.request.frequency_penalty": inputs.frequency_penalty,
|
|
111
|
-
"gen_ai.request.max_tokens": inputs.max_completion_tokens,
|
|
112
|
-
"gen_ai.request.presence_penalty": inputs.presence_penalty,
|
|
113
|
-
"gen_ai.request.stop_sequences": inputs.stop
|
|
114
|
-
? Array.isArray(inputs.stop)
|
|
115
|
-
? inputs.stop
|
|
116
|
-
: [inputs.stop]
|
|
117
|
-
: undefined,
|
|
118
|
-
"gen_ai.request.temperature": inputs.temperature,
|
|
119
|
-
"gen_ai.request.top_p": inputs.top_p,
|
|
120
|
-
});
|
|
121
|
-
}
|
|
122
|
-
if (attributesLevel === "full") {
|
|
123
|
-
Object.assign(attrs, {
|
|
124
|
-
// FUTURE: only construct once
|
|
125
|
-
"gen_ai.system_instructions": inputs.messages
|
|
126
|
-
.filter((m) => m.role === "system")
|
|
127
|
-
.map((m) => JSON.stringify({ parts: [toTextPart(m.content)] })),
|
|
128
|
-
"gen_ai.input.messages": inputs.messages
|
|
129
|
-
.filter((m) => m.role !== "system")
|
|
130
|
-
.map((m) => JSON.stringify({ role: m.role, parts: toMessageParts(m) })),
|
|
131
|
-
"gen_ai.tool.definitions": JSON.stringify(inputs.tools),
|
|
132
|
-
});
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
// FUTURE: implement streaming
|
|
136
|
-
if (result && !(result instanceof ReadableStream)) {
|
|
137
|
-
const completions = result;
|
|
138
|
-
Object.assign(attrs, {
|
|
139
|
-
"gen_ai.response.model": completions.model,
|
|
140
|
-
"gen_ai.response.id": completions.id,
|
|
141
|
-
});
|
|
142
|
-
if (attributesLevel !== "required") {
|
|
143
|
-
Object.assign(attrs, {
|
|
144
|
-
"gen_ai.response.finish_reasons": completions.choices?.map((c) => c.finish_reason),
|
|
145
|
-
"gen_ai.usage.total_tokens": completions.usage?.total_tokens,
|
|
146
|
-
"gen_ai.usage.input_tokens": completions.usage?.prompt_tokens,
|
|
147
|
-
"gen_ai.usage.cached_tokens": completions.usage?.prompt_tokens_details?.cached_tokens,
|
|
148
|
-
"gen_ai.usage.output_tokens": completions.usage?.completion_tokens,
|
|
149
|
-
"gen_ai.usage.reasoning_tokens": completions.usage?.completion_tokens_details?.reasoning_tokens,
|
|
150
|
-
});
|
|
151
|
-
}
|
|
152
|
-
if (attributesLevel === "full") {
|
|
153
|
-
Object.assign(attrs, {
|
|
154
|
-
"gen_ai.output.messages": completions.choices?.map((c) => JSON.stringify({
|
|
155
|
-
role: c.message.role,
|
|
156
|
-
parts: toMessageParts(c.message),
|
|
157
|
-
finish_reason: c.finish_reason,
|
|
158
|
-
})),
|
|
159
|
-
});
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
if (isEmbeddings) {
|
|
164
|
-
if (body) {
|
|
165
|
-
const inputs = body;
|
|
166
|
-
if (attributesLevel !== "required") {
|
|
167
|
-
Object.assign(attrs, {
|
|
168
|
-
"gen_ai.embeddings.dimension.count": inputs.dimensions,
|
|
169
|
-
});
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
if (result) {
|
|
173
|
-
const embeddings = result;
|
|
174
|
-
Object.assign(attrs, {
|
|
175
|
-
"gen_ai.response.model": embeddings.model,
|
|
176
|
-
});
|
|
177
|
-
if (attributesLevel !== "required") {
|
|
178
|
-
Object.assign(attrs, {
|
|
179
|
-
"gen_ai.usage.input_tokens": embeddings.usage?.prompt_tokens,
|
|
180
|
-
"gen_ai.usage.total_tokens": embeddings.usage?.total_tokens,
|
|
181
|
-
});
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
return attrs;
|
|
186
|
-
};
|
|
187
|
-
export const getResponseAttributes = (response, attributesLevel = DEFAULT_ATTRIBUTES_LEVEL) => {
|
|
188
|
-
if (!response)
|
|
189
|
-
return {};
|
|
190
|
-
const attrs = {
|
|
191
|
-
"http.response.status_code": response.status,
|
|
192
|
-
};
|
|
193
|
-
if (attributesLevel === "full") {
|
|
194
|
-
Object.assign(attrs, {
|
|
195
|
-
"http.response.header.content-type": [response.headers.get("content-type") ?? undefined],
|
|
196
|
-
"http.response.header.content-length": [response.headers.get("content-length") ?? undefined],
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
return attrs;
|
|
200
|
-
};
|
|
201
|
-
export const getBaggageAttributes = (request) => {
|
|
202
|
-
const h = request?.headers.get("baggage");
|
|
203
|
-
if (!h)
|
|
204
|
-
return {};
|
|
205
|
-
const attrs = {};
|
|
206
|
-
for (const part of h.split(",")) {
|
|
207
|
-
const [k, v] = part.trim().split("=", 2);
|
|
208
|
-
if (!k || !v)
|
|
209
|
-
continue;
|
|
210
|
-
const [rawValue] = v.split(";", 1);
|
|
211
|
-
if (!rawValue)
|
|
212
|
-
continue;
|
|
213
|
-
let value = rawValue;
|
|
214
|
-
try {
|
|
215
|
-
value = decodeURIComponent(rawValue);
|
|
216
|
-
}
|
|
217
|
-
catch { }
|
|
218
|
-
if (k.startsWith(HEBO_BAGGAGE_PREFIX)) {
|
|
219
|
-
attrs[k.slice(HEBO_BAGGAGE_PREFIX.length)] = value;
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
return attrs;
|
|
223
|
-
};
|