@juspay/neurolink 12.58.1 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -3
- package/dist/auth/anthropicOAuth.js +3 -3
- package/dist/auth/middleware/AuthMiddleware.d.ts +1 -1
- package/dist/auth/sessionManager.js +4 -4
- package/dist/auth/tokenStore.js +4 -4
- package/dist/browser/neurolink.min.js +474 -473
- package/dist/cli/commands/observability.d.ts +1 -6
- package/dist/cli/commands/observability.js +0 -106
- package/dist/cli/commands/proxy.js +9 -18
- package/dist/cli/commands/telemetry.d.ts +3 -13
- package/dist/cli/commands/telemetry.js +7 -354
- package/dist/cli/loop/session.js +1 -1
- package/dist/client/httpClient.js +1 -1
- package/dist/client/sseClient.js +1 -1
- package/dist/client/streamingClient.js +2 -2
- package/dist/client/wsClient.js +1 -1
- package/dist/constants/langfuse.d.ts +25 -0
- package/dist/constants/langfuse.js +38 -0
- package/dist/constants/telemetry.d.ts +20 -0
- package/dist/constants/telemetry.js +34 -0
- package/dist/core/baseProvider.d.ts +2 -1
- package/dist/core/baseProvider.js +4 -11
- package/dist/core/conversationMemoryManager.js +5 -8
- package/dist/core/factory.js +1 -1
- package/dist/core/geminiLoopAdapter.d.ts +8 -1
- package/dist/core/geminiLoopAdapter.js +25 -1
- package/dist/core/loopEngine.js +69 -40
- package/dist/core/modules/MessageBuilder.js +2 -2
- package/dist/core/modules/StreamHandler.js +2 -2
- package/dist/core/modules/TelemetryHandler.d.ts +2 -18
- package/dist/core/modules/TelemetryHandler.js +1 -66
- package/dist/core/modules/ToolsManager.js +40 -31
- package/dist/core/nativeGenerateLoop.js +78 -8
- package/dist/core/redisConversationMemoryManager.js +8 -15
- package/dist/core/toolExecutionGuards.js +1 -13
- package/dist/index.d.ts +5 -12
- package/dist/index.js +5 -37
- package/dist/mcp/batching/requestBatcher.js +1 -1
- package/dist/mcp/externalServerManager.js +3 -21
- package/dist/mcp/toolDiscoveryService.js +4 -16
- package/dist/mcp/toolRegistry.js +2 -5
- package/dist/memory/memoryRetrievalTools.js +1 -1
- package/dist/neurolink.d.ts +16 -27
- package/dist/neurolink.js +788 -792
- package/dist/observability/exporterSpanProcessor.d.ts +50 -0
- package/dist/observability/exporterSpanProcessor.js +181 -0
- package/dist/observability/exporters/langfuseExporter.d.ts +13 -34
- package/dist/observability/exporters/langfuseExporter.js +101 -154
- package/dist/observability/index.d.ts +1 -0
- package/dist/observability/index.js +1 -0
- package/dist/observability/langfuseMasking.d.ts +8 -0
- package/dist/observability/langfuseMasking.js +47 -0
- package/dist/observability/otlpEncoding.d.ts +20 -0
- package/dist/observability/otlpEncoding.js +85 -0
- package/dist/providers/amazonBedrock/client.js +142 -176
- package/dist/providers/amazonBedrock/loopAdapter.js +14 -1
- package/dist/providers/amazonSagemaker.js +118 -80
- package/dist/providers/anthropic/client.js +3 -52
- package/dist/providers/anthropic/loopAdapter.js +24 -1
- package/dist/providers/anthropic/rateLimitCapture.js +9 -12
- package/dist/providers/googleAiStudio/client.d.ts +4 -5
- package/dist/providers/googleAiStudio/client.js +38 -40
- package/dist/providers/googleVertex/client.d.ts +2 -7
- package/dist/providers/googleVertex/client.js +282 -378
- package/dist/providers/googleVertex/constants.js +2 -2
- package/dist/providers/litellm/client.d.ts +2 -9
- package/dist/providers/litellm/client.js +1 -63
- package/dist/providers/openAI/client.d.ts +1 -8
- package/dist/providers/openAI/client.js +0 -65
- package/dist/providers/openaiChatCompletionsBase.d.ts +1 -0
- package/dist/providers/openaiChatCompletionsBase.js +61 -4
- package/dist/providers/replicate.js +22 -4
- package/dist/proxy/codexUsage.js +7 -0
- package/dist/proxy/proxyFetch.js +19 -25
- package/dist/proxy/proxyTracer.d.ts +14 -7
- package/dist/proxy/proxyTracer.js +129 -83
- package/dist/proxy/proxyTranslationEngine.js +0 -1
- package/dist/proxy/requestLogger.d.ts +1 -1
- package/dist/proxy/requestLogger.js +39 -45
- package/dist/server/middleware/common.js +1 -1
- package/dist/server/routes/agentRoutes.js +4 -4
- package/dist/server/routes/claudeProxyRoutes.js +99 -196
- package/dist/server/routes/codexProxyRoutes.js +621 -591
- package/dist/server/routes/healthRoutes.js +6 -6
- package/dist/server/routes/mcpRoutes.js +7 -7
- package/dist/server/routes/memoryRoutes.js +7 -7
- package/dist/server/routes/openApiRoutes.js +3 -3
- package/dist/server/routes/toolRoutes.js +5 -5
- package/dist/server/routes/transcribeRoutes.js +1 -1
- package/dist/server/utils/validation.d.ts +3 -3
- package/dist/services/server/ai/observability/instrumentation.d.ts +34 -201
- package/dist/services/server/ai/observability/instrumentation.js +220 -1267
- package/dist/services/server/ai/observability/langfuseSetup.d.ts +14 -0
- package/dist/services/server/ai/observability/langfuseSetup.js +55 -0
- package/dist/services/server/ai/observability/otlpSetup.d.ts +16 -0
- package/dist/services/server/ai/observability/otlpSetup.js +58 -0
- package/dist/session/globalSessionState.js +0 -11
- package/dist/telemetry/attributes.d.ts +23 -46
- package/dist/telemetry/attributes.js +76 -162
- package/dist/telemetry/chatMessages.d.ts +13 -0
- package/dist/telemetry/chatMessages.js +59 -0
- package/dist/telemetry/index.d.ts +5 -19
- package/dist/telemetry/index.js +5 -26
- package/dist/telemetry/observation.d.ts +37 -0
- package/dist/telemetry/observation.js +112 -0
- package/dist/telemetry/observationNames.d.ts +18 -0
- package/dist/telemetry/observationNames.js +24 -0
- package/dist/telemetry/spanSanitizer.d.ts +39 -0
- package/dist/telemetry/spanSanitizer.js +92 -0
- package/dist/telemetry/spanSerialization.d.ts +13 -0
- package/dist/telemetry/spanSerialization.js +46 -0
- package/dist/telemetry/tracers.d.ts +37 -18
- package/dist/telemetry/tracers.js +97 -19
- package/dist/telemetry/withSpan.d.ts +47 -25
- package/dist/telemetry/withSpan.js +158 -71
- package/dist/types/cli.d.ts +0 -11
- package/dist/types/decision.d.ts +3 -0
- package/dist/types/exporter.d.ts +54 -0
- package/dist/types/exporter.js +0 -4
- package/dist/types/generate.d.ts +4 -1
- package/dist/types/isolatedAgent.d.ts +5 -4
- package/dist/types/loopEngine.d.ts +18 -28
- package/dist/types/observability.d.ts +127 -160
- package/dist/types/providers.d.ts +15 -0
- package/dist/types/proxy.d.ts +41 -2
- package/dist/utils/anthropicTraceSanitizer.d.ts +2 -2
- package/dist/utils/anthropicTraceSanitizer.js +3 -1
- package/dist/utils/conversationMemory.js +4 -4
- package/dist/utils/fileDetector.js +1 -1
- package/dist/utils/messageBuilder.js +1 -1
- package/dist/utils/noOutputSentinel.d.ts +4 -16
- package/dist/utils/noOutputSentinel.js +12 -28
- package/dist/utils/providerRetry.js +4 -4
- package/dist/voice/livekit/brain.js +1 -1
- package/dist/voice/livekit/realtimeMemory.d.ts +1 -1
- package/dist/voice/livekit/realtimeMemory.js +66 -1
- package/dist/voice/livekit/realtimeVoiceAgent.js +1 -1
- package/docs-site/static/search-index.json +42 -63
- package/package.json +10 -4
- package/dist/telemetry/telemetryService.d.ts +0 -79
- package/dist/telemetry/telemetryService.js +0 -572
- package/dist/utils/observabilityHelpers.d.ts +0 -27
- package/dist/utils/observabilityHelpers.js +0 -47
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { scrubbedTracer } from "../../telemetry/tracers.js";
|
|
2
2
|
export const MAX_IMAGE_DOWNLOAD_BYTES = 10 * 1024 * 1024;
|
|
3
|
-
export const streamTracer =
|
|
3
|
+
export const streamTracer = scrubbedTracer("provider.vertex");
|
|
4
4
|
export let cachedCredentialsPath = null;
|
|
5
5
|
export const VERTEX_MODEL_ALIASES = {
|
|
6
6
|
// Claude 4.x shorthand aliases → versioned names
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AIProviderName } from "../../constants/enums.js";
|
|
2
|
-
import type { EmbedInput, OpenAICompatBuildBodyArgs
|
|
2
|
+
import type { EmbedInput, OpenAICompatBuildBodyArgs } from "../../types/index.js";
|
|
3
3
|
import { OpenAIChatCompletionsProvider } from "../openaiChatCompletionsBase.js";
|
|
4
4
|
/** Test hook: reset the discovery caches (state is module-global). */
|
|
5
5
|
export declare function clearLiteLLMModelLimitsCache(): void;
|
|
@@ -28,8 +28,7 @@ export declare function ensureLiteLLMModelLimits(baseURL: string, apiKey: string
|
|
|
28
28
|
*
|
|
29
29
|
* All request/stream/tool-loop orchestration lives in
|
|
30
30
|
* `OpenAIChatCompletionsProvider`. This class adds LiteLLM-specific
|
|
31
|
-
* behaviour:
|
|
32
|
-
* maxTokens skip (`adjustBuildBodyOptions`), ModelAccessDeniedError on
|
|
31
|
+
* behaviour: Gemini 2.5 maxTokens skip (`adjustBuildBodyOptions`), ModelAccessDeniedError on
|
|
33
32
|
* 403, 10-minute model cache (`getAvailableModels`), `LITELLM_FALLBACK_MODELS`
|
|
34
33
|
* env-driven fallback list, and native `/v1/embeddings`.
|
|
35
34
|
*/
|
|
@@ -70,12 +69,6 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
|
|
|
70
69
|
* both streaming and non-streaming paths.
|
|
71
70
|
*/
|
|
72
71
|
protected adjustBuildBodyOptions(modelId: string, opts: OpenAICompatBuildBodyArgs["options"]): OpenAICompatBuildBodyArgs["options"];
|
|
73
|
-
/**
|
|
74
|
-
* Wrap the stream in an OTel span to capture provider-level latency,
|
|
75
|
-
* token usage, finish reason, and cost. Matches the pre-migration
|
|
76
|
-
* behaviour where streamText was wrapped in `neurolink.provider.streamText`.
|
|
77
|
-
*/
|
|
78
|
-
protected onStreamStart(modelId: string): OpenAICompatStreamLifecycleListeners | undefined;
|
|
79
72
|
formatProviderError(error: unknown): Error;
|
|
80
73
|
/**
|
|
81
74
|
* Get available models from LiteLLM proxy `/v1/models` endpoint.
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
|
|
2
1
|
import { registerRuntimeContextWindow, registerRuntimeOutputCeiling, } from "../../constants/contextWindows.js";
|
|
3
2
|
import { createProxyFetch } from "../../proxy/proxyFetch.js";
|
|
4
3
|
import { AuthenticationError, InvalidModelError, ModelAccessDeniedError, NetworkError, ProviderError, RateLimitError, isModelAccessDeniedMessage, parseAllowedModels, } from "../../types/index.js";
|
|
@@ -7,12 +6,10 @@ import { isAbortError } from "../../utils/errorHandling.js";
|
|
|
7
6
|
import { logger } from "../../utils/logger.js";
|
|
8
7
|
import { redactUrlCredentials } from "../../utils/logSanitize.js";
|
|
9
8
|
import { isGemini25Model as isCanonicalGemini25Model } from "../../utils/modelDetection.js";
|
|
10
|
-
import { calculateCost } from "../../utils/pricing.js";
|
|
11
9
|
import { getProviderModel } from "../../utils/providerConfig.js";
|
|
12
10
|
import { createTimeoutController } from "../../utils/timeout.js";
|
|
13
11
|
import { stripTrailingSlash } from "../openaiChatCompletionsClient.js";
|
|
14
12
|
import { OpenAIChatCompletionsProvider } from "../openaiChatCompletionsBase.js";
|
|
15
|
-
const streamTracer = trace.getTracer("neurolink.provider.litellm");
|
|
16
13
|
const FALLBACK_LITELLM_MODEL = "openai/gpt-5.4-mini";
|
|
17
14
|
const getLiteLLMConfig = () => ({
|
|
18
15
|
baseURL: process.env.LITELLM_BASE_URL || "http://localhost:4000",
|
|
@@ -195,8 +192,7 @@ const isGemini25Model = (modelName) => {
|
|
|
195
192
|
*
|
|
196
193
|
* All request/stream/tool-loop orchestration lives in
|
|
197
194
|
* `OpenAIChatCompletionsProvider`. This class adds LiteLLM-specific
|
|
198
|
-
* behaviour:
|
|
199
|
-
* maxTokens skip (`adjustBuildBodyOptions`), ModelAccessDeniedError on
|
|
195
|
+
* behaviour: Gemini 2.5 maxTokens skip (`adjustBuildBodyOptions`), ModelAccessDeniedError on
|
|
200
196
|
* 403, 10-minute model cache (`getAvailableModels`), `LITELLM_FALLBACK_MODELS`
|
|
201
197
|
* env-driven fallback list, and native `/v1/embeddings`.
|
|
202
198
|
*/
|
|
@@ -276,64 +272,6 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
|
|
|
276
272
|
}
|
|
277
273
|
return opts;
|
|
278
274
|
}
|
|
279
|
-
/**
|
|
280
|
-
* Wrap the stream in an OTel span to capture provider-level latency,
|
|
281
|
-
* token usage, finish reason, and cost. Matches the pre-migration
|
|
282
|
-
* behaviour where streamText was wrapped in `neurolink.provider.streamText`.
|
|
283
|
-
*/
|
|
284
|
-
onStreamStart(modelId) {
|
|
285
|
-
const span = streamTracer.startSpan("neurolink.provider.streamText", {
|
|
286
|
-
kind: SpanKind.CLIENT,
|
|
287
|
-
attributes: {
|
|
288
|
-
"gen_ai.system": "litellm",
|
|
289
|
-
"gen_ai.request.model": modelId,
|
|
290
|
-
},
|
|
291
|
-
});
|
|
292
|
-
let spanEnded = false;
|
|
293
|
-
const endSpan = () => {
|
|
294
|
-
if (!spanEnded) {
|
|
295
|
-
spanEnded = true;
|
|
296
|
-
span.end();
|
|
297
|
-
}
|
|
298
|
-
};
|
|
299
|
-
return {
|
|
300
|
-
onUsage: (usage) => {
|
|
301
|
-
// promptTokens is the uncached remainder — the attribute reports the
|
|
302
|
-
// full prompt (uncached + cache read/creation), and the cache fields
|
|
303
|
-
// let calculateCost price tiers.
|
|
304
|
-
span.setAttribute("gen_ai.usage.input_tokens", usage.promptTokens +
|
|
305
|
-
(usage.cacheReadTokens ?? 0) +
|
|
306
|
-
(usage.cacheCreationTokens ?? 0));
|
|
307
|
-
span.setAttribute("gen_ai.usage.output_tokens", usage.completionTokens);
|
|
308
|
-
const cost = calculateCost(this.providerName, modelId, {
|
|
309
|
-
input: usage.promptTokens,
|
|
310
|
-
output: usage.completionTokens,
|
|
311
|
-
total: usage.totalTokens,
|
|
312
|
-
...(usage.cacheReadTokens
|
|
313
|
-
? { cacheReadTokens: usage.cacheReadTokens }
|
|
314
|
-
: {}),
|
|
315
|
-
...(usage.cacheCreationTokens
|
|
316
|
-
? { cacheCreationTokens: usage.cacheCreationTokens }
|
|
317
|
-
: {}),
|
|
318
|
-
});
|
|
319
|
-
if (cost && cost > 0) {
|
|
320
|
-
span.setAttribute("neurolink.cost", cost);
|
|
321
|
-
}
|
|
322
|
-
},
|
|
323
|
-
onFinish: (reason, capturedError) => {
|
|
324
|
-
span.setAttribute("gen_ai.response.finish_reason", reason || "unknown");
|
|
325
|
-
if (reason === "error") {
|
|
326
|
-
span.setStatus({
|
|
327
|
-
code: SpanStatusCode.ERROR,
|
|
328
|
-
message: capturedError instanceof Error
|
|
329
|
-
? capturedError.message
|
|
330
|
-
: String(capturedError ?? "stream error"),
|
|
331
|
-
});
|
|
332
|
-
}
|
|
333
|
-
endSpan();
|
|
334
|
-
},
|
|
335
|
-
};
|
|
336
|
-
}
|
|
337
275
|
formatProviderError(error) {
|
|
338
276
|
// Curator P1-1: detect "team not allowed to access model" responses and
|
|
339
277
|
// surface as ModelAccessDeniedError with the allowed_models array parsed
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AIProviderName } from "../../constants/enums.js";
|
|
2
|
-
import type { EmbedInput, EnhancedGenerateResult, NeurolinkCredentials,
|
|
2
|
+
import type { EmbedInput, EnhancedGenerateResult, NeurolinkCredentials, TextGenerationOptions } from "../../types/index.js";
|
|
3
3
|
import { OpenAIChatCompletionsProvider } from "../openaiChatCompletionsBase.js";
|
|
4
4
|
/**
|
|
5
5
|
* OpenAI Provider — direct HTTP, no AI SDK.
|
|
@@ -7,7 +7,6 @@ import { OpenAIChatCompletionsProvider } from "../openaiChatCompletionsBase.js";
|
|
|
7
7
|
* OpenAI chat completions at api.openai.com/v1. All request / stream /
|
|
8
8
|
* tool-loop orchestration lives in `OpenAIChatCompletionsProvider`; this
|
|
9
9
|
* class adds:
|
|
10
|
-
* - OTel span wrap with cost (`onStreamStart`)
|
|
11
10
|
* - Native `/v1/embeddings` (`embed` / `embedMany`)
|
|
12
11
|
* - Image generation via `/v1/images/generations` (`executeImageGeneration`)
|
|
13
12
|
* - OpenAI-specific error mapping (`formatProviderError`)
|
|
@@ -25,12 +24,6 @@ export declare class OpenAIProvider extends OpenAIChatCompletionsProvider {
|
|
|
25
24
|
protected getProviderName(): AIProviderName;
|
|
26
25
|
protected getDefaultModel(): string;
|
|
27
26
|
formatProviderError(error: unknown): Error;
|
|
28
|
-
/**
|
|
29
|
-
* Wrap the stream in an OTel span to capture provider-level latency,
|
|
30
|
-
* token usage, finish reason, and cost. Mirrors the pre-migration
|
|
31
|
-
* `streamText`-span behaviour.
|
|
32
|
-
*/
|
|
33
|
-
protected onStreamStart(modelId: string): OpenAICompatStreamLifecycleListeners | undefined;
|
|
34
27
|
/**
|
|
35
28
|
* Default embedding model, overridable via OPENAI_EMBEDDING_MODEL env var.
|
|
36
29
|
*/
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
|
|
2
1
|
import { AIProviderName as AIProviderNameEnum } from "../../constants/enums.js";
|
|
3
2
|
import { createProxyFetch } from "../../proxy/proxyFetch.js";
|
|
4
3
|
import { AuthenticationError, InvalidModelError, ProviderError, RateLimitError, } from "../../types/index.js";
|
|
5
4
|
import { logger } from "../../utils/logger.js";
|
|
6
5
|
import { redactUrlCredentials } from "../../utils/logSanitize.js";
|
|
7
|
-
import { calculateCost } from "../../utils/pricing.js";
|
|
8
6
|
import { classifyProviderError, DEFAULT_ERROR_RULES, } from "../../utils/errorClassifier.js";
|
|
9
7
|
import { createOpenAIConfig, getProviderModel, validateApiKey, } from "../../utils/providerConfig.js";
|
|
10
8
|
import { safeDownload } from "../../utils/safeFetch.js";
|
|
@@ -45,14 +43,12 @@ const resolveOpenAIBaseURL = (credentialBaseURL, envBaseURL) => {
|
|
|
45
43
|
};
|
|
46
44
|
const getOpenAIApiKey = () => validateApiKey(createOpenAIConfig());
|
|
47
45
|
const getOpenAIModel = () => getProviderModel("OPENAI_MODEL", "gpt-5.4");
|
|
48
|
-
const streamTracer = trace.getTracer("neurolink.provider.openai");
|
|
49
46
|
/**
|
|
50
47
|
* OpenAI Provider — direct HTTP, no AI SDK.
|
|
51
48
|
*
|
|
52
49
|
* OpenAI chat completions at api.openai.com/v1. All request / stream /
|
|
53
50
|
* tool-loop orchestration lives in `OpenAIChatCompletionsProvider`; this
|
|
54
51
|
* class adds:
|
|
55
|
-
* - OTel span wrap with cost (`onStreamStart`)
|
|
56
52
|
* - Native `/v1/embeddings` (`embed` / `embedMany`)
|
|
57
53
|
* - Image generation via `/v1/images/generations` (`executeImageGeneration`)
|
|
58
54
|
* - OpenAI-specific error mapping (`formatProviderError`)
|
|
@@ -147,67 +143,6 @@ export class OpenAIProvider extends OpenAIChatCompletionsProvider {
|
|
|
147
143
|
return classifyProviderError(error, rules, this.providerName, this.modelName);
|
|
148
144
|
}
|
|
149
145
|
// ===========================================================================
|
|
150
|
-
// Optional hook overrides
|
|
151
|
-
// ===========================================================================
|
|
152
|
-
/**
|
|
153
|
-
* Wrap the stream in an OTel span to capture provider-level latency,
|
|
154
|
-
* token usage, finish reason, and cost. Mirrors the pre-migration
|
|
155
|
-
* `streamText`-span behaviour.
|
|
156
|
-
*/
|
|
157
|
-
onStreamStart(modelId) {
|
|
158
|
-
const span = streamTracer.startSpan("neurolink.provider.streamText", {
|
|
159
|
-
kind: SpanKind.CLIENT,
|
|
160
|
-
attributes: {
|
|
161
|
-
"gen_ai.system": "openai",
|
|
162
|
-
"gen_ai.request.model": modelId,
|
|
163
|
-
},
|
|
164
|
-
});
|
|
165
|
-
let spanEnded = false;
|
|
166
|
-
const endSpan = () => {
|
|
167
|
-
if (!spanEnded) {
|
|
168
|
-
spanEnded = true;
|
|
169
|
-
span.end();
|
|
170
|
-
}
|
|
171
|
-
};
|
|
172
|
-
return {
|
|
173
|
-
onUsage: (usage) => {
|
|
174
|
-
// promptTokens is the uncached remainder — the attribute reports the
|
|
175
|
-
// full prompt (uncached + cache read/creation), and the cache fields
|
|
176
|
-
// let calculateCost price tiers.
|
|
177
|
-
span.setAttribute("gen_ai.usage.input_tokens", usage.promptTokens +
|
|
178
|
-
(usage.cacheReadTokens ?? 0) +
|
|
179
|
-
(usage.cacheCreationTokens ?? 0));
|
|
180
|
-
span.setAttribute("gen_ai.usage.output_tokens", usage.completionTokens);
|
|
181
|
-
const cost = calculateCost(this.providerName, modelId, {
|
|
182
|
-
input: usage.promptTokens,
|
|
183
|
-
output: usage.completionTokens,
|
|
184
|
-
total: usage.totalTokens,
|
|
185
|
-
...(usage.cacheReadTokens
|
|
186
|
-
? { cacheReadTokens: usage.cacheReadTokens }
|
|
187
|
-
: {}),
|
|
188
|
-
...(usage.cacheCreationTokens
|
|
189
|
-
? { cacheCreationTokens: usage.cacheCreationTokens }
|
|
190
|
-
: {}),
|
|
191
|
-
});
|
|
192
|
-
if (cost && cost > 0) {
|
|
193
|
-
span.setAttribute("neurolink.cost", cost);
|
|
194
|
-
}
|
|
195
|
-
},
|
|
196
|
-
onFinish: (reason, capturedError) => {
|
|
197
|
-
span.setAttribute("gen_ai.response.finish_reason", reason || "unknown");
|
|
198
|
-
if (reason === "error") {
|
|
199
|
-
span.setStatus({
|
|
200
|
-
code: SpanStatusCode.ERROR,
|
|
201
|
-
message: capturedError instanceof Error
|
|
202
|
-
? capturedError.message
|
|
203
|
-
: String(capturedError ?? "stream error"),
|
|
204
|
-
});
|
|
205
|
-
}
|
|
206
|
-
endSpan();
|
|
207
|
-
},
|
|
208
|
-
};
|
|
209
|
-
}
|
|
210
|
-
// ===========================================================================
|
|
211
146
|
// Embeddings — native POST /v1/embeddings
|
|
212
147
|
// ===========================================================================
|
|
213
148
|
/**
|
|
@@ -301,6 +301,7 @@ export declare abstract class OpenAIChatCompletionsProvider extends BaseProvider
|
|
|
301
301
|
private resolveStreamStructuredData;
|
|
302
302
|
protected executeStream(options: StreamOptions, _analysisSchema?: ZodUnknownSchema | Schema<unknown>): Promise<StreamResult>;
|
|
303
303
|
private runStreamLoop;
|
|
304
|
+
private streamObservedStep;
|
|
304
305
|
private streamOneStep;
|
|
305
306
|
private executeToolBatch;
|
|
306
307
|
/**
|
|
@@ -28,6 +28,8 @@ import { BaseProvider } from "../core/baseProvider.js";
|
|
|
28
28
|
import { DEFAULT_MAX_STEPS } from "../core/constants.js";
|
|
29
29
|
import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
|
|
30
30
|
import { createProxyFetch } from "../proxy/proxyFetch.js";
|
|
31
|
+
import { observeGeneration } from "../telemetry/observation.js";
|
|
32
|
+
import { StepSpans, withGeneration } from "../telemetry/withSpan.js";
|
|
31
33
|
import { logger } from "../utils/logger.js";
|
|
32
34
|
import { redactUrlCredentials } from "../utils/logSanitize.js";
|
|
33
35
|
import { NoOutputGeneratedError } from "../utils/generationErrors.js";
|
|
@@ -964,6 +966,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
964
966
|
withTools = true) => {
|
|
965
967
|
return runNativeGenerateLoop({
|
|
966
968
|
doGenerate,
|
|
969
|
+
provider: this.providerName,
|
|
967
970
|
...createNativeGenerateGuard({
|
|
968
971
|
provider: this.providerName,
|
|
969
972
|
availableInputTokens: getAvailableInputTokens(this.providerName, modelId, options.maxTokens ?? undefined),
|
|
@@ -1469,6 +1472,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
1469
1472
|
const billed = { inputTokens: 0, outputTokens: 0 };
|
|
1470
1473
|
const runLoop = (conv, format) => runNativeGenerateLoop({
|
|
1471
1474
|
doGenerate,
|
|
1475
|
+
provider: this.providerName,
|
|
1472
1476
|
conversation: conv,
|
|
1473
1477
|
toolsRecord: {},
|
|
1474
1478
|
modelId,
|
|
@@ -2074,6 +2078,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
2074
2078
|
...(reasoningTokens > 0 ? { reasoningTokens } : {}),
|
|
2075
2079
|
};
|
|
2076
2080
|
};
|
|
2081
|
+
const stepSpans = new StepSpans();
|
|
2077
2082
|
try {
|
|
2078
2083
|
// May grow mid-turn: hydrated tools with wire-unsafe names need
|
|
2079
2084
|
// reverse-mapping even when the initial name set required none.
|
|
@@ -2087,6 +2092,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
2087
2092
|
// Completed-step records handed to a caller's prepareStep hook.
|
|
2088
2093
|
const stepRecords = [];
|
|
2089
2094
|
for (let step = 0; step < maxSteps; step++) {
|
|
2095
|
+
stepSpans.begin(step + 1);
|
|
2090
2096
|
const openAIToolChoice = await stepToolChoice(step, stepRecords);
|
|
2091
2097
|
// Mid-turn discovery sync: search_tools (tools.discovery) hydrates
|
|
2092
2098
|
// new tools into toolsRecord between steps. Dispatch already re-reads
|
|
@@ -2141,7 +2147,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
2141
2147
|
conversation.length = 0;
|
|
2142
2148
|
conversation.push(...guarded);
|
|
2143
2149
|
}
|
|
2144
|
-
const stepResult = await this.
|
|
2150
|
+
const stepResult = await stepSpans.run(() => this.streamObservedStep({
|
|
2145
2151
|
modelId,
|
|
2146
2152
|
url,
|
|
2147
2153
|
fetchImpl,
|
|
@@ -2152,7 +2158,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
2152
2158
|
openAIToolChoice,
|
|
2153
2159
|
pushChunk,
|
|
2154
2160
|
responseFormat,
|
|
2155
|
-
});
|
|
2161
|
+
}));
|
|
2156
2162
|
lastObservedPromptTokens = stepResult.usage?.prompt_tokens;
|
|
2157
2163
|
stepFinish = stepResult.finishReason;
|
|
2158
2164
|
if (stepResult.usage) {
|
|
@@ -2170,7 +2176,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
2170
2176
|
if (stepResult.toolCalls.size === 0) {
|
|
2171
2177
|
break;
|
|
2172
2178
|
}
|
|
2173
|
-
await this.executeToolBatch({
|
|
2179
|
+
await stepSpans.run(() => this.executeToolBatch({
|
|
2174
2180
|
stepResult,
|
|
2175
2181
|
conversation,
|
|
2176
2182
|
toolsRecord,
|
|
@@ -2179,7 +2185,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
2179
2185
|
toolExecutionSummaries,
|
|
2180
2186
|
options,
|
|
2181
2187
|
stepIndex: step,
|
|
2182
|
-
});
|
|
2188
|
+
}));
|
|
2183
2189
|
const executed = toolExecutionSummaries.slice(-stepResult.toolCalls.size);
|
|
2184
2190
|
stepRecords.push(toPrepareStepRecord({
|
|
2185
2191
|
stepNumber: step,
|
|
@@ -2243,6 +2249,57 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
|
|
|
2243
2249
|
closeChannel();
|
|
2244
2250
|
throw err;
|
|
2245
2251
|
}
|
|
2252
|
+
finally {
|
|
2253
|
+
stepSpans.end();
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
streamObservedStep(args) {
|
|
2257
|
+
return withGeneration({
|
|
2258
|
+
operation: "stream",
|
|
2259
|
+
provider: this.providerName,
|
|
2260
|
+
model: args.modelId,
|
|
2261
|
+
modelParameters: {
|
|
2262
|
+
temperature: args.options.temperature,
|
|
2263
|
+
maxTokens: args.options.maxTokens,
|
|
2264
|
+
topP: args.options.topP,
|
|
2265
|
+
thinkingLevel: args.options.thinkingConfig?.thinkingLevel,
|
|
2266
|
+
},
|
|
2267
|
+
input: { messages: args.conversation },
|
|
2268
|
+
}, async (span) => {
|
|
2269
|
+
let firstChunkAt;
|
|
2270
|
+
const result = await this.streamOneStep({
|
|
2271
|
+
...args,
|
|
2272
|
+
pushChunk: (chunk) => {
|
|
2273
|
+
firstChunkAt ??= new Date();
|
|
2274
|
+
args.pushChunk(chunk);
|
|
2275
|
+
},
|
|
2276
|
+
});
|
|
2277
|
+
const toolCalls = [...result.toolCalls.values()].map((call) => ({
|
|
2278
|
+
id: call.id,
|
|
2279
|
+
name: call.name,
|
|
2280
|
+
args: call.argsBuffered,
|
|
2281
|
+
}));
|
|
2282
|
+
observeGeneration(span, {
|
|
2283
|
+
provider: this.providerName,
|
|
2284
|
+
model: args.modelId,
|
|
2285
|
+
responseModel: result.model,
|
|
2286
|
+
output: {
|
|
2287
|
+
text: result.text,
|
|
2288
|
+
...(result.reasoning ? { reasoning: result.reasoning } : {}),
|
|
2289
|
+
...(toolCalls.length > 0 ? { toolCalls } : {}),
|
|
2290
|
+
},
|
|
2291
|
+
usage: result.usage && {
|
|
2292
|
+
promptTokens: result.usage.prompt_tokens,
|
|
2293
|
+
completionTokens: result.usage.completion_tokens,
|
|
2294
|
+
prompt_tokens_details: result.usage.prompt_tokens_details,
|
|
2295
|
+
reasoning_tokens: result.usage.completion_tokens_details?.reasoning_tokens,
|
|
2296
|
+
},
|
|
2297
|
+
finishReason: result.finishReason ?? undefined,
|
|
2298
|
+
aborted: args.abortSignal?.aborted,
|
|
2299
|
+
completionStartTime: firstChunkAt,
|
|
2300
|
+
});
|
|
2301
|
+
return result;
|
|
2302
|
+
});
|
|
2246
2303
|
}
|
|
2247
2304
|
async streamOneStep(args) {
|
|
2248
2305
|
// Per-step max_tokens fit: the conversation grows every step of the
|
|
@@ -5,6 +5,8 @@ import { getReplicateAuth } from "../adapters/replicate/auth.js";
|
|
|
5
5
|
import { downloadPredictionOutput, predict, } from "../adapters/replicate/predictionLifecycle.js";
|
|
6
6
|
import { MAX_IMAGE_BYTES } from "../utils/sizeGuard.js";
|
|
7
7
|
import { isNeuroLink } from "../neurolink.js";
|
|
8
|
+
import { observeGeneration } from "../telemetry/observation.js";
|
|
9
|
+
import { withGeneration } from "../telemetry/withSpan.js";
|
|
8
10
|
import { logger } from "../utils/logger.js";
|
|
9
11
|
import { ERROR_CODES, NeuroLinkError } from "../utils/errorHandling.js";
|
|
10
12
|
import { createReplicateConfig, getProviderModel, validateApiKey, } from "../utils/providerConfig.js";
|
|
@@ -227,10 +229,26 @@ export class ReplicateProvider extends BaseProvider {
|
|
|
227
229
|
};
|
|
228
230
|
let prediction;
|
|
229
231
|
try {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
232
|
+
const call = { provider: this.providerName, model: this.modelName };
|
|
233
|
+
prediction = await withGeneration({
|
|
234
|
+
...call,
|
|
235
|
+
modelParameters: {
|
|
236
|
+
temperature: options.temperature,
|
|
237
|
+
maxTokens: options.maxTokens,
|
|
238
|
+
topP: 1,
|
|
239
|
+
},
|
|
240
|
+
input: { prompt },
|
|
241
|
+
}, async (span) => {
|
|
242
|
+
const settled = await predict(auth, {
|
|
243
|
+
model: this.modelName,
|
|
244
|
+
input: replicateInput,
|
|
245
|
+
}, { abortSignal: options.abortSignal });
|
|
246
|
+
observeGeneration(span, {
|
|
247
|
+
...call,
|
|
248
|
+
output: flattenReplicateOutput(settled.output),
|
|
249
|
+
});
|
|
250
|
+
return settled;
|
|
251
|
+
});
|
|
234
252
|
}
|
|
235
253
|
catch (err) {
|
|
236
254
|
throw this.handleProviderError(err);
|
package/dist/proxy/codexUsage.js
CHANGED
|
@@ -273,6 +273,13 @@ export function createCodexUsageTap() {
|
|
|
273
273
|
evidence.firstUsefulOutputAt = Date.now();
|
|
274
274
|
evidence.firstUsefulOutputEvent = String(type);
|
|
275
275
|
}
|
|
276
|
+
if (typeof type === "string" &&
|
|
277
|
+
type.startsWith("response.") &&
|
|
278
|
+
completedResponse &&
|
|
279
|
+
typeof completedResponse === "object" &&
|
|
280
|
+
"output" in completedResponse) {
|
|
281
|
+
evidence.response = completedResponse;
|
|
282
|
+
}
|
|
276
283
|
if (type === "response.completed") {
|
|
277
284
|
evidence.completed = true;
|
|
278
285
|
evidence.terminalBytes = totalBytes;
|
package/dist/proxy/proxyFetch.js
CHANGED
|
@@ -4,22 +4,13 @@
|
|
|
4
4
|
* Lightweight implementation extracted from research of major proxy packages
|
|
5
5
|
*/
|
|
6
6
|
import { logger } from "../utils/logger.js";
|
|
7
|
+
import { getPropagatedAttributesFromContext } from "@langfuse/core";
|
|
8
|
+
import { LangfuseOtelSpanAttributes } from "@langfuse/tracing";
|
|
7
9
|
import { SpanStatusCode, propagation, context } from "@opentelemetry/api";
|
|
8
10
|
import { tracers } from "../telemetry/tracers.js";
|
|
9
11
|
import { shouldBypassProxy } from "./utils/noProxyUtils.js";
|
|
10
12
|
import { createHash } from "node:crypto";
|
|
11
13
|
import { TRANSIENT_NETWORK_CODES } from "../constants/networkErrorCodes.js";
|
|
12
|
-
async function getLangfuseContext() {
|
|
13
|
-
try {
|
|
14
|
-
// Dynamic import to avoid hard dependency — getLangfuseContext is only
|
|
15
|
-
// available when the observability module is loaded.
|
|
16
|
-
const mod = await import("../services/server/ai/observability/instrumentation.js");
|
|
17
|
-
return mod.getLangfuseContext?.();
|
|
18
|
-
}
|
|
19
|
-
catch {
|
|
20
|
-
return undefined;
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
14
|
/**
|
|
24
15
|
* Inject OTel trace context (traceparent/tracestate) and NeuroLink session context
|
|
25
16
|
* into outgoing request headers. This enables:
|
|
@@ -36,19 +27,22 @@ function mergeTraceHeaders(input, init) {
|
|
|
36
27
|
}
|
|
37
28
|
return existingHeaders;
|
|
38
29
|
}
|
|
39
|
-
|
|
30
|
+
function injectTraceContext(input, init) {
|
|
40
31
|
const carrier = {};
|
|
41
32
|
propagation.inject(context.active(), carrier);
|
|
42
|
-
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
33
|
+
const propagated = getPropagatedAttributesFromContext(context.active());
|
|
34
|
+
for (const [header, key] of [
|
|
35
|
+
["x-neurolink-session-id", LangfuseOtelSpanAttributes.TRACE_SESSION_ID],
|
|
36
|
+
["x-neurolink-user-id", LangfuseOtelSpanAttributes.TRACE_USER_ID],
|
|
37
|
+
[
|
|
38
|
+
"x-neurolink-conversation-id",
|
|
39
|
+
`${LangfuseOtelSpanAttributes.TRACE_METADATA}.conversationId`,
|
|
40
|
+
],
|
|
41
|
+
]) {
|
|
42
|
+
const value = propagated[key];
|
|
43
|
+
if (typeof value === "string" && value) {
|
|
44
|
+
carrier[header] = value;
|
|
45
|
+
}
|
|
52
46
|
}
|
|
53
47
|
if (Object.keys(carrier).length === 0) {
|
|
54
48
|
return init ?? {};
|
|
@@ -119,7 +113,7 @@ export function isTransientNetworkError(error) {
|
|
|
119
113
|
*/
|
|
120
114
|
async function fetchWithRetry(url, init, maxRetries = 3, baseDelay = 500) {
|
|
121
115
|
const hostname = extractHostname(url);
|
|
122
|
-
return fetchTracer.startActiveSpan("
|
|
116
|
+
return fetchTracer.startActiveSpan("http.fetchWithRetry", async (span) => {
|
|
123
117
|
span.setAttribute("http.request.max_retries", maxRetries);
|
|
124
118
|
span.setAttribute("http.request.hostname", hostname);
|
|
125
119
|
span.setAttribute("http.request.method", init?.method || "GET");
|
|
@@ -406,7 +400,7 @@ function getTargetUrl(input) {
|
|
|
406
400
|
}
|
|
407
401
|
function createDirectFetchHandler() {
|
|
408
402
|
return async (input, init) => {
|
|
409
|
-
const enrichedInit =
|
|
403
|
+
const enrichedInit = injectTraceContext(input, init);
|
|
410
404
|
const reqId = `req-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
411
405
|
const startTs = Date.now();
|
|
412
406
|
const url = getTargetUrl(input);
|
|
@@ -451,7 +445,7 @@ function createDirectFetchHandler() {
|
|
|
451
445
|
}
|
|
452
446
|
async function executeProxiedFetch(input, init, proxyEnv) {
|
|
453
447
|
const { httpsProxy, httpProxy, allProxy, socksProxy, noProxy } = proxyEnv;
|
|
454
|
-
init =
|
|
448
|
+
init = injectTraceContext(input, init);
|
|
455
449
|
const requestId = `req-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
456
450
|
const requestStartTime = Date.now();
|
|
457
451
|
const targetUrl = getTargetUrl(input);
|
|
@@ -6,19 +6,17 @@
|
|
|
6
6
|
* receive -> account_selection -> upstream (per retry) -> stream -> end
|
|
7
7
|
*
|
|
8
8
|
* Uses the existing instrumentation infrastructure:
|
|
9
|
-
* - getTracer() from instrumentation.ts for span creation
|
|
10
9
|
* - setLangfuseContext() for Langfuse enrichment
|
|
11
|
-
* -
|
|
10
|
+
* - OTel propagation API for context propagation to/from upstream
|
|
12
11
|
* - SpanAttributes from spanTypes.ts for attribute naming
|
|
13
12
|
* - calculateCost() from pricing.ts for cost tracking
|
|
14
|
-
* - TelemetryService for metrics recording
|
|
15
13
|
*/
|
|
16
14
|
import { type Span } from "@opentelemetry/api";
|
|
17
15
|
import type { AccountSelectionContext, ProxyRequestContext, ProxyRequestOrigin, ResponseInfoContext, UpstreamAttemptContext, UsageContext, ProxyUsageAttributionOptions, RuntimeRequestMetadata, ProxyLogTraceContext } from "../types/index.js";
|
|
18
16
|
declare class ProxyTracer {
|
|
19
17
|
private readonly rootSpan;
|
|
18
|
+
private readonly traceContext;
|
|
20
19
|
private readonly proxyTracer;
|
|
21
|
-
private readonly bridge;
|
|
22
20
|
private readonly requestId;
|
|
23
21
|
/**
|
|
24
22
|
* Model used for costing. Starts as the model the client asked for and is
|
|
@@ -42,12 +40,21 @@ declare class ProxyTracer {
|
|
|
42
40
|
private mode;
|
|
43
41
|
/** Where the Claude-shaped request originated, for cache/cost attribution. */
|
|
44
42
|
private requestOrigin;
|
|
43
|
+
private readonly requestedModel;
|
|
44
|
+
private input?;
|
|
45
|
+
private output?;
|
|
46
|
+
private responseModel?;
|
|
47
|
+
private finishReason?;
|
|
48
|
+
private firstOutputAt?;
|
|
49
|
+
private upstreamStartedAt?;
|
|
50
|
+
private cost?;
|
|
51
|
+
private statusMessage?;
|
|
45
52
|
private constructor();
|
|
46
53
|
/**
|
|
47
54
|
* Create a root span for a proxy request and set Langfuse context.
|
|
48
55
|
*
|
|
49
56
|
* If the incoming request carries a `traceparent` header, the root span
|
|
50
|
-
* will be linked to the caller's trace via
|
|
57
|
+
* will be linked to the caller's trace via the OTel propagation API.
|
|
51
58
|
*/
|
|
52
59
|
static startRequest(ctx: ProxyRequestContext, incomingHeaders?: Record<string, string>): ProxyTracer;
|
|
53
60
|
/** Span covering the initial request receive and parse phase. */
|
|
@@ -120,8 +127,8 @@ declare class ProxyTracer {
|
|
|
120
127
|
getUsage(): UsageContext | undefined;
|
|
121
128
|
/** End the root span with final HTTP status and duration, and emit OTEL metrics. */
|
|
122
129
|
end(responseStatus: number, durationMs: number): void;
|
|
123
|
-
|
|
124
|
-
|
|
130
|
+
private levelFor;
|
|
131
|
+
private observeOutcome;
|
|
125
132
|
/**
|
|
126
133
|
* Get trace context headers for propagation to the upstream Anthropic request.
|
|
127
134
|
* Injects the current trace's `traceparent` / `tracestate` into a new header map.
|