@infersec/conduit 1.98.0 → 1.98.2

package/dist/cli.js

@@ -122307,16 +122307,17 @@ const DOWNLOAD_PROGRESS_TIMEOUT = 60000;
 const DOWNLOAD_RETRY_ATTEMPTS_FULL = 3;
 const DOWNLOAD_RETRY_ATTEMPTS_RANGE = 10;
 const ALLOWED_EXTENSIONS = new Set([
+    ".bin",
+    ".gguf",
     ".gitattributes",
+    ".jinja",
     ".json",
     ".merges",
     ".model",
     ".py",
     ".safetensors",
     ".tiktoken",
-    ".txt",
-    ".bin",
-    ".gguf"
+    ".txt"
 ]);
 function isAllowedFile(filePath) {
     const lower = filePath.toLowerCase();
@@ -124186,12 +124187,17 @@ function serializeRequestBody$1(body) {
         };
     }
     const requestPayload = { ...body };
-    const streamOptions = requestPayload.stream_options;
-    const normalizedStreamOptions = isPlainObject$3(streamOptions)
-        ? { ...streamOptions }
-        : {};
-    normalizedStreamOptions.include_usage = true;
-    requestPayload.stream_options = normalizedStreamOptions;
+    if (requestPayload.stream === true) {
+        const streamOptions = requestPayload.stream_options;
+        const normalizedStreamOptions = isPlainObject$3(streamOptions)
+            ? { ...streamOptions }
+            : {};
+        normalizedStreamOptions.include_usage = true;
+        requestPayload.stream_options = normalizedStreamOptions;
+    }
+    else {
+        delete requestPayload.stream_options;
+    }
     if (requestPayload.repetition_penalty === undefined) {
         requestPayload.repetition_penalty = 1.1;
     }
@@ -124201,11 +124207,17 @@ function serializeRequestBody$1(body) {
         payload
     };
 }
-function calculateTokensPerSecond$2({ durationMs, totalTokens }) {
-    if (durationMs <= 0) {
+function calculateTokensPerSecond$1({ completionTokens, durationMs, streamRequested, timeToFirstTokenMs }) {
+    if (completionTokens <= 0 || durationMs <= 0) {
+        return 0;
+    }
+    const decodeMs = streamRequested && timeToFirstTokenMs !== null && timeToFirstTokenMs > 0
+        ? Math.max(0, durationMs - timeToFirstTokenMs)
+        : durationMs;
+    if (decodeMs <= 0) {
         return 0;
     }
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
+    const tokensPerSecond = completionTokens / (decodeMs / 1000);
     if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0) {
         return 0;
     }
@@ -124255,10 +124267,7 @@ async function proxyEmbeddingsRoute({ body, conduitConfiguration, endpointId, lo
             responseBytes,
             successful: upstreamResponseOk && !error,
             timeToFirstTokenMs: null,
-            tokensPerSecond: calculateTokensPerSecond$2({
-                durationMs: latencyMs,
-                totalTokens
-            }),
+            tokensPerSecond: 0,
             totalTokens
         });
     };
@@ -124401,9 +124410,11 @@ async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointI
             responseBytes,
             successful: upstreamResponseOk && !error,
             timeToFirstTokenMs,
-            tokensPerSecond: calculateTokensPerSecond$2({
+            tokensPerSecond: calculateTokensPerSecond$1({
+                completionTokens,
                 durationMs: latencyMs,
-                totalTokens
+                streamRequested,
+                timeToFirstTokenMs
             }),
             totalTokens
         });
@@ -124851,10 +124862,15 @@ function serializeRequestBody(body) {
         payload
     };
 }
-function calculateTokensPerSecond$1({ durationMs, totalTokens }) {
-    if (durationMs <= 0)
+function calculateTokensPerSecond({ completionTokens, durationMs, streamRequested, timeToFirstTokenMs }) {
+    if (completionTokens <= 0 || durationMs <= 0)
+        return 0;
+    const decodeMs = streamRequested && timeToFirstTokenMs !== null && timeToFirstTokenMs > 0
+        ? Math.max(0, durationMs - timeToFirstTokenMs)
+        : durationMs;
+    if (decodeMs <= 0)
         return 0;
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
+    const tokensPerSecond = completionTokens / (decodeMs / 1000);
     if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0)
         return 0;
     return Math.round(tokensPerSecond);
@@ -125032,7 +125048,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
     const targetBody = needsTranslation
         ? translateAnthropicRequestToOpenAI(serializedBody).body
         : serializedBody;
-    const onMonitoringComplete = ({ durationMs, error, responseBytes, usage }) => {
+    const onMonitoringComplete = ({ durationMs, error, responseBytes, timeToFirstTokenMs, usage }) => {
         const promptTokens = normalizeTokenCount(usage?.inputTokens);
         const completionTokens = normalizeTokenCount(usage?.outputTokens);
         const totalTokens = promptTokens + completionTokens;
@@ -125051,10 +125067,12 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             requestPath: "/v1/messages",
             responseBytes,
             successful: !error,
-            timeToFirstTokenMs: null,
-            tokensPerSecond: calculateTokensPerSecond$1({
+            timeToFirstTokenMs,
+            tokensPerSecond: calculateTokensPerSecond({
+                completionTokens,
                 durationMs: latencyMs,
-                totalTokens
+                streamRequested,
+                timeToFirstTokenMs
             }),
             totalTokens
         });
@@ -125151,6 +125169,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
     const passThrough = new PassThrough();
     let responseBytes = 0;
     let completed = false;
+    let firstChunkAt = null;
     const usage = { inputTokens: null, outputTokens: null };
     const upstreamError = response.ok
         ? null
@@ -125163,6 +125182,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             durationMs: Math.max(0, Date.now() - requestStartedAt),
             error,
             responseBytes,
+            timeToFirstTokenMs: firstChunkAt === null ? null : Math.max(0, firstChunkAt - requestStartedAt),
             usage
         });
     }
@@ -125172,6 +125192,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             const chunks = [];
             rawBody.on("data", (chunk) => {
                 const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                if (firstChunkAt === null)
+                    firstChunkAt = Date.now();
                 responseBytes += chunkBuffer.length;
                 chunks.push(chunkBuffer);
             });
@@ -125254,6 +125276,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             let buffer = "";
             rawBody.on("data", (chunk) => {
                 const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                if (firstChunkAt === null)
+                    firstChunkAt = Date.now();
                 responseBytes += chunkBuffer.length;
                 buffer += chunkBuffer.toString("utf8");
                 const lines = buffer.split("\n");
@@ -125324,6 +125348,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
         const chunks = [];
         rawBody.on("data", (chunk) => {
             const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            if (firstChunkAt === null)
+                firstChunkAt = Date.now();
             responseBytes += chunkBuffer.length;
             chunks.push(chunkBuffer);
         });
@@ -125474,7 +125500,7 @@ function createHealthHandler() {
     };
 }
 
-async function handleSSERequests({ apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, signal }) {
+async function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, signal }) {
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/stream`;
     const maxReconnectDelayMs = 30000;
     let reconnectAttempt = 0;
@@ -125516,14 +125542,11 @@ async function handleSSERequests({ apiURL, conduitConfiguration, configuration,
                     handleRequest({
                         activeRequests,
                         apiURL,
-                        conduitConfiguration,
                         configuration,
                         logger,
-                        modelID,
                         onRequest,
                         onRequestEnd,
                         onRequestStart,
-                        reportMetrics,
                         request: payload,
                         signal: effectiveSignal
                     }).catch(error => {
@@ -125560,23 +125583,12 @@ async function handleSSERequests({ apiURL, conduitConfiguration, configuration,
         }
     }
 }
-async function handleRequest({ activeRequests, apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request, signal }) {
-    function reportMetricsSafe(payload) {
-        reportMetrics(payload).catch(error => {
-            logger.warn("Failed to upload LLM prompt metrics", {
-                error: asError(error),
-                requestUrl: request.path
-            });
-        });
-    }
-    const engineType = conduitConfiguration().engineConfig?.type ?? null;
-    const endpointId = request.parameters?.endpointID ?? null;
+async function handleRequest({ activeRequests, apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, request, signal }) {
     const requestStartedAt = Date.now();
-    const requestBytes = calculateRequestBytes(request.body ?? null);
     try {
         await onRequestStart?.(request);
         const response = await onRequest({ request, signal });
-        const responseMetrics = await streamResponse({
+        await streamResponse({
             apiURL,
             configuration,
             logger,
@@ -125585,30 +125597,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             response,
             signal
         });
-        const latencyMs = Math.max(0, Date.now() - requestStartedAt);
-        const totalTokens = 0;
-        const tokensPerSecond = calculateTokensPerSecond({
-            durationMs: latencyMs,
-            totalTokens
-        });
-        reportMetricsSafe({
-            bytes: requestBytes + responseMetrics.responseBytes,
-            completionTokens: 0,
-            engine: engineType,
-            endpointId,
-            latencyMs,
-            modelId: modelID,
-            promptTokens: 0,
-            requestBytes,
-            requestId: request.requestID,
-            requestMethod: request.method,
-            requestPath: request.path,
-            responseBytes: responseMetrics.responseBytes,
-            successful: responseMetrics.status < 400,
-            timeToFirstTokenMs: responseMetrics.timeToFirstTokenMs,
-            tokensPerSecond,
-            totalTokens
-        });
     }
     catch (error) {
         const isCancelled = signal?.aborted;
@@ -125624,8 +125612,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             });
         }
         const failureMessage = "Bad gateway\n\nProxying failed";
-        const failureBytes = Buffer.byteLength(failureMessage, "utf8");
-        const latencyMs = Math.max(0, Date.now() - requestStartedAt);
         if (!isCancelled) {
             const streamHandler = await sendChunkStream({
                 apiURL,
@@ -125645,24 +125631,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             });
             await streamHandler.end();
         }
-        reportMetricsSafe({
-            bytes: isCancelled ? requestBytes : requestBytes + failureBytes,
-            completionTokens: 0,
-            engine: engineType,
-            endpointId,
-            latencyMs,
-            modelId: modelID,
-            promptTokens: 0,
-            requestBytes,
-            requestId: request.requestID,
-            requestMethod: request.method,
-            requestPath: request.path,
-            responseBytes: isCancelled ? 0 : failureBytes,
-            successful: false,
-            timeToFirstTokenMs: isCancelled ? null : latencyMs,
-            tokensPerSecond: 0,
-            totalTokens: 0
-        });
     }
     finally {
         activeRequests.delete(request.requestID);
@@ -125850,25 +125818,6 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
         abort
     };
 }
-function calculateRequestBytes(body) {
-    if (body === null || body === undefined) {
-        return 0;
-    }
-    if (typeof body === "string") {
-        return Buffer.byteLength(body, "utf8");
-    }
-    return Buffer.byteLength(JSON.stringify(body), "utf8");
-}
-function calculateTokensPerSecond({ durationMs, totalTokens }) {
-    if (durationMs <= 0) {
-        return 0;
-    }
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
-    if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0) {
-        return 0;
-    }
-    return Math.round(tokensPerSecond);
-}
 
 const ENGINE_FETCH_TIMEOUT_MS = 7_200_000;
 const LOOPBACK_AGENT = new undiciExports.Agent({
@@ -136041,10 +135990,8 @@ async function createApplication({ abortController, apiClient, configuration, lo
     });
     handleSSERequests({
         apiURL: configuration.apiURL,
-        conduitConfiguration: () => conduitConfiguration,
         configuration,
         logger,
-        modelID: conduitConfiguration.targetModel.id,
         onRequest: async ({ request, signal }) => {
             return proxyRequest({
                 configuration,
@@ -136061,7 +136008,6 @@ async function createApplication({ abortController, apiClient, configuration, lo
             conduitStateManager.incrementActiveRequestCount();
             conduitStateReportManager.reportStateChange();
         },
-        reportMetrics: apiClient.reportPromptMetrics,
         signal: abortController.signal
     }).catch(error => {
         logger.error("SSE handler failed", {

package/dist/cli.sea.cjs

@@ -122322,16 +122322,17 @@ const DOWNLOAD_PROGRESS_TIMEOUT = 60000;
 const DOWNLOAD_RETRY_ATTEMPTS_FULL = 3;
 const DOWNLOAD_RETRY_ATTEMPTS_RANGE = 10;
 const ALLOWED_EXTENSIONS = new Set([
+    ".bin",
+    ".gguf",
     ".gitattributes",
+    ".jinja",
     ".json",
     ".merges",
     ".model",
     ".py",
     ".safetensors",
     ".tiktoken",
-    ".txt",
-    ".bin",
-    ".gguf"
+    ".txt"
 ]);
 function isAllowedFile(filePath) {
     const lower = filePath.toLowerCase();
@@ -124201,12 +124202,17 @@ function serializeRequestBody$1(body) {
         };
     }
     const requestPayload = { ...body };
-    const streamOptions = requestPayload.stream_options;
-    const normalizedStreamOptions = isPlainObject$3(streamOptions)
-        ? { ...streamOptions }
-        : {};
-    normalizedStreamOptions.include_usage = true;
-    requestPayload.stream_options = normalizedStreamOptions;
+    if (requestPayload.stream === true) {
+        const streamOptions = requestPayload.stream_options;
+        const normalizedStreamOptions = isPlainObject$3(streamOptions)
+            ? { ...streamOptions }
+            : {};
+        normalizedStreamOptions.include_usage = true;
+        requestPayload.stream_options = normalizedStreamOptions;
+    }
+    else {
+        delete requestPayload.stream_options;
+    }
     if (requestPayload.repetition_penalty === undefined) {
         requestPayload.repetition_penalty = 1.1;
     }
@@ -124216,11 +124222,17 @@ function serializeRequestBody$1(body) {
         payload
     };
 }
-function calculateTokensPerSecond$2({ durationMs, totalTokens }) {
-    if (durationMs <= 0) {
+function calculateTokensPerSecond$1({ completionTokens, durationMs, streamRequested, timeToFirstTokenMs }) {
+    if (completionTokens <= 0 || durationMs <= 0) {
+        return 0;
+    }
+    const decodeMs = streamRequested && timeToFirstTokenMs !== null && timeToFirstTokenMs > 0
+        ? Math.max(0, durationMs - timeToFirstTokenMs)
+        : durationMs;
+    if (decodeMs <= 0) {
         return 0;
     }
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
+    const tokensPerSecond = completionTokens / (decodeMs / 1000);
     if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0) {
         return 0;
     }
@@ -124270,10 +124282,7 @@ async function proxyEmbeddingsRoute({ body, conduitConfiguration, endpointId, lo
             responseBytes,
             successful: upstreamResponseOk && !error,
             timeToFirstTokenMs: null,
-            tokensPerSecond: calculateTokensPerSecond$2({
-                durationMs: latencyMs,
-                totalTokens
-            }),
+            tokensPerSecond: 0,
             totalTokens
         });
     };
@@ -124416,9 +124425,11 @@ async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointI
             responseBytes,
             successful: upstreamResponseOk && !error,
             timeToFirstTokenMs,
-            tokensPerSecond: calculateTokensPerSecond$2({
+            tokensPerSecond: calculateTokensPerSecond$1({
+                completionTokens,
                 durationMs: latencyMs,
-                totalTokens
+                streamRequested,
+                timeToFirstTokenMs
             }),
             totalTokens
         });
@@ -124866,10 +124877,15 @@ function serializeRequestBody(body) {
         payload
     };
 }
-function calculateTokensPerSecond$1({ durationMs, totalTokens }) {
-    if (durationMs <= 0)
+function calculateTokensPerSecond({ completionTokens, durationMs, streamRequested, timeToFirstTokenMs }) {
+    if (completionTokens <= 0 || durationMs <= 0)
+        return 0;
+    const decodeMs = streamRequested && timeToFirstTokenMs !== null && timeToFirstTokenMs > 0
+        ? Math.max(0, durationMs - timeToFirstTokenMs)
+        : durationMs;
+    if (decodeMs <= 0)
         return 0;
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
+    const tokensPerSecond = completionTokens / (decodeMs / 1000);
     if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0)
         return 0;
     return Math.round(tokensPerSecond);
@@ -125047,7 +125063,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
     const targetBody = needsTranslation
         ? translateAnthropicRequestToOpenAI(serializedBody).body
         : serializedBody;
-    const onMonitoringComplete = ({ durationMs, error, responseBytes, usage }) => {
+    const onMonitoringComplete = ({ durationMs, error, responseBytes, timeToFirstTokenMs, usage }) => {
         const promptTokens = normalizeTokenCount(usage?.inputTokens);
         const completionTokens = normalizeTokenCount(usage?.outputTokens);
         const totalTokens = promptTokens + completionTokens;
@@ -125066,10 +125082,12 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             requestPath: "/v1/messages",
             responseBytes,
             successful: !error,
-            timeToFirstTokenMs: null,
-            tokensPerSecond: calculateTokensPerSecond$1({
+            timeToFirstTokenMs,
+            tokensPerSecond: calculateTokensPerSecond({
+                completionTokens,
                 durationMs: latencyMs,
-                totalTokens
+                streamRequested,
+                timeToFirstTokenMs
             }),
             totalTokens
         });
@@ -125166,6 +125184,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
     const passThrough = new require$$0$8.PassThrough();
     let responseBytes = 0;
     let completed = false;
+    let firstChunkAt = null;
     const usage = { inputTokens: null, outputTokens: null };
     const upstreamError = response.ok
         ? null
@@ -125178,6 +125197,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             durationMs: Math.max(0, Date.now() - requestStartedAt),
             error,
             responseBytes,
+            timeToFirstTokenMs: firstChunkAt === null ? null : Math.max(0, firstChunkAt - requestStartedAt),
             usage
         });
     }
@@ -125187,6 +125207,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             const chunks = [];
             rawBody.on("data", (chunk) => {
                 const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                if (firstChunkAt === null)
+                    firstChunkAt = Date.now();
                 responseBytes += chunkBuffer.length;
                 chunks.push(chunkBuffer);
             });
@@ -125269,6 +125291,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             let buffer = "";
             rawBody.on("data", (chunk) => {
                 const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                if (firstChunkAt === null)
+                    firstChunkAt = Date.now();
                 responseBytes += chunkBuffer.length;
                 buffer += chunkBuffer.toString("utf8");
                 const lines = buffer.split("\n");
@@ -125339,6 +125363,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
         const chunks = [];
         rawBody.on("data", (chunk) => {
             const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            if (firstChunkAt === null)
+                firstChunkAt = Date.now();
             responseBytes += chunkBuffer.length;
             chunks.push(chunkBuffer);
         });
@@ -125489,7 +125515,7 @@ function createHealthHandler() {
     };
 }
 
-async function handleSSERequests({ apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, signal }) {
+async function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, signal }) {
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/stream`;
     const maxReconnectDelayMs = 30000;
     let reconnectAttempt = 0;
@@ -125531,14 +125557,11 @@ async function handleSSERequests({ apiURL, conduitConfiguration, configuration,
                     handleRequest({
                         activeRequests,
                         apiURL,
-                        conduitConfiguration,
                         configuration,
                         logger,
-                        modelID,
                         onRequest,
                         onRequestEnd,
                         onRequestStart,
-                        reportMetrics,
                         request: payload,
                         signal: effectiveSignal
                     }).catch(error => {
@@ -125575,23 +125598,12 @@ async function handleSSERequests({ apiURL, conduitConfiguration, configuration,
         }
     }
 }
-async function handleRequest({ activeRequests, apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request, signal }) {
-    function reportMetricsSafe(payload) {
-        reportMetrics(payload).catch(error => {
-            logger.warn("Failed to upload LLM prompt metrics", {
-                error: asError(error),
-                requestUrl: request.path
-            });
-        });
-    }
-    const engineType = conduitConfiguration().engineConfig?.type ?? null;
-    const endpointId = request.parameters?.endpointID ?? null;
+async function handleRequest({ activeRequests, apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, request, signal }) {
     const requestStartedAt = Date.now();
-    const requestBytes = calculateRequestBytes(request.body ?? null);
     try {
         await onRequestStart?.(request);
         const response = await onRequest({ request, signal });
-        const responseMetrics = await streamResponse({
+        await streamResponse({
             apiURL,
             configuration,
             logger,
@@ -125600,30 +125612,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             response,
             signal
         });
-        const latencyMs = Math.max(0, Date.now() - requestStartedAt);
-        const totalTokens = 0;
-        const tokensPerSecond = calculateTokensPerSecond({
-            durationMs: latencyMs,
-            totalTokens
-        });
-        reportMetricsSafe({
-            bytes: requestBytes + responseMetrics.responseBytes,
-            completionTokens: 0,
-            engine: engineType,
-            endpointId,
-            latencyMs,
-            modelId: modelID,
-            promptTokens: 0,
-            requestBytes,
-            requestId: request.requestID,
-            requestMethod: request.method,
-            requestPath: request.path,
-            responseBytes: responseMetrics.responseBytes,
-            successful: responseMetrics.status < 400,
-            timeToFirstTokenMs: responseMetrics.timeToFirstTokenMs,
-            tokensPerSecond,
-            totalTokens
-        });
     }
     catch (error) {
         const isCancelled = signal?.aborted;
@@ -125639,8 +125627,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             });
         }
         const failureMessage = "Bad gateway\n\nProxying failed";
-        const failureBytes = Buffer.byteLength(failureMessage, "utf8");
-        const latencyMs = Math.max(0, Date.now() - requestStartedAt);
         if (!isCancelled) {
             const streamHandler = await sendChunkStream({
                 apiURL,
@@ -125660,24 +125646,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             });
             await streamHandler.end();
         }
-        reportMetricsSafe({
-            bytes: isCancelled ? requestBytes : requestBytes + failureBytes,
-            completionTokens: 0,
-            engine: engineType,
-            endpointId,
-            latencyMs,
-            modelId: modelID,
-            promptTokens: 0,
-            requestBytes,
-            requestId: request.requestID,
-            requestMethod: request.method,
-            requestPath: request.path,
-            responseBytes: isCancelled ? 0 : failureBytes,
-            successful: false,
-            timeToFirstTokenMs: isCancelled ? null : latencyMs,
-            tokensPerSecond: 0,
-            totalTokens: 0
-        });
     }
     finally {
         activeRequests.delete(request.requestID);
@@ -125865,25 +125833,6 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
         abort
     };
 }
-function calculateRequestBytes(body) {
-    if (body === null || body === undefined) {
-        return 0;
-    }
-    if (typeof body === "string") {
-        return Buffer.byteLength(body, "utf8");
-    }
-    return Buffer.byteLength(JSON.stringify(body), "utf8");
-}
-function calculateTokensPerSecond({ durationMs, totalTokens }) {
-    if (durationMs <= 0) {
-        return 0;
-    }
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
-    if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0) {
-        return 0;
-    }
-    return Math.round(tokensPerSecond);
-}
 
 const ENGINE_FETCH_TIMEOUT_MS = 7_200_000;
 const LOOPBACK_AGENT = new undiciExports.Agent({
@@ -156271,10 +156220,8 @@ async function createApplication({ abortController, apiClient, configuration, lo
     });
     handleSSERequests({
         apiURL: configuration.apiURL,
-        conduitConfiguration: () => conduitConfiguration,
         configuration,
         logger,
-        modelID: conduitConfiguration.targetModel.id,
         onRequest: async ({ request, signal }) => {
             return proxyRequest({
                 configuration,
@@ -156291,7 +156238,6 @@ async function createApplication({ abortController, apiClient, configuration, lo
             conduitStateManager.incrementActiveRequestCount();
             conduitStateReportManager.reportStateChange();
         },
-        reportMetrics: apiClient.reportPromptMetrics,
         signal: abortController.signal
     }).catch(error => {
         logger.error("SSE handler failed", {

package/dist/sse/handler.d.ts

@@ -1,18 +1,15 @@
-import { InferenceAgentConfiguration, InferenceAgentLLMMetricsPayload, type ULID, type APIResponse, type ServerToClientAPIRequest } from "@infersec/definitions";
+import { type APIResponse, type ServerToClientAPIRequest } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
-export declare function handleSSERequests({ apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, signal }: {
+export declare function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, signal }: {
     apiURL: string;
-    conduitConfiguration: () => InferenceAgentConfiguration;
     configuration: Configuration;
     logger: Logger;
-    modelID: ULID;
     onRequest: ({ request, signal }: {
         request: ServerToClientAPIRequest;
         signal?: AbortSignal;
     }) => Promise<APIResponse>;
     onRequestEnd?: (request: ServerToClientAPIRequest) => Promise<void> | void;
     onRequestStart?: (request: ServerToClientAPIRequest) => Promise<void> | void;
-    reportMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;
     signal?: AbortSignal;
 }): Promise<void>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.98.0",
+  "version": "1.98.2",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },