@infersec/conduit 1.98.1 → 1.99.0

package/dist/cli.js

@@ -21381,6 +21381,165 @@ object$1({
     password: RawPasswordSchema
 });
 
+_enum$1(["delete", "read", "write"]);
+const ACCOUNT_MANAGEMENT_OPERATION_METADATA = [
+    {
+        category: "read",
+        description: "List inference endpoints in the account.",
+        name: "list_endpoints"
+    },
+    {
+        category: "read",
+        description: "Get full details for a single inference endpoint, including its sources and tool service.",
+        name: "get_endpoint_details"
+    },
+    { category: "write", description: "Create a new inference endpoint.", name: "create_endpoint" },
+    {
+        category: "write",
+        description: "Update an inference endpoint's name, routing method or sources.",
+        name: "update_endpoint"
+    },
+    {
+        category: "write",
+        description: "Enable or disable an inference endpoint.",
+        name: "toggle_endpoint"
+    },
+    { category: "delete", description: "Delete an inference endpoint.", name: "delete_endpoint" },
+    {
+        category: "read",
+        description: "List inference sources in the account.",
+        name: "list_sources"
+    },
+    {
+        category: "read",
+        description: "Get full details for a single inference source.",
+        name: "get_source_details"
+    },
+    { category: "write", description: "Create a new inference source.", name: "create_source" },
+    {
+        category: "write",
+        description: "Update an inference source's name, engine, model or context length.",
+        name: "update_source"
+    },
+    { category: "delete", description: "Delete an inference source.", name: "delete_source" },
+    { category: "read", description: "List models in the account.", name: "list_models" },
+    {
+        category: "read",
+        description: "Get full details for a single model.",
+        name: "get_model_details"
+    },
+    {
+        category: "read",
+        description: "Search the third-party (HuggingFace) model catalogue for models available to add.",
+        name: "search_3rd_party"
+    },
+    {
+        category: "write",
+        description: "Register a new model in the account.",
+        name: "create_model"
+    },
+    {
+        category: "write",
+        description: "Update a model's name, task type or multimodal flag.",
+        name: "update_model"
+    },
+    { category: "delete", description: "Delete a model.", name: "delete_model" },
+    {
+        category: "read",
+        description: "List hardware machines registered to the account.",
+        name: "list_machines"
+    },
+    {
+        category: "read",
+        description: "Get details for a single hardware machine.",
+        name: "get_machine_details"
+    },
+    { category: "read", description: "List API keys in the account.", name: "list_api_keys" },
+    {
+        category: "write",
+        description: "Create a new API key and return its secret value.",
+        name: "create_api_key"
+    },
+    { category: "delete", description: "Delete an API key.", name: "delete_api_key" },
+    { category: "read", description: "List tools in the account.", name: "list_tools" },
+    {
+        category: "read",
+        description: "Get full details for a single tool.",
+        name: "get_tool_details"
+    },
+    { category: "write", description: "Create a new tool.", name: "create_tool" },
+    {
+        category: "write",
+        description: "Update a tool's name, type or configuration.",
+        name: "update_tool"
+    },
+    { category: "delete", description: "Delete a tool.", name: "delete_tool" },
+    {
+        category: "read",
+        description: "List tool services in the account.",
+        name: "list_tool_services"
+    },
+    {
+        category: "read",
+        description: "Get full details for a single tool service.",
+        name: "get_tool_service_details"
+    },
+    { category: "write", description: "Create a new tool service.", name: "create_tool_service" },
+    {
+        category: "write",
+        description: "Update a tool service's name, enabled flag or public exposure.",
+        name: "update_tool_service"
+    },
+    {
+        category: "write",
+        description: "Enable or disable a tool service.",
+        name: "toggle_tool_service"
+    },
+    { category: "delete", description: "Delete a tool service.", name: "delete_tool_service" },
+    {
+        category: "read",
+        description: "Get the tool service currently attached to an endpoint, if any.",
+        name: "get_endpoint_tool_service"
+    },
+    {
+        category: "write",
+        description: "Attach a tool service to an endpoint (replaces any existing attachment).",
+        name: "attach_tool_service_to_endpoint"
+    },
+    {
+        category: "write",
+        description: "Detach the tool service from an endpoint.",
+        name: "detach_tool_service_from_endpoint"
+    },
+    {
+        category: "write",
+        description: "Add a tool to a tool service.",
+        name: "add_tool_to_service"
+    },
+    {
+        category: "write",
+        description: "Remove a tool from a tool service.",
+        name: "remove_tool_from_service"
+    },
+    {
+        category: "read",
+        description: "List inference engines in the account.",
+        name: "list_engines"
+    },
+    { category: "write", description: "Create a new inference engine.", name: "create_engine" },
+    { category: "write", description: "Update an inference engine.", name: "update_engine" },
+    { category: "delete", description: "Delete an inference engine.", name: "delete_engine" }
+];
+const OPERATION_NAMES = [...ACCOUNT_MANAGEMENT_OPERATION_METADATA.map(op => op.name)];
+const AccountManagementOperationNameSchema = _enum$1(OPERATION_NAMES);
+const AccountManagementConfigSchema = object$1({
+    allowDelete: boolean$1().default(false),
+    allowRead: boolean$1().default(true),
+    allowWrite: boolean$1().default(false),
+    enabledOperations: array(AccountManagementOperationNameSchema).optional(),
+    type: literal("account-management")
+});
+
 object$1({
     description: string$2().optional(),
     inputSchema: record(string$2(), unknown()).optional(),
@@ -21413,6 +21572,7 @@ object$1({
 });
 
 _enum$1([
+    "account-management",
     "local-filesystem",
     "local-mysql",
     "mcp-proxy",
@@ -21449,6 +21609,7 @@ const ModelSourceCallConfigSchema = object$1({
     type: literal("model-source-call")
 });
 discriminatedUnion("type", [
+    AccountManagementConfigSchema,
     LocalFilesystemConfigSchema,
     LocalMysqlConfigSchema,
     MCPProxyConfigSchema,
@@ -124207,11 +124368,17 @@ function serializeRequestBody$1(body) {
         payload
     };
 }
-function calculateTokensPerSecond$2({ durationMs, totalTokens }) {
-    if (durationMs <= 0) {
+function calculateTokensPerSecond$1({ completionTokens, durationMs, streamRequested, timeToFirstTokenMs }) {
+    if (completionTokens <= 0 || durationMs <= 0) {
         return 0;
     }
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
+    const decodeMs = streamRequested && timeToFirstTokenMs !== null && timeToFirstTokenMs > 0
+        ? Math.max(0, durationMs - timeToFirstTokenMs)
+        : durationMs;
+    if (decodeMs <= 0) {
+        return 0;
+    }
+    const tokensPerSecond = completionTokens / (decodeMs / 1000);
     if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0) {
         return 0;
     }
@@ -124261,10 +124428,7 @@ async function proxyEmbeddingsRoute({ body, conduitConfiguration, endpointId, lo
             responseBytes,
             successful: upstreamResponseOk && !error,
             timeToFirstTokenMs: null,
-            tokensPerSecond: calculateTokensPerSecond$2({
-                durationMs: latencyMs,
-                totalTokens
-            }),
+            tokensPerSecond: 0,
             totalTokens
         });
     };
@@ -124407,9 +124571,11 @@ async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointI
             responseBytes,
             successful: upstreamResponseOk && !error,
             timeToFirstTokenMs,
-            tokensPerSecond: calculateTokensPerSecond$2({
+            tokensPerSecond: calculateTokensPerSecond$1({
+                completionTokens,
                 durationMs: latencyMs,
-                totalTokens
+                streamRequested,
+                timeToFirstTokenMs
             }),
             totalTokens
         });
@@ -124857,10 +125023,15 @@ function serializeRequestBody(body) {
         payload
     };
 }
-function calculateTokensPerSecond$1({ durationMs, totalTokens }) {
-    if (durationMs <= 0)
+function calculateTokensPerSecond({ completionTokens, durationMs, streamRequested, timeToFirstTokenMs }) {
+    if (completionTokens <= 0 || durationMs <= 0)
+        return 0;
+    const decodeMs = streamRequested && timeToFirstTokenMs !== null && timeToFirstTokenMs > 0
+        ? Math.max(0, durationMs - timeToFirstTokenMs)
+        : durationMs;
+    if (decodeMs <= 0)
         return 0;
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
+    const tokensPerSecond = completionTokens / (decodeMs / 1000);
     if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0)
         return 0;
     return Math.round(tokensPerSecond);
@@ -125038,7 +125209,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
     const targetBody = needsTranslation
         ? translateAnthropicRequestToOpenAI(serializedBody).body
         : serializedBody;
-    const onMonitoringComplete = ({ durationMs, error, responseBytes, usage }) => {
+    const onMonitoringComplete = ({ durationMs, error, responseBytes, timeToFirstTokenMs, usage }) => {
         const promptTokens = normalizeTokenCount(usage?.inputTokens);
         const completionTokens = normalizeTokenCount(usage?.outputTokens);
         const totalTokens = promptTokens + completionTokens;
@@ -125057,10 +125228,12 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             requestPath: "/v1/messages",
             responseBytes,
             successful: !error,
-            timeToFirstTokenMs: null,
-            tokensPerSecond: calculateTokensPerSecond$1({
+            timeToFirstTokenMs,
+            tokensPerSecond: calculateTokensPerSecond({
+                completionTokens,
                 durationMs: latencyMs,
-                totalTokens
+                streamRequested,
+                timeToFirstTokenMs
             }),
             totalTokens
         });
@@ -125157,6 +125330,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
     const passThrough = new PassThrough();
     let responseBytes = 0;
     let completed = false;
+    let firstChunkAt = null;
     const usage = { inputTokens: null, outputTokens: null };
     const upstreamError = response.ok
         ? null
@@ -125169,6 +125343,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             durationMs: Math.max(0, Date.now() - requestStartedAt),
             error,
             responseBytes,
+            timeToFirstTokenMs: firstChunkAt === null ? null : Math.max(0, firstChunkAt - requestStartedAt),
             usage
         });
     }
@@ -125178,6 +125353,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             const chunks = [];
             rawBody.on("data", (chunk) => {
                 const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                if (firstChunkAt === null)
+                    firstChunkAt = Date.now();
                 responseBytes += chunkBuffer.length;
                 chunks.push(chunkBuffer);
             });
@@ -125260,6 +125437,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             let buffer = "";
             rawBody.on("data", (chunk) => {
                 const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                if (firstChunkAt === null)
+                    firstChunkAt = Date.now();
                 responseBytes += chunkBuffer.length;
                 buffer += chunkBuffer.toString("utf8");
                 const lines = buffer.split("\n");
@@ -125330,6 +125509,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
         const chunks = [];
         rawBody.on("data", (chunk) => {
             const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            if (firstChunkAt === null)
+                firstChunkAt = Date.now();
             responseBytes += chunkBuffer.length;
             chunks.push(chunkBuffer);
         });
@@ -125480,7 +125661,7 @@ function createHealthHandler() {
     };
 }
 
-async function handleSSERequests({ apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, signal }) {
+async function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, signal }) {
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/stream`;
     const maxReconnectDelayMs = 30000;
     let reconnectAttempt = 0;
@@ -125522,14 +125703,11 @@ async function handleSSERequests({ apiURL, conduitConfiguration, configuration,
                     handleRequest({
                         activeRequests,
                         apiURL,
-                        conduitConfiguration,
                         configuration,
                         logger,
-                        modelID,
                         onRequest,
                         onRequestEnd,
                         onRequestStart,
-                        reportMetrics,
                         request: payload,
                         signal: effectiveSignal
                     }).catch(error => {
@@ -125566,23 +125744,12 @@ async function handleSSERequests({ apiURL, conduitConfiguration, configuration,
         }
     }
 }
-async function handleRequest({ activeRequests, apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request, signal }) {
-    function reportMetricsSafe(payload) {
-        reportMetrics(payload).catch(error => {
-            logger.warn("Failed to upload LLM prompt metrics", {
-                error: asError(error),
-                requestUrl: request.path
-            });
-        });
-    }
-    const engineType = conduitConfiguration().engineConfig?.type ?? null;
-    const endpointId = request.parameters?.endpointID ?? null;
+async function handleRequest({ activeRequests, apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, request, signal }) {
     const requestStartedAt = Date.now();
-    const requestBytes = calculateRequestBytes(request.body ?? null);
     try {
         await onRequestStart?.(request);
         const response = await onRequest({ request, signal });
-        const responseMetrics = await streamResponse({
+        await streamResponse({
             apiURL,
             configuration,
             logger,
@@ -125591,30 +125758,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             response,
             signal
         });
-        const latencyMs = Math.max(0, Date.now() - requestStartedAt);
-        const totalTokens = 0;
-        const tokensPerSecond = calculateTokensPerSecond({
-            durationMs: latencyMs,
-            totalTokens
-        });
-        reportMetricsSafe({
-            bytes: requestBytes + responseMetrics.responseBytes,
-            completionTokens: 0,
-            engine: engineType,
-            endpointId,
-            latencyMs,
-            modelId: modelID,
-            promptTokens: 0,
-            requestBytes,
-            requestId: request.requestID,
-            requestMethod: request.method,
-            requestPath: request.path,
-            responseBytes: responseMetrics.responseBytes,
-            successful: responseMetrics.status < 400,
-            timeToFirstTokenMs: responseMetrics.timeToFirstTokenMs,
-            tokensPerSecond,
-            totalTokens
-        });
     }
     catch (error) {
         const isCancelled = signal?.aborted;
@@ -125630,8 +125773,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             });
         }
         const failureMessage = "Bad gateway\n\nProxying failed";
-        const failureBytes = Buffer.byteLength(failureMessage, "utf8");
-        const latencyMs = Math.max(0, Date.now() - requestStartedAt);
         if (!isCancelled) {
             const streamHandler = await sendChunkStream({
                 apiURL,
@@ -125651,24 +125792,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             });
             await streamHandler.end();
         }
-        reportMetricsSafe({
-            bytes: isCancelled ? requestBytes : requestBytes + failureBytes,
-            completionTokens: 0,
-            engine: engineType,
-            endpointId,
-            latencyMs,
-            modelId: modelID,
-            promptTokens: 0,
-            requestBytes,
-            requestId: request.requestID,
-            requestMethod: request.method,
-            requestPath: request.path,
-            responseBytes: isCancelled ? 0 : failureBytes,
-            successful: false,
-            timeToFirstTokenMs: isCancelled ? null : latencyMs,
-            tokensPerSecond: 0,
-            totalTokens: 0
-        });
     }
     finally {
         activeRequests.delete(request.requestID);
@@ -125856,25 +125979,6 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
         abort
     };
 }
-function calculateRequestBytes(body) {
-    if (body === null || body === undefined) {
-        return 0;
-    }
-    if (typeof body === "string") {
-        return Buffer.byteLength(body, "utf8");
-    }
-    return Buffer.byteLength(JSON.stringify(body), "utf8");
-}
-function calculateTokensPerSecond({ durationMs, totalTokens }) {
-    if (durationMs <= 0) {
-        return 0;
-    }
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
-    if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0) {
-        return 0;
-    }
-    return Math.round(tokensPerSecond);
-}
 
 const ENGINE_FETCH_TIMEOUT_MS = 7_200_000;
 const LOOPBACK_AGENT = new undiciExports.Agent({
@@ -136047,10 +136151,8 @@ async function createApplication({ abortController, apiClient, configuration, lo
     });
     handleSSERequests({
         apiURL: configuration.apiURL,
-        conduitConfiguration: () => conduitConfiguration,
         configuration,
         logger,
-        modelID: conduitConfiguration.targetModel.id,
         onRequest: async ({ request, signal }) => {
             return proxyRequest({
                 configuration,
@@ -136067,7 +136169,6 @@ async function createApplication({ abortController, apiClient, configuration, lo
             conduitStateManager.incrementActiveRequestCount();
             conduitStateReportManager.reportStateChange();
         },
-        reportMetrics: apiClient.reportPromptMetrics,
         signal: abortController.signal
     }).catch(error => {
         logger.error("SSE handler failed", {

package/dist/cli.sea.cjs

@@ -21396,6 +21396,165 @@ object$1({
     password: RawPasswordSchema
 });
 
+_enum$1(["delete", "read", "write"]);
+const ACCOUNT_MANAGEMENT_OPERATION_METADATA = [
+    {
+        category: "read",
+        description: "List inference endpoints in the account.",
+        name: "list_endpoints"
+    },
+    {
+        category: "read",
+        description: "Get full details for a single inference endpoint, including its sources and tool service.",
+        name: "get_endpoint_details"
+    },
+    { category: "write", description: "Create a new inference endpoint.", name: "create_endpoint" },
+    {
+        category: "write",
+        description: "Update an inference endpoint's name, routing method or sources.",
+        name: "update_endpoint"
+    },
+    {
+        category: "write",
+        description: "Enable or disable an inference endpoint.",
+        name: "toggle_endpoint"
+    },
+    { category: "delete", description: "Delete an inference endpoint.", name: "delete_endpoint" },
+    {
+        category: "read",
+        description: "List inference sources in the account.",
+        name: "list_sources"
+    },
+    {
+        category: "read",
+        description: "Get full details for a single inference source.",
+        name: "get_source_details"
+    },
+    { category: "write", description: "Create a new inference source.", name: "create_source" },
+    {
+        category: "write",
+        description: "Update an inference source's name, engine, model or context length.",
+        name: "update_source"
+    },
+    { category: "delete", description: "Delete an inference source.", name: "delete_source" },
+    { category: "read", description: "List models in the account.", name: "list_models" },
+    {
+        category: "read",
+        description: "Get full details for a single model.",
+        name: "get_model_details"
+    },
+    {
+        category: "read",
+        description: "Search the third-party (HuggingFace) model catalogue for models available to add.",
+        name: "search_3rd_party"
+    },
+    {
+        category: "write",
+        description: "Register a new model in the account.",
+        name: "create_model"
+    },
+    {
+        category: "write",
+        description: "Update a model's name, task type or multimodal flag.",
+        name: "update_model"
+    },
+    { category: "delete", description: "Delete a model.", name: "delete_model" },
+    {
+        category: "read",
+        description: "List hardware machines registered to the account.",
+        name: "list_machines"
+    },
+    {
+        category: "read",
+        description: "Get details for a single hardware machine.",
+        name: "get_machine_details"
+    },
+    { category: "read", description: "List API keys in the account.", name: "list_api_keys" },
+    {
+        category: "write",
+        description: "Create a new API key and return its secret value.",
+        name: "create_api_key"
+    },
+    { category: "delete", description: "Delete an API key.", name: "delete_api_key" },
+    { category: "read", description: "List tools in the account.", name: "list_tools" },
+    {
+        category: "read",
+        description: "Get full details for a single tool.",
+        name: "get_tool_details"
+    },
+    { category: "write", description: "Create a new tool.", name: "create_tool" },
+    {
+        category: "write",
+        description: "Update a tool's name, type or configuration.",
+        name: "update_tool"
+    },
+    { category: "delete", description: "Delete a tool.", name: "delete_tool" },
+    {
+        category: "read",
+        description: "List tool services in the account.",
+        name: "list_tool_services"
+    },
+    {
+        category: "read",
+        description: "Get full details for a single tool service.",
+        name: "get_tool_service_details"
+    },
+    { category: "write", description: "Create a new tool service.", name: "create_tool_service" },
+    {
+        category: "write",
+        description: "Update a tool service's name, enabled flag or public exposure.",
+        name: "update_tool_service"
+    },
+    {
+        category: "write",
+        description: "Enable or disable a tool service.",
+        name: "toggle_tool_service"
+    },
+    { category: "delete", description: "Delete a tool service.", name: "delete_tool_service" },
+    {
+        category: "read",
+        description: "Get the tool service currently attached to an endpoint, if any.",
+        name: "get_endpoint_tool_service"
+    },
+    {
+        category: "write",
+        description: "Attach a tool service to an endpoint (replaces any existing attachment).",
+        name: "attach_tool_service_to_endpoint"
+    },
+    {
+        category: "write",
+        description: "Detach the tool service from an endpoint.",
+        name: "detach_tool_service_from_endpoint"
+    },
+    {
+        category: "write",
+        description: "Add a tool to a tool service.",
+        name: "add_tool_to_service"
+    },
+    {
+        category: "write",
+        description: "Remove a tool from a tool service.",
+        name: "remove_tool_from_service"
+    },
+    {
+        category: "read",
+        description: "List inference engines in the account.",
+        name: "list_engines"
+    },
+    { category: "write", description: "Create a new inference engine.", name: "create_engine" },
+    { category: "write", description: "Update an inference engine.", name: "update_engine" },
+    { category: "delete", description: "Delete an inference engine.", name: "delete_engine" }
+];
+const OPERATION_NAMES = [...ACCOUNT_MANAGEMENT_OPERATION_METADATA.map(op => op.name)];
+const AccountManagementOperationNameSchema = _enum$1(OPERATION_NAMES);
+const AccountManagementConfigSchema = object$1({
+    allowDelete: boolean$1().default(false),
+    allowRead: boolean$1().default(true),
+    allowWrite: boolean$1().default(false),
+    enabledOperations: array(AccountManagementOperationNameSchema).optional(),
+    type: literal("account-management")
+});
+
 object$1({
     description: string$2().optional(),
     inputSchema: record(string$2(), unknown()).optional(),
@@ -21428,6 +21587,7 @@ object$1({
 });
 
 _enum$1([
+    "account-management",
     "local-filesystem",
     "local-mysql",
     "mcp-proxy",
@@ -21464,6 +21624,7 @@ const ModelSourceCallConfigSchema = object$1({
     type: literal("model-source-call")
 });
 discriminatedUnion("type", [
+    AccountManagementConfigSchema,
     LocalFilesystemConfigSchema,
     LocalMysqlConfigSchema,
     MCPProxyConfigSchema,
@@ -124222,11 +124383,17 @@ function serializeRequestBody$1(body) {
         payload
     };
 }
-function calculateTokensPerSecond$2({ durationMs, totalTokens }) {
-    if (durationMs <= 0) {
+function calculateTokensPerSecond$1({ completionTokens, durationMs, streamRequested, timeToFirstTokenMs }) {
+    if (completionTokens <= 0 || durationMs <= 0) {
         return 0;
     }
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
+    const decodeMs = streamRequested && timeToFirstTokenMs !== null && timeToFirstTokenMs > 0
+        ? Math.max(0, durationMs - timeToFirstTokenMs)
+        : durationMs;
+    if (decodeMs <= 0) {
+        return 0;
+    }
+    const tokensPerSecond = completionTokens / (decodeMs / 1000);
     if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0) {
         return 0;
     }
@@ -124276,10 +124443,7 @@ async function proxyEmbeddingsRoute({ body, conduitConfiguration, endpointId, lo
             responseBytes,
             successful: upstreamResponseOk && !error,
             timeToFirstTokenMs: null,
-            tokensPerSecond: calculateTokensPerSecond$2({
-                durationMs: latencyMs,
-                totalTokens
-            }),
+            tokensPerSecond: 0,
             totalTokens
         });
     };
@@ -124422,9 +124586,11 @@ async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointI
             responseBytes,
             successful: upstreamResponseOk && !error,
             timeToFirstTokenMs,
-            tokensPerSecond: calculateTokensPerSecond$2({
+            tokensPerSecond: calculateTokensPerSecond$1({
+                completionTokens,
                 durationMs: latencyMs,
-                totalTokens
+                streamRequested,
+                timeToFirstTokenMs
             }),
             totalTokens
         });
@@ -124872,10 +125038,15 @@ function serializeRequestBody(body) {
         payload
     };
 }
-function calculateTokensPerSecond$1({ durationMs, totalTokens }) {
-    if (durationMs <= 0)
+function calculateTokensPerSecond({ completionTokens, durationMs, streamRequested, timeToFirstTokenMs }) {
+    if (completionTokens <= 0 || durationMs <= 0)
+        return 0;
+    const decodeMs = streamRequested && timeToFirstTokenMs !== null && timeToFirstTokenMs > 0
+        ? Math.max(0, durationMs - timeToFirstTokenMs)
+        : durationMs;
+    if (decodeMs <= 0)
         return 0;
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
+    const tokensPerSecond = completionTokens / (decodeMs / 1000);
     if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0)
         return 0;
     return Math.round(tokensPerSecond);
@@ -125053,7 +125224,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
     const targetBody = needsTranslation
         ? translateAnthropicRequestToOpenAI(serializedBody).body
         : serializedBody;
-    const onMonitoringComplete = ({ durationMs, error, responseBytes, usage }) => {
+    const onMonitoringComplete = ({ durationMs, error, responseBytes, timeToFirstTokenMs, usage }) => {
         const promptTokens = normalizeTokenCount(usage?.inputTokens);
         const completionTokens = normalizeTokenCount(usage?.outputTokens);
         const totalTokens = promptTokens + completionTokens;
@@ -125072,10 +125243,12 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             requestPath: "/v1/messages",
             responseBytes,
             successful: !error,
-            timeToFirstTokenMs: null,
-            tokensPerSecond: calculateTokensPerSecond$1({
+            timeToFirstTokenMs,
+            tokensPerSecond: calculateTokensPerSecond({
+                completionTokens,
                 durationMs: latencyMs,
-                totalTokens
+                streamRequested,
+                timeToFirstTokenMs
             }),
             totalTokens
         });
@@ -125172,6 +125345,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
     const passThrough = new require$$0$8.PassThrough();
     let responseBytes = 0;
     let completed = false;
+    let firstChunkAt = null;
     const usage = { inputTokens: null, outputTokens: null };
     const upstreamError = response.ok
         ? null
@@ -125184,6 +125358,7 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             durationMs: Math.max(0, Date.now() - requestStartedAt),
             error,
             responseBytes,
+            timeToFirstTokenMs: firstChunkAt === null ? null : Math.max(0, firstChunkAt - requestStartedAt),
             usage
         });
     }
@@ -125193,6 +125368,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             const chunks = [];
             rawBody.on("data", (chunk) => {
                 const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                if (firstChunkAt === null)
+                    firstChunkAt = Date.now();
                 responseBytes += chunkBuffer.length;
                 chunks.push(chunkBuffer);
             });
@@ -125275,6 +125452,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
             let buffer = "";
             rawBody.on("data", (chunk) => {
                 const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                if (firstChunkAt === null)
+                    firstChunkAt = Date.now();
                 responseBytes += chunkBuffer.length;
                 buffer += chunkBuffer.toString("utf8");
                 const lines = buffer.split("\n");
@@ -125345,6 +125524,8 @@ async function proxyAnthropicStreamingRoute({ body, conduitConfiguration, endpoi
         const chunks = [];
         rawBody.on("data", (chunk) => {
             const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            if (firstChunkAt === null)
+                firstChunkAt = Date.now();
             responseBytes += chunkBuffer.length;
             chunks.push(chunkBuffer);
         });
@@ -125495,7 +125676,7 @@ function createHealthHandler() {
     };
 }
 
-async function handleSSERequests({ apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, signal }) {
+async function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, signal }) {
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/stream`;
     const maxReconnectDelayMs = 30000;
     let reconnectAttempt = 0;
@@ -125537,14 +125718,11 @@ async function handleSSERequests({ apiURL, conduitConfiguration, configuration,
                     handleRequest({
                         activeRequests,
                         apiURL,
-                        conduitConfiguration,
                         configuration,
                         logger,
-                        modelID,
                         onRequest,
                         onRequestEnd,
                         onRequestStart,
-                        reportMetrics,
                         request: payload,
                         signal: effectiveSignal
                     }).catch(error => {
@@ -125581,23 +125759,12 @@ async function handleSSERequests({ apiURL, conduitConfiguration, configuration,
         }
     }
 }
-async function handleRequest({ activeRequests, apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request, signal }) {
-    function reportMetricsSafe(payload) {
-        reportMetrics(payload).catch(error => {
-            logger.warn("Failed to upload LLM prompt metrics", {
-                error: asError(error),
-                requestUrl: request.path
-            });
-        });
-    }
-    const engineType = conduitConfiguration().engineConfig?.type ?? null;
-    const endpointId = request.parameters?.endpointID ?? null;
+async function handleRequest({ activeRequests, apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, request, signal }) {
     const requestStartedAt = Date.now();
-    const requestBytes = calculateRequestBytes(request.body ?? null);
     try {
         await onRequestStart?.(request);
         const response = await onRequest({ request, signal });
-        const responseMetrics = await streamResponse({
+        await streamResponse({
             apiURL,
             configuration,
             logger,
@@ -125606,30 +125773,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             response,
             signal
         });
-        const latencyMs = Math.max(0, Date.now() - requestStartedAt);
-        const totalTokens = 0;
-        const tokensPerSecond = calculateTokensPerSecond({
-            durationMs: latencyMs,
-            totalTokens
-        });
-        reportMetricsSafe({
-            bytes: requestBytes + responseMetrics.responseBytes,
-            completionTokens: 0,
-            engine: engineType,
-            endpointId,
-            latencyMs,
-            modelId: modelID,
-            promptTokens: 0,
-            requestBytes,
-            requestId: request.requestID,
-            requestMethod: request.method,
-            requestPath: request.path,
-            responseBytes: responseMetrics.responseBytes,
-            successful: responseMetrics.status < 400,
-            timeToFirstTokenMs: responseMetrics.timeToFirstTokenMs,
-            tokensPerSecond,
-            totalTokens
-        });
     }
     catch (error) {
         const isCancelled = signal?.aborted;
@@ -125645,8 +125788,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             });
         }
         const failureMessage = "Bad gateway\n\nProxying failed";
-        const failureBytes = Buffer.byteLength(failureMessage, "utf8");
-        const latencyMs = Math.max(0, Date.now() - requestStartedAt);
         if (!isCancelled) {
             const streamHandler = await sendChunkStream({
                 apiURL,
@@ -125666,24 +125807,6 @@ async function handleRequest({ activeRequests, apiURL, conduitConfiguration, con
             });
             await streamHandler.end();
         }
-        reportMetricsSafe({
-            bytes: isCancelled ? requestBytes : requestBytes + failureBytes,
-            completionTokens: 0,
-            engine: engineType,
-            endpointId,
-            latencyMs,
-            modelId: modelID,
-            promptTokens: 0,
-            requestBytes,
-            requestId: request.requestID,
-            requestMethod: request.method,
-            requestPath: request.path,
-            responseBytes: isCancelled ? 0 : failureBytes,
-            successful: false,
-            timeToFirstTokenMs: isCancelled ? null : latencyMs,
-            tokensPerSecond: 0,
-            totalTokens: 0
-        });
     }
     finally {
         activeRequests.delete(request.requestID);
@@ -125871,25 +125994,6 @@ async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
         abort
     };
 }
-function calculateRequestBytes(body) {
-    if (body === null || body === undefined) {
-        return 0;
-    }
-    if (typeof body === "string") {
-        return Buffer.byteLength(body, "utf8");
-    }
-    return Buffer.byteLength(JSON.stringify(body), "utf8");
-}
-function calculateTokensPerSecond({ durationMs, totalTokens }) {
-    if (durationMs <= 0) {
-        return 0;
-    }
-    const tokensPerSecond = totalTokens / (durationMs / 1000);
-    if (!Number.isFinite(tokensPerSecond) || tokensPerSecond <= 0) {
-        return 0;
-    }
-    return Math.round(tokensPerSecond);
-}
 
 const ENGINE_FETCH_TIMEOUT_MS = 7_200_000;
 const LOOPBACK_AGENT = new undiciExports.Agent({
@@ -156277,10 +156381,8 @@ async function createApplication({ abortController, apiClient, configuration, lo
     });
     handleSSERequests({
         apiURL: configuration.apiURL,
-        conduitConfiguration: () => conduitConfiguration,
         configuration,
         logger,
-        modelID: conduitConfiguration.targetModel.id,
         onRequest: async ({ request, signal }) => {
             return proxyRequest({
                 configuration,
@@ -156297,7 +156399,6 @@ async function createApplication({ abortController, apiClient, configuration, lo
             conduitStateManager.incrementActiveRequestCount();
             conduitStateReportManager.reportStateChange();
         },
-        reportMetrics: apiClient.reportPromptMetrics,
         signal: abortController.signal
     }).catch(error => {
         logger.error("SSE handler failed", {

package/dist/sse/handler.d.ts

@@ -1,18 +1,15 @@
-import { InferenceAgentConfiguration, InferenceAgentLLMMetricsPayload, type ULID, type APIResponse, type ServerToClientAPIRequest } from "@infersec/definitions";
+import { type APIResponse, type ServerToClientAPIRequest } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
-export declare function handleSSERequests({ apiURL, conduitConfiguration, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, signal }: {
+export declare function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, signal }: {
     apiURL: string;
-    conduitConfiguration: () => InferenceAgentConfiguration;
     configuration: Configuration;
     logger: Logger;
-    modelID: ULID;
     onRequest: ({ request, signal }: {
         request: ServerToClientAPIRequest;
         signal?: AbortSignal;
     }) => Promise<APIResponse>;
     onRequestEnd?: (request: ServerToClientAPIRequest) => Promise<void> | void;
     onRequestStart?: (request: ServerToClientAPIRequest) => Promise<void> | void;
-    reportMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;
     signal?: AbortSignal;
 }): Promise<void>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.98.1",
+  "version": "1.99.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },