@infersec/conduit 1.20.2 → 1.21.0

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-CopKwPN6.js';
+import { a as asError, s as startInferenceAgent } from './start-Cqvc5hOj.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-CopKwPN6.js';
+import { s as startInferenceAgent, a as asError } from './start-Cqvc5hOj.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/{start-CopKwPN6.js → start-Cqvc5hOj.js}

@@ -15056,6 +15056,7 @@ const ChatCompletionCreateParamsSchema = object({
 // ==================== USAGE AND CHOICE SCHEMAS ====================
 const ChatCompletionUsageSchema = object({
     completion_tokens: number$1(),
+    context_usage: number$1().min(0).max(1).optional(),
     prompt_tokens: number$1(),
     total_tokens: number$1()
 });
@@ -15775,6 +15776,9 @@ function joinURL(...parts) {
         .replace("&", "?");
 }
 
+function isTerminatedError(error) {
+    return error instanceof Error && error.message === "terminated" && error.name === "TypeError";
+}
 async function connectSSE(url, options) {
     const response = await fetch(url, {
         headers: {
@@ -108349,18 +108353,23 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
             if (signal?.aborted) {
                 return;
             }
-            logger.error("SSE connection failed", {
-                error: asError(error)
-            });
-        }
-        if (signal?.aborted) {
-            return;
+            const isTerminated = isTerminatedError(error);
+            if (!isTerminated) {
+                logger.error("SSE connection failed", {
+                    error: asError(error)
+                });
+            }
+            if (signal?.aborted) {
+                return;
+            }
+            if (!isTerminated) {
+                const connectionDurationMs = Date.now() - connectionStartedAt;
+                reconnectAttempt = connectionDurationMs > 10000 ? 0 : reconnectAttempt + 1;
+                const reconnectDelayMs = Math.min(maxReconnectDelayMs, Math.max(1000, 1000 * 2 ** Math.min(6, reconnectAttempt)));
+                logger.warn("SSE disconnected, retrying");
+                await sleep(reconnectDelayMs);
+            }
         }
-        const connectionDurationMs = Date.now() - connectionStartedAt;
-        reconnectAttempt = connectionDurationMs > 10000 ? 0 : reconnectAttempt + 1;
-        const reconnectDelayMs = Math.min(maxReconnectDelayMs, Math.max(1000, 1000 * 2 ** Math.min(6, reconnectAttempt)));
-        logger.warn("SSE disconnected, retrying");
-        await sleep(reconnectDelayMs);
     }
 }
 async function handleRequest({ apiURL, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request }) {
@@ -117931,26 +117940,7 @@ function isEngineUsageChunk(value) {
     }
     return true;
 }
-function logEngineMetrics({ agentEngineType, error, level, logger, requestBodyBytes, requestPath, responseBytes, usage }) {
-    const metricsMessage = [
-        "LLM engine stream metrics",
-        `path=${requestPath}`,
-        `bytesTo=${requestBodyBytes}`,
-        `bytesFrom=${responseBytes}`,
-        `promptTokens=${usage?.promptTokens ?? "n/a"}`,
-        `completionTokens=${usage?.completionTokens ?? "n/a"}`,
-        `totalTokens=${usage?.totalTokens ?? "n/a"}`
-    ].join(" ");
-    const attributes = {
-        agentEngineType,
-        requestUrl: requestPath
-    };
-    if (error) {
-        attributes.error = error;
-    }
-    logger[level](metricsMessage, attributes);
-}
-function monitorEngineResponseStream({ agentEngineType, body, logger, onComplete, requestBodyBytes, requestPath, requestStartedAt }) {
+function monitorEngineResponseStream({ agentEngineType, body, contextLength, engine, logger, onComplete, parallelism, requestBodyBytes, requestPath, requestStartedAt }) {
     const startedAt = requestStartedAt ?? Date.now();
     const passThrough = new PassThrough();
     let responseBytes = 0;
@@ -117958,29 +117948,45 @@ function monitorEngineResponseStream({ agentEngineType, body, logger, onComplete
     let usage = null;
     let buffer = "";
     let completed = false;
-    function finalize(error) {
-        if (completed) {
-            return;
-        }
-        completed = true;
-        if (onComplete) {
-            const completion = onComplete({
-                durationMs: Math.max(0, Date.now() - startedAt),
-                error,
-                requestBodyBytes,
-                responseBytes,
-                timeToFirstTokenMs: firstChunkAt === null ? null : Math.max(0, firstChunkAt - startedAt),
-                usage
-            });
-            if (completion && typeof completion.catch === "function") {
-                completion.catch(error => {
-                    logger.error("Engine metrics completion failed", {
-                        error: asError(error),
-                        requestUrl: requestPath
-                    });
-                });
+    function modifyChunkWithUsage(chunk) {
+        const text = chunk.toString("utf8");
+        const lines = text.split("\n");
+        const modifiedLines = [];
+        for (const rawLine of lines) {
+            const line = rawLine.trim();
+            if (!line.startsWith("data:")) {
+                modifiedLines.push(rawLine);
+                continue;
             }
+            const payload = line.slice(5).trim();
+            if (!payload || payload === "[DONE]") {
+                modifiedLines.push(rawLine);
+                continue;
+            }
+            try {
+                const parsed = JSON.parse(payload);
+                if (parsed.usage) {
+                    const usageChunk = parsed.usage;
+                    if (usageChunk.context_usage === undefined &&
+                        usageChunk.prompt_tokens !== undefined &&
+                        contextLength !== null &&
+                        contextLength > 0) {
+                        let totalContextSize = contextLength;
+                        if (engine === "llama.cpp" && parallelism !== null && parallelism > 0) {
+                            totalContextSize = contextLength / parallelism;
+                        }
+                        usageChunk.context_usage = usageChunk.prompt_tokens / totalContextSize;
+                        modifiedLines.push("data: " + JSON.stringify(parsed));
+                        continue;
+                    }
+                }
+            }
+            catch (_error) {
+                // Ignore malformed chunks
+            }
+            modifiedLines.push(rawLine);
         }
+        return Buffer.from(modifiedLines.join("\n"), "utf8");
     }
     function parseUsageFromBuffer() {
         const lines = buffer.split("\n");
@@ -117997,10 +118003,21 @@ function monitorEngineResponseStream({ agentEngineType, body, logger, onComplete
             try {
                 const parsed = JSON.parse(payload);
                 if (isEngineUsageChunk(parsed)) {
+                    const completionTokens = parsed.usage?.completion_tokens ?? null;
+                    const promptTokens = parsed.usage?.prompt_tokens ?? null;
+                    const totalTokens = parsed.usage?.total_tokens ?? null;
+                    let contextUsage = parsed.usage?.context_usage ?? null;
+                    if (contextUsage === null &&
+                        promptTokens !== null &&
+                        contextLength !== null &&
+                        contextLength > 0) {
+                        contextUsage = promptTokens / contextLength;
+                    }
                     usage = {
-                        completionTokens: parsed.usage?.completion_tokens ?? null,
-                        promptTokens: parsed.usage?.prompt_tokens ?? null,
-                        totalTokens: parsed.usage?.total_tokens ?? null
+                        completionTokens,
+                        contextUsage,
+                        promptTokens,
+                        totalTokens
                     };
                 }
             }
@@ -118009,6 +118026,30 @@ function monitorEngineResponseStream({ agentEngineType, body, logger, onComplete
             }
         }
     }
+    function finalize(error) {
+        if (completed) {
+            return;
+        }
+        completed = true;
+        if (onComplete) {
+            const completion = onComplete({
+                durationMs: Math.max(0, Date.now() - startedAt),
+                error,
+                requestBodyBytes,
+                responseBytes,
+                timeToFirstTokenMs: firstChunkAt === null ? null : Math.max(0, firstChunkAt - startedAt),
+                usage
+            });
+            if (completion && typeof completion.catch === "function") {
+                completion.catch(error => {
+                    logger.error("Engine metrics completion failed", {
+                        error: asError(error),
+                        requestUrl: requestPath
+                    });
+                });
+            }
+        }
+    }
     body.on("data", (chunk) => {
         if (firstChunkAt === null) {
             firstChunkAt = Date.now();
@@ -118016,7 +118057,7 @@ function monitorEngineResponseStream({ agentEngineType, body, logger, onComplete
         responseBytes += chunk.length;
         buffer += chunk.toString("utf8");
         parseUsageFromBuffer();
-        passThrough.write(chunk);
+        passThrough.write(modifyChunkWithUsage(chunk));
     });
     body.once("error", err => {
         logEngineMetrics({
@@ -118073,6 +118114,26 @@ function monitorEngineResponseStream({ agentEngineType, body, logger, onComplete
         stream: passThrough
     };
 }
+function logEngineMetrics({ agentEngineType, error, level, logger, requestBodyBytes, requestPath, responseBytes, usage }) {
+    const metricsMessage = [
+        "LLM engine stream metrics",
+        `path=${requestPath}`,
+        `bytesTo=${requestBodyBytes}`,
+        `bytesFrom=${responseBytes}`,
+        `promptTokens=${usage?.promptTokens ?? "n/a"}`,
+        `completionTokens=${usage?.completionTokens ?? "n/a"}`,
+        `totalTokens=${usage?.totalTokens ?? "n/a"}`,
+        `contextUsage=${usage?.contextUsage ?? "n/a"}`
+    ].join(" ");
+    const attributes = {
+        agentEngineType,
+        requestUrl: requestPath
+    };
+    if (error) {
+        attributes.error = error;
+    }
+    logger[level](metricsMessage, attributes);
+}
 
 function isPlainObject(value) {
     return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -118222,6 +118283,8 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
     const monitoredResponse = monitorEngineResponseStream({
         agentEngineType: configuration.agentEngineType,
         body: Readable.fromWeb(response.body),
+        contextLength: modelManager.contextLength,
+        engine: configuration.agentEngineType,
         logger,
         onComplete: ({ durationMs, error, responseBytes, timeToFirstTokenMs, usage }) => {
             const completionTokens = normalizeTokenCount(usage?.completionTokens);
@@ -118250,6 +118313,7 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
                 totalTokens
             });
         },
+        parallelism: modelManager.parallelism,
         requestBodyBytes,
         requestPath: path,
         requestStartedAt

package/dist/utils/engineMetrics.d.ts

@@ -3,6 +3,7 @@ import { LLMEngine } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 export interface EngineUsageMetrics {
     completionTokens: number | null;
+    contextUsage: number | null;
     promptTokens: number | null;
     totalTokens: number | null;
 }
@@ -22,7 +23,10 @@ interface EngineMetricsCompletion {
 }
 interface MonitorEngineResponseOptions extends EngineMetricsLoggerOptions {
     body: Readable;
+    contextLength: number | null;
+    engine: LLMEngine;
     onComplete?: (result: EngineMetricsCompletion) => void | Promise<void>;
+    parallelism: number | null;
     requestStartedAt?: number;
 }
 interface EngineMetricsLogOptions extends EngineMetricsLoggerOptions {
@@ -34,6 +38,6 @@ interface EngineMetricsLogOptions extends EngineMetricsLoggerOptions {
 interface MonitorEngineResponseResult {
     stream: Readable;
 }
+export declare function monitorEngineResponseStream({ agentEngineType, body, contextLength, engine, logger, onComplete, parallelism, requestBodyBytes, requestPath, requestStartedAt }: MonitorEngineResponseOptions): MonitorEngineResponseResult;
 export declare function logEngineMetrics({ agentEngineType, error, level, logger, requestBodyBytes, requestPath, responseBytes, usage }: EngineMetricsLogOptions): void;
-export declare function monitorEngineResponseStream({ agentEngineType, body, logger, onComplete, requestBodyBytes, requestPath, requestStartedAt }: MonitorEngineResponseOptions): MonitorEngineResponseResult;
 export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.20.2",
+  "version": "1.21.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },