@infersec/conduit 1.24.3 → 1.25.0

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-DiikacLf.js';
+import { a as asError, s as startInferenceAgent } from './start-DBk2G4SP.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-DiikacLf.js';
+import { s as startInferenceAgent, a as asError } from './start-DBk2G4SP.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/{start-DiikacLf.js → start-DBk2G4SP.js}

@@ -14914,6 +14914,23 @@ const API_SERVICE_CONDUIT_API_REFERENCE = {
                 type: "rest"
             }
         }
+    },
+    "/conduit/api/v1/source/:sourceID/requests/:requestID/stream": {
+        POST: {
+            auth: {
+                type: "api-key"
+            },
+            parameters: {
+                requestID: ULIDSchema,
+                sourceID: ULIDSchema
+            },
+            response: {
+                schema: object({
+                    acknowledged: literal(true)
+                }),
+                type: "rest"
+            }
+        }
     }
 };
 
@@ -17601,6 +17618,9 @@ function createLogger({ attributes = {}, name }) {
         child: (attributes) => {
             return buildLogger(logger.child(processAttributes(attributes)));
         },
+        debug: (message, attributes) => {
+            logger.debug(processAttributes(attributes ?? {}), message);
+        },
         error: (message, attributes) => {
             logger.error(processAttributes(attributes ?? {}), message);
         },
@@ -97840,7 +97860,6 @@ function implementSingleEndpoint({ endpoint, handler, method, mount, route }) {
         : []), (async (req, res) => {
         res.locals.requestID = ulid$2();
         try {
-            console.log("HANDLE REQ", method, route, req.params);
             // Extract and validate parameters with proper type assertion
             const parameters = endpoint.parameters
                 ? validateAndExtract("params", req.params, endpoint.parameters)
@@ -97888,7 +97907,6 @@ function implementSingleEndpoint({ endpoint, handler, method, mount, route }) {
                 res.status(output.status).send(output.statusText);
                 return;
             }
-            console.log("GOT RESPONSE", method, route, output.status, typeof output.body);
             res.status(output.status);
             if (endpoint.response.type === "text-stream") {
                 if (!res.getHeader("content-type")) {
@@ -108360,7 +108378,8 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
                         onRequestEnd,
                         onRequestStart,
                         reportMetrics,
-                        request: payload
+                        request: payload,
+                        signal
                     }).catch(error => {
                         logger.error("SSE request handler failed", {
                             error: asError(error),
@@ -108394,7 +108413,7 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
         }
     }
 }
-async function handleRequest({ apiURL, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request }) {
+async function handleRequest({ apiURL, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request, signal }) {
     function reportMetricsSafe(payload) {
         reportMetrics(payload).catch(error => {
             logger.warn("Failed to upload LLM prompt metrics", {
@@ -108414,7 +108433,8 @@ async function handleRequest({ apiURL, configuration, logger, modelID, onRequest
             logger,
             requestID: request.requestID,
             requestStartedAt,
-            response
+            response,
+            signal
         });
         const latencyMs = Math.max(0, Date.now() - requestStartedAt);
         const totalTokens = 0;
@@ -108454,26 +108474,23 @@ async function handleRequest({ apiURL, configuration, logger, modelID, onRequest
             durationMs: latencyMs,
             totalTokens
         });
-        await postChunk({
+        const streamHandler = await sendChunkStream({
             apiURL,
             configuration,
-            payload: {
-                data: encodeBinaryChunk(Buffer.from(failureMessage)),
-                sequence: 0,
-                status: 502
-            },
-            requestID: request.requestID
+            requestID: request.requestID,
+            logger
         });
-        await postChunk({
-            apiURL,
-            configuration,
-            payload: {
-                data: null,
-                sequence: 1,
-                status: 502
-            },
-            requestID: request.requestID
+        await streamHandler.sendChunk({
+            data: encodeBinaryChunk(Buffer.from(failureMessage)),
+            sequence: 0,
+            status: 502
+        });
+        await streamHandler.sendChunk({
+            data: null,
+            sequence: 1,
+            status: 502
         });
+        await streamHandler.end();
         reportMetricsSafe({
             bytes: requestBytes + failureBytes,
             completionTokens: 0,
@@ -108497,12 +108514,22 @@ async function handleRequest({ apiURL, configuration, logger, modelID, onRequest
         await onRequestEnd?.(request);
     }
 }
-async function streamResponse({ apiURL, configuration, logger, requestID, requestStartedAt, response }) {
+async function streamResponse({ apiURL, configuration, logger, requestID, requestStartedAt, response, signal }) {
     let sequence = 0;
     let responseBytes = 0;
     let timeToFirstTokenMs = null;
+    const streamHandler = await sendChunkStream({
+        apiURL,
+        configuration,
+        requestID,
+        logger
+    });
     if (response.body instanceof Readable) {
         for await (const chunk of response.body) {
+            if (signal?.aborted) {
+                streamHandler.abort();
+                throw new Error("Request cancelled");
+            }
             const buffer = Buffer.isBuffer(chunk)
                 ? chunk
                 : Buffer.from(chunk);
@@ -108510,28 +108537,19 @@ async function streamResponse({ apiURL, configuration, logger, requestID, reques
                 timeToFirstTokenMs = Math.max(0, Date.now() - requestStartedAt);
             }
             responseBytes += buffer.length;
-            await postChunk({
-                apiURL,
-                configuration,
-                payload: {
-                    data: encodeBinaryChunk(buffer),
-                    sequence,
-                    status: response.status
-                },
-                requestID
+            await streamHandler.sendChunk({
+                data: encodeBinaryChunk(buffer),
+                sequence,
+                status: response.status
             });
             sequence += 1;
         }
-        await postChunk({
-            apiURL,
-            configuration,
-            payload: {
-                data: null,
-                sequence,
-                status: response.status
-            },
-            requestID
+        await streamHandler.sendChunk({
+            data: null,
+            sequence,
+            status: response.status
         });
+        await streamHandler.end();
         return {
             responseBytes,
             status: response.status,
@@ -108547,27 +108565,18 @@ async function streamResponse({ apiURL, configuration, logger, requestID, reques
         responseBytes = Buffer.byteLength(responsePayload, "utf8");
         timeToFirstTokenMs = Math.max(0, Date.now() - requestStartedAt);
     }
-    await postChunk({
-        apiURL,
-        configuration,
-        payload: {
-            data: encodeBinaryChunk(Buffer.from(responsePayload)),
-            headers: response.headers,
-            sequence,
-            status: response.status
-        },
-        requestID
+    await streamHandler.sendChunk({
+        data: encodeBinaryChunk(Buffer.from(responsePayload)),
+        headers: response.headers,
+        sequence,
+        status: response.status
     });
-    await postChunk({
-        apiURL,
-        configuration,
-        payload: {
-            data: null,
-            sequence: sequence + 1,
-            status: response.status
-        },
-        requestID
+    await streamHandler.sendChunk({
+        data: null,
+        sequence: sequence + 1,
+        status: response.status
     });
+    await streamHandler.end();
     logger.info("SSE response queued", {
         requestMethod: requestID
     });
@@ -108577,28 +108586,101 @@ async function streamResponse({ apiURL, configuration, logger, requestID, reques
         timeToFirstTokenMs
     };
 }
-async function postChunk({ apiURL, configuration, payload, requestID }) {
-    const response = ClientToServerAPIResponseSchema.parse({
-        data: payload.data,
-        headers: payload.headers,
-        requestID,
-        status: payload.status
-    });
-    await fetch(`${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/${requestID}/chunk`, {
-        body: JSON.stringify({
-            ...response,
-            sequence: payload.sequence
-        }),
-        headers: {
-            "content-type": "application/json",
-            "x-api-key": configuration.apiKey
-        },
-        method: "POST"
-    });
-}
 function encodeBinaryChunk(chunk) {
     return chunk.toString("base64");
 }
+async function sendChunkStream({ apiURL, configuration, requestID, logger }) {
+    const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/${requestID}/stream`;
+    const maxFlushAttempts = 3;
+    let isAborted = false;
+    let isClosed = false;
+    let activeAbortController = null;
+    const chunks = [];
+    const sendChunk = async (payload) => {
+        if (isAborted || isClosed) {
+            return;
+        }
+        const response = ClientToServerAPIResponseSchema.parse({
+            data: payload.data,
+            headers: payload.headers,
+            requestID,
+            status: payload.status
+        });
+        const chunk = JSON.stringify({
+            ...response,
+            sequence: payload.sequence
+        });
+        chunks.push(Buffer.from(chunk + "\n"));
+        if (chunks.length >= 10) {
+            await flushChunks();
+        }
+    };
+    const flushChunks = async () => {
+        if (chunks.length === 0 || isAborted) {
+            return;
+        }
+        const batch = chunks.splice(0, chunks.length);
+        const body = Buffer.concat(batch);
+        for (let attempt = 1; attempt <= maxFlushAttempts; attempt += 1) {
+            try {
+                activeAbortController = new AbortController();
+                const response = await fetch(streamURL, {
+                    body: body.toString(),
+                    headers: {
+                        "content-type": "application/json",
+                        "x-api-key": configuration.apiKey
+                    },
+                    method: "POST",
+                    signal: activeAbortController.signal
+                });
+                if (!response.ok) {
+                    throw new Error(`Chunk stream flush failed with status ${response.status}`);
+                }
+                return;
+            }
+            catch (error) {
+                if (isAborted) {
+                    return;
+                }
+                if (attempt >= maxFlushAttempts) {
+                    chunks.unshift(...batch);
+                    throw asError(error);
+                }
+                logger.warn("Failed to send chunk batch", {
+                    error: asError(error)
+                });
+                await sleep(100 * attempt);
+            }
+            finally {
+                activeAbortController = null;
+            }
+        }
+    };
+    const end = async () => {
+        if (isClosed || isAborted) {
+            return;
+        }
+        await flushChunks();
+        isClosed = true;
+    };
+    const abort = (error) => {
+        isAborted = true;
+        if (activeAbortController) {
+            activeAbortController.abort();
+        }
+        chunks.length = 0;
+        if (error) {
+            logger.error("Chunk stream aborted", {
+                error: asError(error)
+            });
+        }
+    };
+    return {
+        sendChunk,
+        end,
+        abort
+    };
+}
 function calculateRequestBytes(body) {
     if (body === null || body === undefined) {
         return 0;
@@ -118119,13 +118201,14 @@ function monitorEngineResponseStream({ agentEngineType, body, contextLength, eng
         }
     }
     body.on("data", (chunk) => {
+        const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
         if (firstChunkAt === null) {
             firstChunkAt = Date.now();
         }
-        responseBytes += chunk.length;
-        buffer += chunk.toString("utf8");
+        responseBytes += chunkBuffer.length;
+        buffer += chunkBuffer.toString("utf8");
         parseUsageFromBuffer();
-        passThrough.write(modifyChunkWithUsage(chunk));
+        passThrough.write(modifyChunkWithUsage(chunkBuffer));
     });
     body.once("error", err => {
         logEngineMetrics({
@@ -118182,6 +118265,148 @@ function monitorEngineResponseStream({ agentEngineType, body, contextLength, eng
         stream: passThrough
     };
 }
+function monitorEngineResponseSingle({ agentEngineType, body, contextLength, engine, logger, onComplete, parallelism, requestBodyBytes, requestPath, requestStartedAt }) {
+    const maxUsageCaptureBytes = 1024 * 1024;
+    const startedAt = requestStartedAt ?? Date.now();
+    const passThrough = new PassThrough();
+    let responseBytes = 0;
+    let firstChunkAt = null;
+    let usage = null;
+    const usageChunks = [];
+    let usageBytes = 0;
+    let usageCaptureEnabled = true;
+    let completed = false;
+    function finalize(error) {
+        if (completed) {
+            return;
+        }
+        completed = true;
+        if (onComplete) {
+            const completion = onComplete({
+                durationMs: Math.max(0, Date.now() - startedAt),
+                error,
+                requestBodyBytes,
+                responseBytes,
+                timeToFirstTokenMs: firstChunkAt === null ? null : Math.max(0, firstChunkAt - startedAt),
+                usage
+            });
+            if (completion && typeof completion.catch === "function") {
+                completion.catch(error => {
+                    logger.error("Engine metrics completion failed", {
+                        error: asError(error),
+                        requestUrl: requestPath
+                    });
+                });
+            }
+        }
+    }
+    body.on("data", (chunk) => {
+        const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        if (firstChunkAt === null) {
+            firstChunkAt = Date.now();
+        }
+        responseBytes += chunkBuffer.length;
+        if (usageCaptureEnabled) {
+            const nextSize = usageBytes + chunkBuffer.length;
+            if (nextSize <= maxUsageCaptureBytes) {
+                usageChunks.push(chunkBuffer);
+                usageBytes = nextSize;
+            }
+            else {
+                usageCaptureEnabled = false;
+                usageChunks.length = 0;
+            }
+        }
+        passThrough.write(chunkBuffer);
+    });
+    body.once("error", err => {
+        logEngineMetrics({
+            agentEngineType,
+            error: err,
+            level: "error",
+            logger,
+            requestBodyBytes,
+            requestPath,
+            responseBytes,
+            usage
+        });
+        finalize(err);
+        passThrough.destroy(err);
+    });
+    body.once("end", () => {
+        if (usageCaptureEnabled) {
+            try {
+                const parsed = JSON.parse(Buffer.concat(usageChunks).toString("utf8"));
+                if (parsed.usage) {
+                    const usageChunk = parsed.usage;
+                    const completionTokens = usageChunk.completion_tokens ?? null;
+                    const promptTokens = usageChunk.prompt_tokens ?? null;
+                    const totalTokens = usageChunk.total_tokens ?? null;
+                    let contextUsage = usageChunk.context_usage ?? null;
+                    const effectiveContext = getEffectiveContextLength({
+                        contextLength,
+                        engine,
+                        parallelism
+                    });
+                    if (contextUsage === null &&
+                        promptTokens !== null &&
+                        effectiveContext !== null) {
+                        contextUsage = promptTokens / effectiveContext;
+                    }
+                    usage = {
+                        completionTokens,
+                        contextUsage,
+                        promptTokens,
+                        totalTokens
+                    };
+                }
+            }
+            catch (error) {
+                logger.error("Failed to parse engine response body", {
+                    error: asError(error),
+                    requestUrl: requestPath
+                });
+            }
+        }
+        logEngineMetrics({
+            agentEngineType,
+            level: "info",
+            logger,
+            requestBodyBytes,
+            requestPath,
+            responseBytes,
+            usage
+        });
+        finalize(null);
+        passThrough.end();
+    });
+    body.once("close", () => {
+        if (completed) {
+            if (!passThrough.writableEnded) {
+                passThrough.end();
+            }
+            return;
+        }
+        const closeError = new Error("Engine response stream closed before completion");
+        logEngineMetrics({
+            agentEngineType,
+            error: closeError,
+            level: "error",
+            logger,
+            requestBodyBytes,
+            requestPath,
+            responseBytes,
+            usage
+        });
+        finalize(closeError);
+        if (!passThrough.writableEnded) {
+            passThrough.end();
+        }
+    });
+    return {
+        stream: passThrough
+    };
+}
 function logEngineMetrics({ agentEngineType, error, level, logger, requestBodyBytes, requestPath, responseBytes, usage }) {
     const metricsMessage = [
         "LLM engine stream metrics",
@@ -118254,6 +118479,35 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
     }
     const { bytes: requestBodyBytes, payload: serializedBody } = serializeRequestBody(body);
     const requestStartedAt = Date.now();
+    const requestBody = JSON.parse(serializedBody);
+    const streamRequested = requestBody.stream === true;
+    const onMonitoringComplete = ({ durationMs, error, responseBytes, timeToFirstTokenMs, usage }) => {
+        const completionTokens = normalizeTokenCount(usage?.completionTokens);
+        const promptTokens = normalizeTokenCount(usage?.promptTokens);
+        const totalTokens = normalizeTokenCount(usage?.totalTokens ?? completionTokens + promptTokens);
+        const latencyMs = Math.max(0, durationMs);
+        reportMetricsSafe({
+            bytes: requestBodyBytes + responseBytes,
+            completionTokens,
+            engine: configuration.agentEngineType,
+            endpointId: null,
+            latencyMs,
+            modelId: modelID,
+            promptTokens,
+            requestBytes: requestBodyBytes,
+            requestId: null,
+            requestMethod: "POST",
+            requestPath: path,
+            responseBytes,
+            successful: !error,
+            timeToFirstTokenMs,
+            tokensPerSecond: calculateTokensPerSecond({
+                durationMs: latencyMs,
+                totalTokens
+            }),
+            totalTokens
+        });
+    };
     const response = await modelManager
         .fetchOpenAI(path, {
         body: serializedBody,
@@ -118348,44 +118602,31 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
             statusText: responseStatusText
         };
     }
-    const monitoredResponse = monitorEngineResponseStream({
-        agentEngineType: configuration.agentEngineType,
-        body: Readable.fromWeb(response.body),
-        contextLength: modelManager.contextLength,
-        engine: configuration.agentEngineType,
-        logger,
-        onComplete: ({ durationMs, error, responseBytes, timeToFirstTokenMs, usage }) => {
-            const completionTokens = normalizeTokenCount(usage?.completionTokens);
-            const promptTokens = normalizeTokenCount(usage?.promptTokens);
-            const totalTokens = normalizeTokenCount(usage?.totalTokens ?? completionTokens + promptTokens);
-            const latencyMs = Math.max(0, durationMs);
-            reportMetricsSafe({
-                bytes: requestBodyBytes + responseBytes,
-                completionTokens,
-                engine: configuration.agentEngineType,
-                endpointId: null,
-                latencyMs,
-                modelId: modelID,
-                promptTokens,
-                requestBytes: requestBodyBytes,
-                requestId: null,
-                requestMethod: "POST",
-                requestPath: path,
-                responseBytes,
-                successful: !error,
-                timeToFirstTokenMs,
-                tokensPerSecond: calculateTokensPerSecond({
-                    durationMs: latencyMs,
-                    totalTokens
-                }),
-                totalTokens
-            });
-        },
-        parallelism: modelManager.parallelism,
-        requestBodyBytes,
-        requestPath: path,
-        requestStartedAt
-    });
+    const monitoredResponse = streamRequested
+        ? monitorEngineResponseStream({
+            agentEngineType: configuration.agentEngineType,
+            body: Readable.fromWeb(response.body),
+            contextLength: modelManager.contextLength,
+            engine: configuration.agentEngineType,
+            logger,
+            onComplete: onMonitoringComplete,
+            parallelism: modelManager.parallelism,
+            requestBodyBytes,
+            requestPath: path,
+            requestStartedAt
+        })
+        : monitorEngineResponseSingle({
+            agentEngineType: configuration.agentEngineType,
+            body: Readable.fromWeb(response.body),
+            contextLength: modelManager.contextLength,
+            engine: configuration.agentEngineType,
+            logger,
+            onComplete: onMonitoringComplete,
+            parallelism: modelManager.parallelism,
+            requestBodyBytes,
+            requestPath: path,
+            requestStartedAt
+        });
     return {
         body: monitoredResponse.stream,
         headers: Object.fromEntries(response.headers.entries()),

package/dist/utils/engineMetrics.d.ts

@@ -39,5 +39,6 @@ interface MonitorEngineResponseResult {
     stream: Readable;
 }
 export declare function monitorEngineResponseStream({ agentEngineType, body, contextLength, engine, logger, onComplete, parallelism, requestBodyBytes, requestPath, requestStartedAt }: MonitorEngineResponseOptions): MonitorEngineResponseResult;
+export declare function monitorEngineResponseSingle({ agentEngineType, body, contextLength, engine, logger, onComplete, parallelism, requestBodyBytes, requestPath, requestStartedAt }: MonitorEngineResponseOptions): MonitorEngineResponseResult;
 export declare function logEngineMetrics({ agentEngineType, error, level, logger, requestBodyBytes, requestPath, responseBytes, usage }: EngineMetricsLogOptions): void;
 export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.24.3",
+  "version": "1.25.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },
@@ -27,7 +27,7 @@
     "test:format": "prettier --check .",
     "test:lint": "eslint source/**/*.ts",
     "test:types": "tsc -p tsconfig.json --noEmit",
-    "test:unit": "vitest run"
+    "test:unit": "vitest -c vitest.config.ts run"
   },
   "prettier": "@infersec/prettier",
   "publishConfig": {

package/dist/utils/__tests__/engineMetrics.test.d.ts

@@ -1 +0,0 @@
-export {};