@infersec/conduit 1.40.0 → 1.41.0

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-Dw1tdbuz.js';
+import { a as asError, s as startInferenceAgent } from './start-CPrgh7rN.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-Dw1tdbuz.js';
+import { s as startInferenceAgent, a as asError } from './start-CPrgh7rN.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/sse/handler.d.ts

@@ -6,7 +6,10 @@ export declare function handleSSERequests({ apiURL, configuration, logger, model
     configuration: Configuration;
     logger: Logger;
     modelID: ULID;
-    onRequest: (request: ServerToClientAPIRequest) => Promise<APIResponse>;
+    onRequest: ({ request, signal }: {
+        request: ServerToClientAPIRequest;
+        signal?: AbortSignal;
+    }) => Promise<APIResponse>;
     onRequestEnd?: (request: ServerToClientAPIRequest) => Promise<void> | void;
     onRequestStart?: (request: ServerToClientAPIRequest) => Promise<void> | void;
     reportMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;

package/dist/sse/requestProxy.d.ts

@@ -3,7 +3,8 @@ import type { Configuration } from "../configuration.js";
 /**
  * Proxy server requests to the local inference HTTP server.
  */
-export declare function proxyRequest({ configuration, request }: {
+export declare function proxyRequest({ configuration, request, signal }: {
     configuration: Configuration;
     request: ServerToClientAPIRequest;
+    signal?: AbortSignal;
 }): Promise<APIResponse>;

package/dist/{start-Dw1tdbuz.js → start-CPrgh7rN.js}

@@ -108660,7 +108660,9 @@ class ModelManager extends EventEmitter {
                 const timeout = setTimeout(() => {
                     controller.abort(new Error("Inference request timeout"));
                 }, ENGINE_FETCH_TIMEOUT_MS);
-                const effectiveSignal = callerSignal ?? controller.signal;
+                const effectiveSignal = callerSignal
+                    ? AbortSignal.any([callerSignal, controller.signal])
+                    : controller.signal;
                 try {
                     return await undiciExports.fetch(joinURL(`http://localhost:${this.enginePort}`, path), {
                         ...opts,
@@ -109240,11 +109242,14 @@ function monitorEngineResponseStream({ agentEngineType, body, contextLength, eng
             }
             return;
         }
-        const closeError = new Error("Engine response stream closed before completion");
+        const aborted = body.destroyed && body.errored?.name === "AbortError";
+        const closeError = aborted
+            ? null
+            : new Error("Engine response stream closed before completion");
         logEngineMetrics({
             agentEngineType,
-            error: closeError,
-            level: "error",
+            error: closeError ?? undefined,
+            level: aborted ? "info" : "error",
             logger,
             requestBodyBytes,
             requestPath,
@@ -109382,11 +109387,14 @@ function monitorEngineResponseSingle({ agentEngineType, body, contextLength, eng
             }
             return;
         }
-        const closeError = new Error("Engine response stream closed before completion");
+        const aborted = body.destroyed && body.errored?.name === "AbortError";
+        const closeError = aborted
+            ? null
+            : new Error("Engine response stream closed before completion");
         logEngineMetrics({
             agentEngineType,
-            error: closeError,
-            level: "error",
+            error: closeError ?? undefined,
+            level: aborted ? "info" : "error",
             logger,
             requestBodyBytes,
             requestPath,
@@ -109457,7 +109465,7 @@ function calculateTokensPerSecond$2({ durationMs, totalTokens }) {
     }
     return Math.round(tokensPerSecond);
 }
-async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID, modelManager, path, reportMetrics }) {
+async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID, modelManager, path, reportMetrics, signal }) {
     function normalizeTokenCount(value) {
         if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
             return value;
@@ -109509,7 +109517,8 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
         headers: {
             "Content-Type": "application/json"
         },
-        method: "POST"
+        method: "POST",
+        signal
     })
         .catch(error => {
         logEngineMetrics({
@@ -109631,9 +109640,13 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
 function createConduitOpenAIAPIReferenceHandlers({ apiClient, configuration, getModelID, getModelManager, logger, startup }) {
     return {
         "/v1/chat/completions": {
-            POST: async ({ body }) => {
+            POST: async ({ body, res }) => {
                 const modelID = getModelID();
                 const modelManager = getModelManager();
+                const abortController = new AbortController();
+                res.on("close", () => {
+                    abortController.abort();
+                });
                 return proxyOpenAIStreamingRoute({
                     body,
                     configuration,
@@ -109641,14 +109654,19 @@ function createConduitOpenAIAPIReferenceHandlers({ apiClient, configuration, get
                     modelID,
                     modelManager,
                     path: "/v1/chat/completions",
-                    reportMetrics: apiClient.reportPromptMetrics
+                    reportMetrics: apiClient.reportPromptMetrics,
+                    signal: abortController.signal
                 });
             }
         },
         "/v1/completions": {
-            POST: async ({ body }) => {
+            POST: async ({ body, res }) => {
                 const modelID = getModelID();
                 const modelManager = getModelManager();
+                const abortController = new AbortController();
+                res.on("close", () => {
+                    abortController.abort();
+                });
                 return proxyOpenAIStreamingRoute({
                     body,
                     configuration,
@@ -109656,7 +109674,8 @@ function createConduitOpenAIAPIReferenceHandlers({ apiClient, configuration, get
                     modelID,
                     modelManager,
                     path: "/v1/completions",
-                    reportMetrics: apiClient.reportPromptMetrics
+                    reportMetrics: apiClient.reportPromptMetrics,
+                    signal: abortController.signal
                 });
             }
         },
@@ -109770,7 +109789,7 @@ function extractAnthropicNonStreamUsage(body) {
         return null;
     }
 }
-async function proxyAnthropicStreamingRoute({ body, configuration, logger, modelID, modelManager, reportMetrics }) {
+async function proxyAnthropicStreamingRoute({ body, configuration, logger, modelID, modelManager, reportMetrics, signal }) {
     function reportMetricsSafe(payload) {
         reportMetrics(payload).catch(error => {
             logger.warn("Failed to upload LLM prompt metrics", {
@@ -109816,7 +109835,8 @@ async function proxyAnthropicStreamingRoute({ body, configuration, logger, model
         headers: {
             "Content-Type": "application/json"
         },
-        method: "POST"
+        method: "POST",
+        signal
     })
         .catch(error => {
         logEngineMetrics({
@@ -110060,16 +110080,21 @@ async function proxyAnthropicStreamingRoute({ body, configuration, logger, model
 function createConduitAnthropicAPIReferenceHandlers({ apiClient, configuration, getModelID, getModelManager, logger }) {
     return {
         "/v1/messages": {
-            POST: async ({ body }) => {
+            POST: async ({ body, res }) => {
                 const modelID = getModelID();
                 const modelManager = getModelManager();
+                const abortController = new AbortController();
+                res.on("close", () => {
+                    abortController.abort();
+                });
                 return proxyAnthropicStreamingRoute({
                     body,
                     configuration,
                     logger,
                     modelID,
                     modelManager,
-                    reportMetrics: apiClient.reportPromptMetrics
+                    reportMetrics: apiClient.reportPromptMetrics,
+                    signal: abortController.signal
                 });
             }
         }
@@ -110089,6 +110114,7 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/stream`;
     const maxReconnectDelayMs = 30000;
     let reconnectAttempt = 0;
+    const activeRequests = new Map();
     while (!signal?.aborted) {
         const connectionStartedAt = Date.now();
         try {
@@ -110102,11 +110128,26 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
                     });
                 },
                 onMessage: (message) => {
+                    if (message.event === "cancel") {
+                        const { requestID } = JSON.parse(message.data);
+                        const controller = activeRequests.get(requestID);
+                        if (controller) {
+                            logger.info("Cancelling active request", {
+                                requestID
+                            });
+                            controller.abort();
+                            activeRequests.delete(requestID);
+                        }
+                        return;
+                    }
                     if (message.event !== "request") {
                         return;
                     }
                     const payload = ServerToClientAPIRequestSchema.parse(JSON.parse(message.data));
+                    const perRequestController = new AbortController();
+                    activeRequests.set(payload.requestID, perRequestController);
                     handleRequest({
+                        activeRequests,
                         apiURL,
                         configuration,
                         logger,
@@ -110116,7 +110157,7 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
                         onRequestStart,
                         reportMetrics,
                         request: payload,
-                        signal
+                        signal: perRequestController.signal
                     }).catch(error => {
                         logger.error("SSE request handler failed", {
                             error: asError(error),
@@ -110151,7 +110192,7 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
         }
     }
 }
-async function handleRequest({ apiURL, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request, signal }) {
+async function handleRequest({ activeRequests, apiURL, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request, signal }) {
     function reportMetricsSafe(payload) {
         reportMetrics(payload).catch(error => {
             logger.warn("Failed to upload LLM prompt metrics", {
@@ -110164,7 +110205,7 @@ async function handleRequest({ apiURL, configuration, logger, modelID, onRequest
     const requestBytes = calculateRequestBytes(request.body ?? null);
     try {
         await onRequestStart?.(request);
-        const response = await onRequest(request);
+        const response = await onRequest({ request, signal });
         const responseMetrics = await streamResponse({
             apiURL,
             configuration,
@@ -110200,37 +110241,42 @@ async function handleRequest({ apiURL, configuration, logger, modelID, onRequest
         });
     }
     catch (error) {
-        logger.error("SSE request failed", {
-            error: asError(error),
-            requestMethod: request.requestID
-        });
+        const isCancelled = signal?.aborted;
+        if (isCancelled) {
+            logger.info("SSE request cancelled", {
+                requestID: request.requestID
+            });
+        }
+        else {
+            logger.error("SSE request failed", {
+                error: asError(error),
+                requestMethod: request.requestID
+            });
+        }
         const failureMessage = "Bad gateway\n\nProxying failed";
         const failureBytes = Buffer.byteLength(failureMessage, "utf8");
         const latencyMs = Math.max(0, Date.now() - requestStartedAt);
-        const totalTokens = 0;
-        const tokensPerSecond = calculateTokensPerSecond({
-            durationMs: latencyMs,
-            totalTokens
-        });
-        const streamHandler = await sendChunkStream({
-            apiURL,
-            configuration,
-            requestID: request.requestID,
-            logger
-        });
-        await streamHandler.sendChunk({
-            data: encodeBinaryChunk(Buffer.from(failureMessage)),
-            sequence: 0,
-            status: 502
-        });
-        await streamHandler.sendChunk({
-            data: null,
-            sequence: 1,
-            status: 502
-        });
-        await streamHandler.end();
+        if (!isCancelled) {
+            const streamHandler = await sendChunkStream({
+                apiURL,
+                configuration,
+                requestID: request.requestID,
+                logger
+            });
+            await streamHandler.sendChunk({
+                data: encodeBinaryChunk(Buffer.from(failureMessage)),
+                sequence: 0,
+                status: 502
+            });
+            await streamHandler.sendChunk({
+                data: null,
+                sequence: 1,
+                status: 502
+            });
+            await streamHandler.end();
+        }
         reportMetricsSafe({
-            bytes: requestBytes + failureBytes,
+            bytes: isCancelled ? requestBytes : requestBytes + failureBytes,
             completionTokens: 0,
             engine: configuration.agentEngineType,
             endpointId: null,
@@ -110241,14 +110287,15 @@ async function handleRequest({ apiURL, configuration, logger, modelID, onRequest
             requestId: request.requestID,
             requestMethod: request.method,
             requestPath: request.path,
-            responseBytes: failureBytes,
+            responseBytes: isCancelled ? 0 : failureBytes,
             successful: false,
-            timeToFirstTokenMs: latencyMs,
-            tokensPerSecond,
-            totalTokens
+            timeToFirstTokenMs: isCancelled ? null : latencyMs,
+            tokensPerSecond: 0,
+            totalTokens: 0
         });
     }
     finally {
+        activeRequests.delete(request.requestID);
         await onRequestEnd?.(request);
     }
 }
@@ -110446,7 +110493,7 @@ function calculateTokensPerSecond({ durationMs, totalTokens }) {
 /**
  * Proxy server requests to the local inference HTTP server.
  */
-async function proxyRequest({ configuration, request }) {
+async function proxyRequest({ configuration, request, signal }) {
     let finalPath = request.path;
     if (request.parameters) {
         Object.entries(request.parameters).forEach(([key, value]) => {
@@ -110463,6 +110510,9 @@ async function proxyRequest({ configuration, request }) {
         method: request.method,
         headers: request.headers
     };
+    if (signal) {
+        fetchOptions.signal = signal;
+    }
     if (request.body) {
         fetchOptions.body =
             typeof request.body === "object" ? JSON.stringify(request.body) : request.body;
@@ -120361,10 +120411,11 @@ async function createApplication({ abortController, apiClient, configuration, lo
         configuration,
         logger,
         modelID: conduitConfiguration.targetModel.id,
-        onRequest: async (request) => {
+        onRequest: async ({ request, signal }) => {
             return proxyRequest({
                 configuration,
-                request
+                request,
+                signal
             });
         },
         onRequestEnd: () => {

package/dist/utils/anthropic.d.ts

@@ -3,13 +3,14 @@ import { InferenceAgentLLMMetricsPayload, type ULID } from "@infersec/definition
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
 import { ModelManager } from "../modelManagement/ModelManager.js";
-export declare function proxyAnthropicStreamingRoute({ body, configuration, logger, modelID, modelManager, reportMetrics }: {
+export declare function proxyAnthropicStreamingRoute({ body, configuration, logger, modelID, modelManager, reportMetrics, signal }: {
     body: unknown;
     configuration: Configuration;
     logger: Logger;
     modelID: ULID;
     modelManager: ModelManager;
     reportMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;
+    signal?: AbortSignal;
 }): Promise<{
     body: Readable;
     headers: Record<string, string>;

package/dist/utils/openai.d.ts

@@ -3,7 +3,7 @@ import { InferenceAgentLLMMetricsPayload, type ULID } from "@infersec/definition
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
 import { ModelManager } from "../modelManagement/ModelManager.js";
-export declare function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID, modelManager, path, reportMetrics }: {
+export declare function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID, modelManager, path, reportMetrics, signal }: {
     body: unknown;
     configuration: Configuration;
     logger: Logger;
@@ -11,6 +11,7 @@ export declare function proxyOpenAIStreamingRoute({ body, configuration, logger,
     modelManager: ModelManager;
     path: "/v1/chat/completions" | "/v1/completions";
     reportMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;
+    signal?: AbortSignal;
 }): Promise<{
     body: Readable;
     headers: Record<string, string>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.40.0",
+  "version": "1.41.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },