@infersec/conduit 1.40.0 → 1.42.0

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-Dw1tdbuz.js';
+import { a as asError, s as startInferenceAgent } from './start-DGdf6ycx.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-Dw1tdbuz.js';
+import { s as startInferenceAgent, a as asError } from './start-DGdf6ycx.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/requestHandlers/createConduitAnthropicAPIReferenceHandlers.d.ts

@@ -1,5 +1,5 @@
 import { API_CLIENT_CONDUIT_ANTHROPIC_REFERENCE } from "@infersec/definitions";
-import { implementAPIReference } from "@infersec/fetch";
+import { implementAPIReference, type APIRequest } from "@infersec/fetch";
 import { Logger } from "@infersec/logger";
 import { APIClient } from "../apiClient/index.js";
 import { Configuration } from "../configuration.js";
@@ -19,7 +19,7 @@ export declare function createPostMessagesHandler(options: {
     getModelManager: () => ModelManager;
     logger: Logger;
 }): (params: {
-    req: import("@infersec/fetch").APIRequest;
+    req: APIRequest;
     res: import("@infersec/fetch").APIResponse;
     parameters: Record<string, never>;
     query: Record<string, never>;

package/dist/requestHandlers/createConduitOpenAIAPIReferenceHandlers.d.ts

@@ -1,5 +1,5 @@
 import { API_CLIENT_CONDUIT_OPENAI_REFERENCE } from "@infersec/definitions";
-import { implementAPIReference } from "@infersec/fetch";
+import { implementAPIReference, type APIRequest } from "@infersec/fetch";
 import { Logger } from "@infersec/logger";
 import { APIClient } from "../apiClient/index.js";
 import { Configuration } from "../configuration.js";
@@ -21,7 +21,7 @@ export declare function createGetModelsHandler(options: {
     logger: Logger;
     startup: number;
 }): (params: {
-    req: import("@infersec/fetch").APIRequest;
+    req: APIRequest;
     res: import("@infersec/fetch").APIResponse;
     parameters: Record<string, never>;
     query: Record<string, never>;
@@ -66,7 +66,7 @@ export declare function createPostChatCompletionsHandler(options: {
     logger: Logger;
     startup: number;
 }): (params: {
-    req: import("@infersec/fetch").APIRequest;
+    req: APIRequest;
     res: import("@infersec/fetch").APIResponse;
     parameters: Record<string, never>;
     query: Record<string, never>;
@@ -172,7 +172,7 @@ export declare function createPostCompletionsHandler(options: {
     logger: Logger;
     startup: number;
 }): (params: {
-    req: import("@infersec/fetch").APIRequest;
+    req: APIRequest;
     res: import("@infersec/fetch").APIResponse;
     parameters: Record<string, never>;
     query: Record<string, never>;

package/dist/sse/handler.d.ts

@@ -6,7 +6,10 @@ export declare function handleSSERequests({ apiURL, configuration, logger, model
     configuration: Configuration;
     logger: Logger;
     modelID: ULID;
-    onRequest: (request: ServerToClientAPIRequest) => Promise<APIResponse>;
+    onRequest: ({ request, signal }: {
+        request: ServerToClientAPIRequest;
+        signal?: AbortSignal;
+    }) => Promise<APIResponse>;
     onRequestEnd?: (request: ServerToClientAPIRequest) => Promise<void> | void;
     onRequestStart?: (request: ServerToClientAPIRequest) => Promise<void> | void;
     reportMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;

package/dist/sse/requestProxy.d.ts

@@ -3,7 +3,8 @@ import type { Configuration } from "../configuration.js";
 /**
  * Proxy server requests to the local inference HTTP server.
  */
-export declare function proxyRequest({ configuration, request }: {
+export declare function proxyRequest({ configuration, request, signal }: {
     configuration: Configuration;
     request: ServerToClientAPIRequest;
+    signal?: AbortSignal;
 }): Promise<APIResponse>;

package/dist/{start-Dw1tdbuz.js → start-DGdf6ycx.js}

@@ -15622,21 +15622,30 @@ const ServerToClientAPIRequestSchema = APIRequestSchema.extend({
 
 _enum(["inf-end", "inf-src", "storage"]);
 
-const PlanSlugSchema = _enum(["basic", "business", "free-trial", "open", "team"]);
-object({
-    href: string$1(),
-    limits: object({
-        endpoints: number$1().int().nonnegative(),
-        sources: number$1().int().nonnegative(),
-        users: number$1().int().nonnegative()
-    }),
-    monthlyCostEUR: number$1().int().nonnegative(),
-    name: string$1(),
-    slug: PlanSlugSchema,
-    trialDays: number$1().int().positive().nullable(),
-    websiteFeatures: array(string$1()),
-    websiteIconPath: string$1()
+const ConnectedSourceMetaSchema = object({
+    chargeName: literal("connected_source"),
+    unitCost: number$1(),
+    units: number$1()
+});
+const TokenInputMetaSchema = object({
+    chargeName: literal("tokens_input"),
+    modelId: string$1().nullable(),
+    requestId: string$1().nullable(),
+    unitCost: number$1(),
+    units: number$1()
 });
+const TokenOutputMetaSchema = object({
+    chargeName: literal("tokens_output"),
+    modelId: string$1().nullable(),
+    requestId: string$1().nullable(),
+    unitCost: number$1(),
+    units: number$1()
+});
+discriminatedUnion("chargeName", [
+    ConnectedSourceMetaSchema,
+    TokenInputMetaSchema,
+    TokenOutputMetaSchema
+]);
 
 function readEnvIntegerOptional({ defaultValue, name }) {
     const str = readEnvStringOptional(name, `${defaultValue}`).trim();
@@ -108660,7 +108669,9 @@ class ModelManager extends EventEmitter {
                 const timeout = setTimeout(() => {
                     controller.abort(new Error("Inference request timeout"));
                 }, ENGINE_FETCH_TIMEOUT_MS);
-                const effectiveSignal = callerSignal ?? controller.signal;
+                const effectiveSignal = callerSignal
+                    ? AbortSignal.any([callerSignal, controller.signal])
+                    : controller.signal;
                 try {
                     return await undiciExports.fetch(joinURL(`http://localhost:${this.enginePort}`, path), {
                         ...opts,
@@ -109240,11 +109251,14 @@ function monitorEngineResponseStream({ agentEngineType, body, contextLength, eng
             }
             return;
         }
-        const closeError = new Error("Engine response stream closed before completion");
+        const aborted = body.destroyed && body.errored?.name === "AbortError";
+        const closeError = aborted
+            ? null
+            : new Error("Engine response stream closed before completion");
         logEngineMetrics({
             agentEngineType,
-            error: closeError,
-            level: "error",
+            error: closeError ?? undefined,
+            level: aborted ? "info" : "error",
             logger,
             requestBodyBytes,
             requestPath,
@@ -109382,11 +109396,14 @@ function monitorEngineResponseSingle({ agentEngineType, body, contextLength, eng
             }
             return;
         }
-        const closeError = new Error("Engine response stream closed before completion");
+        const aborted = body.destroyed && body.errored?.name === "AbortError";
+        const closeError = aborted
+            ? null
+            : new Error("Engine response stream closed before completion");
         logEngineMetrics({
             agentEngineType,
-            error: closeError,
-            level: "error",
+            error: closeError ?? undefined,
+            level: aborted ? "info" : "error",
             logger,
             requestBodyBytes,
             requestPath,
@@ -109457,7 +109474,7 @@ function calculateTokensPerSecond$2({ durationMs, totalTokens }) {
     }
     return Math.round(tokensPerSecond);
 }
-async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID, modelManager, path, reportMetrics }) {
+async function proxyOpenAIStreamingRoute({ body, configuration, endpointId, logger, modelID, modelManager, path, reportMetrics, signal }) {
     function normalizeTokenCount(value) {
         if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
             return value;
@@ -109485,7 +109502,7 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
             bytes: requestBodyBytes + responseBytes,
             completionTokens,
             engine: configuration.agentEngineType,
-            endpointId: null,
+            endpointId: endpointId ?? null,
             latencyMs,
             modelId: modelID,
             promptTokens,
@@ -109509,7 +109526,8 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
         headers: {
             "Content-Type": "application/json"
         },
-        method: "POST"
+        method: "POST",
+        signal
     })
         .catch(error => {
         logEngineMetrics({
@@ -109527,7 +109545,7 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
             bytes: requestBodyBytes,
             completionTokens: 0,
             engine: configuration.agentEngineType,
-            endpointId: null,
+            endpointId: endpointId ?? null,
             latencyMs,
             modelId: modelID,
             promptTokens: 0,
@@ -109577,7 +109595,7 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
             bytes: requestBodyBytes,
             completionTokens: 0,
             engine: configuration.agentEngineType,
-            endpointId: null,
+            endpointId: endpointId ?? null,
             latencyMs,
             modelId: modelID,
             promptTokens: 0,
@@ -109628,35 +109646,52 @@ async function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID,
     };
 }
 
+function extractEndpointId$1(req) {
+    const value = req.headers["x-endpoint-id"];
+    const raw = typeof value === "string" ? value : Array.isArray(value) ? value[0] : null;
+    return raw && isValid(raw) ? raw : null;
+}
 function createConduitOpenAIAPIReferenceHandlers({ apiClient, configuration, getModelID, getModelManager, logger, startup }) {
     return {
         "/v1/chat/completions": {
-            POST: async ({ body }) => {
+            POST: async ({ body, req, res }) => {
                 const modelID = getModelID();
                 const modelManager = getModelManager();
+                const abortController = new AbortController();
+                res.on("close", () => {
+                    abortController.abort();
+                });
                 return proxyOpenAIStreamingRoute({
                     body,
                     configuration,
+                    endpointId: extractEndpointId$1(req),
                     logger,
                     modelID,
                     modelManager,
                     path: "/v1/chat/completions",
-                    reportMetrics: apiClient.reportPromptMetrics
+                    reportMetrics: apiClient.reportPromptMetrics,
+                    signal: abortController.signal
                 });
             }
         },
         "/v1/completions": {
-            POST: async ({ body }) => {
+            POST: async ({ body, req, res }) => {
                 const modelID = getModelID();
                 const modelManager = getModelManager();
+                const abortController = new AbortController();
+                res.on("close", () => {
+                    abortController.abort();
+                });
                 return proxyOpenAIStreamingRoute({
                     body,
                     configuration,
+                    endpointId: extractEndpointId$1(req),
                     logger,
                     modelID,
                     modelManager,
                     path: "/v1/completions",
-                    reportMetrics: apiClient.reportPromptMetrics
+                    reportMetrics: apiClient.reportPromptMetrics,
+                    signal: abortController.signal
                 });
             }
         },
@@ -109770,7 +109805,7 @@ function extractAnthropicNonStreamUsage(body) {
         return null;
     }
 }
-async function proxyAnthropicStreamingRoute({ body, configuration, logger, modelID, modelManager, reportMetrics }) {
+async function proxyAnthropicStreamingRoute({ body, configuration, endpointId, logger, modelID, modelManager, reportMetrics, signal }) {
     function reportMetricsSafe(payload) {
         reportMetrics(payload).catch(error => {
             logger.warn("Failed to upload LLM prompt metrics", {
@@ -109792,7 +109827,7 @@ async function proxyAnthropicStreamingRoute({ body, configuration, logger, model
             bytes: requestBodyBytes + responseBytes,
             completionTokens,
             engine: configuration.agentEngineType,
-            endpointId: null,
+            endpointId: endpointId ?? null,
             latencyMs,
             modelId: modelID,
             promptTokens,
@@ -109816,7 +109851,8 @@ async function proxyAnthropicStreamingRoute({ body, configuration, logger, model
         headers: {
             "Content-Type": "application/json"
         },
-        method: "POST"
+        method: "POST",
+        signal
     })
         .catch(error => {
         logEngineMetrics({
@@ -109834,7 +109870,7 @@ async function proxyAnthropicStreamingRoute({ body, configuration, logger, model
             bytes: requestBodyBytes,
             completionTokens: 0,
             engine: configuration.agentEngineType,
-            endpointId: null,
+            endpointId: endpointId ?? null,
             latencyMs,
             modelId: modelID,
             promptTokens: 0,
@@ -109879,7 +109915,7 @@ async function proxyAnthropicStreamingRoute({ body, configuration, logger, model
             bytes: requestBodyBytes,
             completionTokens: 0,
             engine: configuration.agentEngineType,
-            endpointId: null,
+            endpointId: endpointId ?? null,
             latencyMs,
             modelId: modelID,
             promptTokens: 0,
@@ -110057,19 +110093,30 @@ async function proxyAnthropicStreamingRoute({ body, configuration, logger, model
     };
 }
 
+function extractEndpointId(req) {
+    const value = req.headers["x-endpoint-id"];
+    const raw = typeof value === "string" ? value : Array.isArray(value) ? value[0] : null;
+    return raw && isValid(raw) ? raw : null;
+}
 function createConduitAnthropicAPIReferenceHandlers({ apiClient, configuration, getModelID, getModelManager, logger }) {
     return {
         "/v1/messages": {
-            POST: async ({ body }) => {
+            POST: async ({ body, req, res }) => {
                 const modelID = getModelID();
                 const modelManager = getModelManager();
+                const abortController = new AbortController();
+                res.on("close", () => {
+                    abortController.abort();
+                });
                 return proxyAnthropicStreamingRoute({
                     body,
                     configuration,
+                    endpointId: extractEndpointId(req),
                     logger,
                     modelID,
                     modelManager,
-                    reportMetrics: apiClient.reportPromptMetrics
+                    reportMetrics: apiClient.reportPromptMetrics,
+                    signal: abortController.signal
                 });
             }
         }
@@ -110089,6 +110136,7 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/stream`;
     const maxReconnectDelayMs = 30000;
     let reconnectAttempt = 0;
+    const activeRequests = new Map();
     while (!signal?.aborted) {
         const connectionStartedAt = Date.now();
         try {
@@ -110102,11 +110150,29 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
                     });
                 },
                 onMessage: (message) => {
+                    if (message.event === "cancel") {
+                        const { requestID } = JSON.parse(message.data);
+                        const controller = activeRequests.get(requestID);
+                        if (controller) {
+                            logger.info("Cancelling active request", {
+                                requestID
+                            });
+                            controller.abort();
+                            activeRequests.delete(requestID);
+                        }
+                        return;
+                    }
                     if (message.event !== "request") {
                         return;
                     }
                     const payload = ServerToClientAPIRequestSchema.parse(JSON.parse(message.data));
+                    const perRequestController = new AbortController();
+                    activeRequests.set(payload.requestID, perRequestController);
+                    const effectiveSignal = signal
+                        ? AbortSignal.any([perRequestController.signal, signal])
+                        : perRequestController.signal;
                     handleRequest({
+                        activeRequests,
                         apiURL,
                         configuration,
                         logger,
@@ -110116,7 +110182,7 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
                         onRequestStart,
                         reportMetrics,
                         request: payload,
-                        signal
+                        signal: effectiveSignal
                     }).catch(error => {
                         logger.error("SSE request handler failed", {
                             error: asError(error),
@@ -110151,7 +110217,7 @@ async function handleSSERequests({ apiURL, configuration, logger, modelID, onReq
         }
     }
 }
-async function handleRequest({ apiURL, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request, signal }) {
+async function handleRequest({ activeRequests, apiURL, configuration, logger, modelID, onRequest, onRequestEnd, onRequestStart, reportMetrics, request, signal }) {
     function reportMetricsSafe(payload) {
         reportMetrics(payload).catch(error => {
             logger.warn("Failed to upload LLM prompt metrics", {
@@ -110160,11 +110226,12 @@ async function handleRequest({ apiURL, configuration, logger, modelID, onRequest
             });
         });
     }
+    const endpointId = request.parameters?.endpointID ?? null;
     const requestStartedAt = Date.now();
     const requestBytes = calculateRequestBytes(request.body ?? null);
     try {
         await onRequestStart?.(request);
-        const response = await onRequest(request);
+        const response = await onRequest({ request, signal });
         const responseMetrics = await streamResponse({
             apiURL,
             configuration,
@@ -110184,7 +110251,7 @@ async function handleRequest({ apiURL, configuration, logger, modelID, onRequest
             bytes: requestBytes + responseMetrics.responseBytes,
             completionTokens: 0,
             engine: configuration.agentEngineType,
-            endpointId: null,
+            endpointId,
             latencyMs,
             modelId: modelID,
             promptTokens: 0,
@@ -110200,40 +110267,45 @@ async function handleRequest({ apiURL, configuration, logger, modelID, onRequest
         });
     }
     catch (error) {
-        logger.error("SSE request failed", {
-            error: asError(error),
-            requestMethod: request.requestID
-        });
+        const isCancelled = signal?.aborted;
+        if (isCancelled) {
+            logger.info("SSE request cancelled", {
+                requestID: request.requestID
+            });
+        }
+        else {
+            logger.error("SSE request failed", {
+                error: asError(error),
+                requestMethod: request.requestID
+            });
+        }
         const failureMessage = "Bad gateway\n\nProxying failed";
         const failureBytes = Buffer.byteLength(failureMessage, "utf8");
         const latencyMs = Math.max(0, Date.now() - requestStartedAt);
-        const totalTokens = 0;
-        const tokensPerSecond = calculateTokensPerSecond({
-            durationMs: latencyMs,
-            totalTokens
-        });
-        const streamHandler = await sendChunkStream({
-            apiURL,
-            configuration,
-            requestID: request.requestID,
-            logger
-        });
-        await streamHandler.sendChunk({
-            data: encodeBinaryChunk(Buffer.from(failureMessage)),
-            sequence: 0,
-            status: 502
-        });
-        await streamHandler.sendChunk({
-            data: null,
-            sequence: 1,
-            status: 502
-        });
-        await streamHandler.end();
+        if (!isCancelled) {
+            const streamHandler = await sendChunkStream({
+                apiURL,
+                configuration,
+                requestID: request.requestID,
+                logger
+            });
+            await streamHandler.sendChunk({
+                data: encodeBinaryChunk(Buffer.from(failureMessage)),
+                sequence: 0,
+                status: 502
+            });
+            await streamHandler.sendChunk({
+                data: null,
+                sequence: 1,
+                status: 502
+            });
+            await streamHandler.end();
+        }
         reportMetricsSafe({
-            bytes: requestBytes + failureBytes,
+            bytes: isCancelled ? requestBytes : requestBytes + failureBytes,
             completionTokens: 0,
             engine: configuration.agentEngineType,
-            endpointId: null,
+            endpointId,
             latencyMs,
             modelId: modelID,
             promptTokens: 0,
@@ -110241,14 +110313,15 @@ async function handleRequest({ apiURL, configuration, logger, modelID, onRequest
             requestId: request.requestID,
             requestMethod: request.method,
             requestPath: request.path,
-            responseBytes: failureBytes,
+            responseBytes: isCancelled ? 0 : failureBytes,
             successful: false,
-            timeToFirstTokenMs: latencyMs,
-            tokensPerSecond,
-            totalTokens
+            timeToFirstTokenMs: isCancelled ? null : latencyMs,
+            tokensPerSecond: 0,
+            totalTokens: 0
         });
     }
     finally {
+        activeRequests.delete(request.requestID);
         await onRequestEnd?.(request);
     }
 }
@@ -110446,7 +110519,7 @@ function calculateTokensPerSecond({ durationMs, totalTokens }) {
 /**
  * Proxy server requests to the local inference HTTP server.
  */
-async function proxyRequest({ configuration, request }) {
+async function proxyRequest({ configuration, request, signal }) {
     let finalPath = request.path;
     if (request.parameters) {
         Object.entries(request.parameters).forEach(([key, value]) => {
@@ -110461,8 +110534,16 @@ async function proxyRequest({ configuration, request }) {
     }
     const fetchOptions = {
         method: request.method,
-        headers: request.headers
+        headers: {
+            ...request.headers,
+            ...(request.parameters?.endpointID
+                ? { "x-endpoint-id": request.parameters.endpointID }
+                : undefined)
+        }
     };
+    if (signal) {
+        fetchOptions.signal = signal;
+    }
     if (request.body) {
         fetchOptions.body =
             typeof request.body === "object" ? JSON.stringify(request.body) : request.body;
@@ -120361,10 +120442,11 @@ async function createApplication({ abortController, apiClient, configuration, lo
         configuration,
         logger,
         modelID: conduitConfiguration.targetModel.id,
-        onRequest: async (request) => {
+        onRequest: async ({ request, signal }) => {
             return proxyRequest({
                 configuration,
-                request
+                request,
+                signal
             });
         },
         onRequestEnd: () => {

package/dist/utils/anthropic.d.ts

@@ -3,13 +3,15 @@ import { InferenceAgentLLMMetricsPayload, type ULID } from "@infersec/definition
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
 import { ModelManager } from "../modelManagement/ModelManager.js";
-export declare function proxyAnthropicStreamingRoute({ body, configuration, logger, modelID, modelManager, reportMetrics }: {
+export declare function proxyAnthropicStreamingRoute({ body, configuration, endpointId, logger, modelID, modelManager, reportMetrics, signal }: {
     body: unknown;
     configuration: Configuration;
+    endpointId?: ULID | null;
     logger: Logger;
     modelID: ULID;
     modelManager: ModelManager;
     reportMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;
+    signal?: AbortSignal;
 }): Promise<{
     body: Readable;
     headers: Record<string, string>;

package/dist/utils/openai.d.ts

@@ -3,14 +3,16 @@ import { InferenceAgentLLMMetricsPayload, type ULID } from "@infersec/definition
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
 import { ModelManager } from "../modelManagement/ModelManager.js";
-export declare function proxyOpenAIStreamingRoute({ body, configuration, logger, modelID, modelManager, path, reportMetrics }: {
+export declare function proxyOpenAIStreamingRoute({ body, configuration, endpointId, logger, modelID, modelManager, path, reportMetrics, signal }: {
     body: unknown;
     configuration: Configuration;
+    endpointId?: ULID | null;
     logger: Logger;
     modelID: ULID;
     modelManager: ModelManager;
     path: "/v1/chat/completions" | "/v1/completions";
     reportMetrics: (payload: InferenceAgentLLMMetricsPayload) => Promise<void>;
+    signal?: AbortSignal;
 }): Promise<{
     body: Readable;
     headers: Record<string, string>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.40.0",
+  "version": "1.42.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },