@infersec/conduit 1.8.0 → 1.8.2

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-CVgqDYvF.js';
+import { a as asError, s as startInferenceAgent } from './start-LWLy96m_.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-CVgqDYvF.js';
+import { s as startInferenceAgent, a as asError } from './start-LWLy96m_.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/modelManagement/ModelManager.d.ts

@@ -30,5 +30,7 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
         onDownloadProgress?: (update: ModelDownloadProgressUpdate) => void;
     }): Promise<void>;
     start(): Promise<void>;
+    private isEngineReady;
+    private waitForEngineReady;
 }
 export {};

package/dist/sse/handler.d.ts

@@ -1,11 +1,12 @@
 import { type APIResponse, type ServerToClientAPIRequest } from "@infersec/definitions";
 import { Logger } from "@infersec/logger";
 import { Configuration } from "../configuration.js";
-export declare function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart }: {
+export declare function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, signal }: {
     apiURL: string;
     configuration: Configuration;
     logger: Logger;
     onRequest: (request: ServerToClientAPIRequest) => Promise<APIResponse>;
     onRequestEnd?: (request: ServerToClientAPIRequest) => Promise<void> | void;
     onRequestStart?: (request: ServerToClientAPIRequest) => Promise<void> | void;
+    signal?: AbortSignal;
 }): Promise<void>;

package/dist/{start-CVgqDYvF.js → start-LWLy96m_.js}

@@ -105091,42 +105091,112 @@ class ModelManager extends EventEmitter {
         this.logger.info("Started LLM engine", {
             agentEngineType: this.engine
         });
+        try {
+            await this.waitForEngineReady();
+        }
+        catch (error) {
+            const err = error instanceof Error ? error : new Error(String(error));
+            this.emit("engineError", err);
+            throw err;
+        }
         this.emit("engineReady");
     }
+    async isEngineReady() {
+        switch (this.engine) {
+            case "llama.cpp":
+            case "vllm": {
+                try {
+                    const response = await this.fetchOpenAI("/v1/models", {
+                        method: "GET",
+                        signal: AbortSignal.timeout(5000)
+                    });
+                    return response.ok;
+                }
+                catch (_error) {
+                    return false;
+                }
+            }
+            default:
+                return true;
+        }
+    }
+    async waitForEngineReady() {
+        const maxWaitMs = 5 * 60 * 1000;
+        const pollIntervalMs = 2000;
+        const start = Date.now();
+        while (Date.now() - start < maxWaitMs) {
+            const ready = await this.isEngineReady();
+            if (ready) {
+                return;
+            }
+            await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
+        }
+        throw new Error("LLM engine failed readiness checks within timeout");
+    }
+}
+
+function sleep(ms) {
+    return new Promise(resolve => {
+        setTimeout(() => resolve(), ms);
+    });
 }
 
-async function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart }) {
+async function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, signal }) {
     const streamURL = `${apiURL}/conduit/api/v1/source/${configuration.inferenceSourceID}/requests/stream`;
-    await connectSSE(streamURL, {
-        headers: {
-            "x-api-key": configuration.apiKey
-        },
-        onError: (error) => {
-            logger.error("SSE connection error", {
-                error
+    const maxReconnectDelayMs = 30000;
+    let reconnectAttempt = 0;
+    while (!signal?.aborted) {
+        const connectionStartedAt = Date.now();
+        try {
+            await connectSSE(streamURL, {
+                headers: {
+                    "x-api-key": configuration.apiKey
+                },
+                onError: (error) => {
+                    logger.error("SSE connection error", {
+                        error
+                    });
+                },
+                onMessage: (message) => {
+                    if (message.event !== "request") {
+                        return;
+                    }
+                    const payload = ServerToClientAPIRequestSchema.parse(JSON.parse(message.data));
+                    handleRequest({
+                        apiURL,
+                        configuration,
+                        logger,
+                        onRequest,
+                        onRequestEnd,
+                        onRequestStart,
+                        request: payload
+                    }).catch(error => {
+                        logger.error("SSE request handler failed", {
+                            error: asError(error),
+                            requestMethod: payload.requestID
+                        });
+                    });
+                },
+                signal
             });
-        },
-        onMessage: (message) => {
-            if (message.event !== "request") {
+        }
+        catch (error) {
+            if (signal?.aborted) {
                 return;
             }
-            const payload = ServerToClientAPIRequestSchema.parse(JSON.parse(message.data));
-            handleRequest({
-                apiURL,
-                configuration,
-                logger,
-                onRequestEnd,
-                onRequestStart,
-                onRequest,
-                request: payload
-            }).catch(error => {
-                logger.error("SSE request handler failed", {
-                    error: asError(error),
-                    requestMethod: payload.requestID
-                });
+            logger.error("SSE connection failed", {
+                error: asError(error)
             });
         }
-    });
+        if (signal?.aborted) {
+            return;
+        }
+        const connectionDurationMs = Date.now() - connectionStartedAt;
+        reconnectAttempt = connectionDurationMs > 10000 ? 0 : reconnectAttempt + 1;
+        const reconnectDelayMs = Math.min(maxReconnectDelayMs, Math.max(1000, 1000 * 2 ** Math.min(6, reconnectAttempt)));
+        logger.warn("SSE disconnected, retrying");
+        await sleep(reconnectDelayMs);
+    }
 }
 async function handleRequest({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart, request }) {
     try {
@@ -114784,7 +114854,6 @@ async function createApplication({ abortController, apiClient, configuration, lo
     });
     const modelFileName = getConduitModelFileName(conduitConfiguration);
     const modelName = getConduitModelName(conduitConfiguration);
-    const idleReason = "Awaiting requests";
     const startup = Date.now();
     // Initialise model manager
     const modelManager = new ModelManager({
@@ -114795,28 +114864,6 @@ async function createApplication({ abortController, apiClient, configuration, lo
         parallelism: conduitConfiguration.parallelism ?? null,
         root: configuration.rootDirectory
     });
-    modelManager.on("engineError", err => {
-        logger.error("LLM engine error", {
-            error: err
-        });
-        conduitStateManager.setState({
-            error: err.message,
-            state: "error"
-        });
-        abortController.abort(err);
-    });
-    modelManager.on("engineTerminated", () => {
-        conduitStateManager.setState({
-            state: "offline"
-        });
-        abortController.abort();
-    });
-    modelManager.on("engineReady", () => {
-        conduitStateManager.setState({
-            reason: idleReason,
-            state: "idle"
-        });
-    });
     conduitStateManager.setState({
         modelFileName,
         modelName,
@@ -114931,40 +114978,54 @@ async function createApplication({ abortController, apiClient, configuration, lo
         });
     }, CONDUIT_STATE_INTERVAL_MS);
     let activeRequests = 0;
-    const setIdleState = () => {
-        conduitStateManager.setState({
-            reason: idleReason,
-            state: "idle"
-        });
-    };
     const setOnlineState = () => {
         conduitStateManager.setState({
             modelName,
             state: "online"
         });
     };
+    modelManager.on("engineError", err => {
+        logger.error("LLM engine error", {
+            error: err
+        });
+        conduitStateManager.setState({
+            error: err.message,
+            state: "error"
+        });
+        abortController.abort(err);
+    });
+    modelManager.on("engineTerminated", () => {
+        conduitStateManager.setState({
+            state: "offline"
+        });
+        abortController.abort();
+    });
+    modelManager.on("engineReady", () => {
+        setOnlineState();
+    });
     handleSSERequests({
         apiURL: configuration.apiURL,
         configuration,
         logger,
-        onRequestEnd: () => {
-            activeRequests = Math.max(0, activeRequests - 1);
-            if (activeRequests === 0) {
-                setIdleState();
-            }
-        },
         onRequest: async (request) => {
             return proxyRequest({
                 configuration,
                 request
             });
         },
+        onRequestEnd: () => {
+            activeRequests = Math.max(0, activeRequests - 1);
+            if (activeRequests === 0) {
+                setOnlineState();
+            }
+        },
         onRequestStart: () => {
             activeRequests += 1;
             if (activeRequests === 1) {
                 setOnlineState();
             }
-        }
+        },
+        signal: abortController.signal
     }).catch(error => {
         logger.error("SSE handler failed", {
             error: asError(error)
@@ -115004,12 +115065,6 @@ function getConfiguration({ overrides } = {}) {
     };
 }
 
-function sleep(ms) {
-    return new Promise(resolve => {
-        setTimeout(() => resolve(), ms);
-    });
-}
-
 async function startInferenceAgent({ configurationOverrides }) {
     const abortController = new AbortController();
     const configuration = getConfiguration({ overrides: configurationOverrides });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.8.0",
+  "version": "1.8.2",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },