@infersec/conduit 1.46.0 → 1.47.0

package/dist/cli.js

@@ -18502,22 +18502,6 @@ const ULIDSchema = string$1().refine(isValid, { message: "Invalid ULID" });
 //
 //  irid://SERVICE/ACCOUNT_ID/RESOURCE_ID::RESOURCE_SUB_SPECIFIER
 const IRID_MAX_LENGTH = 2048;
-// export const IRIDSchema = z.custom<`irid://${string}/${string}/${string}:${string}`>((irid) => {
-//     if (typeof irid !== "string") return false;
-//     if (!/^irid:\/\//.test(irid)) return false;
-//     const prefixStripped = irid.replace(/^irid:\/\//, "");
-//     if (irid.length > IRID_MAX_LENGTH) {
-//         return false;
-//     }
-//     if (!/:[a-z0-9:.-]+$/.test(prefixStripped)) return false;
-//     const [components] = prefixStripped.split(":");
-//     const [service, accountID, resourceID] = components.split("/");
-//     if (!/^[a-z0-9-]+$/.test(service)) return false;
-//     if (!isValid(accountID)) return false;
-//     if (!/^[a-z0-9-]+$/.test(resourceID)) return false;
-//     return true;
-// }, "Invalid IRID value");
-// export const IRIDSchema = z.string().brand<`irid://${string}/${string}/${string}:${string}`>();
 const IRIDSchema = string$1()
     .refine(irid => /^irid:\/\//.test(irid), { message: "Bad IRID prefix" })
     .refine(irid => irid.length <= IRID_MAX_LENGTH, {
@@ -18965,6 +18949,7 @@ const ConduitStateSchema = z
     })
 ])
     .and(z.object({
+    activeRequestCount: z.number().int().nonnegative().optional(),
     timestamp: z.iso.datetime()
 }));
 const ConduitState = z.preprocess(value => {
@@ -19666,6 +19651,17 @@ const API_CLIENT_CONDUIT_OPENAI_REFERENCE = {
     }
 });
 
+var RoutingMethod;
+(function (RoutingMethod) {
+    RoutingMethod["FirstAvailable"] = "first-available";
+    RoutingMethod["RoundRobin"] = "round-robin";
+})(RoutingMethod || (RoutingMethod = {}));
+RoutingMethod.FirstAvailable;
+({
+    [RoutingMethod.FirstAvailable]: "First Available",
+    [RoutingMethod.RoundRobin]: "Round Robin"
+});
+
 object({
     accountID: ULIDSchema.optional(),
     email: string$1().email(),
@@ -19741,7 +19737,7 @@ custom((perm) => {
 });
 // #endregion
 
-// @ts-ignore
+// @ts-expect-error Cannot find name "window" in VSCode
 const isBrowser = typeof window !== "undefined";
 
 const APIRequestSchema = object({
@@ -112782,6 +112778,7 @@ class ModelManager extends EventEmitter {
     contextLength;
     logger;
     engineProcess = null;
+    healthPollInterval = null;
     lifecycleState = "stopped";
     stopRequested = false;
     modelsDirectory;
@@ -112919,6 +112916,9 @@ class ModelManager extends EventEmitter {
             }
             this.lifecycleState = "errored";
             this.emit("engineError", err);
+            if (this.engineProcess) {
+                this.startHealthPoll();
+            }
             throw err;
         }
         this.lifecycleState = "running";
@@ -112930,16 +112930,18 @@ class ModelManager extends EventEmitter {
                 message: "Cannot stop LLM engine: already stopping"
             });
         }
-        if (this.lifecycleState !== "running" && this.lifecycleState !== "starting") {
+        if (this.lifecycleState !== "running" &&
+            this.lifecycleState !== "starting" &&
+            this.lifecycleState !== "errored") {
             throw new UnknownError({
                 message: `Cannot stop LLM engine: Invalid state: ${this.lifecycleState}`
             });
         }
+        this.clearHealthPoll();
         const processManager = this.engineProcess;
         if (!processManager) {
-            throw new UnknownError({
-                message: "Cannot stop LLM engine: engine process not running"
-            });
+            this.lifecycleState = "stopped";
+            return;
         }
         this.lifecycleState = "stopping";
         this.stopRequested = true;
@@ -112949,32 +112951,63 @@ class ModelManager extends EventEmitter {
         return this.lifecycleState === "stopped";
     }
     get canStop() {
-        return this.lifecycleState === "running" || this.lifecycleState === "starting";
+        return (this.lifecycleState === "running" ||
+            this.lifecycleState === "starting" ||
+            this.lifecycleState === "errored");
     }
     get state() {
         return this.lifecycleState;
     }
-    async isEngineReady() {
+    async checkEngineReadiness() {
         switch (this.engine) {
-            case "llama.cpp":
+            case "llama.cpp": {
+                return this.checkLlamacppReadiness();
+            }
             case "vllm": {
-                try {
-                    const response = await this.fetchOpenAI("/v1/models", {
-                        method: "GET",
-                        signal: AbortSignal.timeout(5000)
-                    });
-                    return response.ok;
-                }
-                catch (_error) {
-                    return false;
-                }
+                return this.checkVLLMReadiness();
             }
             default:
-                return true;
+                return "ready";
+        }
+    }
+    async checkLlamacppReadiness() {
+        try {
+            const response = await undiciExports.fetch(joinURL(`http://localhost:${this.enginePort}`, "/health"), {
+                method: "GET",
+                signal: AbortSignal.timeout(5000)
+            });
+            if (response.status === 503) {
+                return "loading";
+            }
+            if (response.ok) {
+                return "ready";
+            }
+            return "loading";
+        }
+        catch (_error) {
+            return "unreachable";
+        }
+    }
+    async checkVLLMReadiness() {
+        try {
+            const response = await undiciExports.fetch(joinURL(`http://localhost:${this.enginePort}`, "/v1/models"), {
+                method: "GET",
+                signal: AbortSignal.timeout(5000)
+            });
+            if (response.status === 503) {
+                return "loading";
+            }
+            if (response.ok) {
+                return "ready";
+            }
+            return "loading";
+        }
+        catch (_error) {
+            return "unreachable";
         }
     }
     async waitForEngineReady() {
-        const maxWaitMs = 5 * 60 * 1000;
+        const maxWaitMs = 15 * 60 * 1000;
         const pollIntervalMs = 2000;
         const start = Date.now();
         while (Date.now() - start < maxWaitMs) {
@@ -112984,14 +113017,41 @@ class ModelManager extends EventEmitter {
             if (!this.engineProcess) {
                 throw new Error("LLM engine process exited before readiness checks completed");
             }
-            const ready = await this.isEngineReady();
-            if (ready) {
+            const readiness = await this.checkEngineReadiness();
+            if (readiness === "ready") {
                 return;
             }
             await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
         }
         throw new Error("LLM engine failed readiness checks within timeout");
     }
+    clearHealthPoll() {
+        if (this.healthPollInterval) {
+            clearInterval(this.healthPollInterval);
+            this.healthPollInterval = null;
+        }
+    }
+    startHealthPoll() {
+        this.clearHealthPoll();
+        this.logger.info("Starting background health poll for errored engine");
+        this.healthPollInterval = setInterval(() => {
+            if (!this.engineProcess) {
+                this.clearHealthPoll();
+                return;
+            }
+            this.checkEngineReadiness()
+                .then(readiness => {
+                if (readiness === "ready") {
+                    this.clearHealthPoll();
+                    this.lifecycleState = "running";
+                    this.emit("engineReady");
+                }
+            })
+                .catch(() => {
+                // keep polling
+            });
+        }, 15_000);
+    }
     bindEngineProcessEvents(processManager) {
         let hasTerminated = false;
         processManager.on("stderr", line => {
@@ -113005,6 +113065,7 @@ class ModelManager extends EventEmitter {
                 return;
             }
             hasTerminated = true;
+            this.clearHealthPoll();
             if (this.stopRequested) {
                 this.engineProcess = null;
                 this.lifecycleState = "stopped";
@@ -113026,6 +113087,7 @@ class ModelManager extends EventEmitter {
                 return;
             }
             hasTerminated = true;
+            this.clearHealthPoll();
             this.engineProcess = null;
             if (this.stopRequested) {
                 this.lifecycleState = "stopped";
@@ -114844,6 +114906,7 @@ class ConduitStateReportManager {
 }
 
 class ConduitStateManager {
+    activeRequestCount = 0;
     currentState;
     constructor({ initialState }) {
         this.currentState = {
@@ -114851,22 +114914,31 @@ class ConduitStateManager {
             timestamp: new Date().toISOString()
         };
     }
+    decrementActiveRequestCount() {
+        this.activeRequestCount = Math.max(0, this.activeRequestCount - 1);
+    }
     getState() {
-        return this.currentState;
+        return {
+            ...this.currentState,
+            activeRequestCount: this.activeRequestCount
+        };
+    }
+    incrementActiveRequestCount() {
+        this.activeRequestCount += 1;
     }
     setState(state) {
         this.currentState = {
             ...state,
             timestamp: new Date().toISOString()
         };
-        return this.currentState;
+        return this.getState();
     }
     touch() {
         this.currentState = {
             ...this.currentState,
             timestamp: new Date().toISOString()
         };
-        return this.currentState;
+        return this.getState();
     }
 }
 
@@ -124642,10 +124714,12 @@ async function createApplication({ abortController, apiClient, configuration, lo
             });
         },
         onRequestEnd: () => {
-            return;
+            conduitStateManager.decrementActiveRequestCount();
+            conduitStateReportManager.reportStateChange();
         },
         onRequestStart: () => {
-            return;
+            conduitStateManager.incrementActiveRequestCount();
+            conduitStateReportManager.reportStateChange();
         },
         reportMetrics: apiClient.reportPromptMetrics,
         signal: abortController.signal

package/dist/modelManagement/ModelManager.d.ts

@@ -18,6 +18,7 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
     readonly contextLength: number | null;
     protected readonly logger: Logger;
     private engineProcess;
+    private healthPollInterval;
     private lifecycleState;
     private stopRequested;
     protected readonly modelsDirectory: string;
@@ -39,8 +40,12 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
     get canStart(): boolean;
     get canStop(): boolean;
     get state(): EngineLifecycleState;
-    private isEngineReady;
+    private checkEngineReadiness;
+    private checkLlamacppReadiness;
+    private checkVLLMReadiness;
     private waitForEngineReady;
+    private clearHealthPoll;
+    private startHealthPoll;
     private bindEngineProcessEvents;
     private startEngineProcess;
 }

package/dist/state/ConduitStateManager.d.ts

@@ -3,11 +3,14 @@ type ConduitStateInput = Record<string, unknown> & {
     state: ConduitState["state"];
 };
 export declare class ConduitStateManager {
+    private activeRequestCount;
     private currentState;
     constructor({ initialState }: {
         initialState: ConduitStateInput;
     });
+    decrementActiveRequestCount(): void;
     getState(): ConduitState;
+    incrementActiveRequestCount(): void;
     setState(state: ConduitStateInput): ConduitState;
     touch(): ConduitState;
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.46.0",
+  "version": "1.47.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },
@@ -22,6 +22,8 @@
     "format": "prettier --write .",
     "prepublishOnly": "npm run build",
     "start": "npm run build && node ./dist/cli.js inference start",
+    "start-2": "npm run build && node ./dist/cli.js inference start --source 01k1p3rgt1wdq8k43rev1dg4mh --port 9506",
+    "start-3": "npm run build && node ./dist/cli.js inference start --source 01k1p3rgt1wdq8k43rev1dg4mj --port 9507",
     "test": "npm run test:types && npm run test:lint && npm run test:format && npm run test:unit",
     "test:format": "prettier --check .",
     "test:lint": "eslint source/**/*.ts",