@infersec/conduit 1.7.0 → 1.8.1

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-BF-CnHXg.js';
+import { a as asError, s as startInferenceAgent } from './start-ItMOqpI1.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-BF-CnHXg.js';
+import { s as startInferenceAgent, a as asError } from './start-ItMOqpI1.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/modelManagement/ModelManager.d.ts

@@ -30,5 +30,7 @@ export declare class ModelManager extends EventEmitter<ModelManagerEvents> {
         onDownloadProgress?: (update: ModelDownloadProgressUpdate) => void;
     }): Promise<void>;
     start(): Promise<void>;
+    private isEngineReady;
+    private waitForEngineReady;
 }
 export {};

package/dist/{start-BF-CnHXg.js → start-ItMOqpI1.js}

@@ -104881,8 +104881,9 @@ const glob = Object.assign(glob_, {
 });
 glob.glob = glob;
 
+const DEFAULT_LLAMACPP_GPU_LAYERS = 999;
 const LLAMACPP_START_ARGS = ["--host", "0.0.0.0", "--port", "8000", "--jinja"];
-const LLAMACPP_EXECUTABLE = "llama-server";
+const LLAMACPP_EXECUTABLE = process.env.LLAMACPP_EXECUTABLE ?? "llama-server";
 const DEFAULT_LLAMACPP_CONTEXT_LENGTH = 131072;
 async function findQuantizedModelTarget({ model, path }) {
     if (model.source.type === "storage") {
@@ -104918,6 +104919,10 @@ async function startLlamacpp({ targetDirectory }) {
     const contextLength = Math.max(1, this.contextLength ?? DEFAULT_LLAMACPP_CONTEXT_LENGTH);
     const parallelism = this.parallelism;
     const args = [...LLAMACPP_START_ARGS, "--model", target, "--ctx-size", String(contextLength)];
+    const gpuLayers = Number.parseInt(process.env.LLAMACPP_GPU_LAYERS ?? String(DEFAULT_LLAMACPP_GPU_LAYERS), 10);
+    if (Number.isFinite(gpuLayers) && gpuLayers > 0) {
+        args.push("--n-gpu-layers", String(gpuLayers));
+    }
     if (typeof parallelism === "number") {
         args.push("--parallel", String(Math.max(1, parallelism)));
     }
@@ -105086,8 +105091,48 @@ class ModelManager extends EventEmitter {
         this.logger.info("Started LLM engine", {
             agentEngineType: this.engine
         });
+        try {
+            await this.waitForEngineReady();
+        }
+        catch (error) {
+            const err = error instanceof Error ? error : new Error(String(error));
+            this.emit("engineError", err);
+            throw err;
+        }
         this.emit("engineReady");
     }
+    async isEngineReady() {
+        switch (this.engine) {
+            case "llama.cpp":
+            case "vllm": {
+                try {
+                    const response = await this.fetchOpenAI("/v1/models", {
+                        method: "GET",
+                        signal: AbortSignal.timeout(5000)
+                    });
+                    return response.ok;
+                }
+                catch (_error) {
+                    return false;
+                }
+            }
+            default:
+                return true;
+        }
+    }
+    async waitForEngineReady() {
+        const maxWaitMs = 5 * 60 * 1000;
+        const pollIntervalMs = 2000;
+        const start = Date.now();
+        while (Date.now() - start < maxWaitMs) {
+            const ready = await this.isEngineReady();
+            if (ready) {
+                return;
+            }
+            await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
+        }
+        throw new Error("LLM engine failed readiness checks within timeout");
+    }
 }
 
 async function handleSSERequests({ apiURL, configuration, logger, onRequest, onRequestEnd, onRequestStart }) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.7.0",
+  "version": "1.8.1",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },