@infersec/conduit 1.31.0 → 1.31.1

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-CC3HzuZU.js';
+import { a as asError, s as startInferenceAgent } from './start-DvkkeCrj.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-CC3HzuZU.js';
+import { s as startInferenceAgent, a as asError } from './start-DvkkeCrj.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/{start-CC3HzuZU.js → start-DvkkeCrj.js}

@@ -108429,6 +108429,8 @@ async function startLlamacpp({ targetDirectory }) {
     return processManager;
 }
 
+// 2 hours
+const ENGINE_FETCH_TIMEOUT_MS = 7200000;
 class ModelManager extends EventEmitter {
     engine;
     model;
@@ -108472,9 +108474,28 @@ class ModelManager extends EventEmitter {
     async fetchOpenAI(path, opts) {
         switch (this.engine) {
             case "llama.cpp":
-            case "vllm":
-                console.log("FETCH", path, opts);
-                return undiciExports.fetch(joinURL("http://localhost:8000", path), opts);
+            case "vllm": {
+                this.logger.debug(`Fetching from engine: ${path}`);
+                const callerSignal = opts?.signal;
+                const controller = new AbortController();
+                const timeout = setTimeout(() => {
+                    controller.abort(new Error("Inference request timeout"));
+                }, ENGINE_FETCH_TIMEOUT_MS);
+                const effectiveSignal = callerSignal ?? controller.signal;
+                try {
+                    return await undiciExports.fetch(joinURL("http://localhost:8000", path), {
+                        ...opts,
+                        headers: {
+                            ...opts?.headers,
+                            Connection: "keep-alive"
+                        },
+                        signal: effectiveSignal
+                    });
+                }
+                finally {
+                    clearTimeout(timeout);
+                }
+            }
             // case "ollama":
             //     console.log("FETCH", path, opts);
             //     return fetch(

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.31.0",
+  "version": "1.31.1",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },