@infersec/conduit 1.6.4 → 1.8.0

package/dist/cli.js

@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
 
 import { parseArgs } from 'node:util';
 import 'node:crypto';
-import { a as asError, s as startInferenceAgent } from './start-D9p7R0lT.js';
+import { a as asError, s as startInferenceAgent } from './start-CVgqDYvF.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/index.js

@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
 const __dirname = __pathDirname(__filename);
 
 import 'node:crypto';
-import { s as startInferenceAgent, a as asError } from './start-D9p7R0lT.js';
+import { s as startInferenceAgent, a as asError } from './start-CVgqDYvF.js';
 import 'argon2';
 import 'node:child_process';
 import 'node:stream';

package/dist/{start-D9p7R0lT.js → start-CVgqDYvF.js}

@@ -104881,8 +104881,9 @@ const glob = Object.assign(glob_, {
 });
 glob.glob = glob;
 
+const DEFAULT_LLAMACPP_GPU_LAYERS = 999;
 const LLAMACPP_START_ARGS = ["--host", "0.0.0.0", "--port", "8000", "--jinja"];
-const LLAMACPP_EXECUTABLE = "llama-server";
+const LLAMACPP_EXECUTABLE = process.env.LLAMACPP_EXECUTABLE ?? "llama-server";
 const DEFAULT_LLAMACPP_CONTEXT_LENGTH = 131072;
 async function findQuantizedModelTarget({ model, path }) {
     if (model.source.type === "storage") {
@@ -104902,21 +104903,26 @@ async function findQuantizedModelTarget({ model, path }) {
     if (modelFiles.length <= 0) {
         throw new Error(`No models found for format: ${model.format}`);
     }
+    modelFiles.sort((left, right) => left.localeCompare(right, undefined, { sensitivity: "base" }));
     if (!variant) {
         // Just return the first
         return modelFiles[0];
     }
-    const target = modelFiles.find(fileName => fileName.toLowerCase().includes(variant.toLowerCase()));
-    if (!target) {
+    const matches = modelFiles.filter(fileName => fileName.toLowerCase().includes(variant.toLowerCase()));
+    if (matches.length === 0) {
         throw new Error(`No model found for format and variant: ${model.format} / ${variant}`);
     }
-    return target;
+    return matches[0];
 }
 async function startLlamacpp({ targetDirectory }) {
     const target = await findQuantizedModelTarget({ model: this.model, path: targetDirectory });
     const contextLength = Math.max(1, this.contextLength ?? DEFAULT_LLAMACPP_CONTEXT_LENGTH);
     const parallelism = this.parallelism;
     const args = [...LLAMACPP_START_ARGS, "--model", target, "--ctx-size", String(contextLength)];
+    const gpuLayers = Number.parseInt(process.env.LLAMACPP_GPU_LAYERS ?? String(DEFAULT_LLAMACPP_GPU_LAYERS), 10);
+    if (Number.isFinite(gpuLayers) && gpuLayers > 0) {
+        args.push("--n-gpu-layers", String(gpuLayers));
+    }
     if (typeof parallelism === "number") {
         args.push("--parallel", String(Math.max(1, parallelism)));
     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.6.4",
+  "version": "1.8.0",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },