@infersec/conduit 1.101.0 → 1.102.1

package/dist/cli.js

@@ -4345,35 +4345,6 @@ function ulid$2(seedTime, prng) {
     return encodeTime(seed, TIME_LEN) + encodeRandom(RANDOM_LEN, currentPRNG);
 }
 
-function getEffectiveContextLength({ contextLength, engineConfig, engineType }) {
-    if (contextLength === null || contextLength <= 0) {
-        return null;
-    }
-    if (!engineConfig) {
-        return contextLength;
-    }
-    switch (engineType) {
-        case "llama.cpp": {
-            const parallelism = engineConfig.parallelism;
-            if (typeof parallelism === "number" && parallelism > 0) {
-                return contextLength / parallelism;
-            }
-            return contextLength;
-        }
-        case "sglang":
-        case "tensorrt-llm":
-        case "vllm": {
-            const tensorParallelSize = engineConfig.tensorParallelSize;
-            if (typeof tensorParallelSize === "number" && tensorParallelSize > 0) {
-                return contextLength / tensorParallelSize;
-            }
-            return contextLength;
-        }
-        default:
-            return contextLength;
-    }
-}
-
 function asError(error) {
     if (error instanceof Error) {
         return error;
@@ -19914,58 +19885,10 @@ const LLMEngineSchema = _enum$1([
     "tensorrt-llm",
     "vllm"
 ]);
-const LlamacppEngineConfigSchema = object$1({
-    batchSize: number$1().int().positive().nullable().optional(),
-    cacheTypeK: string$2().nullable().optional(),
-    cacheTypeV: string$2().nullable().optional(),
-    extraArgs: array(string$2()).optional(),
-    flashAttn: boolean$1().optional(),
-    gpuLayers: number$1().int().min(0).optional(),
-    mainGpu: number$1().int().min(0).nullable().optional(),
-    parallelism: number$1().int().positive().optional(),
-    tensorSplit: string$2().nullable().optional(),
-    ubatchSize: number$1().int().positive().nullable().optional()
-});
-const VLLMEngineConfigSchema = object$1({
-    device: string$2().optional(),
-    dtype: string$2().optional(),
-    extraArgs: array(string$2()).optional(),
-    tensorParallelSize: number$1().int().positive().optional()
-});
-const SGLangEngineConfigSchema = object$1({
-    device: string$2().optional(),
-    dtype: string$2().optional(),
-    extraArgs: array(string$2()).optional(),
-    tensorParallelSize: number$1().int().positive().optional()
-});
-const TensorRTLLMEngineConfigSchema = object$1({
-    backend: _enum$1(["_autodeploy", "pytorch", "tensorrt"]).optional(),
-    dtype: string$2().optional(),
-    extraArgs: array(string$2()).optional(),
-    tensorParallelSize: number$1().int().positive().optional()
-});
-const Exllamav3EngineConfigSchema = object$1({
-    cacheMode: _enum$1(["fp16", "q4", "q6", "q8"]).optional(),
-    extraArgs: array(string$2()).optional(),
-    gpuSplit: string$2().optional(),
-    maxSeqLen: number$1().int().positive().optional()
-});
-const MLXLMEngineConfigSchema = object$1({
-    extraArgs: array(string$2()).optional(),
-    maxKvSize: number$1().int().positive().optional(),
-    trustRemoteCode: boolean$1().optional()
+const EngineConfigSchema = object$1({
+    extraArgs: array(string$2()),
+    type: LLMEngineSchema
 });
-const EngineConfigSchema = discriminatedUnion("type", [
-    object$1({ config: Exllamav3EngineConfigSchema, type: literal("exllamav3") }),
-    object$1({ config: LlamacppEngineConfigSchema, type: literal("llama.cpp") }),
-    object$1({ config: MLXLMEngineConfigSchema, type: literal("mlx-lm") }),
-    object$1({ config: SGLangEngineConfigSchema, type: literal("sglang") }),
-    object$1({
-        config: TensorRTLLMEngineConfigSchema,
-        type: literal("tensorrt-llm")
-    }),
-    object$1({ config: VLLMEngineConfigSchema, type: literal("vllm") })
-]);
 const LLMModelFormatSchema = _enum$1([
     // VLLM / SGLang / TensorRT-LLM
     "safetensors",
@@ -21161,105 +21084,21 @@ object$1({
 });
 const EngineOutputSchema = object$1({
     created: string$2(),
-    exllamav3CacheMode: string$2().nullable(),
-    exllamav3ExtraArgs: array(string$2()),
-    exllamav3GpuSplit: string$2().nullable(),
-    exllamav3MaxSeqLen: number$1().nullable(),
+    extraArgs: array(string$2()),
     id: ULIDSchema,
-    llamacppBatchSize: number$1().nullable(),
-    llamacppCacheTypeK: string$2().nullable(),
-    llamacppCacheTypeV: string$2().nullable(),
-    llamacppExtraArgs: array(string$2()),
-    llamacppFlashAttn: boolean$1(),
-    llamacppGpuLayers: number$1(),
-    llamacppMainGpu: number$1().nullable(),
-    llamacppParallelism: number$1(),
-    llamacppTensorSplit: string$2().nullable(),
-    llamacppUbatchSize: number$1().nullable(),
-    mlxlmExtraArgs: array(string$2()),
-    mlxlmMaxKvSize: number$1().nullable(),
-    mlxlmTrustRemoteCode: boolean$1(),
     name: string$2(),
-    sglangDevice: string$2().nullable(),
-    sglangDtype: string$2().nullable(),
-    sglangExtraArgs: array(string$2()),
-    sglangTensorParallelSize: number$1(),
-    trtllmBackend: string$2().nullable(),
-    trtllmDtype: string$2().nullable(),
-    trtllmExtraArgs: array(string$2()),
-    trtllmTensorParallelSize: number$1(),
     type: LLMEngineSchema,
-    updated: string$2(),
-    vllmDevice: string$2().nullable(),
-    vllmDtype: string$2().nullable(),
-    vllmExtraArgs: array(string$2()),
-    vllmTensorParallelSize: number$1()
+    updated: string$2()
 });
 object$1({
-    exllamav3CacheMode: string$2().nullable().optional(),
-    exllamav3ExtraArgs: array(string$2()).optional(),
-    exllamav3GpuSplit: string$2().nullable().optional(),
-    exllamav3MaxSeqLen: number$1().int().positive().nullable().optional(),
-    llamacppBatchSize: number$1().int().positive().nullable().optional(),
-    llamacppCacheTypeK: string$2().nullable().optional(),
-    llamacppCacheTypeV: string$2().nullable().optional(),
-    llamacppExtraArgs: array(string$2()).optional(),
-    llamacppFlashAttn: boolean$1().optional(),
-    llamacppGpuLayers: number$1().int().min(0).optional(),
-    llamacppMainGpu: number$1().int().min(0).nullable().optional(),
-    llamacppParallelism: number$1().int().positive().optional(),
-    llamacppTensorSplit: string$2().nullable().optional(),
-    llamacppUbatchSize: number$1().int().positive().nullable().optional(),
-    mlxlmExtraArgs: array(string$2()).optional(),
-    mlxlmMaxKvSize: number$1().int().positive().nullable().optional(),
-    mlxlmTrustRemoteCode: boolean$1().optional(),
+    extraArgs: array(string$2()).optional(),
     name: ResourceNameSchema,
-    sglangDevice: string$2().nullable().optional(),
-    sglangDtype: string$2().nullable().optional(),
-    sglangExtraArgs: array(string$2()).optional(),
-    sglangTensorParallelSize: number$1().int().positive().optional(),
-    trtllmBackend: string$2().nullable().optional(),
-    trtllmDtype: string$2().nullable().optional(),
-    trtllmExtraArgs: array(string$2()).optional(),
-    trtllmTensorParallelSize: number$1().int().positive().optional(),
-    type: LLMEngineSchema,
-    vllmDevice: string$2().nullable().optional(),
-    vllmDtype: string$2().nullable().optional(),
-    vllmExtraArgs: array(string$2()).optional(),
-    vllmTensorParallelSize: number$1().int().positive().optional()
+    type: LLMEngineSchema
 });
 object$1({
-    exllamav3CacheMode: string$2().nullable().optional(),
-    exllamav3ExtraArgs: array(string$2()).optional(),
-    exllamav3GpuSplit: string$2().nullable().optional(),
-    exllamav3MaxSeqLen: number$1().int().positive().nullable().optional(),
-    llamacppBatchSize: number$1().int().positive().nullable().optional(),
-    llamacppCacheTypeK: string$2().nullable().optional(),
-    llamacppCacheTypeV: string$2().nullable().optional(),
-    llamacppExtraArgs: array(string$2()).optional(),
-    llamacppFlashAttn: boolean$1().optional(),
-    llamacppGpuLayers: number$1().int().min(0).optional(),
-    llamacppMainGpu: number$1().int().min(0).nullable().optional(),
-    llamacppParallelism: number$1().int().positive().optional(),
-    llamacppTensorSplit: string$2().nullable().optional(),
-    llamacppUbatchSize: number$1().int().positive().nullable().optional(),
-    mlxlmExtraArgs: array(string$2()).optional(),
-    mlxlmMaxKvSize: number$1().int().positive().nullable().optional(),
-    mlxlmTrustRemoteCode: boolean$1().optional(),
+    extraArgs: array(string$2()).optional(),
     name: ResourceNameSchema.optional(),
-    sglangDevice: string$2().nullable().optional(),
-    sglangDtype: string$2().nullable().optional(),
-    sglangExtraArgs: array(string$2()).optional(),
-    sglangTensorParallelSize: number$1().int().positive().optional(),
-    trtllmBackend: string$2().nullable().optional(),
-    trtllmDtype: string$2().nullable().optional(),
-    trtllmExtraArgs: array(string$2()).optional(),
-    trtllmTensorParallelSize: number$1().int().positive().optional(),
-    type: LLMEngineSchema.optional(),
-    vllmDevice: string$2().nullable().optional(),
-    vllmDtype: string$2().nullable().optional(),
-    vllmExtraArgs: array(string$2()).optional(),
-    vllmTensorParallelSize: number$1().int().positive().optional()
+    type: LLMEngineSchema.optional()
 });
 object$1({
     results: array(EngineOutputSchema)
@@ -21336,6 +21175,12 @@ object$1({
     }
 });
 
+object$1({
+    config: EngineConfigSchema,
+    description: string$2().optional(),
+    name: ResourceNameSchema
+});
+
 const ENGINE_API_COMPATIBILITY = {
     exllamav3: {
         nativeAnthropicMessages: false,
@@ -21754,6 +21599,105 @@ discriminatedUnion("chargeName", [
     ToolServiceCallMetaSchema
 ]);
 
+function parseExtraArgs(extraArgs) {
+    if (!Array.isArray(extraArgs) ||
+        !extraArgs.every((value) => typeof value === "string")) {
+        return [];
+    }
+    return extraArgs.flatMap(tokenizeShellLine);
+}
+function findArgValue(args, flag) {
+    let found = null;
+    for (let index = 0; index < args.length; index += 1) {
+        const token = args[index];
+        if (token === flag) {
+            const value = args[index + 1];
+            if (value !== undefined && !value.startsWith("--")) {
+                found = value;
+            }
+            else {
+                found = null;
+            }
+        }
+        else if (token.startsWith(`${flag}=`)) {
+            found = token.slice(flag.length + 1);
+        }
+    }
+    return found;
+}
+function tokenizeShellLine(input) {
+    const tokens = [];
+    let buffer = "";
+    let inQuote = null;
+    let hasBuffer = false;
+    for (let index = 0; index < input.length; index += 1) {
+        const char = input[index];
+        if (inQuote) {
+            if (char === inQuote) {
+                inQuote = null;
+            }
+            else {
+                buffer += char;
+            }
+            hasBuffer = true;
+        }
+        else if (char === '"' || char === "'") {
+            inQuote = char;
+            hasBuffer = true;
+        }
+        else if (char === " " || char === "\t") {
+            if (hasBuffer) {
+                tokens.push(buffer);
+                buffer = "";
+                hasBuffer = false;
+            }
+        }
+        else {
+            buffer += char;
+            hasBuffer = true;
+        }
+    }
+    if (hasBuffer) {
+        tokens.push(buffer);
+    }
+    return tokens;
+}
+
+function getEffectiveContextLength({ contextLength, engineConfig, engineType }) {
+    if (contextLength === null || contextLength <= 0) {
+        return null;
+    }
+    if (!engineConfig) {
+        return contextLength;
+    }
+    let divisorFlag = null;
+    switch (engineType) {
+        case "llama.cpp":
+            divisorFlag = "--parallel";
+            break;
+        case "sglang":
+        case "tensorrt-llm":
+            divisorFlag = "--tp-size";
+            break;
+        case "vllm":
+            divisorFlag = "--tensor-parallel-size";
+            break;
+    }
+    if (!divisorFlag) {
+        return contextLength;
+    }
+    const tokens = parseExtraArgs(engineConfig.extraArgs);
+    const rawValue = findArgValue(tokens, divisorFlag);
+    if (rawValue === null) {
+        return contextLength;
+    }
+    const divisor = Number(rawValue);
+    if (Number.isSafeInteger(divisor) && divisor > 0) {
+        return contextLength / divisor;
+    }
+    return contextLength;
+}
+
 const ENV_BOOL_TRUE = ["true", "1", "yes"];
 const ENV_BOOL_FALSE = ["false", "0", "no"];
 function readEnvBoolean(name) {
@@ -22237,41 +22181,6 @@ class ProcessManager extends EventEmitter {
     }
 }
 
-function watchStreamProgress(emitIntervalBytes) {
-    let bytesProcessed = 0;
-    let lastEmittedAt = 0;
-    const emitter = new EventEmitter();
-    const stream = new Transform({
-        transform(chunk, _encoding, callback) {
-            bytesProcessed += chunk.length;
-            // Emit progress updates at intervals
-            if (bytesProcessed - lastEmittedAt >= emitIntervalBytes) {
-                emitter.emit("progress", bytesProcessed);
-                lastEmittedAt = bytesProcessed;
-            }
-            // Pass chunk through unchanged
-            callback(null, chunk);
-        },
-        flush(callback) {
-            emitter.emit("progress", bytesProcessed);
-            callback();
-        }
-    });
-    stream.progress = emitter;
-    return stream;
-}
-
-function joinURL(...parts) {
-    return parts
-        .join("/")
-        .replace(/[\/]+/g, "/")
-        .replace(/^(.+):\//, "$1://")
-        .replace(/^file:/, "file:/")
-        .replace(/\/(\?|&|#[^!])/g, "$1")
-        .replace(/\?/g, "&")
-        .replace("&", "?");
-}
-
 function isTerminatedError(error) {
     return error instanceof Error && error.message === "terminated" && error.name === "TypeError";
 }
@@ -22355,6 +22264,41 @@ function parseSSEEvent(rawEvent) {
     };
 }
 
+function watchStreamProgress(emitIntervalBytes) {
+    let bytesProcessed = 0;
+    let lastEmittedAt = 0;
+    const emitter = new EventEmitter();
+    const stream = new Transform({
+        transform(chunk, _encoding, callback) {
+            bytesProcessed += chunk.length;
+            // Emit progress updates at intervals
+            if (bytesProcessed - lastEmittedAt >= emitIntervalBytes) {
+                emitter.emit("progress", bytesProcessed);
+                lastEmittedAt = bytesProcessed;
+            }
+            // Pass chunk through unchanged
+            callback(null, chunk);
+        },
+        flush(callback) {
+            emitter.emit("progress", bytesProcessed);
+            callback();
+        }
+    });
+    stream.progress = emitter;
+    return stream;
+}
+
+function joinURL(...parts) {
+    return parts
+        .join("/")
+        .replace(/[\/]+/g, "/")
+        .replace(/^(.+):\//, "$1://")
+        .replace(/^file:/, "file:/")
+        .replace(/\/(\?|&|#[^!])/g, "$1")
+        .replace(/\?/g, "&")
+        .replace("&", "?");
+}
+
 function buildConfigurationOverrides(options) {
     const configurationOverrides = {};
     if (options.apiUrl) {
@@ -114221,51 +114165,6 @@ async function createEngineProcess({ args, bin, logger }) {
     return processManager;
 }
 
-function parseExtraArgs(extraArgs) {
-    if (!Array.isArray(extraArgs) ||
-        !extraArgs.every((value) => typeof value === "string")) {
-        return [];
-    }
-    return extraArgs.flatMap(tokenizeShellLine);
-}
-function tokenizeShellLine(input) {
-    const tokens = [];
-    let buffer = "";
-    let inQuote = null;
-    let hasBuffer = false;
-    for (let index = 0; index < input.length; index += 1) {
-        const char = input[index];
-        if (inQuote) {
-            if (char === inQuote) {
-                inQuote = null;
-            }
-            else {
-                buffer += char;
-            }
-            hasBuffer = true;
-        }
-        else if (char === '"' || char === "'") {
-            inQuote = char;
-            hasBuffer = true;
-        }
-        else if (char === " " || char === "\t") {
-            if (hasBuffer) {
-                tokens.push(buffer);
-                buffer = "";
-                hasBuffer = false;
-            }
-        }
-        else {
-            buffer += char;
-            hasBuffer = true;
-        }
-    }
-    if (hasBuffer) {
-        tokens.push(buffer);
-    }
-    return tokens;
-}
-
 const balanced = (a, b, str) => {
     const ma = a instanceof RegExp ? maybeMatch(a, str) : a;
     const mb = b instanceof RegExp ? maybeMatch(b, str) : b;
@@ -122669,9 +122568,8 @@ async function startVLLM({ enginePort, targetDirectory }) {
         modelPath = await findQuantizedModelTarget({ model: this.model, path: targetDirectory });
     }
     const engineConfig = this.engineConfig;
-    const device = typeof engineConfig?.device === "string" ? engineConfig.device : process.env.VLLM_DEVICE;
-    const dtype = typeof engineConfig?.dtype === "string" ? engineConfig.dtype : process.env.VLLM_DTYPE;
-    const tensorParallelSize = typeof engineConfig?.tensorParallelSize === "number" ? engineConfig.tensorParallelSize : 1;
+    const device = process.env.VLLM_DEVICE;
+    const dtype = process.env.VLLM_DTYPE;
     const args = [
         ...VLLM_START_ARGS,
         "--port",
@@ -122681,9 +122579,7 @@ async function startVLLM({ enginePort, targetDirectory }) {
         "--served-model-name",
         SERVED_MODEL_NAME,
         "--max-model-len",
-        String(contextLength),
-        "--tensor-parallel-size",
-        String(tensorParallelSize)
+        String(contextLength)
     ];
     if (this.model.taskType === "embeddings") {
         args.push("--task", "embed");
@@ -123253,9 +123149,6 @@ const DEFAULT_EXLLAMAV3_CONTEXT_LENGTH = 4096;
 async function startExllamav3({ enginePort, targetDirectory }) {
     const contextLength = Math.max(1, this.contextLength ?? DEFAULT_EXLLAMAV3_CONTEXT_LENGTH);
     const engineConfig = this.engineConfig;
-    const cacheMode = typeof engineConfig?.cacheMode === "string" ? engineConfig.cacheMode : "q4";
-    const gpuSplit = typeof engineConfig?.gpuSplit === "string" ? engineConfig.gpuSplit : null;
-    const maxSeqLen = typeof engineConfig?.maxSeqLen === "number" ? engineConfig.maxSeqLen : contextLength;
     const args = [
         SERVER_SCRIPT,
         "--model",
@@ -123264,20 +123157,15 @@ async function startExllamav3({ enginePort, targetDirectory }) {
         "127.0.0.1",
         "--port",
         String(enginePort),
-        "--cache-mode",
-        cacheMode,
         "--max-seq-len",
-        String(maxSeqLen)
+        String(contextLength)
     ];
-    if (gpuSplit) {
-        args.push("--gpu-split", gpuSplit);
-    }
     args.push(...parseExtraArgs(engineConfig?.extraArgs));
     return createEngineProcess({ args, bin: EXLLAMAV3_EXECUTABLE, logger: this.logger });
 }
 
 const DEFAULT_LLAMACPP_GPU_LAYERS = 999;
-const LLAMACPP_START_ARGS = ["--host", "0.0.0.0", "--jinja"];
+const LLAMACPP_START_ARGS = ["--host", "0.0.0.0", "--jinja", "--flash-attn", "on"];
 const LLAMACPP_EXECUTABLE = process.env.LLAMACPP_EXECUTABLE ?? "llama-server";
 const DEFAULT_LLAMACPP_CONTEXT_LENGTH = 131072;
 async function findMultimodalProjector({ path }) {
@@ -123299,7 +123187,6 @@ async function startLlamacpp({ enginePort, targetDirectory }) {
     const target = await findQuantizedModelTarget({ model: this.model, path: targetDirectory });
     const contextLength = Math.max(1, this.contextLength ?? DEFAULT_LLAMACPP_CONTEXT_LENGTH);
     const engineConfig = this.engineConfig;
-    const parallelism = typeof engineConfig?.parallelism === "number" ? engineConfig.parallelism : null;
     const args = [
         ...LLAMACPP_START_ARGS,
         "--port",
@@ -123317,43 +123204,10 @@ async function startLlamacpp({ enginePort, targetDirectory }) {
     if (this.model.taskType === "embeddings") {
         args.push("--embedding");
     }
-    const gpuLayers = typeof engineConfig?.gpuLayers === "number"
-        ? engineConfig.gpuLayers
-        : Number.parseInt(process.env.LLAMACPP_GPU_LAYERS ?? String(DEFAULT_LLAMACPP_GPU_LAYERS), 10);
+    const gpuLayers = Number.parseInt(process.env.LLAMACPP_GPU_LAYERS ?? String(DEFAULT_LLAMACPP_GPU_LAYERS), 10);
     if (Number.isFinite(gpuLayers) && gpuLayers > 0) {
         args.push("--n-gpu-layers", String(gpuLayers));
     }
-    if (typeof parallelism === "number") {
-        args.push("--parallel", String(Math.max(1, parallelism)));
-    }
-    const flashAttn = engineConfig?.flashAttn;
-    if (flashAttn === true || flashAttn === undefined) {
-        args.push("--flash-attn", "on");
-    }
-    const cacheTypeK = typeof engineConfig?.cacheTypeK === "string" ? engineConfig.cacheTypeK : null;
-    if (cacheTypeK) {
-        args.push("--cache-type-k", cacheTypeK);
-    }
-    const cacheTypeV = typeof engineConfig?.cacheTypeV === "string" ? engineConfig.cacheTypeV : null;
-    if (cacheTypeV) {
-        args.push("--cache-type-v", cacheTypeV);
-    }
-    const batchSize = typeof engineConfig?.batchSize === "number" ? engineConfig.batchSize : null;
-    if (batchSize !== null) {
-        args.push("--batch-size", String(batchSize));
-    }
-    const ubatchSize = typeof engineConfig?.ubatchSize === "number" ? engineConfig.ubatchSize : null;
-    if (ubatchSize !== null) {
-        args.push("--ubatch-size", String(ubatchSize));
-    }
-    const tensorSplit = typeof engineConfig?.tensorSplit === "string" ? engineConfig.tensorSplit : null;
-    if (tensorSplit) {
-        args.push("--tensor-split", tensorSplit);
-    }
-    const mainGpu = typeof engineConfig?.mainGpu === "number" ? engineConfig.mainGpu : null;
-    if (mainGpu !== null) {
-        args.push("--main-gpu", String(mainGpu));
-    }
     args.push(...parseExtraArgs(engineConfig?.extraArgs));
     if (this.model.multimodalEnabled) {
         const projector = await findMultimodalProjector({ path: targetDirectory });
@@ -123392,12 +123246,7 @@ async function startMLXLM({ enginePort, targetDirectory }) {
         "--context-length",
         String(contextLength)
     ];
-    const maxKvSize = typeof engineConfig?.maxKvSize === "number" ? engineConfig.maxKvSize : null;
-    if (maxKvSize !== null) {
-        args.push("--max-kv-size", String(maxKvSize));
-    }
-    const trustRemoteCode = engineConfig?.trustRemoteCode === true;
-    if (trustRemoteCode || process.env.MLXLM_TRUST_REMOTE_CODE === "true") {
+    if (process.env.MLXLM_TRUST_REMOTE_CODE === "true") {
         args.push("--trust-remote-code");
     }
     args.push(...parseExtraArgs(engineConfig?.extraArgs));
@@ -123434,9 +123283,6 @@ const DEFAULT_SGLANG_CONTEXT_LENGTH = 2048;
 async function startSGLang({ enginePort, targetDirectory }) {
     const contextLength = Math.max(1, this.contextLength ?? DEFAULT_SGLANG_CONTEXT_LENGTH);
     const engineConfig = this.engineConfig;
-    const device = typeof engineConfig?.device === "string" ? engineConfig.device : undefined;
-    const dtype = typeof engineConfig?.dtype === "string" ? engineConfig.dtype : undefined;
-    const tensorParallelSize = typeof engineConfig?.tensorParallelSize === "number" ? engineConfig.tensorParallelSize : 1;
     const args = [
         ...SGLANG_START_ARGS,
         "--port",
@@ -123446,9 +123292,7 @@ async function startSGLang({ enginePort, targetDirectory }) {
         "--served-model-name",
         SERVED_MODEL_NAME,
         "--context-length",
-        String(contextLength),
-        "--tp-size",
-        String(tensorParallelSize)
+        String(contextLength)
     ];
     if (this.model.taskType === "embeddings") {
         args.push("--task", "embed");
@@ -123458,12 +123302,6 @@ async function startSGLang({ enginePort, targetDirectory }) {
         model: this.model,
         targetDirectory
     })));
-    if (device) {
-        args.push("--device", device);
-    }
-    if (dtype) {
-        args.push("--dtype", dtype);
-    }
     args.push(...parseExtraArgs(engineConfig?.extraArgs));
     if (this.model.multimodalEnabled) {
         args.push("--limit-mm-per-prompt", process.env.SGLANG_MM_LIMIT ?? '{"image":5}');
@@ -123479,9 +123317,6 @@ const DEFAULT_TRTLLM_CONTEXT_LENGTH = 2048;
 async function startTensorRTLLM({ enginePort, targetDirectory }) {
     const contextLength = Math.max(1, this.contextLength ?? DEFAULT_TRTLLM_CONTEXT_LENGTH);
     const engineConfig = this.engineConfig;
-    const backend = typeof engineConfig?.backend === "string" ? engineConfig.backend : "pytorch";
-    const dtype = typeof engineConfig?.dtype === "string" ? engineConfig.dtype : undefined;
-    const tensorParallelSize = typeof engineConfig?.tensorParallelSize === "number" ? engineConfig.tensorParallelSize : 1;
     const args = [
         "serve",
         targetDirectory,
@@ -123489,19 +123324,12 @@ async function startTensorRTLLM({ enginePort, targetDirectory }) {
         "127.0.0.1",
         "--port",
         String(enginePort),
-        "--backend",
-        backend,
         "--max-seq-len",
-        String(contextLength),
-        "--tp-size",
-        String(tensorParallelSize)
+        String(contextLength)
     ];
     if (this.model.taskType === "embeddings") {
         args.push("--task", "embed");
     }
-    if (dtype) {
-        args.push("--dtype", dtype);
-    }
     args.push(...parseExtraArgs(engineConfig?.extraArgs));
     return createEngineProcess({ args, bin: TRTLLM_EXECUTABLE, logger: this.logger });
 }
@@ -124720,7 +124548,7 @@ async function proxyEmbeddingsRoute({ body, conduitConfiguration, endpointId, lo
         });
     }
     const engineType = conduitConfiguration.engineConfig?.type ?? null;
-    const engineConfig = conduitConfiguration.engineConfig?.config ?? null;
+    const engineConfig = conduitConfiguration.engineConfig ?? null;
     const serializedBody = isPlainObject$3(body)
         ? JSON.stringify(body)
         : typeof body === "string"
@@ -124864,7 +124692,7 @@ async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointI
         });
     }
     const engineType = conduitConfiguration.engineConfig?.type ?? null;
-    const engineConfig = conduitConfiguration.engineConfig?.config ?? null;
+    const engineConfig = conduitConfiguration.engineConfig ?? null;
     const effectiveBody = modelManager.model.multimodalEnabled ? body : stripImagesFromBody(body);
     const { bytes: requestBodyBytes, payload: serializedBody } = serializeRequestBody$1(effectiveBody, { model: modelManager.model, path });
     const requestStartedAt = Date.now();
@@ -125113,7 +124941,7 @@ function createConduitOpenAIAPIReferenceHandlers({ apiClient, conduitConfigurati
                 const currentConfig = conduitConfiguration();
                 const effectiveContextLength = getEffectiveContextLength({
                     contextLength: modelManager.contextLength,
-                    engineConfig: currentConfig.engineConfig?.config ?? null,
+                    engineConfig: currentConfig.engineConfig ?? null,
                     engineType: currentConfig.engineConfig?.type ?? null
                 });
                 return {
@@ -136580,7 +136408,9 @@ function createModelManagerFromConfig(conduitConfiguration, configuration, logge
     const engineConfig = conduitConfiguration.engineConfig;
     return new ModelManager({
         contextLength: conduitConfiguration.contextLength ?? null,
-        engineConfig: engineConfig?.config ?? null,
+        engineConfig: engineConfig
+            ? { extraArgs: engineConfig.extraArgs, type: engineConfig.type }
+            : null,
         enginePort: configuration.enginePort,
         engineType: engineConfig?.type ?? "llama.cpp",
         logger,

package/dist/cli.sea.cjs

@@ -4360,35 +4360,6 @@ function ulid$2(seedTime, prng) {
     return encodeTime(seed, TIME_LEN) + encodeRandom(RANDOM_LEN, currentPRNG);
 }
 
-function getEffectiveContextLength({ contextLength, engineConfig, engineType }) {
-    if (contextLength === null || contextLength <= 0) {
-        return null;
-    }
-    if (!engineConfig) {
-        return contextLength;
-    }
-    switch (engineType) {
-        case "llama.cpp": {
-            const parallelism = engineConfig.parallelism;
-            if (typeof parallelism === "number" && parallelism > 0) {
-                return contextLength / parallelism;
-            }
-            return contextLength;
-        }
-        case "sglang":
-        case "tensorrt-llm":
-        case "vllm": {
-            const tensorParallelSize = engineConfig.tensorParallelSize;
-            if (typeof tensorParallelSize === "number" && tensorParallelSize > 0) {
-                return contextLength / tensorParallelSize;
-            }
-            return contextLength;
-        }
-        default:
-            return contextLength;
-    }
-}
-
 function asError(error) {
     if (error instanceof Error) {
         return error;
@@ -19929,58 +19900,10 @@ const LLMEngineSchema = _enum$1([
     "tensorrt-llm",
     "vllm"
 ]);
-const LlamacppEngineConfigSchema = object$1({
-    batchSize: number$1().int().positive().nullable().optional(),
-    cacheTypeK: string$2().nullable().optional(),
-    cacheTypeV: string$2().nullable().optional(),
-    extraArgs: array(string$2()).optional(),
-    flashAttn: boolean$1().optional(),
-    gpuLayers: number$1().int().min(0).optional(),
-    mainGpu: number$1().int().min(0).nullable().optional(),
-    parallelism: number$1().int().positive().optional(),
-    tensorSplit: string$2().nullable().optional(),
-    ubatchSize: number$1().int().positive().nullable().optional()
-});
-const VLLMEngineConfigSchema = object$1({
-    device: string$2().optional(),
-    dtype: string$2().optional(),
-    extraArgs: array(string$2()).optional(),
-    tensorParallelSize: number$1().int().positive().optional()
-});
-const SGLangEngineConfigSchema = object$1({
-    device: string$2().optional(),
-    dtype: string$2().optional(),
-    extraArgs: array(string$2()).optional(),
-    tensorParallelSize: number$1().int().positive().optional()
-});
-const TensorRTLLMEngineConfigSchema = object$1({
-    backend: _enum$1(["_autodeploy", "pytorch", "tensorrt"]).optional(),
-    dtype: string$2().optional(),
-    extraArgs: array(string$2()).optional(),
-    tensorParallelSize: number$1().int().positive().optional()
-});
-const Exllamav3EngineConfigSchema = object$1({
-    cacheMode: _enum$1(["fp16", "q4", "q6", "q8"]).optional(),
-    extraArgs: array(string$2()).optional(),
-    gpuSplit: string$2().optional(),
-    maxSeqLen: number$1().int().positive().optional()
-});
-const MLXLMEngineConfigSchema = object$1({
-    extraArgs: array(string$2()).optional(),
-    maxKvSize: number$1().int().positive().optional(),
-    trustRemoteCode: boolean$1().optional()
+const EngineConfigSchema = object$1({
+    extraArgs: array(string$2()),
+    type: LLMEngineSchema
 });
-const EngineConfigSchema = discriminatedUnion("type", [
-    object$1({ config: Exllamav3EngineConfigSchema, type: literal("exllamav3") }),
-    object$1({ config: LlamacppEngineConfigSchema, type: literal("llama.cpp") }),
-    object$1({ config: MLXLMEngineConfigSchema, type: literal("mlx-lm") }),
-    object$1({ config: SGLangEngineConfigSchema, type: literal("sglang") }),
-    object$1({
-        config: TensorRTLLMEngineConfigSchema,
-        type: literal("tensorrt-llm")
-    }),
-    object$1({ config: VLLMEngineConfigSchema, type: literal("vllm") })
-]);
 const LLMModelFormatSchema = _enum$1([
     // VLLM / SGLang / TensorRT-LLM
     "safetensors",
@@ -21176,105 +21099,21 @@ object$1({
 });
 const EngineOutputSchema = object$1({
     created: string$2(),
-    exllamav3CacheMode: string$2().nullable(),
-    exllamav3ExtraArgs: array(string$2()),
-    exllamav3GpuSplit: string$2().nullable(),
-    exllamav3MaxSeqLen: number$1().nullable(),
+    extraArgs: array(string$2()),
     id: ULIDSchema,
-    llamacppBatchSize: number$1().nullable(),
-    llamacppCacheTypeK: string$2().nullable(),
-    llamacppCacheTypeV: string$2().nullable(),
-    llamacppExtraArgs: array(string$2()),
-    llamacppFlashAttn: boolean$1(),
-    llamacppGpuLayers: number$1(),
-    llamacppMainGpu: number$1().nullable(),
-    llamacppParallelism: number$1(),
-    llamacppTensorSplit: string$2().nullable(),
-    llamacppUbatchSize: number$1().nullable(),
-    mlxlmExtraArgs: array(string$2()),
-    mlxlmMaxKvSize: number$1().nullable(),
-    mlxlmTrustRemoteCode: boolean$1(),
     name: string$2(),
-    sglangDevice: string$2().nullable(),
-    sglangDtype: string$2().nullable(),
-    sglangExtraArgs: array(string$2()),
-    sglangTensorParallelSize: number$1(),
-    trtllmBackend: string$2().nullable(),
-    trtllmDtype: string$2().nullable(),
-    trtllmExtraArgs: array(string$2()),
-    trtllmTensorParallelSize: number$1(),
     type: LLMEngineSchema,
-    updated: string$2(),
-    vllmDevice: string$2().nullable(),
-    vllmDtype: string$2().nullable(),
-    vllmExtraArgs: array(string$2()),
-    vllmTensorParallelSize: number$1()
+    updated: string$2()
 });
 object$1({
-    exllamav3CacheMode: string$2().nullable().optional(),
-    exllamav3ExtraArgs: array(string$2()).optional(),
-    exllamav3GpuSplit: string$2().nullable().optional(),
-    exllamav3MaxSeqLen: number$1().int().positive().nullable().optional(),
-    llamacppBatchSize: number$1().int().positive().nullable().optional(),
-    llamacppCacheTypeK: string$2().nullable().optional(),
-    llamacppCacheTypeV: string$2().nullable().optional(),
-    llamacppExtraArgs: array(string$2()).optional(),
-    llamacppFlashAttn: boolean$1().optional(),
-    llamacppGpuLayers: number$1().int().min(0).optional(),
-    llamacppMainGpu: number$1().int().min(0).nullable().optional(),
-    llamacppParallelism: number$1().int().positive().optional(),
-    llamacppTensorSplit: string$2().nullable().optional(),
-    llamacppUbatchSize: number$1().int().positive().nullable().optional(),
-    mlxlmExtraArgs: array(string$2()).optional(),
-    mlxlmMaxKvSize: number$1().int().positive().nullable().optional(),
-    mlxlmTrustRemoteCode: boolean$1().optional(),
+    extraArgs: array(string$2()).optional(),
     name: ResourceNameSchema,
-    sglangDevice: string$2().nullable().optional(),
-    sglangDtype: string$2().nullable().optional(),
-    sglangExtraArgs: array(string$2()).optional(),
-    sglangTensorParallelSize: number$1().int().positive().optional(),
-    trtllmBackend: string$2().nullable().optional(),
-    trtllmDtype: string$2().nullable().optional(),
-    trtllmExtraArgs: array(string$2()).optional(),
-    trtllmTensorParallelSize: number$1().int().positive().optional(),
-    type: LLMEngineSchema,
-    vllmDevice: string$2().nullable().optional(),
-    vllmDtype: string$2().nullable().optional(),
-    vllmExtraArgs: array(string$2()).optional(),
-    vllmTensorParallelSize: number$1().int().positive().optional()
+    type: LLMEngineSchema
 });
 object$1({
-    exllamav3CacheMode: string$2().nullable().optional(),
-    exllamav3ExtraArgs: array(string$2()).optional(),
-    exllamav3GpuSplit: string$2().nullable().optional(),
-    exllamav3MaxSeqLen: number$1().int().positive().nullable().optional(),
-    llamacppBatchSize: number$1().int().positive().nullable().optional(),
-    llamacppCacheTypeK: string$2().nullable().optional(),
-    llamacppCacheTypeV: string$2().nullable().optional(),
-    llamacppExtraArgs: array(string$2()).optional(),
-    llamacppFlashAttn: boolean$1().optional(),
-    llamacppGpuLayers: number$1().int().min(0).optional(),
-    llamacppMainGpu: number$1().int().min(0).nullable().optional(),
-    llamacppParallelism: number$1().int().positive().optional(),
-    llamacppTensorSplit: string$2().nullable().optional(),
-    llamacppUbatchSize: number$1().int().positive().nullable().optional(),
-    mlxlmExtraArgs: array(string$2()).optional(),
-    mlxlmMaxKvSize: number$1().int().positive().nullable().optional(),
-    mlxlmTrustRemoteCode: boolean$1().optional(),
+    extraArgs: array(string$2()).optional(),
     name: ResourceNameSchema.optional(),
-    sglangDevice: string$2().nullable().optional(),
-    sglangDtype: string$2().nullable().optional(),
-    sglangExtraArgs: array(string$2()).optional(),
-    sglangTensorParallelSize: number$1().int().positive().optional(),
-    trtllmBackend: string$2().nullable().optional(),
-    trtllmDtype: string$2().nullable().optional(),
-    trtllmExtraArgs: array(string$2()).optional(),
-    trtllmTensorParallelSize: number$1().int().positive().optional(),
-    type: LLMEngineSchema.optional(),
-    vllmDevice: string$2().nullable().optional(),
-    vllmDtype: string$2().nullable().optional(),
-    vllmExtraArgs: array(string$2()).optional(),
-    vllmTensorParallelSize: number$1().int().positive().optional()
+    type: LLMEngineSchema.optional()
 });
 object$1({
     results: array(EngineOutputSchema)
@@ -21351,6 +21190,12 @@ object$1({
     }
 });
 
+object$1({
+    config: EngineConfigSchema,
+    description: string$2().optional(),
+    name: ResourceNameSchema
+});
+
 const ENGINE_API_COMPATIBILITY = {
     exllamav3: {
         nativeAnthropicMessages: false,
@@ -21769,6 +21614,105 @@ discriminatedUnion("chargeName", [
     ToolServiceCallMetaSchema
 ]);
 
+function parseExtraArgs(extraArgs) {
+    if (!Array.isArray(extraArgs) ||
+        !extraArgs.every((value) => typeof value === "string")) {
+        return [];
+    }
+    return extraArgs.flatMap(tokenizeShellLine);
+}
+function findArgValue(args, flag) {
+    let found = null;
+    for (let index = 0; index < args.length; index += 1) {
+        const token = args[index];
+        if (token === flag) {
+            const value = args[index + 1];
+            if (value !== undefined && !value.startsWith("--")) {
+                found = value;
+            }
+            else {
+                found = null;
+            }
+        }
+        else if (token.startsWith(`${flag}=`)) {
+            found = token.slice(flag.length + 1);
+        }
+    }
+    return found;
+}
+function tokenizeShellLine(input) {
+    const tokens = [];
+    let buffer = "";
+    let inQuote = null;
+    let hasBuffer = false;
+    for (let index = 0; index < input.length; index += 1) {
+        const char = input[index];
+        if (inQuote) {
+            if (char === inQuote) {
+                inQuote = null;
+            }
+            else {
+                buffer += char;
+            }
+            hasBuffer = true;
+        }
+        else if (char === '"' || char === "'") {
+            inQuote = char;
+            hasBuffer = true;
+        }
+        else if (char === " " || char === "\t") {
+            if (hasBuffer) {
+                tokens.push(buffer);
+                buffer = "";
+                hasBuffer = false;
+            }
+        }
+        else {
+            buffer += char;
+            hasBuffer = true;
+        }
+    }
+    if (hasBuffer) {
+        tokens.push(buffer);
+    }
+    return tokens;
+}
+
+function getEffectiveContextLength({ contextLength, engineConfig, engineType }) {
+    if (contextLength === null || contextLength <= 0) {
+        return null;
+    }
+    if (!engineConfig) {
+        return contextLength;
+    }
+    let divisorFlag = null;
+    switch (engineType) {
+        case "llama.cpp":
+            divisorFlag = "--parallel";
+            break;
+        case "sglang":
+        case "tensorrt-llm":
+            divisorFlag = "--tp-size";
+            break;
+        case "vllm":
+            divisorFlag = "--tensor-parallel-size";
+            break;
+    }
+    if (!divisorFlag) {
+        return contextLength;
+    }
+    const tokens = parseExtraArgs(engineConfig.extraArgs);
+    const rawValue = findArgValue(tokens, divisorFlag);
+    if (rawValue === null) {
+        return contextLength;
+    }
+    const divisor = Number(rawValue);
+    if (Number.isSafeInteger(divisor) && divisor > 0) {
+        return contextLength / divisor;
+    }
+    return contextLength;
+}
+
 const ENV_BOOL_TRUE = ["true", "1", "yes"];
 const ENV_BOOL_FALSE = ["false", "0", "no"];
 function readEnvBoolean(name) {
@@ -22252,41 +22196,6 @@ class ProcessManager extends EventEmitter {
     }
 }
 
-function watchStreamProgress(emitIntervalBytes) {
-    let bytesProcessed = 0;
-    let lastEmittedAt = 0;
-    const emitter = new EventEmitter();
-    const stream = new require$$0$8.Transform({
-        transform(chunk, _encoding, callback) {
-            bytesProcessed += chunk.length;
-            // Emit progress updates at intervals
-            if (bytesProcessed - lastEmittedAt >= emitIntervalBytes) {
-                emitter.emit("progress", bytesProcessed);
-                lastEmittedAt = bytesProcessed;
-            }
-            // Pass chunk through unchanged
-            callback(null, chunk);
-        },
-        flush(callback) {
-            emitter.emit("progress", bytesProcessed);
-            callback();
-        }
-    });
-    stream.progress = emitter;
-    return stream;
-}
-
-function joinURL(...parts) {
-    return parts
-        .join("/")
-        .replace(/[\/]+/g, "/")
-        .replace(/^(.+):\//, "$1://")
-        .replace(/^file:/, "file:/")
-        .replace(/\/(\?|&|#[^!])/g, "$1")
-        .replace(/\?/g, "&")
-        .replace("&", "?");
-}
-
 function isTerminatedError(error) {
     return error instanceof Error && error.message === "terminated" && error.name === "TypeError";
 }
@@ -22370,6 +22279,41 @@ function parseSSEEvent(rawEvent) {
     };
 }
 
+function watchStreamProgress(emitIntervalBytes) {
+    let bytesProcessed = 0;
+    let lastEmittedAt = 0;
+    const emitter = new EventEmitter();
+    const stream = new require$$0$8.Transform({
+        transform(chunk, _encoding, callback) {
+            bytesProcessed += chunk.length;
+            // Emit progress updates at intervals
+            if (bytesProcessed - lastEmittedAt >= emitIntervalBytes) {
+                emitter.emit("progress", bytesProcessed);
+                lastEmittedAt = bytesProcessed;
+            }
+            // Pass chunk through unchanged
+            callback(null, chunk);
+        },
+        flush(callback) {
+            emitter.emit("progress", bytesProcessed);
+            callback();
+        }
+    });
+    stream.progress = emitter;
+    return stream;
+}
+
+function joinURL(...parts) {
+    return parts
+        .join("/")
+        .replace(/[\/]+/g, "/")
+        .replace(/^(.+):\//, "$1://")
+        .replace(/^file:/, "file:/")
+        .replace(/\/(\?|&|#[^!])/g, "$1")
+        .replace(/\?/g, "&")
+        .replace("&", "?");
+}
+
 function buildConfigurationOverrides(options) {
     const configurationOverrides = {};
     if (options.apiUrl) {
@@ -114236,51 +114180,6 @@ async function createEngineProcess({ args, bin, logger }) {
     return processManager;
 }
 
-function parseExtraArgs(extraArgs) {
-    if (!Array.isArray(extraArgs) ||
-        !extraArgs.every((value) => typeof value === "string")) {
-        return [];
-    }
-    return extraArgs.flatMap(tokenizeShellLine);
-}
-function tokenizeShellLine(input) {
-    const tokens = [];
-    let buffer = "";
-    let inQuote = null;
-    let hasBuffer = false;
-    for (let index = 0; index < input.length; index += 1) {
-        const char = input[index];
-        if (inQuote) {
-            if (char === inQuote) {
-                inQuote = null;
-            }
-            else {
-                buffer += char;
-            }
-            hasBuffer = true;
-        }
-        else if (char === '"' || char === "'") {
-            inQuote = char;
-            hasBuffer = true;
-        }
-        else if (char === " " || char === "\t") {
-            if (hasBuffer) {
-                tokens.push(buffer);
-                buffer = "";
-                hasBuffer = false;
-            }
-        }
-        else {
-            buffer += char;
-            hasBuffer = true;
-        }
-    }
-    if (hasBuffer) {
-        tokens.push(buffer);
-    }
-    return tokens;
-}
-
 const balanced = (a, b, str) => {
     const ma = a instanceof RegExp ? maybeMatch(a, str) : a;
     const mb = b instanceof RegExp ? maybeMatch(b, str) : b;
@@ -122684,9 +122583,8 @@ async function startVLLM({ enginePort, targetDirectory }) {
         modelPath = await findQuantizedModelTarget({ model: this.model, path: targetDirectory });
     }
     const engineConfig = this.engineConfig;
-    const device = typeof engineConfig?.device === "string" ? engineConfig.device : process.env.VLLM_DEVICE;
-    const dtype = typeof engineConfig?.dtype === "string" ? engineConfig.dtype : process.env.VLLM_DTYPE;
-    const tensorParallelSize = typeof engineConfig?.tensorParallelSize === "number" ? engineConfig.tensorParallelSize : 1;
+    const device = process.env.VLLM_DEVICE;
+    const dtype = process.env.VLLM_DTYPE;
     const args = [
         ...VLLM_START_ARGS,
         "--port",
@@ -122696,9 +122594,7 @@ async function startVLLM({ enginePort, targetDirectory }) {
         "--served-model-name",
         SERVED_MODEL_NAME,
         "--max-model-len",
-        String(contextLength),
-        "--tensor-parallel-size",
-        String(tensorParallelSize)
+        String(contextLength)
     ];
     if (this.model.taskType === "embeddings") {
         args.push("--task", "embed");
@@ -123268,9 +123164,6 @@ const DEFAULT_EXLLAMAV3_CONTEXT_LENGTH = 4096;
 async function startExllamav3({ enginePort, targetDirectory }) {
     const contextLength = Math.max(1, this.contextLength ?? DEFAULT_EXLLAMAV3_CONTEXT_LENGTH);
     const engineConfig = this.engineConfig;
-    const cacheMode = typeof engineConfig?.cacheMode === "string" ? engineConfig.cacheMode : "q4";
-    const gpuSplit = typeof engineConfig?.gpuSplit === "string" ? engineConfig.gpuSplit : null;
-    const maxSeqLen = typeof engineConfig?.maxSeqLen === "number" ? engineConfig.maxSeqLen : contextLength;
     const args = [
         SERVER_SCRIPT,
         "--model",
@@ -123279,20 +123172,15 @@ async function startExllamav3({ enginePort, targetDirectory }) {
         "127.0.0.1",
         "--port",
         String(enginePort),
-        "--cache-mode",
-        cacheMode,
         "--max-seq-len",
-        String(maxSeqLen)
+        String(contextLength)
     ];
-    if (gpuSplit) {
-        args.push("--gpu-split", gpuSplit);
-    }
     args.push(...parseExtraArgs(engineConfig?.extraArgs));
     return createEngineProcess({ args, bin: EXLLAMAV3_EXECUTABLE, logger: this.logger });
 }
 
 const DEFAULT_LLAMACPP_GPU_LAYERS = 999;
-const LLAMACPP_START_ARGS = ["--host", "0.0.0.0", "--jinja"];
+const LLAMACPP_START_ARGS = ["--host", "0.0.0.0", "--jinja", "--flash-attn", "on"];
 const LLAMACPP_EXECUTABLE = process.env.LLAMACPP_EXECUTABLE ?? "llama-server";
 const DEFAULT_LLAMACPP_CONTEXT_LENGTH = 131072;
 async function findMultimodalProjector({ path }) {
@@ -123314,7 +123202,6 @@ async function startLlamacpp({ enginePort, targetDirectory }) {
     const target = await findQuantizedModelTarget({ model: this.model, path: targetDirectory });
     const contextLength = Math.max(1, this.contextLength ?? DEFAULT_LLAMACPP_CONTEXT_LENGTH);
     const engineConfig = this.engineConfig;
-    const parallelism = typeof engineConfig?.parallelism === "number" ? engineConfig.parallelism : null;
     const args = [
         ...LLAMACPP_START_ARGS,
         "--port",
@@ -123332,43 +123219,10 @@ async function startLlamacpp({ enginePort, targetDirectory }) {
     if (this.model.taskType === "embeddings") {
         args.push("--embedding");
     }
-    const gpuLayers = typeof engineConfig?.gpuLayers === "number"
-        ? engineConfig.gpuLayers
-        : Number.parseInt(process.env.LLAMACPP_GPU_LAYERS ?? String(DEFAULT_LLAMACPP_GPU_LAYERS), 10);
+    const gpuLayers = Number.parseInt(process.env.LLAMACPP_GPU_LAYERS ?? String(DEFAULT_LLAMACPP_GPU_LAYERS), 10);
     if (Number.isFinite(gpuLayers) && gpuLayers > 0) {
         args.push("--n-gpu-layers", String(gpuLayers));
     }
-    if (typeof parallelism === "number") {
-        args.push("--parallel", String(Math.max(1, parallelism)));
-    }
-    const flashAttn = engineConfig?.flashAttn;
-    if (flashAttn === true || flashAttn === undefined) {
-        args.push("--flash-attn", "on");
-    }
-    const cacheTypeK = typeof engineConfig?.cacheTypeK === "string" ? engineConfig.cacheTypeK : null;
-    if (cacheTypeK) {
-        args.push("--cache-type-k", cacheTypeK);
-    }
-    const cacheTypeV = typeof engineConfig?.cacheTypeV === "string" ? engineConfig.cacheTypeV : null;
-    if (cacheTypeV) {
-        args.push("--cache-type-v", cacheTypeV);
-    }
-    const batchSize = typeof engineConfig?.batchSize === "number" ? engineConfig.batchSize : null;
-    if (batchSize !== null) {
-        args.push("--batch-size", String(batchSize));
-    }
-    const ubatchSize = typeof engineConfig?.ubatchSize === "number" ? engineConfig.ubatchSize : null;
-    if (ubatchSize !== null) {
-        args.push("--ubatch-size", String(ubatchSize));
-    }
-    const tensorSplit = typeof engineConfig?.tensorSplit === "string" ? engineConfig.tensorSplit : null;
-    if (tensorSplit) {
-        args.push("--tensor-split", tensorSplit);
-    }
-    const mainGpu = typeof engineConfig?.mainGpu === "number" ? engineConfig.mainGpu : null;
-    if (mainGpu !== null) {
-        args.push("--main-gpu", String(mainGpu));
-    }
     args.push(...parseExtraArgs(engineConfig?.extraArgs));
     if (this.model.multimodalEnabled) {
         const projector = await findMultimodalProjector({ path: targetDirectory });
@@ -123407,12 +123261,7 @@ async function startMLXLM({ enginePort, targetDirectory }) {
         "--context-length",
         String(contextLength)
     ];
-    const maxKvSize = typeof engineConfig?.maxKvSize === "number" ? engineConfig.maxKvSize : null;
-    if (maxKvSize !== null) {
-        args.push("--max-kv-size", String(maxKvSize));
-    }
-    const trustRemoteCode = engineConfig?.trustRemoteCode === true;
-    if (trustRemoteCode || process.env.MLXLM_TRUST_REMOTE_CODE === "true") {
+    if (process.env.MLXLM_TRUST_REMOTE_CODE === "true") {
         args.push("--trust-remote-code");
     }
     args.push(...parseExtraArgs(engineConfig?.extraArgs));
@@ -123449,9 +123298,6 @@ const DEFAULT_SGLANG_CONTEXT_LENGTH = 2048;
 async function startSGLang({ enginePort, targetDirectory }) {
     const contextLength = Math.max(1, this.contextLength ?? DEFAULT_SGLANG_CONTEXT_LENGTH);
     const engineConfig = this.engineConfig;
-    const device = typeof engineConfig?.device === "string" ? engineConfig.device : undefined;
-    const dtype = typeof engineConfig?.dtype === "string" ? engineConfig.dtype : undefined;
-    const tensorParallelSize = typeof engineConfig?.tensorParallelSize === "number" ? engineConfig.tensorParallelSize : 1;
     const args = [
         ...SGLANG_START_ARGS,
         "--port",
@@ -123461,9 +123307,7 @@ async function startSGLang({ enginePort, targetDirectory }) {
         "--served-model-name",
         SERVED_MODEL_NAME,
         "--context-length",
-        String(contextLength),
-        "--tp-size",
-        String(tensorParallelSize)
+        String(contextLength)
     ];
     if (this.model.taskType === "embeddings") {
         args.push("--task", "embed");
@@ -123473,12 +123317,6 @@ async function startSGLang({ enginePort, targetDirectory }) {
         model: this.model,
         targetDirectory
     })));
-    if (device) {
-        args.push("--device", device);
-    }
-    if (dtype) {
-        args.push("--dtype", dtype);
-    }
     args.push(...parseExtraArgs(engineConfig?.extraArgs));
     if (this.model.multimodalEnabled) {
         args.push("--limit-mm-per-prompt", process.env.SGLANG_MM_LIMIT ?? '{"image":5}');
@@ -123494,9 +123332,6 @@ const DEFAULT_TRTLLM_CONTEXT_LENGTH = 2048;
 async function startTensorRTLLM({ enginePort, targetDirectory }) {
     const contextLength = Math.max(1, this.contextLength ?? DEFAULT_TRTLLM_CONTEXT_LENGTH);
     const engineConfig = this.engineConfig;
-    const backend = typeof engineConfig?.backend === "string" ? engineConfig.backend : "pytorch";
-    const dtype = typeof engineConfig?.dtype === "string" ? engineConfig.dtype : undefined;
-    const tensorParallelSize = typeof engineConfig?.tensorParallelSize === "number" ? engineConfig.tensorParallelSize : 1;
     const args = [
         "serve",
         targetDirectory,
@@ -123504,19 +123339,12 @@ async function startTensorRTLLM({ enginePort, targetDirectory }) {
         "127.0.0.1",
         "--port",
         String(enginePort),
-        "--backend",
-        backend,
         "--max-seq-len",
-        String(contextLength),
-        "--tp-size",
-        String(tensorParallelSize)
+        String(contextLength)
     ];
     if (this.model.taskType === "embeddings") {
         args.push("--task", "embed");
     }
-    if (dtype) {
-        args.push("--dtype", dtype);
-    }
     args.push(...parseExtraArgs(engineConfig?.extraArgs));
     return createEngineProcess({ args, bin: TRTLLM_EXECUTABLE, logger: this.logger });
 }
@@ -124735,7 +124563,7 @@ async function proxyEmbeddingsRoute({ body, conduitConfiguration, endpointId, lo
         });
     }
     const engineType = conduitConfiguration.engineConfig?.type ?? null;
-    const engineConfig = conduitConfiguration.engineConfig?.config ?? null;
+    const engineConfig = conduitConfiguration.engineConfig ?? null;
     const serializedBody = isPlainObject$3(body)
         ? JSON.stringify(body)
         : typeof body === "string"
@@ -124879,7 +124707,7 @@ async function proxyOpenAIStreamingRoute({ body, conduitConfiguration, endpointI
         });
     }
     const engineType = conduitConfiguration.engineConfig?.type ?? null;
-    const engineConfig = conduitConfiguration.engineConfig?.config ?? null;
+    const engineConfig = conduitConfiguration.engineConfig ?? null;
     const effectiveBody = modelManager.model.multimodalEnabled ? body : stripImagesFromBody(body);
     const { bytes: requestBodyBytes, payload: serializedBody } = serializeRequestBody$1(effectiveBody, { model: modelManager.model, path });
     const requestStartedAt = Date.now();
@@ -125128,7 +124956,7 @@ function createConduitOpenAIAPIReferenceHandlers({ apiClient, conduitConfigurati
                 const currentConfig = conduitConfiguration();
                 const effectiveContextLength = getEffectiveContextLength({
                     contextLength: modelManager.contextLength,
-                    engineConfig: currentConfig.engineConfig?.config ?? null,
+                    engineConfig: currentConfig.engineConfig ?? null,
                     engineType: currentConfig.engineConfig?.type ?? null
                 });
                 return {
@@ -156810,7 +156638,9 @@ function createModelManagerFromConfig(conduitConfiguration, configuration, logge
     const engineConfig = conduitConfiguration.engineConfig;
     return new ModelManager({
         contextLength: conduitConfiguration.contextLength ?? null,
-        engineConfig: engineConfig?.config ?? null,
+        engineConfig: engineConfig
+            ? { extraArgs: engineConfig.extraArgs, type: engineConfig.type }
+            : null,
         enginePort: configuration.enginePort,
         engineType: engineConfig?.type ?? "llama.cpp",
         logger,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@infersec/conduit",
   "description": "End user conduit agent for connecting local LLMs to the cloud.",
-  "version": "1.101.0",
+  "version": "1.102.1",
   "bin": {
     "infersec-conduit": "./dist/cli.js"
   },

package/dist/modelManagement/extraArgs.d.ts

@@ -1 +0,0 @@
-export declare function parseExtraArgs(extraArgs: unknown): Array<string>;