@infersec/conduit 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js
CHANGED
|
@@ -6,7 +6,7 @@ const __dirname = __pathDirname(__filename);
|
|
|
6
6
|
|
|
7
7
|
import { parseArgs } from 'node:util';
|
|
8
8
|
import 'node:crypto';
|
|
9
|
-
import { a as asError, s as startInferenceAgent } from './start-
|
|
9
|
+
import { a as asError, s as startInferenceAgent } from './start-CVgqDYvF.js';
|
|
10
10
|
import 'argon2';
|
|
11
11
|
import 'node:child_process';
|
|
12
12
|
import 'node:stream';
|
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ const __filename = __fileURLToPath(import.meta.url);
|
|
|
5
5
|
const __dirname = __pathDirname(__filename);
|
|
6
6
|
|
|
7
7
|
import 'node:crypto';
|
|
8
|
-
import { s as startInferenceAgent, a as asError } from './start-
|
|
8
|
+
import { s as startInferenceAgent, a as asError } from './start-CVgqDYvF.js';
|
|
9
9
|
import 'argon2';
|
|
10
10
|
import 'node:child_process';
|
|
11
11
|
import 'node:stream';
|
|
@@ -104881,8 +104881,9 @@ const glob = Object.assign(glob_, {
|
|
|
104881
104881
|
});
|
|
104882
104882
|
glob.glob = glob;
|
|
104883
104883
|
|
|
104884
|
+
const DEFAULT_LLAMACPP_GPU_LAYERS = 999;
|
|
104884
104885
|
const LLAMACPP_START_ARGS = ["--host", "0.0.0.0", "--port", "8000", "--jinja"];
|
|
104885
|
-
const LLAMACPP_EXECUTABLE = "llama-server";
|
|
104886
|
+
const LLAMACPP_EXECUTABLE = process.env.LLAMACPP_EXECUTABLE ?? "llama-server";
|
|
104886
104887
|
const DEFAULT_LLAMACPP_CONTEXT_LENGTH = 131072;
|
|
104887
104888
|
async function findQuantizedModelTarget({ model, path }) {
|
|
104888
104889
|
if (model.source.type === "storage") {
|
|
@@ -104918,6 +104919,10 @@ async function startLlamacpp({ targetDirectory }) {
|
|
|
104918
104919
|
const contextLength = Math.max(1, this.contextLength ?? DEFAULT_LLAMACPP_CONTEXT_LENGTH);
|
|
104919
104920
|
const parallelism = this.parallelism;
|
|
104920
104921
|
const args = [...LLAMACPP_START_ARGS, "--model", target, "--ctx-size", String(contextLength)];
|
|
104922
|
+
const gpuLayers = Number.parseInt(process.env.LLAMACPP_GPU_LAYERS ?? String(DEFAULT_LLAMACPP_GPU_LAYERS), 10);
|
|
104923
|
+
if (Number.isFinite(gpuLayers) && gpuLayers > 0) {
|
|
104924
|
+
args.push("--n-gpu-layers", String(gpuLayers));
|
|
104925
|
+
}
|
|
104921
104926
|
if (typeof parallelism === "number") {
|
|
104922
104927
|
args.push("--parallel", String(Math.max(1, parallelism)));
|
|
104923
104928
|
}
|