node-llama-cpp 3.3.2 → 3.4.0

package/dist/cli/commands/inspect/commands/InspectMeasureCommand.js

@@ -2,8 +2,8 @@ import path from "path";
 import process from "process";
 import { fileURLToPath } from "url";
 import { fork } from "node:child_process";
+import os from "os";
 import chalk from "chalk";
-import bytes from "bytes";
 import stripAnsi from "strip-ansi";
 import { readGgufFileInfo } from "../../../../gguf/readGgufFileInfo.js";
 import { resolveCommandGgufPath } from "../../../utils/resolveCommandGgufPath.js";
@@ -17,6 +17,7 @@ import { getPrettyBuildGpuName } from "../../../../bindings/consts.js";
 import { getReadablePath } from "../../../utils/getReadablePath.js";
 import { withCliCommandDescriptionDocsUrl } from "../../../utils/withCliCommandDescriptionDocsUrl.js";
 import { documentationPageUrls } from "../../../../config.js";
+import { toBytes } from "../../../utils/toBytes.js";
 export const InspectMeasureCommand = {
     command: "measure [modelPath]",
     describe: withCliCommandDescriptionDocsUrl("Measure VRAM consumption of a GGUF model file with all possible combinations of gpu layers and context sizes", documentationPageUrls.CLI.Inspect.Measure),
@@ -82,6 +83,17 @@ export const InspectMeasureCommand = {
             type: "number",
             default: 10,
             description: "Number of context size measures to take for each gpu layers count"
+        })
+            .option("memory", {
+            type: "string",
+            choices: ["vram", "ram", "all"],
+            default: "vram",
+            description: "Type of memory to measure"
+        })
+            .option("noMmap", {
+            type: "boolean",
+            default: false,
+            description: "Disable mmap (memory-mapped file) usage"
         })
             .option("printHeaderBeforeEachLayer", {
             alias: "ph",
@@ -101,13 +113,14 @@ export const InspectMeasureCommand = {
             description: "Number of times to repeat the evaluation text before sending it for evaluation, in order to make it longer"
         });
     },
-    async handler({ modelPath: ggufPath, header: headerArg, gpu, minLayers, maxLayers, minContextSize, maxContextSize, flashAttention, measures = 10, printHeaderBeforeEachLayer = true, evaluateText, repeatEvaluateText }) {
+    async handler({ modelPath: ggufPath, header: headerArg, gpu, minLayers, maxLayers, minContextSize, maxContextSize, flashAttention, measures = 10, memory: measureMemoryType, noMmap, printHeaderBeforeEachLayer = true, evaluateText, repeatEvaluateText }) {
         if (maxLayers === -1)
             maxLayers = undefined;
         if (maxContextSize === -1)
             maxContextSize = undefined;
         if (minLayers < 1)
             minLayers = 1;
+        const exitAfterEachMeasurement = measureMemoryType === "ram" || measureMemoryType === "all";
         const headers = resolveHeaderFlag(headerArg);
         // ensure a llama build is available
         const llama = gpu == null
@@ -118,17 +131,29 @@ export const InspectMeasureCommand = {
                 gpu,
                 logLevel: LlamaLogLevel.error
             });
-        const resolvedGgufPath = await resolveCommandGgufPath(ggufPath, llama, headers);
+        const useMmap = !noMmap && llama.supportsMmap;
+        const resolvedGgufPath = await resolveCommandGgufPath(ggufPath, llama, headers, {
+            flashAttention, useMmap
+        });
         console.info(`${chalk.yellow("File:")} ${getReadablePath(resolvedGgufPath)}`);
         console.info(`${chalk.yellow("GPU:")} ${getPrettyBuildGpuName(llama.gpu)}${gpu == null ? chalk.gray(" (last build)") : ""}`);
+        console.info(chalk.yellow("mmap:") + " " + (!llama.supportsMmap
+            ? "unsupported"
+            : useMmap
+                ? "enabled"
+                : "disabled"));
+        if (measureMemoryType === "ram" || measureMemoryType === "all")
+            console.warn(chalk.yellow("RAM measurements are greatly inaccurate due to OS optimizations that prevent released memory from being immediately available"));
         console.info();
         const ggufMetadata = await readGgufFileInfo(resolvedGgufPath, {
             sourceType: "filesystem"
         });
         const ggufInsights = await GgufInsights.from(ggufMetadata, llama);
         const totalVram = (await llama.getVramState()).total;
+        const totalRam = os.totalmem();
         let lastGpuLayers = maxLayers ?? ggufInsights.totalLayers;
         let previousContextSizeCheck = undefined;
+        const measureTable = getMeasureTable(measureMemoryType);
         measureTable.logHeader({ drawRowSeparator: !printHeaderBeforeEachLayer });
         while (lastGpuLayers >= (minLayers ?? 0)) {
             let printedAlreadyWithThisProcess = false;
@@ -141,6 +166,7 @@ export const InspectMeasureCommand = {
             };
             const done = await measureModel({
                 modelPath: resolvedGgufPath,
+                useMmap,
                 gpu: gpu == null
                     ? undefined
                     : llama.gpu,
@@ -154,6 +180,7 @@ export const InspectMeasureCommand = {
                 evaluateText: evaluateText == null
                     ? undefined
                     : evaluateText.repeat(repeatEvaluateText ?? 1),
+                exitAfterMeasurement: exitAfterEachMeasurement,
                 onInfo({ gpuLayers, result }) {
                     if (lastGpuLayers !== gpuLayers) {
                         lastGpuLayers = gpuLayers;
@@ -196,26 +223,45 @@ export const InspectMeasureCommand = {
                     else if (result.type === "success") {
                         previousContextSizeCheck = result.contextSize;
                         hadSuccessInThisProcess = true;
-                        const modelVramEstimation = ggufInsights.estimateModelResourceRequirements({ gpuLayers: lastGpuLayers }).gpuVram;
+                        const modelResourceEstimation = ggufInsights.estimateModelResourceRequirements({
+                            gpuLayers: lastGpuLayers,
+                            useMmap
+                        });
+                        const modelVramEstimation = modelResourceEstimation.gpuVram;
                         const modelVramEstimationDiffBytes = (modelVramEstimation < result.modelVramUsage ? "-" : "") +
-                            bytes(Math.abs(result.modelVramUsage - modelVramEstimation));
+                            toBytes(Math.abs(result.modelVramUsage - modelVramEstimation));
                         const modelVramEstimationDiffText = modelVramEstimationDiffBytes.padEnd(9, " ") + " " +
                             padStartAnsi("(" + renderDiffPercentageWithColors(((modelVramEstimation / result.modelVramUsage) - 1) * 100) + ")", 9);
-                        const contextVramEstimation = previousContextSizeCheck == null
+                        const modelRamEstimation = modelResourceEstimation.cpuRam;
+                        const modelRamEstimationDiffBytes = (modelRamEstimation < result.modelRamUsage ? "-" : "") +
+                            toBytes(Math.abs(result.modelRamUsage - modelRamEstimation));
+                        const modelRamEstimationDiffText = modelRamEstimationDiffBytes.padEnd(9, " ") + " " +
+                            padStartAnsi("(" + renderDiffPercentageWithColors(((modelRamEstimation / result.modelRamUsage) - 1) * 100) + ")", 9);
+                        const contextResourceEstimation = previousContextSizeCheck == null
                             ? undefined
                             : ggufInsights.estimateContextResourceRequirements({
                                 contextSize: previousContextSizeCheck,
                                 modelGpuLayers: lastGpuLayers,
                                 flashAttention
-                            }).gpuVram;
+                            });
+                        const contextVramEstimation = contextResourceEstimation?.gpuVram;
                         const contextVramEstimationDiffBytes = (result.contextVramUsage == null || contextVramEstimation == null)
                             ? undefined
                             : ((contextVramEstimation < result.contextVramUsage ? "-" : "") +
-                                bytes(Math.abs(result.contextVramUsage - contextVramEstimation)));
+                                toBytes(Math.abs(result.contextVramUsage - contextVramEstimation)));
                         const contextVramEstimationDiffText = (contextVramEstimation == null || contextVramEstimationDiffBytes == null || result.contextVramUsage == null)
                             ? undefined
                             : (contextVramEstimationDiffBytes.padEnd(9, " ") + " " +
                                 padStartAnsi("(" + renderDiffPercentageWithColors(((contextVramEstimation / result.contextVramUsage) - 1) * 100) + ")", 9));
+                        const contextRamEstimation = contextResourceEstimation?.cpuRam;
+                        const contextRamEstimationDiffBytes = (result.contextRamUsage == null || contextRamEstimation == null)
+                            ? undefined
+                            : ((contextRamEstimation < result.contextRamUsage ? "-" : "") +
+                                toBytes(Math.abs(result.contextRamUsage - contextRamEstimation)));
+                        const contextRamEstimationDiffText = (contextRamEstimation == null || contextRamEstimationDiffBytes == null || result.contextRamUsage == null)
+                            ? undefined
+                            : (contextRamEstimationDiffBytes.padEnd(9, " ") + " " +
+                                padStartAnsi("(" + renderDiffPercentageWithColors(((contextRamEstimation / result.contextRamUsage) - 1) * 100) + ")", 9));
                         measureTable.logLine({
                             newProcess: getNewProccessValue(),
                             type: previousContextSizeCheck == null
@@ -225,18 +271,30 @@ export const InspectMeasureCommand = {
                             contextSize: previousContextSizeCheck != null
                                 ? String(previousContextSizeCheck)
                                 : undefined,
-                            estimatedModelVram: bytes(modelVramEstimation),
-                            actualModelVram: bytes(result.modelVramUsage),
-                            modelEstimationDiff: modelVramEstimationDiffText,
+                            estimatedModelVram: toBytes(modelVramEstimation),
+                            actualModelVram: toBytes(result.modelVramUsage),
+                            modelVramEstimationDiff: modelVramEstimationDiffText,
+                            estimatedModelRam: toBytes(modelRamEstimation),
+                            actualModelRam: toBytes(result.modelRamUsage),
+                            modelRamEstimationDiff: modelRamEstimationDiffText,
                             estimatedContextVram: contextVramEstimation == null
                                 ? undefined
-                                : bytes(contextVramEstimation),
+                                : toBytes(contextVramEstimation),
                             actualContextVram: result.contextVramUsage == null
                                 ? undefined
-                                : bytes(result.contextVramUsage),
-                            contextEstimationDiff: contextVramEstimationDiffText,
+                                : toBytes(result.contextVramUsage),
+                            contextVramEstimationDiff: contextVramEstimationDiffText,
                             totalVramUsage: ((result.totalVramUsage / totalVram) * 100).toFixed(2).padStart(5, "0") + "% " +
-                                chalk.gray("(" + bytes(result.totalVramUsage) + "/" + bytes(totalVram) + ")")
+                                chalk.gray("(" + toBytes(result.totalVramUsage) + "/" + toBytes(totalVram) + ")"),
+                            estimatedContextRam: contextRamEstimation == null
+                                ? undefined
+                                : toBytes(contextRamEstimation),
+                            actualContextRam: result.contextRamUsage == null
+                                ? undefined
+                                : toBytes(result.contextRamUsage),
+                            contextRamEstimationDiff: contextRamEstimationDiffText,
+                            totalRamUsage: ((result.totalRamUsage / totalRam) * 100).toFixed(2).padStart(5, "0") + "% " +
+                                chalk.gray("(" + toBytes(result.totalRamUsage) + "/" + toBytes(totalRam) + ")")
                         });
                     }
                 }
@@ -246,55 +304,100 @@ export const InspectMeasureCommand = {
         }
     }
 };
-const measureTable = new ConsoleTable([{
-        key: "newProcess",
-        title: " ",
-        width: 1
-    }, {
-        key: "type",
-        title: "Type",
-        width: Math.max("Type".length, "Model".length, "Context".length),
-        canSpanOverEmptyColumns: true
-    }, {
-        key: "gpuLayers",
-        title: "Layers",
-        width: "Layers".length,
-        canSpanOverEmptyColumns: true
-    }, {
-        key: "contextSize",
-        title: "Context size",
-        width: "Context size".length,
-        canSpanOverEmptyColumns: true
-    }, {
-        key: "estimatedModelVram",
-        title: "Estimated model VRAM",
-        width: "Estimated model VRAM".length,
-        canSpanOverEmptyColumns: true
-    }, {
-        key: "actualModelVram",
-        title: "Model VRAM",
-        width: "Model VRAM".length
-    }, {
-        key: "modelEstimationDiff",
-        title: "Diff",
-        width: Math.max("Diff".length, 9 + 1 + 9)
-    }, {
-        key: "estimatedContextVram",
-        title: "Estimated context VRAM",
-        width: "Estimated context VRAM".length
-    }, {
-        key: "actualContextVram",
-        title: "Context VRAM",
-        width: "Context VRAM".length
-    }, {
-        key: "contextEstimationDiff",
-        title: "Diff",
-        width: Math.max("Diff".length, 9 + 1 + 9)
-    }, {
-        key: "totalVramUsage",
-        title: "VRAM usage",
-        width: Math.max("VRAM usage".length, 8 + 1 + 8 + 1 + 8)
-    }]);
+function getMeasureTable(memoryType) {
+    return new ConsoleTable([{
+            key: "newProcess",
+            title: " ",
+            width: 1
+        }, {
+            key: "type",
+            title: "Type",
+            width: Math.max("Type".length, "Model".length, "Context".length),
+            canSpanOverEmptyColumns: true
+        }, {
+            key: "gpuLayers",
+            title: "Layers",
+            width: "Layers".length,
+            canSpanOverEmptyColumns: true
+        }, {
+            key: "contextSize",
+            title: "Context size",
+            width: "Context size".length,
+            canSpanOverEmptyColumns: true
+        }, {
+            key: "estimatedModelVram",
+            visible: memoryType === "vram" || memoryType === "all",
+            title: "Estimated model VRAM",
+            width: "Estimated model VRAM".length,
+            canSpanOverEmptyColumns: true
+        }, {
+            key: "actualModelVram",
+            visible: memoryType === "vram" || memoryType === "all",
+            title: "Model VRAM",
+            width: "Model VRAM".length
+        }, {
+            key: "modelVramEstimationDiff",
+            visible: memoryType === "vram" || memoryType === "all",
+            title: "Diff",
+            width: Math.max("Diff".length, 9 + 1 + 9)
+        }, {
+            key: "estimatedModelRam",
+            visible: memoryType === "ram" || memoryType === "all",
+            title: "Estimated model RAM",
+            width: "Estimated model RAM".length,
+            canSpanOverEmptyColumns: true
+        }, {
+            key: "actualModelRam",
+            visible: memoryType === "ram" || memoryType === "all",
+            title: "Model RAM",
+            width: "Model RAM".length
+        }, {
+            key: "modelRamEstimationDiff",
+            visible: memoryType === "ram" || memoryType === "all",
+            title: "Diff",
+            width: Math.max("Diff".length, 9 + 1 + 9)
+        }, {
+            key: "estimatedContextVram",
+            visible: memoryType === "vram" || memoryType === "all",
+            title: "Estimated context VRAM",
+            width: "Estimated context VRAM".length
+        }, {
+            key: "actualContextVram",
+            visible: memoryType === "vram" || memoryType === "all",
+            title: "Context VRAM",
+            width: "Context VRAM".length
+        }, {
+            key: "contextVramEstimationDiff",
+            visible: memoryType === "vram" || memoryType === "all",
+            title: "Diff",
+            width: Math.max("Diff".length, 9 + 1 + 9)
+        }, {
+            key: "totalVramUsage",
+            visible: memoryType === "vram" || memoryType === "all",
+            title: "VRAM usage",
+            width: Math.max("VRAM usage".length, 8 + 1 + 8 + 1 + 8)
+        }, {
+            key: "estimatedContextRam",
+            visible: memoryType === "ram" || memoryType === "all",
+            title: "Estimated context RAM",
+            width: "Estimated context RAM".length
+        }, {
+            key: "actualContextRam",
+            visible: memoryType === "ram" || memoryType === "all",
+            title: "Context RAM",
+            width: "Context RAM".length
+        }, {
+            key: "contextRamEstimationDiff",
+            visible: memoryType === "ram" || memoryType === "all",
+            title: "Diff",
+            width: Math.max("Diff".length, 9 + 1 + 9)
+        }, {
+            key: "totalRamUsage",
+            visible: memoryType === "ram" || memoryType === "all",
+            title: "RAM usage",
+            width: Math.max("RAM usage".length, 8 + 1 + 8 + 1 + 8)
+        }]);
+}
 function renderDiffPercentageWithColors(percentage, { greenBright = 2, green = 6, yellow = 10, yellowBright = 14 } = {}) {
     const percentageText = percentage.toFixed(2).padStart(5, "0") + "%";
     const absPercentage = Math.abs(percentage);
@@ -311,7 +414,7 @@ function renderDiffPercentageWithColors(percentage, { greenBright = 2, green = 6
 const __filename = fileURLToPath(import.meta.url);
 const detectedFileName = path.basename(__filename);
 const expectedFileName = "InspectMeasureCommand";
-async function measureModel({ modelPath, gpu, tests, initialMaxContextSize, maxContextSize, minContextSize, maxGpuLayers, minGpuLayers, flashAttention, evaluateText, onInfo }) {
+async function measureModel({ modelPath, useMmap, gpu, tests, initialMaxContextSize, maxContextSize, minContextSize, maxGpuLayers, minGpuLayers, flashAttention, evaluateText, exitAfterMeasurement = false, onInfo }) {
     if (!detectedFileName.startsWith(expectedFileName)) {
         console.warn(getConsoleLogPrefix() +
             `"${expectedFileName}.js" file is not independent, so running sub-process tests cannot be done with it\n` +
@@ -331,6 +434,7 @@ async function measureModel({ modelPath, gpu, tests, initialMaxContextSize, maxC
         }
     });
     let isPlannedExit = false;
+    let isDone = false;
     let forkSucceeded = false;
     let timeoutHandle = null;
     const processCreationTimeout = 1000 * 60 * 5;
@@ -363,8 +467,8 @@ async function measureModel({ modelPath, gpu, tests, initialMaxContextSize, maxC
             function done() {
                 if (!forkSucceeded)
                     reject(new Error(`Measuring a model failed to run a sub-process via file "${__filename}"`));
-                else
-                    resolve(isPlannedExit);
+                else if (isPlannedExit)
+                    resolve(isPlannedExit && isDone);
                 cleanup();
             }
             subProcess.on("message", (message) => {
@@ -373,6 +477,7 @@ async function measureModel({ modelPath, gpu, tests, initialMaxContextSize, maxC
                     subProcess.send({
                         type: "start",
                         modelPath,
+                        useMmap,
                         tests,
                         initialMaxContextSize,
                         maxContextSize,
@@ -380,7 +485,8 @@ async function measureModel({ modelPath, gpu, tests, initialMaxContextSize, maxC
                         maxGpuLayers,
                         minGpuLayers,
                         flashAttention,
-                        evaluateText
+                        evaluateText,
+                        exitAfterMeasurement
                     });
                     if (timeoutHandle != null) {
                         clearTimeout(timeoutHandle);
@@ -388,6 +494,11 @@ async function measureModel({ modelPath, gpu, tests, initialMaxContextSize, maxC
                     }
                 }
                 else if (message.type === "done") {
+                    isPlannedExit = true;
+                    isDone = true;
+                    subProcess.send({ type: "exit" });
+                }
+                else if (message.type === "exit") {
                     isPlannedExit = true;
                     subProcess.send({ type: "exit" });
                 }
@@ -409,10 +520,13 @@ async function measureModel({ modelPath, gpu, tests, initialMaxContextSize, maxC
                         result: {
                             type: "success",
                             modelVramUsage: message.modelVramUsage,
+                            modelRamUsage: message.modelRamUsage,
                             contextSize: message.contextSize,
                             contextVramUsage: message.contextVramUsage,
+                            contextRamUsage: message.contextRamUsage,
                             contextStateSize: message.contextStateSize,
-                            totalVramUsage: message.totalVramUsage
+                            totalVramUsage: message.totalVramUsage,
+                            totalRamUsage: message.totalRamUsage
                         }
                     });
                 }
@@ -462,7 +576,8 @@ async function runTestWorkerLogic() {
             process.exit(1);
         process.send(info);
     }
-    async function testContextSizes({ model, modelVramUsage, startContextSize, maxContextSize, minContextSize, tests, flashAttention, evaluateText }) {
+    async function testContextSizes({ model, modelVramUsage, modelRamUsage, startContextSize, maxContextSize, minContextSize, tests, flashAttention, evaluateText, exitAfterMeasurement = false }) {
+        let measurementsDone = 0;
         const contextSizeCheckPlan = getContextSizesCheckPlan(maxContextSize != null
             ? Math.min(model.trainContextSize, maxContextSize)
             : model.trainContextSize, tests, minContextSize);
@@ -474,6 +589,7 @@ async function runTestWorkerLogic() {
                 currentContextSizeCheck = null;
             try {
                 const preContextVramUsage = (await llama.getVramState()).used;
+                const preContextRamUsage = getMemoryUsage(llama);
                 const context = await model.createContext({
                     contextSize: currentContextSizeCheck ?? (maxContextSize != null
                         ? { max: maxContextSize }
@@ -487,14 +603,19 @@ async function runTestWorkerLogic() {
                     await sequence.evaluateWithoutGeneratingNewTokens(model.tokenize(evaluateText));
                 }
                 const postContextVramUsage = (await llama.getVramState()).used;
+                const postContextRamUsage = getMemoryUsage(llama);
+                measurementsDone++;
                 sendInfoBack({
                     type: "stats",
                     gpuLayers: model.gpuLayers,
                     modelVramUsage,
+                    modelRamUsage,
                     contextSize: context.contextSize,
                     contextVramUsage: postContextVramUsage - preContextVramUsage,
+                    contextRamUsage: postContextRamUsage - preContextRamUsage,
                     contextStateSize: context.stateSize,
-                    totalVramUsage: postContextVramUsage
+                    totalVramUsage: postContextVramUsage,
+                    totalRamUsage: postContextRamUsage
                 });
                 currentContextSizeCheck = context.contextSize;
                 await context.dispose();
@@ -514,35 +635,46 @@ async function runTestWorkerLogic() {
                 }
             }
             currentContextSizeCheck = getNextItemInCheckContextSizesPlan(contextSizeCheckPlan, currentContextSizeCheck);
+            if (exitAfterMeasurement)
+                return measurementsDone;
         }
+        return measurementsDone;
     }
-    async function testWithGpuLayers({ modelPath, gpuLayers, tests, startContextSize, maxContextSize, minContextSize, flashAttention, evaluateText }) {
+    async function testWithGpuLayers({ modelPath, useMmap, gpuLayers, tests, startContextSize, maxContextSize, minContextSize, flashAttention, evaluateText, exitAfterMeasurement = false }) {
         try {
             const preModelVramUsage = (await llama.getVramState()).used;
+            const preModelRamUsage = getMemoryUsage(llama);
             const model = await llama.loadModel({
                 modelPath,
+                useMmap,
                 gpuLayers,
                 defaultContextFlashAttention: flashAttention,
                 ignoreMemorySafetyChecks: true
             });
             const postModelVramUsage = (await llama.getVramState()).used;
+            const postModelRamUsage = getMemoryUsage(llama);
             sendInfoBack({
                 type: "stats",
                 gpuLayers: model.gpuLayers,
                 modelVramUsage: postModelVramUsage - preModelVramUsage,
-                totalVramUsage: postModelVramUsage
+                modelRamUsage: postModelRamUsage - preModelRamUsage,
+                totalVramUsage: postModelVramUsage,
+                totalRamUsage: postModelRamUsage
             });
-            await testContextSizes({
+            const measurementsDone = await testContextSizes({
                 model,
                 modelVramUsage: postModelVramUsage - preModelVramUsage,
+                modelRamUsage: postModelRamUsage - preModelRamUsage,
                 startContextSize,
                 maxContextSize,
                 minContextSize,
                 flashAttention,
                 tests,
-                evaluateText
+                evaluateText,
+                exitAfterMeasurement
             });
             await model.dispose();
+            return measurementsDone;
         }
         catch (err) {
             sendInfoBack({
@@ -551,12 +683,23 @@ async function runTestWorkerLogic() {
                 gpuLayers: gpuLayers
             });
         }
+        return 0;
     }
     process.on("message", async (message) => {
         if (message.type === "start") {
             for (let gpuLayers = message.maxGpuLayers; gpuLayers >= (message.minGpuLayers ?? 0); gpuLayers--) {
-                await testWithGpuLayers({
+                if (gpuLayers == message.maxGpuLayers && message.initialMaxContextSize != null) {
+                    const ggufInsights = await GgufInsights.from(await readGgufFileInfo(message.modelPath), llama);
+                    const contextSizeCheckPlan = getContextSizesCheckPlan(message.maxContextSize != null
+                        ? Math.min(ggufInsights.trainContextSize ?? 4096, message.maxContextSize)
+                        : ggufInsights.trainContextSize ?? 4096, message.tests, message.minContextSize);
+                    const firstContextSizeCheck = getNextItemInCheckContextSizesPlan(contextSizeCheckPlan, message.initialMaxContextSize);
+                    if (firstContextSizeCheck == null)
+                        continue;
+                }
+                const measurementsDone = await testWithGpuLayers({
                     modelPath: message.modelPath,
+                    useMmap: message.useMmap,
                     gpuLayers,
                     tests: message.tests,
                     startContextSize: gpuLayers == message.maxGpuLayers
@@ -565,8 +708,13 @@ async function runTestWorkerLogic() {
                     maxContextSize: message.maxContextSize,
                     minContextSize: message.minContextSize,
                     flashAttention: message.flashAttention,
-                    evaluateText: message.evaluateText
+                    evaluateText: message.evaluateText,
+                    exitAfterMeasurement: message.exitAfterMeasurement
                 });
+                if (measurementsDone > 0 && message.exitAfterMeasurement) {
+                    sendInfoBack({ type: "exit" });
+                    return;
+                }
             }
             sendInfoBack({ type: "done" });
         }
@@ -626,4 +774,13 @@ function padStartAnsi(text, length, padChar = " ") {
     const textWithoutAnsi = stripAnsi(text);
     return padChar.repeat(Math.max(0, length - textWithoutAnsi.length)) + text;
 }
+function getMemoryUsage(llama) {
+    const totalMemoryUsage = llama._bindings.getMemoryInfo().total;
+    const vramUsage = llama._bindings.getGpuVramInfo();
+    let memoryUsage = totalMemoryUsage;
+    const unifiedMemoryVramUsage = Math.min(vramUsage.unifiedSize, vramUsage.used);
+    if (unifiedMemoryVramUsage <= memoryUsage)
+        memoryUsage -= unifiedMemoryVramUsage;
+    return memoryUsage;
+}
 //# sourceMappingURL=InspectMeasureCommand.js.map