metrillm-mcp 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -100,8 +100,13 @@ function toBenchmarkFailureLabel(err) {
100
100
  return compact.length > maxLen ? `ERROR: ${compact.slice(0, maxLen - 1)}\u2026` : `ERROR: ${compact}`;
101
101
  }
102
102
  function stripThinkTags(text) {
103
- const stripped = text.replace(/<think(?:ing)?[\s>][\s\S]*?<\/think(?:ing)?>/gi, "");
104
- return stripped.trim();
103
+ const withoutThinking = text.replace(/<think(?:ing)?[\s>][\s\S]*?<\/think(?:ing)?>/gi, "");
104
+ const withoutTrailingWhitespace = withoutThinking.trimEnd();
105
+ const withoutTrailingControlTokens = withoutTrailingWhitespace.replace(
106
+ /(?:\s*(?:<\|(?:im_end|eot_id|end_of_text|eom_id|end)\|>|<\/s>))+$/gi,
107
+ ""
108
+ );
109
+ return withoutTrailingControlTokens.trim();
105
110
  }
106
111
  function hasThinkingContent(response, thinkingField) {
107
112
  if (thinkingField && thinkingField.trim().length > 0) return true;
@@ -182,18 +187,20 @@ function stripTypeAnnotations(code) {
182
187
  return code;
183
188
  } catch {
184
189
  }
185
- try {
186
- const mod = __require("module");
187
- if (typeof mod.stripTypeScriptTypes === "function") {
188
- const stripped = mod.stripTypeScriptTypes(code);
189
- try {
190
- new vm.Script(`${stripped}
190
+ if (process.env.METRILLM_ENABLE_EXPERIMENTAL_TS_STRIP === "1") {
191
+ try {
192
+ const mod = __require("module");
193
+ if (typeof mod.stripTypeScriptTypes === "function") {
194
+ const stripped = mod.stripTypeScriptTypes(code);
195
+ try {
196
+ new vm.Script(`${stripped}
191
197
  ;`);
192
- return stripped;
193
- } catch {
198
+ return stripped;
199
+ } catch {
200
+ }
194
201
  }
202
+ } catch {
195
203
  }
196
- } catch {
197
204
  }
198
205
  let result = code;
199
206
  result = stripInterfaceDeclarations(result);
@@ -506,7 +513,7 @@ function extractCodeBlock(text, preferredFunctionName) {
506
513
  // ../src/core/ollama-client.ts
507
514
  var client = new Ollama();
508
515
  var DEFAULT_OLLAMA_HOST = "http://127.0.0.1:11434";
509
- var OLLAMA_INIT_TIMEOUT_MS = 15e3;
516
+ var OLLAMA_INIT_TIMEOUT_MS = 12e4;
510
517
  var STREAM_STALL_TIMEOUT_MS = 3e4;
511
518
  function getOllamaBaseUrl() {
512
519
  const configured = process.env.OLLAMA_HOST?.trim();
@@ -541,7 +548,8 @@ async function listModels() {
541
548
  size: m.size,
542
549
  parameterSize: m.details?.parameter_size,
543
550
  quantization: m.details?.quantization_level,
544
- family: m.details?.family
551
+ family: m.details?.family,
552
+ modelFormat: "gguf"
545
553
  }));
546
554
  }
547
555
  async function listRunningModels() {
@@ -578,6 +586,7 @@ async function generateStream(model, prompt, callbacks, options) {
578
586
  let fullResponse = "";
579
587
  let fullThinking = "";
580
588
  let result = null;
589
+ let firstChunkSeen = false;
581
590
  let stallTimer = null;
582
591
  const resetStallTimer = () => {
583
592
  if (stallTimer) clearTimeout(stallTimer);
@@ -589,6 +598,10 @@ async function generateStream(model, prompt, callbacks, options) {
589
598
  resetStallTimer();
590
599
  for await (const chunk of stream) {
591
600
  resetStallTimer();
601
+ if (!firstChunkSeen) {
602
+ firstChunkSeen = true;
603
+ callbacks?.onFirstChunk?.();
604
+ }
592
605
  const chunkAny = chunk;
593
606
  if (chunkAny.thinking) {
594
607
  fullThinking += String(chunkAny.thinking);
@@ -639,7 +652,706 @@ function abortOngoingRequests() {
639
652
  client.abort();
640
653
  }
641
654
 
655
+ // ../src/core/lm-studio-client.ts
656
+ import os from "os";
657
+ import path from "path";
658
+ import { promises as fs } from "fs";
659
+ var DEFAULT_LM_STUDIO_BASE_URL = "http://127.0.0.1:1234";
660
+ var LM_STUDIO_INIT_TIMEOUT_MS = 15e3;
661
+ var LM_STUDIO_METADATA_TIMEOUT_MS = 2e3;
662
+ var DEFAULT_STREAM_STALL_TIMEOUT_MS = 18e4;
663
+ var DEFAULT_LM_STUDIO_HOME_DIR = path.join(os.homedir(), ".lmstudio");
664
+ var DEFAULT_LM_STUDIO_MODELS_DIR = path.join(DEFAULT_LM_STUDIO_HOME_DIR, "models");
665
+ var LM_STUDIO_HOME_DIR_ENV = "LM_STUDIO_HOME_DIR";
666
+ var LM_STUDIO_MODELS_DIR_ENV = "LM_STUDIO_MODELS_DIR";
667
+ var defaultKeepAlive2;
668
+ var activeAbortControllers = /* @__PURE__ */ new Set();
669
+ var directorySizeCache = /* @__PURE__ */ new Map();
670
+ var modelDefinitionCache = /* @__PURE__ */ new Map();
671
+ function buildThinkingConfig(think) {
672
+ if (think === void 0) return {};
673
+ const effort = think ? "high" : "low";
674
+ return {
675
+ include_reasoning: think,
676
+ reasoning_effort: effort,
677
+ reasoning: { effort }
678
+ };
679
+ }
680
+ function parseNonNegativeInt(value) {
681
+ if (!/^\d+$/.test(value)) return null;
682
+ const parsed = Number.parseInt(value, 10);
683
+ if (!Number.isSafeInteger(parsed) || parsed < 0) return null;
684
+ return parsed;
685
+ }
686
+ function resolveStreamStallTimeoutMs(override) {
687
+ if (override !== void 0) {
688
+ if (!Number.isFinite(override) || override < 0) return DEFAULT_STREAM_STALL_TIMEOUT_MS;
689
+ return override === 0 ? void 0 : Math.trunc(override);
690
+ }
691
+ const configured = process.env.LM_STUDIO_STREAM_STALL_TIMEOUT_MS?.trim();
692
+ if (!configured) return DEFAULT_STREAM_STALL_TIMEOUT_MS;
693
+ const parsed = parseNonNegativeInt(configured);
694
+ if (parsed === null) return DEFAULT_STREAM_STALL_TIMEOUT_MS;
695
+ return parsed === 0 ? void 0 : parsed;
696
+ }
697
+ function getLMStudioBaseUrl() {
698
+ const configured = process.env.LM_STUDIO_BASE_URL?.trim();
699
+ if (!configured) return DEFAULT_LM_STUDIO_BASE_URL;
700
+ const candidate = /^https?:\/\//i.test(configured) ? configured : `http://${configured}`;
701
+ try {
702
+ return new URL(candidate).toString();
703
+ } catch {
704
+ return DEFAULT_LM_STUDIO_BASE_URL;
705
+ }
706
+ }
707
+ function getLMStudioHeaders() {
708
+ const headers = {
709
+ "Content-Type": "application/json"
710
+ };
711
+ const apiKey = process.env.LM_STUDIO_API_KEY?.trim();
712
+ if (apiKey) {
713
+ headers.Authorization = `Bearer ${apiKey}`;
714
+ }
715
+ return headers;
716
+ }
717
+ function extractUsage(payload) {
718
+ if (typeof payload !== "object" || payload === null) return void 0;
719
+ const usage = payload.usage;
720
+ if (!usage) return void 0;
721
+ return usage;
722
+ }
723
+ function extractChoice2(payload) {
724
+ if (typeof payload !== "object" || payload === null) return void 0;
725
+ const choices = payload.choices;
726
+ if (!choices || choices.length === 0) return void 0;
727
+ return choices[0];
728
+ }
729
+ function extractContent(choice) {
730
+ const content = choice?.delta?.content ?? choice?.message?.content;
731
+ return typeof content === "string" ? content : "";
732
+ }
733
+ function extractReasoning(choice) {
734
+ const reasoning = choice?.delta?.reasoning_content ?? choice?.delta?.reasoning ?? choice?.message?.reasoning_content ?? choice?.message?.reasoning;
735
+ return typeof reasoning === "string" ? reasoning : "";
736
+ }
737
+ function asNonEmptyString(value) {
738
+ if (typeof value !== "string") return void 0;
739
+ const trimmed = value.trim();
740
+ return trimmed.length > 0 ? trimmed : void 0;
741
+ }
742
+ function stripOptionalQuotes(value) {
743
+ const trimmed = value.trim();
744
+ return trimmed.replace(/^["']|["']$/g, "").trim();
745
+ }
746
+ function normalizeToken(value) {
747
+ return (value ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "");
748
+ }
749
+ function tokenizeModelName(value) {
750
+ return value.toLowerCase().split(/[^a-z0-9]+/g).map((token) => token.trim()).filter((token) => token.length >= 3 && /[a-z]/.test(token));
751
+ }
752
+ function getLMStudioHomeDir() {
753
+ const fromEnv = process.env[LM_STUDIO_HOME_DIR_ENV]?.trim();
754
+ if (!fromEnv) return DEFAULT_LM_STUDIO_HOME_DIR;
755
+ if (fromEnv.startsWith("~")) {
756
+ return path.join(os.homedir(), fromEnv.slice(1));
757
+ }
758
+ return fromEnv;
759
+ }
760
+ async function pathIsDirectory(targetPath) {
761
+ try {
762
+ const stat = await fs.stat(targetPath);
763
+ return stat.isDirectory();
764
+ } catch {
765
+ return false;
766
+ }
767
+ }
768
+ async function loadHubModelDefinition(modelId) {
769
+ if (modelDefinitionCache.has(modelId)) {
770
+ return modelDefinitionCache.get(modelId) ?? null;
771
+ }
772
+ const [publisher, modelName] = modelId.split("/", 2);
773
+ if (!publisher || !modelName) {
774
+ modelDefinitionCache.set(modelId, null);
775
+ return null;
776
+ }
777
+ const yamlPath = path.join(getLMStudioHomeDir(), "hub", "models", publisher, modelName, "model.yaml");
778
+ let content = "";
779
+ try {
780
+ content = await fs.readFile(yamlPath, "utf8");
781
+ } catch {
782
+ modelDefinitionCache.set(modelId, null);
783
+ return null;
784
+ }
785
+ const sourcePattern = /user:\s*([^\n#]+?)\s*(?:\r?\n)\s*repo:\s*([^\n#]+?)\s*(?:\r?\n|$)/g;
786
+ const sourceMap = /* @__PURE__ */ new Map();
787
+ let sourceMatch = null;
788
+ while ((sourceMatch = sourcePattern.exec(content)) !== null) {
789
+ const user = stripOptionalQuotes(sourceMatch[1] ?? "");
790
+ const repo = stripOptionalQuotes(sourceMatch[2] ?? "");
791
+ if (!user || !repo) continue;
792
+ sourceMap.set(`${user}/${repo}`, { user, repo });
793
+ }
794
+ const paramsPattern = /paramsStrings:\s*(?:\r?\n)\s*-\s*([^\n#]+)/;
795
+ const paramsMatch = paramsPattern.exec(content);
796
+ const parameterSize = paramsMatch?.[1] ? stripOptionalQuotes(paramsMatch[1]) : void 0;
797
+ const definition = {
798
+ parameterSize: parameterSize || void 0,
799
+ sources: Array.from(sourceMap.values())
800
+ };
801
+ modelDefinitionCache.set(modelId, definition);
802
+ return definition;
803
+ }
804
+ async function readDirectorySizeBytes(dirPath) {
805
+ const cached = directorySizeCache.get(dirPath);
806
+ if (cached !== void 0) return cached;
807
+ let total = 0;
808
+ const queue = [dirPath];
809
+ while (queue.length > 0) {
810
+ const current = queue.pop();
811
+ if (!current) continue;
812
+ let entries = [];
813
+ try {
814
+ entries = await fs.readdir(current, { withFileTypes: true });
815
+ } catch {
816
+ continue;
817
+ }
818
+ for (const entry of entries) {
819
+ if (entry.isSymbolicLink()) continue;
820
+ const fullPath = path.join(current, entry.name);
821
+ if (entry.isDirectory()) {
822
+ queue.push(fullPath);
823
+ continue;
824
+ }
825
+ if (!entry.isFile()) continue;
826
+ try {
827
+ const stat = await fs.stat(fullPath);
828
+ total += stat.size;
829
+ } catch {
830
+ }
831
+ }
832
+ }
833
+ directorySizeCache.set(dirPath, total);
834
+ return total;
835
+ }
836
+ function scoreModelSourceCandidate(source, modelId, apiModel) {
837
+ const repoToken = normalizeToken(source.repo);
838
+ let score = 0;
839
+ const quantToken = normalizeToken(apiModel?.quantization);
840
+ if (quantToken && repoToken.includes(quantToken)) score += 8;
841
+ const compatibilityToken = normalizeToken(apiModel?.compatibility_type);
842
+ if (compatibilityToken && repoToken.includes(compatibilityToken)) score += 4;
843
+ const [, shortModelName] = modelId.split("/", 2);
844
+ const modelToken = normalizeToken(shortModelName ?? modelId);
845
+ if (modelToken && repoToken.includes(modelToken)) score += 2;
846
+ return score;
847
+ }
848
+ async function resolveModelsRootDir() {
849
+ const fromEnv = process.env[LM_STUDIO_MODELS_DIR_ENV]?.trim();
850
+ if (fromEnv) {
851
+ if (fromEnv.startsWith("~")) {
852
+ return path.join(os.homedir(), fromEnv.slice(1));
853
+ }
854
+ return fromEnv;
855
+ }
856
+ const settingsCandidates = [
857
+ path.join(getLMStudioHomeDir(), "settings.json"),
858
+ path.join(os.homedir(), "Library", "Application Support", "LM Studio", "settings.json")
859
+ ];
860
+ for (const settingsPath of settingsCandidates) {
861
+ try {
862
+ const content = await fs.readFile(settingsPath, "utf8");
863
+ const parsed = JSON.parse(content);
864
+ const downloadsFolder = asNonEmptyString(parsed.downloadsFolder);
865
+ if (downloadsFolder) {
866
+ return downloadsFolder.startsWith("~") ? path.join(os.homedir(), downloadsFolder.slice(1)) : downloadsFolder;
867
+ }
868
+ } catch {
869
+ }
870
+ }
871
+ return DEFAULT_LM_STUDIO_MODELS_DIR;
872
+ }
873
+ async function listImmediateSubdirs(dirPath) {
874
+ let entries = [];
875
+ try {
876
+ entries = await fs.readdir(dirPath, { withFileTypes: true });
877
+ } catch {
878
+ return [];
879
+ }
880
+ return entries.filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()).map((entry) => path.join(dirPath, entry.name));
881
+ }
882
+ async function findPublisherDirs(rootDir, publisher) {
883
+ const normalizedPublisher = normalizeToken(publisher);
884
+ if (!normalizedPublisher) return [];
885
+ const dirs = await listImmediateSubdirs(rootDir);
886
+ return dirs.filter((dir) => normalizeToken(path.basename(dir)) === normalizedPublisher);
887
+ }
888
+ function scorePublisherModelCandidate(modelDirName, modelId, apiModel) {
889
+ const dirToken = normalizeToken(modelDirName);
890
+ if (!dirToken) return 0;
891
+ let score = 0;
892
+ const idToken = normalizeToken(modelId);
893
+ if (idToken && dirToken.includes(idToken)) score += 24;
894
+ const [, shortModelName] = modelId.split("/", 2);
895
+ const shortToken = normalizeToken(shortModelName ?? modelId);
896
+ if (shortToken && dirToken.includes(shortToken)) score += 16;
897
+ for (const token of tokenizeModelName(shortModelName ?? modelId)) {
898
+ if (dirToken.includes(token)) score += 3;
899
+ }
900
+ const quantToken = normalizeToken(apiModel?.quantization);
901
+ if (quantToken && dirToken.includes(quantToken)) score += 6;
902
+ const compatibilityToken = normalizeToken(apiModel?.compatibility_type);
903
+ if (compatibilityToken && dirToken.includes(compatibilityToken)) score += 4;
904
+ return score;
905
+ }
906
+ async function resolvePublisherModelMetadata(modelId, apiModel, modelsRootDir) {
907
+ const explicitPublisher = asNonEmptyString(apiModel?.publisher);
908
+ const [publisherFromId] = modelId.split("/", 1);
909
+ const publisher = explicitPublisher ?? publisherFromId;
910
+ if (!publisher) {
911
+ return { size: 0, parameterSize: inferParameterSizeFromModelId(modelId) };
912
+ }
913
+ const bundledModelsDir = path.join(getLMStudioHomeDir(), ".internal", "bundled-models");
914
+ const roots = Array.from(/* @__PURE__ */ new Set([modelsRootDir, bundledModelsDir]));
915
+ const candidates = [];
916
+ for (const root of roots) {
917
+ const publisherDirs = await findPublisherDirs(root, publisher);
918
+ for (const publisherDir of publisherDirs) {
919
+ const modelDirs = await listImmediateSubdirs(publisherDir);
920
+ if (modelDirs.length === 0) {
921
+ candidates.push({
922
+ fullPath: publisherDir,
923
+ score: scorePublisherModelCandidate(path.basename(publisherDir), modelId, apiModel)
924
+ });
925
+ continue;
926
+ }
927
+ for (const modelDir of modelDirs) {
928
+ candidates.push({
929
+ fullPath: modelDir,
930
+ score: scorePublisherModelCandidate(path.basename(modelDir), modelId, apiModel)
931
+ });
932
+ }
933
+ }
934
+ }
935
+ if (candidates.length === 0) {
936
+ return { size: 0, parameterSize: inferParameterSizeFromModelId(modelId) };
937
+ }
938
+ candidates.sort((a, b) => {
939
+ const scoreDiff = b.score - a.score;
940
+ if (scoreDiff !== 0) return scoreDiff;
941
+ return a.fullPath.localeCompare(b.fullPath);
942
+ });
943
+ let bestSize = 0;
944
+ for (const candidate of candidates) {
945
+ const size = await readDirectorySizeBytes(candidate.fullPath);
946
+ if (size > bestSize) bestSize = size;
947
+ if (size > 0 && candidate.score > 0) {
948
+ return { size, parameterSize: inferParameterSizeFromModelId(modelId) };
949
+ }
950
+ }
951
+ return { size: bestSize, parameterSize: inferParameterSizeFromModelId(modelId) };
952
+ }
953
+ async function resolveLocalModelMetadata(modelId, apiModel, modelsRootDir) {
954
+ const definition = await loadHubModelDefinition(modelId);
955
+ if (!definition) {
956
+ return resolvePublisherModelMetadata(modelId, apiModel, modelsRootDir);
957
+ }
958
+ const installedSources = [];
959
+ for (const source of definition.sources) {
960
+ const fullPath = path.join(modelsRootDir, source.user, source.repo);
961
+ if (await pathIsDirectory(fullPath)) {
962
+ installedSources.push({ ...source, fullPath });
963
+ }
964
+ }
965
+ if (installedSources.length === 0) {
966
+ const fallback2 = await resolvePublisherModelMetadata(modelId, apiModel, modelsRootDir);
967
+ if (fallback2.size > 0) {
968
+ return {
969
+ size: fallback2.size,
970
+ parameterSize: definition.parameterSize ?? fallback2.parameterSize
971
+ };
972
+ }
973
+ return { size: 0, parameterSize: definition.parameterSize };
974
+ }
975
+ installedSources.sort((a, b) => {
976
+ const diff = scoreModelSourceCandidate(b, modelId, apiModel) - scoreModelSourceCandidate(a, modelId, apiModel);
977
+ if (diff !== 0) return diff;
978
+ return a.repo.localeCompare(b.repo);
979
+ });
980
+ let bestSize = 0;
981
+ for (const source of installedSources) {
982
+ const size = await readDirectorySizeBytes(source.fullPath);
983
+ if (size > bestSize) bestSize = size;
984
+ if (size > 0) {
985
+ return { size, parameterSize: definition.parameterSize };
986
+ }
987
+ }
988
+ if (bestSize > 0) {
989
+ return { size: bestSize, parameterSize: definition.parameterSize };
990
+ }
991
+ const fallback = await resolvePublisherModelMetadata(modelId, apiModel, modelsRootDir);
992
+ if (fallback.size > 0) {
993
+ return {
994
+ size: fallback.size,
995
+ parameterSize: definition.parameterSize ?? fallback.parameterSize
996
+ };
997
+ }
998
+ return { size: 0, parameterSize: definition.parameterSize ?? fallback.parameterSize };
999
+ }
1000
+ function parseSizeBytes(model) {
1001
+ if (!model) return 0;
1002
+ const candidates = [
1003
+ model.size_bytes,
1004
+ model.model_size_bytes,
1005
+ model.file_size_bytes,
1006
+ model.bytes,
1007
+ model.size
1008
+ ];
1009
+ for (const candidate of candidates) {
1010
+ if (typeof candidate !== "number" || !Number.isFinite(candidate)) continue;
1011
+ if (candidate > 0) return Math.trunc(candidate);
1012
+ }
1013
+ return 0;
1014
+ }
1015
+ function normalizeModelNumber(value) {
1016
+ return value.replace(/\.0+$/, "");
1017
+ }
1018
+ function inferParameterSizeFromModelId(modelId) {
1019
+ const id = modelId.toLowerCase();
1020
+ const mixtureMatch = id.match(/\b(\d+(?:\.\d+)?)\s*[x*]\s*(\d+(?:\.\d+)?)\s*b\b/);
1021
+ if (mixtureMatch) {
1022
+ const experts = normalizeModelNumber(mixtureMatch[1] ?? "");
1023
+ const perExpert = normalizeModelNumber(mixtureMatch[2] ?? "");
1024
+ if (experts && perExpert) return `${experts}x${perExpert}B`;
1025
+ }
1026
+ const billionMatch = id.match(/\b(\d+(?:\.\d+)?)\s*b\b/);
1027
+ if (billionMatch?.[1]) {
1028
+ return `${normalizeModelNumber(billionMatch[1])}B`;
1029
+ }
1030
+ const millionMatch = id.match(/\b(\d+(?:\.\d+)?)\s*m\b/);
1031
+ if (millionMatch?.[1]) {
1032
+ return `${normalizeModelNumber(millionMatch[1])}M`;
1033
+ }
1034
+ return void 0;
1035
+ }
1036
+ function isLoadedState(state) {
1037
+ if (!state) return false;
1038
+ const normalized = state.trim().toLowerCase();
1039
+ if (!normalized || normalized.includes("not-loaded")) return false;
1040
+ if (normalized === "loaded" || normalized === "ready") return true;
1041
+ return normalized.includes("loaded");
1042
+ }
1043
+ async function fetchApiModels() {
1044
+ try {
1045
+ const resp = await fetchWithTimeout(
1046
+ "/api/v0/models",
1047
+ { method: "GET", headers: getLMStudioHeaders() },
1048
+ LM_STUDIO_METADATA_TIMEOUT_MS,
1049
+ "LM Studio API metadata"
1050
+ );
1051
+ if (!resp.ok) return null;
1052
+ const payload = await resp.json();
1053
+ return Array.isArray(payload.data) ? payload.data : [];
1054
+ } catch {
1055
+ return null;
1056
+ }
1057
+ }
1058
+ async function fetchWithTimeout(path2, init, timeoutMs, label) {
1059
+ const controller = new AbortController();
1060
+ const timeout = setTimeout(() => controller.abort(), timeoutMs);
1061
+ const baseUrl = getLMStudioBaseUrl();
1062
+ try {
1063
+ const url = new URL(path2, baseUrl);
1064
+ return await fetch(url, { ...init, signal: controller.signal });
1065
+ } catch (err) {
1066
+ if (err instanceof Error && err.name === "AbortError") {
1067
+ throw new Error(`${label} timed out after ${timeoutMs}ms`);
1068
+ }
1069
+ throw err;
1070
+ } finally {
1071
+ clearTimeout(timeout);
1072
+ }
1073
+ }
1074
+ async function resolveLocalLMStudioVersion() {
1075
+ const historicalVersionPath = path.join(
1076
+ getLMStudioHomeDir(),
1077
+ ".internal",
1078
+ "historical-version-info.json"
1079
+ );
1080
+ try {
1081
+ const content = await fs.readFile(historicalVersionPath, "utf8");
1082
+ const parsed = JSON.parse(content);
1083
+ const version = asNonEmptyString(parsed.lastRecordedAppVersion) ?? asNonEmptyString(parsed.lastRecorderdAppVersion);
1084
+ if (!version) return null;
1085
+ const build = asNonEmptyString(parsed.lastRecordedAppBuildVersion);
1086
+ return build ? `${version}+${build}` : version;
1087
+ } catch {
1088
+ return null;
1089
+ }
1090
+ }
1091
+ async function getLMStudioVersion() {
1092
+ const localVersion = await resolveLocalLMStudioVersion();
1093
+ try {
1094
+ const resp = await fetchWithTimeout(
1095
+ "/v1/models",
1096
+ { method: "GET", headers: getLMStudioHeaders() },
1097
+ 5e3,
1098
+ "LM Studio version check"
1099
+ );
1100
+ if (!resp.ok) return localVersion ?? "unknown";
1101
+ const fromHeader = asNonEmptyString(resp.headers.get("x-lmstudio-version"));
1102
+ if (fromHeader) return fromHeader;
1103
+ return localVersion ?? "unknown";
1104
+ } catch {
1105
+ return localVersion ?? "unknown";
1106
+ }
1107
+ }
1108
+ async function listModels2() {
1109
+ const resp = await fetchWithTimeout(
1110
+ "/v1/models",
1111
+ { method: "GET", headers: getLMStudioHeaders() },
1112
+ LM_STUDIO_INIT_TIMEOUT_MS,
1113
+ "LM Studio list models"
1114
+ );
1115
+ if (!resp.ok) {
1116
+ throw new Error(`LM Studio list models failed (${resp.status} ${resp.statusText})`);
1117
+ }
1118
+ const data = await resp.json();
1119
+ const ids = (data.data ?? []).map((m) => m.id?.trim()).filter((id) => Boolean(id));
1120
+ const apiModels = await fetchApiModels();
1121
+ const apiById = /* @__PURE__ */ new Map();
1122
+ for (const model of apiModels ?? []) {
1123
+ const id = asNonEmptyString(model.id);
1124
+ if (!id) continue;
1125
+ apiById.set(id, model);
1126
+ }
1127
+ const modelsRootDir = await resolveModelsRootDir();
1128
+ const localMetadataById = /* @__PURE__ */ new Map();
1129
+ for (const id of ids) {
1130
+ const localMetadata = await resolveLocalModelMetadata(id, apiById.get(id), modelsRootDir);
1131
+ localMetadataById.set(id, localMetadata);
1132
+ }
1133
+ return ids.map((id) => {
1134
+ const apiModel = apiById.get(id);
1135
+ const localMetadata = localMetadataById.get(id);
1136
+ const apiSize = parseSizeBytes(apiModel);
1137
+ return {
1138
+ name: id,
1139
+ size: apiSize > 0 ? apiSize : localMetadata?.size ?? 0,
1140
+ parameterSize: localMetadata?.parameterSize ?? inferParameterSizeFromModelId(id),
1141
+ quantization: asNonEmptyString(apiModel?.quantization),
1142
+ runtimeStatus: asNonEmptyString(apiModel?.state),
1143
+ modelFormat: asNonEmptyString(apiModel?.compatibility_type),
1144
+ family: asNonEmptyString(apiModel?.arch) ?? asNonEmptyString(apiModel?.type) ?? asNonEmptyString(apiModel?.publisher)
1145
+ };
1146
+ });
1147
+ }
1148
+ async function listRunningModels2() {
1149
+ const apiModels = await fetchApiModels();
1150
+ if (!apiModels) return [];
1151
+ return apiModels.filter((model) => isLoadedState(model.state)).map((model) => ({
1152
+ name: model.id ?? "",
1153
+ size: parseSizeBytes(model),
1154
+ vramUsed: 0
1155
+ })).filter((model) => model.name.trim().length > 0);
1156
+ }
1157
+ function setDefaultKeepAlive2(keepAlive) {
1158
+ defaultKeepAlive2 = keepAlive;
1159
+ void defaultKeepAlive2;
1160
+ }
1161
+ async function generate2(model, prompt, options) {
1162
+ const start = Date.now();
1163
+ const controller = new AbortController();
1164
+ activeAbortControllers.add(controller);
1165
+ try {
1166
+ const baseUrl = getLMStudioBaseUrl();
1167
+ const url = new URL("/v1/chat/completions", baseUrl);
1168
+ const resp = await fetch(url, {
1169
+ method: "POST",
1170
+ headers: getLMStudioHeaders(),
1171
+ body: JSON.stringify({
1172
+ model,
1173
+ messages: [{ role: "user", content: prompt }],
1174
+ temperature: options?.temperature ?? 0,
1175
+ max_tokens: options?.num_predict ?? 512,
1176
+ stream: false,
1177
+ ...buildThinkingConfig(options?.think)
1178
+ }),
1179
+ signal: controller.signal
1180
+ });
1181
+ if (!resp.ok) {
1182
+ const body = await resp.text().catch(() => "");
1183
+ throw new Error(`LM Studio generate failed (${resp.status} ${resp.statusText}) ${body}`.trim());
1184
+ }
1185
+ const payload = await resp.json();
1186
+ const choice = extractChoice2(payload);
1187
+ const response = extractContent(choice);
1188
+ const reasoning = extractReasoning(choice);
1189
+ const usage = extractUsage(payload);
1190
+ const totalDuration = Math.max(0, Date.now() - start) * 1e6;
1191
+ return {
1192
+ response,
1193
+ ...reasoning ? { thinking: reasoning } : {},
1194
+ totalDuration,
1195
+ loadDuration: 0,
1196
+ promptEvalCount: usage?.prompt_tokens ?? 0,
1197
+ promptEvalDuration: 0,
1198
+ evalCount: usage?.completion_tokens ?? 0,
1199
+ evalDuration: totalDuration
1200
+ };
1201
+ } catch (err) {
1202
+ if (err instanceof Error && err.name === "AbortError") {
1203
+ throw new Error("LM Studio generate request aborted");
1204
+ }
1205
+ throw err;
1206
+ } finally {
1207
+ activeAbortControllers.delete(controller);
1208
+ }
1209
+ }
1210
+ async function generateStream2(model, prompt, callbacks, options) {
1211
+ const start = Date.now();
1212
+ const controller = new AbortController();
1213
+ activeAbortControllers.add(controller);
1214
+ const stallTimeoutMs = resolveStreamStallTimeoutMs(options?.stall_timeout_ms);
1215
+ let abortedByStallTimeout = false;
1216
+ const baseUrl = getLMStudioBaseUrl();
1217
+ const url = new URL("/v1/chat/completions", baseUrl);
1218
+ let stallTimer = null;
1219
+ const resetStallTimer = () => {
1220
+ if (stallTimeoutMs === void 0) return;
1221
+ if (stallTimer) clearTimeout(stallTimer);
1222
+ stallTimer = setTimeout(() => {
1223
+ abortedByStallTimeout = true;
1224
+ controller.abort();
1225
+ }, stallTimeoutMs);
1226
+ };
1227
+ try {
1228
+ resetStallTimer();
1229
+ const resp = await fetch(url, {
1230
+ method: "POST",
1231
+ headers: getLMStudioHeaders(),
1232
+ body: JSON.stringify({
1233
+ model,
1234
+ messages: [{ role: "user", content: prompt }],
1235
+ temperature: options?.temperature ?? 0,
1236
+ max_tokens: options?.num_predict ?? 512,
1237
+ stream: true,
1238
+ stream_options: { include_usage: true },
1239
+ ...buildThinkingConfig(options?.think)
1240
+ }),
1241
+ signal: controller.signal
1242
+ });
1243
+ if (!resp.ok) {
1244
+ const body = await resp.text().catch(() => "");
1245
+ throw new Error(`LM Studio stream failed (${resp.status} ${resp.statusText}) ${body}`.trim());
1246
+ }
1247
+ if (!resp.body) {
1248
+ throw new Error("LM Studio stream response body is empty");
1249
+ }
1250
+ const reader = resp.body.getReader();
1251
+ const decoder = new TextDecoder();
1252
+ let buffered = "";
1253
+ let doneReceived = false;
1254
+ let fullResponse = "";
1255
+ let fullThinking = "";
1256
+ let usage;
1257
+ let firstChunkSeen = false;
1258
+ let firstTokenTime = null;
1259
+ let lastTokenTime = null;
1260
+ const processDataLine = (rawLine) => {
1261
+ const line = rawLine.trim();
1262
+ if (!line.startsWith("data:")) return;
1263
+ const dataStr = line.slice(5).trim();
1264
+ if (!dataStr) return;
1265
+ if (dataStr === "[DONE]") {
1266
+ doneReceived = true;
1267
+ return;
1268
+ }
1269
+ let payload;
1270
+ try {
1271
+ payload = JSON.parse(dataStr);
1272
+ } catch {
1273
+ return;
1274
+ }
1275
+ const choice = extractChoice2(payload);
1276
+ const content = extractContent(choice);
1277
+ const reasoning = extractReasoning(choice);
1278
+ const chunkUsage = extractUsage(payload);
1279
+ if (chunkUsage) usage = chunkUsage;
1280
+ if (reasoning) {
1281
+ fullThinking += reasoning;
1282
+ }
1283
+ if (content) {
1284
+ const now = Date.now();
1285
+ if (firstTokenTime === null) firstTokenTime = now;
1286
+ lastTokenTime = now;
1287
+ fullResponse += content;
1288
+ callbacks?.onToken?.(content);
1289
+ }
1290
+ };
1291
+ while (true) {
1292
+ const { value, done } = await reader.read();
1293
+ if (done) break;
1294
+ resetStallTimer();
1295
+ if (!firstChunkSeen) {
1296
+ firstChunkSeen = true;
1297
+ callbacks?.onFirstChunk?.();
1298
+ }
1299
+ buffered += decoder.decode(value, { stream: true });
1300
+ const lines = buffered.split("\n");
1301
+ buffered = lines.pop() ?? "";
1302
+ for (const rawLine of lines) {
1303
+ processDataLine(rawLine);
1304
+ }
1305
+ }
1306
+ if (buffered.trim().length > 0) {
1307
+ processDataLine(buffered);
1308
+ }
1309
+ if (stallTimer) clearTimeout(stallTimer);
1310
+ if (!doneReceived && !fullResponse && !fullThinking) {
1311
+ throw new Error("LM Studio stream ended without content");
1312
+ }
1313
+ const totalDuration = Math.max(0, Date.now() - start) * 1e6;
1314
+ const evalDurationMs = firstTokenTime !== null && lastTokenTime !== null && lastTokenTime > firstTokenTime ? lastTokenTime - firstTokenTime : Date.now() - start;
1315
+ const result = {
1316
+ response: fullResponse,
1317
+ ...fullThinking ? { thinking: fullThinking } : {},
1318
+ totalDuration,
1319
+ loadDuration: 0,
1320
+ promptEvalCount: usage?.prompt_tokens ?? 0,
1321
+ promptEvalDuration: firstTokenTime !== null ? (firstTokenTime - start) * 1e6 : 0,
1322
+ evalCount: usage?.completion_tokens ?? 0,
1323
+ evalDuration: Math.max(1, evalDurationMs) * 1e6
1324
+ };
1325
+ callbacks?.onDone?.(result);
1326
+ return result;
1327
+ } catch (err) {
1328
+ if (err instanceof Error && err.name === "AbortError") {
1329
+ if (abortedByStallTimeout && stallTimeoutMs !== void 0) {
1330
+ throw new Error(`LM Studio stream timed out after ${stallTimeoutMs}ms`);
1331
+ }
1332
+ throw new Error("LM Studio stream request aborted");
1333
+ }
1334
+ throw err;
1335
+ } finally {
1336
+ if (stallTimer) clearTimeout(stallTimer);
1337
+ activeAbortControllers.delete(controller);
1338
+ }
1339
+ }
1340
+ async function unloadModel2(_model) {
1341
+ }
1342
+ function abortOngoingRequests2() {
1343
+ for (const controller of activeAbortControllers) {
1344
+ controller.abort();
1345
+ }
1346
+ activeAbortControllers.clear();
1347
+ }
1348
+
642
1349
  // ../src/core/runtime.ts
1350
+ var SUPPORTED_RUNTIME_BACKENDS = ["ollama", "lm-studio"];
1351
+ var RUNTIME_LABELS = {
1352
+ ollama: "Ollama",
1353
+ "lm-studio": "LM Studio"
1354
+ };
643
1355
  var OllamaRuntime = class {
644
1356
  name = "ollama";
645
1357
  modelFormat = "gguf";
@@ -668,29 +1380,105 @@ var OllamaRuntime = class {
668
1380
  abortOngoingRequests();
669
1381
  }
670
1382
  };
1383
+ var LMStudioRuntime = class {
1384
+ name = "lm-studio";
1385
+ modelFormat = "gguf";
1386
+ generate(model, prompt, opts) {
1387
+ return generate2(model, prompt, opts);
1388
+ }
1389
+ generateStream(model, prompt, callbacks, opts) {
1390
+ return generateStream2(model, prompt, callbacks, opts);
1391
+ }
1392
+ listModels() {
1393
+ return listModels2();
1394
+ }
1395
+ listRunningModels() {
1396
+ return listRunningModels2();
1397
+ }
1398
+ getVersion() {
1399
+ return getLMStudioVersion();
1400
+ }
1401
+ unloadModel(model) {
1402
+ return unloadModel2(model);
1403
+ }
1404
+ setKeepAlive(keepAlive) {
1405
+ setDefaultKeepAlive2(keepAlive);
1406
+ }
1407
+ abort() {
1408
+ abortOngoingRequests2();
1409
+ }
1410
+ };
671
1411
  var activeRuntime = new OllamaRuntime();
672
- function generate2(model, prompt, opts) {
1412
+ function createRuntime(backend) {
1413
+ if (backend === "lm-studio") return new LMStudioRuntime();
1414
+ return new OllamaRuntime();
1415
+ }
1416
+ function normalizeRuntimeBackend(value) {
1417
+ const candidate = (value ?? "ollama").trim().toLowerCase();
1418
+ if (candidate === "ollama" || candidate === "lm-studio") {
1419
+ return candidate;
1420
+ }
1421
+ throw new Error(
1422
+ `Unsupported backend "${value}". Supported backends: ${SUPPORTED_RUNTIME_BACKENDS.join(", ")}`
1423
+ );
1424
+ }
1425
+ function setRuntimeByName(backend) {
1426
+ const normalized = normalizeRuntimeBackend(backend);
1427
+ if (activeRuntime.name === normalized) return normalized;
1428
+ activeRuntime = createRuntime(normalized);
1429
+ return normalized;
1430
+ }
1431
+ function getRuntimeDisplayName(runtimeName = activeRuntime.name) {
1432
+ if (runtimeName === "lm-studio") return RUNTIME_LABELS["lm-studio"];
1433
+ if (runtimeName === "ollama") return RUNTIME_LABELS.ollama;
1434
+ return runtimeName;
1435
+ }
1436
+ function getRuntimeModelInstallHint(runtimeName = activeRuntime.name) {
1437
+ if (runtimeName === "lm-studio") {
1438
+ return "Download/select a model in LM Studio, then load it in the local server.";
1439
+ }
1440
+ if (runtimeName === "ollama") {
1441
+ return "Pull one with: ollama pull <model>";
1442
+ }
1443
+ return "Add at least one model in your selected runtime.";
1444
+ }
1445
+ function getRuntimeSetupHints(runtimeName = activeRuntime.name) {
1446
+ if (runtimeName === "lm-studio") {
1447
+ return [
1448
+ "Start LM Studio local server (Developer tab -> Local Server).",
1449
+ "Optionally set LM_STUDIO_BASE_URL if your server is not on http://127.0.0.1:1234."
1450
+ ];
1451
+ }
1452
+ if (runtimeName === "ollama") {
1453
+ return [
1454
+ "Start it with: ollama serve",
1455
+ "Install it at: https://ollama.com"
1456
+ ];
1457
+ }
1458
+ return ["Verify the runtime is reachable and configured correctly."];
1459
+ }
1460
+ function generate3(model, prompt, opts) {
673
1461
  return activeRuntime.generate(model, prompt, opts);
674
1462
  }
675
- function generateStream2(model, prompt, callbacks, opts) {
1463
+ function generateStream3(model, prompt, callbacks, opts) {
676
1464
  return activeRuntime.generateStream(model, prompt, callbacks, opts);
677
1465
  }
678
- function listModels2() {
1466
+ function listModels3() {
679
1467
  return activeRuntime.listModels();
680
1468
  }
681
- function listRunningModels2() {
1469
+ function listRunningModels3() {
682
1470
  return activeRuntime.listRunningModels();
683
1471
  }
684
1472
  function getRuntimeVersion() {
685
1473
  return activeRuntime.getVersion();
686
1474
  }
687
- function unloadModel2(model) {
1475
+ function unloadModel3(model) {
688
1476
  return activeRuntime.unloadModel(model);
689
1477
  }
690
1478
  function setRuntimeKeepAlive(keepAlive) {
691
1479
  activeRuntime.setKeepAlive(keepAlive);
692
1480
  }
693
- function abortOngoingRequests2() {
1481
+ function abortOngoingRequests3() {
694
1482
  activeRuntime.abort();
695
1483
  }
696
1484
  function getRuntimeName() {
@@ -706,7 +1494,7 @@ import chalk8 from "chalk";
706
1494
 
707
1495
  // ../src/core/hardware.ts
708
1496
  import si from "systeminformation";
709
- import os from "os";
1497
+ import os2 from "os";
710
1498
  import { execFile as execFile2 } from "child_process";
711
1499
  import { readFile } from "fs/promises";
712
1500
  function execCommand(cmd, args, timeoutMs = 3e3) {
@@ -782,6 +1570,56 @@ async function detectPowerMode() {
782
1570
  return "unknown";
783
1571
  }
784
1572
  }
1573
+ async function detectThermalPressure() {
1574
+ try {
1575
+ if (process.platform === "darwin") {
1576
+ const output4 = await execCommand("pmset", ["-g", "therm"]);
1577
+ if (!output4) return "unknown";
1578
+ const match = output4.match(/CPU_Speed_Limit\s*=\s*(\d+)/i);
1579
+ if (!match) return "unknown";
1580
+ const limit = parseInt(match[1], 10);
1581
+ if (limit >= 100) return "nominal";
1582
+ if (limit >= 80) return "moderate";
1583
+ if (limit >= 50) return "heavy";
1584
+ return "critical";
1585
+ }
1586
+ const temp = await si.cpuTemperature();
1587
+ const main2 = temp.main;
1588
+ if (main2 == null || main2 <= 0) return "unknown";
1589
+ if (main2 < 80) return "nominal";
1590
+ if (main2 < 90) return "moderate";
1591
+ if (main2 < 100) return "heavy";
1592
+ return "critical";
1593
+ } catch {
1594
+ return "unknown";
1595
+ }
1596
+ }
1597
+ async function detectBatteryPowered() {
1598
+ try {
1599
+ if (process.platform === "darwin") {
1600
+ const output4 = await execCommand("pmset", ["-g", "ps"]);
1601
+ if (!output4) return void 0;
1602
+ if (output4.includes("Battery Power")) return true;
1603
+ if (output4.includes("AC Power")) return false;
1604
+ return void 0;
1605
+ }
1606
+ const battery = await si.battery();
1607
+ if (!battery.hasBattery) return void 0;
1608
+ if (typeof battery.acConnected === "boolean") return !battery.acConnected;
1609
+ if (typeof battery.isCharging === "boolean") return !battery.isCharging;
1610
+ return void 0;
1611
+ } catch {
1612
+ return void 0;
1613
+ }
1614
+ }
1615
+ async function getSwapUsedGB() {
1616
+ try {
1617
+ const mem = await si.mem();
1618
+ return +(mem.swapused / 1024 / 1024 / 1024).toFixed(2);
1619
+ } catch {
1620
+ return 0;
1621
+ }
1622
+ }
785
1623
  async function getHardwareInfo() {
786
1624
  const [cpu, mem, graphics, osInfo, memLayout, powerMode, cpuSpeed, machineModel] = await Promise.all([
787
1625
  si.cpu(),
@@ -813,12 +1651,20 @@ async function getHardwareInfo() {
813
1651
  gpuCores: gpuCores && !isNaN(gpuCores) ? gpuCores : null,
814
1652
  gpuVramMB: gpuController?.vram ?? null,
815
1653
  os: `${osInfo.distro} ${osInfo.release}`,
816
- arch: os.arch(),
1654
+ arch: os2.arch(),
817
1655
  machineModel: machineModel || null,
818
1656
  powerMode,
819
1657
  cpuCurrentSpeedGHz: cpuSpeed?.avg ?? null
820
1658
  };
821
1659
  }
1660
+ async function getCpuLoad() {
1661
+ try {
1662
+ const load = await si.currentLoad();
1663
+ return +load.currentLoad.toFixed(1);
1664
+ } catch {
1665
+ return -1;
1666
+ }
1667
+ }
822
1668
  async function getMemoryUsage() {
823
1669
  const mem = await si.mem();
824
1670
  const totalGB = mem.total / 1024 / 1024 / 1024;
@@ -979,8 +1825,22 @@ var BENCH_PROMPTS = [
979
1825
  "Write a detailed step-by-step explanation of how a compiler transforms source code into machine code, covering lexing, parsing, semantic analysis, optimization, and code generation.",
980
1826
  "Implement a function in pseudocode that checks whether a given string of parentheses, brackets, and braces is balanced. Explain the time and space complexity."
981
1827
  ];
982
- var DEFAULT_WARMUP_TIMEOUT_MS = 12e4;
983
- var DEFAULT_PROMPT_TIMEOUT_MS = 6e4;
1828
+ var DEFAULT_WARMUP_TIMEOUT_MS = 3e5;
1829
+ var DEFAULT_PROMPT_TIMEOUT_MS = 12e4;
1830
+ async function optionalProbe(probe, fallback) {
1831
+ try {
1832
+ return await probe();
1833
+ } catch {
1834
+ return fallback;
1835
+ }
1836
+ }
1837
+ async function optionalProbeWithAvailability(probe, fallback) {
1838
+ try {
1839
+ return { value: await probe(), available: true };
1840
+ } catch {
1841
+ return { value: fallback, available: false };
1842
+ }
1843
+ }
984
1844
  async function runPerformanceBench(model, options = {}) {
985
1845
  const spinner = createSpinner("Warming up model...");
986
1846
  spinner.start();
@@ -989,21 +1849,39 @@ async function runPerformanceBench(model, options = {}) {
989
1849
  const promptTimeoutMs = options.promptTimeoutMs ?? DEFAULT_PROMPT_TIMEOUT_MS;
990
1850
  const minSuccessfulPrompts = options.minSuccessfulPrompts ?? Math.max(1, Math.ceil(BENCH_PROMPTS.length / 2));
991
1851
  const failOnPromptError = options.failOnPromptError ?? false;
992
- const memBefore = await getMemoryUsage();
1852
+ const [memBefore, thermalBefore, swapBeforeResult, batteryPowered] = await Promise.all([
1853
+ getMemoryUsage(),
1854
+ optionalProbe(() => detectThermalPressure(), "unknown"),
1855
+ optionalProbeWithAvailability(() => getSwapUsedGB(), 0),
1856
+ optionalProbe(() => detectBatteryPowered(), void 0)
1857
+ ]);
993
1858
  const warmup = await withTimeout(
994
- generateStream2(model, WARMUP_PROMPT, void 0, {
1859
+ generateStream3(model, WARMUP_PROMPT, void 0, {
995
1860
  num_predict: 32,
996
- think: options.think
1861
+ think: options.think,
1862
+ stall_timeout_ms: options.streamStallTimeoutMs
997
1863
  }),
998
1864
  warmupTimeoutMs,
999
1865
  "Model warmup",
1000
- abortOngoingRequests2
1866
+ abortOngoingRequests3
1001
1867
  );
1868
+ const runtimeName = getRuntimeName();
1869
+ const loadTimeAvailable = !(runtimeName === "lm-studio" && warmup.loadDuration === 0);
1002
1870
  const loadTime = warmup.loadDuration / 1e6;
1003
- const runningModels = await listRunningModels2();
1871
+ const runningModels = await listRunningModels3();
1004
1872
  const thisModel = runningModels.find((m) => m.name === model);
1873
+ let installedModelSizeBytes = 0;
1874
+ try {
1875
+ const availableModels = await listModels3();
1876
+ const listedModel = availableModels.find((m) => m.name === model);
1877
+ if (listedModel && Number.isFinite(listedModel.size) && listedModel.size > 0) {
1878
+ installedModelSizeBytes = listedModel.size;
1879
+ }
1880
+ } catch {
1881
+ }
1005
1882
  spinner.succeed("Model loaded");
1006
1883
  const tpsValues = [];
1884
+ const firstChunkValues = [];
1007
1885
  const ttftValues = [];
1008
1886
  let totalPromptTokens = 0;
1009
1887
  let totalCompletionTokens = 0;
@@ -1013,33 +1891,47 @@ async function runPerformanceBench(model, options = {}) {
1013
1891
  let failedPrompts = 0;
1014
1892
  let thinkingDetected = false;
1015
1893
  let totalThinkingTokens = 0;
1894
+ const cpuLoadSamples = [];
1016
1895
  for (let i = 0; i < BENCH_PROMPTS.length; i++) {
1017
1896
  spinner.start(`Running performance test ${i + 1}/${BENCH_PROMPTS.length}...`);
1897
+ let firstChunkTime = null;
1018
1898
  let firstTokenTime = null;
1019
1899
  const startTime = Date.now();
1020
1900
  try {
1021
1901
  const result = await withTimeout(
1022
- generateStream2(
1902
+ generateStream3(
1023
1903
  model,
1024
1904
  BENCH_PROMPTS[i],
1025
1905
  {
1906
+ onFirstChunk: () => {
1907
+ if (firstChunkTime === null) {
1908
+ firstChunkTime = Date.now() - startTime;
1909
+ }
1910
+ },
1026
1911
  onToken: () => {
1027
1912
  if (firstTokenTime === null) {
1028
1913
  firstTokenTime = Date.now() - startTime;
1029
1914
  }
1030
1915
  }
1031
1916
  },
1032
- { num_predict: 256, think: options.think }
1917
+ {
1918
+ num_predict: 256,
1919
+ think: options.think,
1920
+ stall_timeout_ms: options.streamStallTimeoutMs
1921
+ }
1033
1922
  ),
1034
1923
  promptTimeoutMs,
1035
1924
  "Performance benchmark",
1036
- abortOngoingRequests2
1925
+ abortOngoingRequests3
1037
1926
  );
1038
1927
  const evalDurationSec = result.evalDuration / 1e9;
1039
1928
  const tps = evalDurationSec > 0 ? result.evalCount / evalDurationSec : 0;
1040
1929
  tpsValues.push(tps);
1041
1930
  totalEvalCount += result.evalCount;
1042
1931
  totalEvalDurationNs += result.evalDuration;
1932
+ if (firstChunkTime !== null) {
1933
+ firstChunkValues.push(firstChunkTime);
1934
+ }
1043
1935
  if (firstTokenTime !== null) {
1044
1936
  ttftValues.push(firstTokenTime);
1045
1937
  }
@@ -1052,8 +1944,10 @@ async function runPerformanceBench(model, options = {}) {
1052
1944
  totalCompletionTokens += result.evalCount;
1053
1945
  successfulPrompts++;
1054
1946
  subStep(
1055
- ` Prompt ${i + 1}: ${tps.toFixed(1)} tok/s, TTFT ${firstTokenTime ?? "?"}ms`
1947
+ ` Prompt ${i + 1}: ${tps.toFixed(1)} tok/s, first chunk ${firstChunkTime ?? "?"}ms, TTFT ${firstTokenTime ?? "?"}ms`
1056
1948
  );
1949
+ const cpuSample = await optionalProbe(() => getCpuLoad(), -1);
1950
+ if (cpuSample >= 0) cpuLoadSamples.push(cpuSample);
1057
1951
  } catch (err) {
1058
1952
  failedPrompts++;
1059
1953
  const message = err instanceof Error ? err.message : String(err);
@@ -1073,24 +1967,43 @@ async function runPerformanceBench(model, options = {}) {
1073
1967
  ` Completed with ${failedPrompts} failed prompt(s); metrics use ${successfulPrompts} successful prompt(s).`
1074
1968
  );
1075
1969
  }
1076
- const memAfter = await getMemoryUsage();
1970
+ const [memAfter, thermalAfter, swapAfterResult] = await Promise.all([
1971
+ getMemoryUsage(),
1972
+ optionalProbe(() => detectThermalPressure(), thermalBefore),
1973
+ optionalProbeWithAvailability(() => getSwapUsedGB(), swapBeforeResult.value)
1974
+ ]);
1077
1975
  let memoryUsedGB;
1078
1976
  let memoryPercent;
1079
- if (thisModel && thisModel.size > 0) {
1080
- memoryUsedGB = thisModel.size / 1024 ** 3;
1977
+ const loadedModelSizeBytes = thisModel && thisModel.size > 0 ? thisModel.size : installedModelSizeBytes;
1978
+ if (loadedModelSizeBytes > 0) {
1979
+ memoryUsedGB = loadedModelSizeBytes / 1024 ** 3;
1081
1980
  memoryPercent = memoryUsedGB / memAfter.totalGB * 100;
1082
1981
  } else {
1083
1982
  memoryUsedGB = Math.max(0, memAfter.usedGB - memBefore.usedGB);
1084
1983
  memoryPercent = Math.max(0, memAfter.percent - memBefore.percent);
1085
1984
  }
1086
1985
  spinner.succeed("Performance benchmark complete");
1986
+ const firstChunkMs = firstChunkValues.length > 0 ? avg(firstChunkValues) : void 0;
1087
1987
  const ttft = ttftValues.length > 0 ? avg(ttftValues) : -1;
1988
+ const swapDeltaGB = swapBeforeResult.available && swapAfterResult.available ? +(swapAfterResult.value - swapBeforeResult.value).toFixed(2) : void 0;
1989
+ const cpuAvgLoad = cpuLoadSamples.length > 0 ? +(cpuLoadSamples.reduce((a, b) => a + b, 0) / cpuLoadSamples.length).toFixed(1) : void 0;
1990
+ const cpuPeakLoad = cpuLoadSamples.length > 0 ? +Math.max(...cpuLoadSamples).toFixed(1) : void 0;
1991
+ const benchEnvironment = {
1992
+ thermalPressureBefore: thermalBefore,
1993
+ thermalPressureAfter: thermalAfter,
1994
+ ...swapDeltaGB !== void 0 && swapDeltaGB > 0 ? { swapDeltaGB } : {},
1995
+ ...batteryPowered != null ? { batteryPowered } : {},
1996
+ ...cpuAvgLoad !== void 0 ? { cpuAvgLoad } : {},
1997
+ ...cpuPeakLoad !== void 0 ? { cpuPeakLoad } : {}
1998
+ };
1088
1999
  return {
1089
2000
  metrics: {
1090
2001
  tokensPerSecond: totalEvalDurationNs > 0 ? totalEvalCount / (totalEvalDurationNs / 1e9) : avg(tpsValues),
2002
+ ...firstChunkMs !== void 0 ? { firstChunkMs } : {},
1091
2003
  ttft: ttft >= 0 ? ttft : 3e4,
1092
2004
  // Fallback: 30s if no TTFT measured
1093
2005
  loadTime,
2006
+ loadTimeAvailable,
1094
2007
  totalTokens: totalPromptTokens + totalCompletionTokens,
1095
2008
  promptTokens: totalPromptTokens,
1096
2009
  completionTokens: totalCompletionTokens,
@@ -1101,7 +2014,8 @@ async function runPerformanceBench(model, options = {}) {
1101
2014
  tpsStdDev: tpsValues.length >= 2 ? stddev(tpsValues) : void 0,
1102
2015
  ...totalThinkingTokens > 0 ? { thinkingTokensEstimate: totalThinkingTokens } : {}
1103
2016
  },
1104
- thinkingDetected
2017
+ thinkingDetected,
2018
+ benchEnvironment
1105
2019
  };
1106
2020
  } catch (err) {
1107
2021
  if (spinner.isSpinning) {
@@ -1467,11 +2381,13 @@ var reasoning_default = [
1467
2381
 
1468
2382
  // ../src/benchmarks/reasoning.ts
1469
2383
  var questions = reasoning_default;
2384
+ var DEFAULT_REASONING_TIMEOUT_MS = 12e4;
1470
2385
  async function runReasoningBench(model, opts) {
1471
2386
  const spinner = createSpinner("Running reasoning benchmark...");
1472
2387
  spinner.start();
1473
2388
  const details = [];
1474
2389
  let correct = 0;
2390
+ const timeoutMs = opts?.timeoutMs ?? DEFAULT_REASONING_TIMEOUT_MS;
1475
2391
  try {
1476
2392
  for (let i = 0; i < questions.length; i++) {
1477
2393
  const q = questions[i];
@@ -1485,10 +2401,10 @@ Answer:`;
1485
2401
  const startTime = Date.now();
1486
2402
  try {
1487
2403
  const result = await withTimeout(
1488
- generate2(model, prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
1489
- 6e4,
2404
+ generate3(model, prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
2405
+ timeoutMs,
1490
2406
  "Reasoning question",
1491
- abortOngoingRequests2
2407
+ abortOngoingRequests3
1492
2408
  );
1493
2409
  const answer = stripThinkTags(result.response);
1494
2410
  const actual = extractChoice(answer) ?? answer.trim();
@@ -1795,11 +2711,13 @@ var math_default = [
1795
2711
 
1796
2712
  // ../src/benchmarks/math.ts
1797
2713
  var problems = math_default;
2714
+ var DEFAULT_MATH_TIMEOUT_MS = 12e4;
1798
2715
  async function runMathBench(model, opts) {
1799
2716
  const spinner = createSpinner("Running math benchmark...");
1800
2717
  spinner.start();
1801
2718
  const details = [];
1802
2719
  let correct = 0;
2720
+ const timeoutMs = opts?.timeoutMs ?? DEFAULT_MATH_TIMEOUT_MS;
1803
2721
  try {
1804
2722
  for (let i = 0; i < problems.length; i++) {
1805
2723
  const p = problems[i];
@@ -1812,10 +2730,10 @@ Answer:`;
1812
2730
  const startTime = Date.now();
1813
2731
  try {
1814
2732
  const result = await withTimeout(
1815
- generate2(model, prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
1816
- 6e4,
2733
+ generate3(model, prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
2734
+ timeoutMs,
1817
2735
  "Math problem",
1818
- abortOngoingRequests2
2736
+ abortOngoingRequests3
1819
2737
  );
1820
2738
  const actual = extractNumber(stripThinkTags(result.response));
1821
2739
  const tolerance = p.tolerance ?? 0;
@@ -5366,6 +6284,7 @@ var DIFFICULTY_WEIGHT = {
5366
6284
  var SANDBOX_TIMEOUT_MS = 5e3;
5367
6285
  var ISOLATED_WALL_TIMEOUT_MIN_MS = 8e3;
5368
6286
  var ISOLATED_WALL_TIMEOUT_MAX_MS = 6e4;
6287
+ var DEFAULT_CODING_TIMEOUT_MS = 24e4;
5369
6288
  var MAX_SANDBOX_STDOUT_BYTES = 64 * 1024;
5370
6289
  var MAX_SANDBOX_STDERR_BYTES = 64 * 1024;
5371
6290
  function runTestsInSandbox(code, task, sandboxTimeoutMs, vmModule) {
@@ -5668,6 +6587,7 @@ async function runCodingBench(model, opts) {
5668
6587
  const details = [];
5669
6588
  let totalPassed = 0;
5670
6589
  let totalTests = 0;
6590
+ const timeoutMs = opts?.timeoutMs ?? DEFAULT_CODING_TIMEOUT_MS;
5671
6591
  try {
5672
6592
  for (let i = 0; i < tasks.length; i++) {
5673
6593
  const task = tasks[i];
@@ -5693,10 +6613,10 @@ Reply with ONLY the function code, no explanation.`;
5693
6613
  const startTime = Date.now();
5694
6614
  try {
5695
6615
  const result = await withTimeout(
5696
- generate2(model, prompt, { temperature: 0, num_predict: 2048, think: opts?.think }),
5697
- 12e4,
6616
+ generate3(model, prompt, { temperature: 0, num_predict: 2048, think: opts?.think }),
6617
+ timeoutMs,
5698
6618
  "Coding task",
5699
- abortOngoingRequests2
6619
+ abortOngoingRequests3
5700
6620
  );
5701
6621
  const rawCode = extractCodeBlock(stripThinkTags(result.response), task.functionName);
5702
6622
  const code = stripTypeAnnotations(rawCode);
@@ -5893,6 +6813,7 @@ var instruction_following_default = [
5893
6813
 
5894
6814
  // ../src/benchmarks/instruction-following.ts
5895
6815
  var questions2 = instruction_following_default;
6816
+ var DEFAULT_INSTRUCTION_FOLLOWING_TIMEOUT_MS = 12e4;
5896
6817
  function escapeRegex(value) {
5897
6818
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
5898
6819
  }
@@ -6038,6 +6959,7 @@ async function runInstructionFollowingBench(model, opts) {
6038
6959
  spinner.start();
6039
6960
  const details = [];
6040
6961
  let correct = 0;
6962
+ const timeoutMs = opts?.timeoutMs ?? DEFAULT_INSTRUCTION_FOLLOWING_TIMEOUT_MS;
6041
6963
  try {
6042
6964
  for (let i = 0; i < questions2.length; i++) {
6043
6965
  const q = questions2[i];
@@ -6046,10 +6968,10 @@ async function runInstructionFollowingBench(model, opts) {
6046
6968
  const startTime = Date.now();
6047
6969
  try {
6048
6970
  const result = await withTimeout(
6049
- generate2(model, prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
6050
- 6e4,
6971
+ generate3(model, prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
6972
+ timeoutMs,
6051
6973
  "Instruction following task",
6052
- abortOngoingRequests2
6974
+ abortOngoingRequests3
6053
6975
  );
6054
6976
  const answer = stripThinkTags(result.response);
6055
6977
  const isCorrect = validateInstructionFollowingResponse(answer, q);
@@ -6197,6 +7119,7 @@ var structured_output_default = [
6197
7119
 
6198
7120
  // ../src/benchmarks/structured-output.ts
6199
7121
  var questions3 = structured_output_default;
7122
+ var DEFAULT_STRUCTURED_OUTPUT_TIMEOUT_MS = 12e4;
6200
7123
  function tryParseJson(text) {
6201
7124
  try {
6202
7125
  return { ok: true, value: JSON.parse(text) };
@@ -6225,9 +7148,9 @@ function checkType(value, expectedType) {
6225
7148
  if (expectedType === "object") return typeof value === "object" && value !== null && !Array.isArray(value);
6226
7149
  return typeof value === expectedType;
6227
7150
  }
6228
- function getNestedValue(obj, path) {
7151
+ function getNestedValue(obj, path2) {
6229
7152
  let current = obj;
6230
- for (const key of path) {
7153
+ for (const key of path2) {
6231
7154
  if (typeof current !== "object" || current === null) return void 0;
6232
7155
  current = current[key];
6233
7156
  }
@@ -6303,8 +7226,8 @@ function validateStructuredOutputResponse(response, q) {
6303
7226
  const parsed = safeParse(response);
6304
7227
  if (!parsed) return false;
6305
7228
  const paths = ["path1", "path2", "path3", "path4"].filter((k) => k in p).map((k) => p[k]);
6306
- return paths.every((path) => {
6307
- const val = getNestedValue(parsed, path);
7229
+ return paths.every((path2) => {
7230
+ const val = getNestedValue(parsed, path2);
6308
7231
  return val !== void 0 && val !== null;
6309
7232
  });
6310
7233
  }
@@ -6423,6 +7346,7 @@ async function runStructuredOutputBench(model, opts) {
6423
7346
  spinner.start();
6424
7347
  const details = [];
6425
7348
  let correct = 0;
7349
+ const timeoutMs = opts?.timeoutMs ?? DEFAULT_STRUCTURED_OUTPUT_TIMEOUT_MS;
6426
7350
  try {
6427
7351
  for (let i = 0; i < questions3.length; i++) {
6428
7352
  const q = questions3[i];
@@ -6430,10 +7354,10 @@ async function runStructuredOutputBench(model, opts) {
6430
7354
  const startTime = Date.now();
6431
7355
  try {
6432
7356
  const result = await withTimeout(
6433
- generate2(model, q.prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
6434
- 6e4,
7357
+ generate3(model, q.prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
7358
+ timeoutMs,
6435
7359
  "Structured output task",
6436
- abortOngoingRequests2
7360
+ abortOngoingRequests3
6437
7361
  );
6438
7362
  const answer = stripThinkTags(result.response);
6439
7363
  const isCorrect = validateStructuredOutputResponse(answer, q);
@@ -6616,6 +7540,7 @@ var multilingual_default = [
6616
7540
 
6617
7541
  // ../src/benchmarks/multilingual.ts
6618
7542
  var questions4 = multilingual_default;
7543
+ var DEFAULT_MULTILINGUAL_TIMEOUT_MS = 12e4;
6619
7544
  var NEGATION_PATTERNS = [
6620
7545
  /\b(?:not|never|no|non|pas|nicht|kein|keine|ningun|ninguna|nunca|jamas)\b/u,
6621
7546
  /\bn'?est\s+pas\b/u,
@@ -6680,6 +7605,7 @@ async function runMultilingualBench(model, opts) {
6680
7605
  spinner.start();
6681
7606
  const details = [];
6682
7607
  let correct = 0;
7608
+ const timeoutMs = opts?.timeoutMs ?? DEFAULT_MULTILINGUAL_TIMEOUT_MS;
6683
7609
  try {
6684
7610
  for (let i = 0; i < questions4.length; i++) {
6685
7611
  const q = questions4[i];
@@ -6687,10 +7613,10 @@ async function runMultilingualBench(model, opts) {
6687
7613
  const startTime = Date.now();
6688
7614
  try {
6689
7615
  const result = await withTimeout(
6690
- generate2(model, q.prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
6691
- 6e4,
7616
+ generate3(model, q.prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
7617
+ timeoutMs,
6692
7618
  "Multilingual task",
6693
- abortOngoingRequests2
7619
+ abortOngoingRequests3
6694
7620
  );
6695
7621
  const answer = stripThinkTags(result.response);
6696
7622
  const isCorrect = validateMultilingualResponse(answer, q);
@@ -6793,24 +7719,24 @@ function deriveHardwareFitTuning(hardware) {
6793
7719
  };
6794
7720
  }
6795
7721
  function scoreSpeed(tps, tuning) {
6796
- if (tps >= tuning.speed.excellent) return 40;
7722
+ if (tps >= tuning.speed.excellent) return 50;
6797
7723
  if (tps >= tuning.speed.good) {
6798
- return lerp(tps, tuning.speed.good, tuning.speed.excellent, 25, 40);
7724
+ return lerp(tps, tuning.speed.good, tuning.speed.excellent, 30, 50);
6799
7725
  }
6800
7726
  if (tps >= tuning.speed.marginal) {
6801
- return lerp(tps, tuning.speed.marginal, tuning.speed.good, 10, 25);
7727
+ return lerp(tps, tuning.speed.marginal, tuning.speed.good, 12, 30);
6802
7728
  }
6803
- return lerp(tps, 0, tuning.speed.marginal, 0, 10);
7729
+ return lerp(tps, 0, tuning.speed.marginal, 0, 12);
6804
7730
  }
6805
7731
  function scoreTTFT(ttft, tuning) {
6806
- if (ttft <= tuning.ttft.excellentMs) return 30;
7732
+ if (ttft <= tuning.ttft.excellentMs) return 20;
6807
7733
  if (ttft <= tuning.ttft.goodMs) {
6808
- return lerp(ttft, tuning.ttft.excellentMs, tuning.ttft.goodMs, 30, 20);
7734
+ return lerp(ttft, tuning.ttft.excellentMs, tuning.ttft.goodMs, 20, 13);
6809
7735
  }
6810
7736
  if (ttft <= tuning.ttft.marginalMs) {
6811
- return lerp(ttft, tuning.ttft.goodMs, tuning.ttft.marginalMs, 20, 10);
7737
+ return lerp(ttft, tuning.ttft.goodMs, tuning.ttft.marginalMs, 13, 6);
6812
7738
  }
6813
- return lerp(ttft, tuning.ttft.marginalMs, tuning.ttft.hardMaxMs, 10, 0);
7739
+ return lerp(ttft, tuning.ttft.marginalMs, tuning.ttft.hardMaxMs, 6, 0);
6814
7740
  }
6815
7741
  function scoreMemory(memPercent) {
6816
7742
  if (memPercent <= 30) return 30;
@@ -6918,12 +7844,12 @@ function normalizeRawScore(value) {
6918
7844
  if (!Number.isFinite(value)) return 0;
6919
7845
  return clamp(value, 0, 100);
6920
7846
  }
6921
- function computeFitness(perf, quality, hardware) {
7847
+ function computeFitness(perf, quality, hardware, benchEnv) {
6922
7848
  const tuning = deriveHardwareFitTuning(hardware);
6923
7849
  const performanceScore = computePerformanceScore(perf, hardware);
6924
7850
  const qualityScore = quality ? computeQualityScore(quality) : null;
6925
7851
  const hardwareFitScore = Math.round(performanceScore.total);
6926
- const globalScore = qualityScore ? clamp(Math.round(0.4 * hardwareFitScore + 0.6 * qualityScore.total), 0, 100) : null;
7852
+ const globalScore = qualityScore ? clamp(Math.round(0.3 * hardwareFitScore + 0.7 * qualityScore.total), 0, 100) : null;
6927
7853
  const categoryLabels = quality ? [
6928
7854
  { category: "Reasoning", key: "reasoning", result: quality.reasoning },
6929
7855
  { category: "Coding", key: "coding", result: quality.coding },
@@ -7015,6 +7941,26 @@ function computeFitness(perf, quality, hardware) {
7015
7941
  `CPU appears throttled (${hardware.cpuCurrentSpeedGHz.toFixed(1)} GHz current vs ${hardware.cpuFreqGHz.toFixed(1)} GHz nominal, ${ratio}%).`
7016
7942
  );
7017
7943
  }
7944
+ if (benchEnv?.thermalPressureAfter === "heavy" || benchEnv?.thermalPressureAfter === "critical") {
7945
+ warnings.push(
7946
+ `Thermal throttling detected after benchmark (${benchEnv.thermalPressureAfter}). Results may be degraded \u2014 consider cooling or shorter runs.`
7947
+ );
7948
+ }
7949
+ if (benchEnv?.swapDeltaGB != null && benchEnv.swapDeltaGB > 0.5) {
7950
+ warnings.push(
7951
+ `Significant swap activity during benchmark (+${benchEnv.swapDeltaGB.toFixed(1)} GB). Model may exceed available RAM \u2014 results are severely degraded.`
7952
+ );
7953
+ }
7954
+ if (benchEnv?.batteryPowered) {
7955
+ warnings.push(
7956
+ `Running on battery power \u2014 performance may be reduced.`
7957
+ );
7958
+ }
7959
+ if (benchEnv?.cpuAvgLoad != null && benchEnv.cpuAvgLoad > 90) {
7960
+ warnings.push(
7961
+ `High CPU load during inference (avg ${benchEnv.cpuAvgLoad.toFixed(0)}%). System may feel unresponsive \u2014 GPU-accelerated runtimes (MLX) can reduce CPU pressure.`
7962
+ );
7963
+ }
7018
7964
  return {
7019
7965
  verdict,
7020
7966
  globalScore,
@@ -7101,7 +8047,7 @@ function printHardwareTable(hw) {
7101
8047
  }
7102
8048
  console.log(table.toString());
7103
8049
  }
7104
- function printPerformanceTable(perf) {
8050
+ function printPerformanceTable(perf, benchEnvironment) {
7105
8051
  const table = new Table({
7106
8052
  head: [chalk3.bold("Metric"), chalk3.bold("Value")],
7107
8053
  style: { head: [], border: [] }
@@ -7111,8 +8057,15 @@ function printPerformanceTable(perf) {
7111
8057
  const memColor = perf.memoryPercent < 50 ? chalk3.green : perf.memoryPercent < 80 ? chalk3.yellow : chalk3.red;
7112
8058
  table.push(
7113
8059
  ["Tokens/sec", tpsColor(`${perf.tokensPerSecond.toFixed(1)} tok/s`)],
8060
+ [
8061
+ "First Chunk Latency",
8062
+ perf.firstChunkMs !== void 0 ? formatDuration(perf.firstChunkMs) : chalk3.dim("N/A (stream metric unavailable)")
8063
+ ],
7114
8064
  ["Time to First Token", ttftColor(formatDuration(perf.ttft))],
7115
- ["Model Load Time", formatDuration(perf.loadTime)],
8065
+ [
8066
+ "Model Load Time",
8067
+ perf.loadTimeAvailable === false ? chalk3.dim("N/A (runtime metric unavailable)") : formatDuration(perf.loadTime)
8068
+ ],
7116
8069
  ["Total Tokens", String(perf.totalTokens)],
7117
8070
  ["Prompt Tokens", String(perf.promptTokens)],
7118
8071
  ["Completion Tokens", String(perf.completionTokens)],
@@ -7133,6 +8086,13 @@ function printPerformanceTable(perf) {
7133
8086
  chalk3.magenta(`~${perf.thinkingTokensEstimate} tokens`)
7134
8087
  ]);
7135
8088
  }
8089
+ if (benchEnvironment?.cpuAvgLoad != null && benchEnvironment.cpuAvgLoad >= 0) {
8090
+ const cpuColor = benchEnvironment.cpuAvgLoad < 50 ? chalk3.green : benchEnvironment.cpuAvgLoad < 80 ? chalk3.yellow : chalk3.red;
8091
+ table.push([
8092
+ "CPU Load During Bench",
8093
+ cpuColor(`avg ${benchEnvironment.cpuAvgLoad.toFixed(0)}%` + (benchEnvironment.cpuPeakLoad != null ? ` (peak ${benchEnvironment.cpuPeakLoad.toFixed(0)}%)` : ""))
8094
+ ]);
8095
+ }
7136
8096
  console.log(table.toString());
7137
8097
  }
7138
8098
  function printQualityTable(quality, timePenalties) {
@@ -7179,7 +8139,7 @@ function printSummaryTable(results) {
7179
8139
  const compact = termWidth < 100;
7180
8140
  console.log(
7181
8141
  chalk3.dim(
7182
- "Global = 40% Hardware Fit + 60% Quality. Hardware Fit = host compatibility. Quality = model capability."
8142
+ "Global = 30% Hardware Fit + 70% Quality. Hardware Fit = host compatibility. Quality = model capability."
7183
8143
  )
7184
8144
  );
7185
8145
  const head = [
@@ -7362,7 +8322,7 @@ function printVerdict(model, fitness) {
7362
8322
  );
7363
8323
  sectionText(`Active profile: ${fitness.tuning.profile}`, fitBorder, chalk4.white);
7364
8324
  sectionText(
7365
- "Formula: Speed (/40) + TTFT (/30) + Memory (/30) = Hardware Fit (/100).",
8325
+ "Formula: Speed (/50) + TTFT (/20) + Memory (/30) = Hardware Fit (/100).",
7366
8326
  fitBorder
7367
8327
  );
7368
8328
  sectionText(
@@ -7380,7 +8340,7 @@ function printVerdict(model, fitness) {
7380
8340
  sectionText("", fitBorder);
7381
8341
  scoreRow("Performance", fitness.performanceScore.total, fitBorder);
7382
8342
  sectionText(
7383
- ` Breakdown: Speed ${fitness.performanceScore.speed}/40 TTFT ${fitness.performanceScore.ttft}/30 Memory ${fitness.performanceScore.memory}/30`,
8343
+ ` Breakdown: Speed ${fitness.performanceScore.speed}/50 TTFT ${fitness.performanceScore.ttft}/20 Memory ${fitness.performanceScore.memory}/30`,
7384
8344
  fitBorder
7385
8345
  );
7386
8346
  scoreRow("Hardware Fit", fitness.hardwareFitScore, fitBorder);
@@ -7449,7 +8409,7 @@ function printVerdict(model, fitness) {
7449
8409
  sectionStart("C) Global Score", globalBorder);
7450
8410
  if (fitness.globalScore !== null) {
7451
8411
  sectionText(
7452
- "Formula: Global = 40% Hardware Fit + 60% Quality.",
8412
+ "Formula: Global = 30% Hardware Fit + 70% Quality.",
7453
8413
  globalBorder,
7454
8414
  chalk4.white
7455
8415
  );
@@ -7519,6 +8479,12 @@ var DEFAULT_CONFIG = {
7519
8479
  async function ensureDirs() {
7520
8480
  await mkdir(RESULTS_DIR, { recursive: true });
7521
8481
  }
8482
+ function parseRuntimeBackend(value) {
8483
+ if (value === "ollama" || value === "lm-studio") {
8484
+ return value;
8485
+ }
8486
+ return void 0;
8487
+ }
7522
8488
  function resultFilename(result) {
7523
8489
  const ts = result.timestamp.replace(/[:.]/g, "-").replace("T", "_").replace(/[^0-9_-]/g, "");
7524
8490
  const model = result.model.replace(/[^a-zA-Z0-9._-]/g, "_");
@@ -7540,7 +8506,15 @@ async function loadConfig() {
7540
8506
  const telemetry = typeof parsed.telemetry === "boolean" ? parsed.telemetry : void 0;
7541
8507
  const submitterNickname = typeof parsed.submitterNickname === "string" && isValidNickname(parsed.submitterNickname) ? normalizeNickname(parsed.submitterNickname) : void 0;
7542
8508
  const submitterEmail = typeof parsed.submitterEmail === "string" && isValidEmail(parsed.submitterEmail) ? normalizeEmail(parsed.submitterEmail) : void 0;
7543
- return { ...DEFAULT_CONFIG, autoShare, telemetry, submitterNickname, submitterEmail };
8509
+ const runtimeBackend = parseRuntimeBackend(parsed.runtimeBackend);
8510
+ return {
8511
+ ...DEFAULT_CONFIG,
8512
+ autoShare,
8513
+ telemetry,
8514
+ submitterNickname,
8515
+ submitterEmail,
8516
+ runtimeBackend
8517
+ };
7544
8518
  } catch {
7545
8519
  return { ...DEFAULT_CONFIG };
7546
8520
  }
@@ -7552,24 +8526,40 @@ async function saveConfig(config) {
7552
8526
 
7553
8527
  // ../src/core/uploader.ts
7554
8528
  import { createClient } from "@supabase/supabase-js";
8529
+ var SUPABASE_URL_DEFAULT = "https://phvvzbgasxobjzjnkewf.supabase.co";
8530
+ var SUPABASE_ANON_KEY_DEFAULT = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InBodnZ6Ymdhc3hvYmp6am5rZXdmIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NzIyOTMxNTksImV4cCI6MjA4Nzg2OTE1OX0.-Wn0V8f-Uv_x_krNUnjmFRcqfAOPQEVc_Qx1kZ2ppgk";
7555
8531
  var SUPABASE_URL_PLACEHOLDER = "https://YOUR_SUPABASE_PROJECT.supabase.co";
8532
+ var SUPABASE_URL_PLACEHOLDER_ALT = "https://your-project.supabase.co";
7556
8533
  var SUPABASE_ANON_KEY_PLACEHOLDER = "YOUR_SUPABASE_ANON_KEY";
8534
+ var SUPABASE_ANON_KEY_PLACEHOLDER_ALT = "your-supabase-anon-key";
8535
+ var PUBLIC_RESULT_BASE_URL_DEFAULT = "https://metrillm.dev";
7557
8536
  var PUBLIC_RESULT_BASE_URL_PLACEHOLDER = "https://YOUR_DASHBOARD_DOMAIN";
8537
+ var PUBLIC_RESULT_BASE_URL_PLACEHOLDER_ALT = "https://your-dashboard-domain";
7558
8538
  function resolveUploaderConfig() {
8539
+ const configuredPublicBaseUrl = process.env.METRILLM_PUBLIC_RESULT_BASE_URL?.trim();
8540
+ const publicResultBaseUrl = !configuredPublicBaseUrl || hasPlaceholder(configuredPublicBaseUrl) ? PUBLIC_RESULT_BASE_URL_DEFAULT : configuredPublicBaseUrl;
7559
8541
  return {
7560
- supabaseUrl: process.env.METRILLM_SUPABASE_URL ?? SUPABASE_URL_PLACEHOLDER,
7561
- supabaseAnonKey: process.env.METRILLM_SUPABASE_ANON_KEY ?? SUPABASE_ANON_KEY_PLACEHOLDER,
7562
- publicResultBaseUrl: process.env.METRILLM_PUBLIC_RESULT_BASE_URL ?? PUBLIC_RESULT_BASE_URL_PLACEHOLDER
8542
+ supabaseUrl: !process.env.METRILLM_SUPABASE_URL || hasPlaceholder(process.env.METRILLM_SUPABASE_URL) ? SUPABASE_URL_DEFAULT : process.env.METRILLM_SUPABASE_URL,
8543
+ supabaseAnonKey: !process.env.METRILLM_SUPABASE_ANON_KEY || hasPlaceholder(process.env.METRILLM_SUPABASE_ANON_KEY) ? SUPABASE_ANON_KEY_DEFAULT : process.env.METRILLM_SUPABASE_ANON_KEY,
8544
+ publicResultBaseUrl
7563
8545
  };
7564
8546
  }
7565
8547
  function hasPlaceholder(value) {
7566
- return value.includes("YOUR_");
8548
+ const normalized = value.trim();
8549
+ if (normalized.length === 0) return true;
8550
+ return [
8551
+ SUPABASE_URL_PLACEHOLDER,
8552
+ SUPABASE_URL_PLACEHOLDER_ALT,
8553
+ SUPABASE_ANON_KEY_PLACEHOLDER,
8554
+ SUPABASE_ANON_KEY_PLACEHOLDER_ALT,
8555
+ PUBLIC_RESULT_BASE_URL_PLACEHOLDER,
8556
+ PUBLIC_RESULT_BASE_URL_PLACEHOLDER_ALT
8557
+ ].some((placeholder) => placeholder.toLowerCase() === normalized.toLowerCase());
7567
8558
  }
7568
8559
  function assertUploaderConfig(config) {
7569
8560
  const missing = [];
7570
- if (hasPlaceholder(config.supabaseUrl)) missing.push("METRILLM_SUPABASE_URL");
7571
- if (hasPlaceholder(config.supabaseAnonKey)) missing.push("METRILLM_SUPABASE_ANON_KEY");
7572
- if (hasPlaceholder(config.publicResultBaseUrl)) missing.push("METRILLM_PUBLIC_RESULT_BASE_URL");
8561
+ if (!config.supabaseUrl.trim()) missing.push("METRILLM_SUPABASE_URL");
8562
+ if (!config.supabaseAnonKey.trim()) missing.push("METRILLM_SUPABASE_ANON_KEY");
7573
8563
  if (missing.length > 0) {
7574
8564
  throw new Error(
7575
8565
  `Upload is not configured. Set these variables first: ${missing.join(", ")}`
@@ -8021,6 +9011,13 @@ async function promptThinkingMode() {
8021
9011
  var BENCHMARK_SPEC_VERSION = "0.2.0";
8022
9012
  var PROMPT_PACK_VERSION = "0.1.0";
8023
9013
  async function benchCommand(options) {
9014
+ if (options.backend !== void 0) {
9015
+ setRuntimeByName(options.backend);
9016
+ }
9017
+ const runtimeName = getRuntimeName();
9018
+ const runtimeDisplayName = getRuntimeDisplayName(runtimeName);
9019
+ const runtimeModelHint = getRuntimeModelInstallHint(runtimeName);
9020
+ const runtimeSetupHints = getRuntimeSetupHints(runtimeName);
8024
9021
  const shouldSetExitCode = options.setExitCode !== false;
8025
9022
  const silent = options.json === true;
8026
9023
  const shouldUnloadAfterModel = options.unloadAfterBench ?? (!options.model || options.ciNoMenu === true);
@@ -8053,7 +9050,7 @@ async function benchCommand(options) {
8053
9050
  runtimeVersion = await getRuntimeVersion();
8054
9051
  } catch (err) {
8055
9052
  if (!silent) {
8056
- warnMsg("Could not detect Ollama version (continuing without it).");
9053
+ warnMsg(`Could not detect ${runtimeDisplayName} version (continuing without it).`);
8057
9054
  if (err instanceof Error) warnMsg(err.message);
8058
9055
  }
8059
9056
  }
@@ -8062,7 +9059,7 @@ async function benchCommand(options) {
8062
9059
  if (options.model) {
8063
9060
  modelNames = [options.model];
8064
9061
  try {
8065
- allModels = await listModels2();
9062
+ allModels = await listModels3();
8066
9063
  } catch {
8067
9064
  }
8068
9065
  } else {
@@ -8070,22 +9067,23 @@ async function benchCommand(options) {
8070
9067
  const spinnerModels = createSpinner("Fetching model list...");
8071
9068
  if (!silent) spinnerModels.start();
8072
9069
  try {
8073
- allModels = await listModels2();
9070
+ allModels = await listModels3();
8074
9071
  modelNames = allModels.map((m) => m.name);
8075
9072
  if (!silent) spinnerModels.succeed(`Found ${allModels.length} model(s)`);
8076
9073
  } catch (err) {
8077
9074
  if (!silent) {
8078
- spinnerModels.fail("Cannot connect to Ollama");
8079
- errorMsg("Make sure Ollama is installed and running.");
8080
- errorMsg(" \u2022 Start it with: ollama serve");
8081
- errorMsg(" \u2022 Install it at: https://ollama.com");
9075
+ spinnerModels.fail(`Cannot connect to ${runtimeDisplayName}`);
9076
+ errorMsg(`Make sure ${runtimeDisplayName} is installed and running.`);
9077
+ for (const hint of runtimeSetupHints) {
9078
+ errorMsg(` \u2022 ${hint}`);
9079
+ }
8082
9080
  if (err instanceof Error) errorMsg(err.message);
8083
9081
  }
8084
9082
  if (shouldSetExitCode) process.exitCode = 1;
8085
9083
  return { results: [], failedModels: [] };
8086
9084
  }
8087
9085
  if (modelNames.length === 0) {
8088
- if (!silent) errorMsg("No models found. Pull one with: ollama pull <model>");
9086
+ if (!silent) errorMsg(`No models found. ${runtimeModelHint}`);
8089
9087
  if (shouldSetExitCode) process.exitCode = 1;
8090
9088
  return { results: [], failedModels: [] };
8091
9089
  }
@@ -8097,6 +9095,8 @@ async function benchCommand(options) {
8097
9095
  thinkEnabled = options.thinking;
8098
9096
  } else if (!silent && !options.ciNoMenu) {
8099
9097
  thinkEnabled = await promptThinkingMode();
9098
+ } else {
9099
+ thinkEnabled = false;
8100
9100
  }
8101
9101
  if (!silent && thinkEnabled) {
8102
9102
  infoMsg("Thinking mode enabled \u2014 models that support it will use extended reasoning.");
@@ -8129,29 +9129,38 @@ ${tl}${h.repeat(innerWidth)}${tr}`));
8129
9129
  promptTimeoutMs: options.perfPromptTimeoutMs,
8130
9130
  minSuccessfulPrompts: options.perfMinSuccessfulPrompts,
8131
9131
  failOnPromptError: options.perfStrict,
8132
- think: thinkEnabled
9132
+ think: thinkEnabled,
9133
+ streamStallTimeoutMs: options.lmStudioStreamStallTimeoutMs
8133
9134
  });
8134
9135
  const perf = perfResult.metrics;
8135
- const thinkingDetected = perfResult.thinkingDetected;
8136
- if (!silent) printPerformanceTable(perf);
9136
+ const benchEnvironment = perfResult.benchEnvironment;
9137
+ if (!silent) printPerformanceTable(perf, benchEnvironment);
8137
9138
  let quality = null;
8138
9139
  if (!options.perfOnly) {
8139
- const qualityOpts = thinkEnabled !== void 0 ? { think: thinkEnabled } : void 0;
9140
+ const qualityTimeoutMs = options.qualityTimeoutMs;
9141
+ const codingTimeoutMs = options.codingTimeoutMs ?? options.qualityTimeoutMs;
9142
+ const commonQualityOpts = {
9143
+ think: thinkEnabled,
9144
+ ...qualityTimeoutMs !== void 0 ? { timeoutMs: qualityTimeoutMs } : {}
9145
+ };
8140
9146
  if (!silent) stepHeader("Quality Benchmark \u2014 Reasoning");
8141
- const reasoning = await runReasoningBench(modelName, qualityOpts);
9147
+ const reasoning = await runReasoningBench(modelName, commonQualityOpts);
8142
9148
  if (!silent) stepHeader("Quality Benchmark \u2014 Math");
8143
- const math = await runMathBench(modelName, qualityOpts);
9149
+ const math = await runMathBench(modelName, commonQualityOpts);
8144
9150
  if (!silent) stepHeader("Quality Benchmark \u2014 Coding");
8145
- const coding = await runCodingBench(modelName, qualityOpts);
9151
+ const coding = await runCodingBench(modelName, {
9152
+ think: thinkEnabled,
9153
+ ...codingTimeoutMs !== void 0 ? { timeoutMs: codingTimeoutMs } : {}
9154
+ });
8146
9155
  if (!silent) stepHeader("Quality Benchmark \u2014 Instruction Following");
8147
- const instructionFollowing = await runInstructionFollowingBench(modelName, qualityOpts);
9156
+ const instructionFollowing = await runInstructionFollowingBench(modelName, commonQualityOpts);
8148
9157
  if (!silent) stepHeader("Quality Benchmark \u2014 Structured Output");
8149
- const structuredOutput = await runStructuredOutputBench(modelName, qualityOpts);
9158
+ const structuredOutput = await runStructuredOutputBench(modelName, commonQualityOpts);
8150
9159
  if (!silent) stepHeader("Quality Benchmark \u2014 Multilingual");
8151
- const multilingual = await runMultilingualBench(modelName, qualityOpts);
9160
+ const multilingual = await runMultilingualBench(modelName, commonQualityOpts);
8152
9161
  quality = { reasoning, math, coding, instructionFollowing, structuredOutput, multilingual };
8153
9162
  }
8154
- const fitness = computeFitness(perf, quality, hardware);
9163
+ const fitness = computeFitness(perf, quality, hardware, benchEnvironment);
8155
9164
  if (!silent) {
8156
9165
  if (quality) {
8157
9166
  printQualityTable(quality, fitness.qualityScore?.timePenalties);
@@ -8163,8 +9172,9 @@ ${tl}${h.repeat(innerWidth)}${tr}`));
8163
9172
  parameterSize: matchedModel.parameterSize,
8164
9173
  quantization: matchedModel.quantization,
8165
9174
  family: matchedModel.family,
8166
- ...thinkingDetected ? { thinkingDetected } : {}
8167
- } : thinkingDetected ? { thinkingDetected } : void 0;
9175
+ // Persist the configured benchmark mode (not model auto-detection).
9176
+ thinkingDetected: thinkEnabled
9177
+ } : { thinkingDetected: thinkEnabled };
8168
9178
  const partialResult = {
8169
9179
  model: modelName,
8170
9180
  modelInfo,
@@ -8172,13 +9182,14 @@ ${tl}${h.repeat(innerWidth)}${tr}`));
8172
9182
  performance: perf,
8173
9183
  quality,
8174
9184
  fitness,
9185
+ benchEnvironment,
8175
9186
  timestamp: (/* @__PURE__ */ new Date()).toISOString(),
8176
9187
  metadata: {
8177
9188
  benchmarkSpecVersion: BENCHMARK_SPEC_VERSION,
8178
9189
  promptPackVersion: PROMPT_PACK_VERSION,
8179
9190
  runtimeVersion,
8180
9191
  runtimeBackend: getRuntimeName(),
8181
- modelFormat: getRuntimeModelFormat()
9192
+ modelFormat: matchedModel?.modelFormat ?? getRuntimeModelFormat()
8182
9193
  }
8183
9194
  };
8184
9195
  const rawLogHash = createHash3("sha256").update(JSON.stringify(partialResult)).digest("hex");
@@ -8233,7 +9244,7 @@ ${tl}${h.repeat(innerWidth)}${tr}`));
8233
9244
  } finally {
8234
9245
  if (shouldUnloadAfterModel) {
8235
9246
  try {
8236
- await unloadModel2(modelName);
9247
+ await unloadModel3(modelName);
8237
9248
  } catch (err) {
8238
9249
  if (!silent) {
8239
9250
  warnMsg(`Could not unload model ${modelName} after benchmark.`);
@@ -8330,8 +9341,8 @@ ${tl}${h.repeat(innerWidth)}${tr}`));
8330
9341
  }
8331
9342
 
8332
9343
  // src/tools.ts
8333
- var SUPPORTED_RUNTIMES = ["ollama"];
8334
- var runtimeSchema = z.enum(SUPPORTED_RUNTIMES).optional().default("ollama").describe("Inference runtime to use. Currently only 'ollama' is supported.");
9344
+ var SUPPORTED_RUNTIMES = ["ollama", "lm-studio"];
9345
+ var runtimeSchema = z.enum(SUPPORTED_RUNTIMES).optional().default("ollama").describe("Inference runtime to use (ollama | lm-studio).");
8335
9346
  var listModelsSchema = z.object({
8336
9347
  runtime: runtimeSchema
8337
9348
  });
@@ -8365,18 +9376,24 @@ var toolDefinitions = [
8365
9376
  },
8366
9377
  {
8367
9378
  name: "share_result",
8368
- description: "Upload a benchmark result to the public MetriLLM leaderboard. Requires METRILLM_SUPABASE_URL and METRILLM_SUPABASE_ANON_KEY environment variables. The resultFile must be an absolute path to a JSON file in ~/.metrillm/results/.",
9379
+ description: "Upload a benchmark result to the public MetriLLM leaderboard. Uses official upload defaults; METRILLM_* environment variables can override for self-hosted deployments. The resultFile must be an absolute path to a JSON file in ~/.metrillm/results/.",
8369
9380
  inputSchema: shareResultSchema
8370
9381
  }
8371
9382
  ];
8372
9383
  function assertRuntime(runtime) {
8373
- if (runtime !== "ollama") {
9384
+ if (!SUPPORTED_RUNTIMES.includes(runtime)) {
8374
9385
  throw new Error(
8375
9386
  `Runtime "${runtime}" is not yet supported. Currently supported: ${SUPPORTED_RUNTIMES.join(", ")}. New runtimes will be added as they become available in the MetriLLM CLI.`
8376
9387
  );
8377
9388
  }
8378
9389
  }
8379
9390
  var RESULTS_DIR2 = join2(homedir2(), ".metrillm", "results");
9391
+ function normalizeResultRuntimeBackend(result) {
9392
+ const runtimeBackend = result.metadata?.runtimeBackend?.trim().toLowerCase();
9393
+ if (runtimeBackend === "lm-studio") return "lm-studio";
9394
+ if (runtimeBackend === "ollama") return "ollama";
9395
+ return "ollama";
9396
+ }
8380
9397
  var benchLock = Promise.resolve();
8381
9398
  function withBenchLock(fn) {
8382
9399
  const prev = benchLock;
@@ -8388,29 +9405,37 @@ function withBenchLock(fn) {
8388
9405
  }
8389
9406
  async function handleListModels(args) {
8390
9407
  assertRuntime(args.runtime);
8391
- const models = await listModels2();
8392
- if (models.length === 0) {
9408
+ return withBenchLock(async () => {
9409
+ setRuntimeByName(args.runtime);
9410
+ const runtimeDisplayName = getRuntimeDisplayName(args.runtime);
9411
+ const runtimeModelHint = getRuntimeModelInstallHint(args.runtime);
9412
+ const models = await listModels3();
9413
+ if (models.length === 0) {
9414
+ return JSON.stringify({
9415
+ models: [],
9416
+ message: `No models found on ${runtimeDisplayName}. ${runtimeModelHint}`
9417
+ });
9418
+ }
8393
9419
  return JSON.stringify({
8394
- models: [],
8395
- message: "No models found. Pull one with: ollama pull <model>"
9420
+ models: models.map((m) => ({
9421
+ name: m.name,
9422
+ size: m.size,
9423
+ parameterSize: m.parameterSize ?? null,
9424
+ quantization: m.quantization ?? null,
9425
+ family: m.family ?? null
9426
+ })),
9427
+ count: models.length
8396
9428
  });
8397
- }
8398
- return JSON.stringify({
8399
- models: models.map((m) => ({
8400
- name: m.name,
8401
- size: m.size,
8402
- parameterSize: m.parameterSize ?? null,
8403
- quantization: m.quantization ?? null,
8404
- family: m.family ?? null
8405
- })),
8406
- count: models.length
8407
9429
  });
8408
9430
  }
8409
9431
  async function handleRunBenchmark(args) {
8410
9432
  assertRuntime(args.runtime);
9433
+ const runtimeDisplayName = getRuntimeDisplayName(args.runtime);
8411
9434
  return withBenchLock(async () => {
9435
+ setRuntimeByName(args.runtime);
8412
9436
  const outcome = await benchCommand({
8413
9437
  model: args.model,
9438
+ backend: args.runtime,
8414
9439
  perfOnly: args.perfOnly,
8415
9440
  json: true,
8416
9441
  // suppress all UI output
@@ -8428,7 +9453,7 @@ async function handleRunBenchmark(args) {
8428
9453
  if (outcome.results.length === 0) {
8429
9454
  return JSON.stringify({
8430
9455
  success: false,
8431
- error: "No results produced. Is Ollama running? Is the model pulled?"
9456
+ error: `No results produced. Is ${runtimeDisplayName} running?`
8432
9457
  });
8433
9458
  }
8434
9459
  const result = outcome.results[0];
@@ -8467,6 +9492,10 @@ async function handleGetResults(args) {
8467
9492
  try {
8468
9493
  const content = await readFile3(join2(RESULTS_DIR2, file), "utf8");
8469
9494
  const result = JSON.parse(content);
9495
+ const resultRuntime = normalizeResultRuntimeBackend(result);
9496
+ if (resultRuntime !== args.runtime) {
9497
+ continue;
9498
+ }
8470
9499
  if (args.model && !result.model.toLowerCase().includes(args.model.toLowerCase())) {
8471
9500
  continue;
8472
9501
  }