metrillm-mcp 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -7
- package/dist/index.js +1153 -124
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -100,8 +100,13 @@ function toBenchmarkFailureLabel(err) {
|
|
|
100
100
|
return compact.length > maxLen ? `ERROR: ${compact.slice(0, maxLen - 1)}\u2026` : `ERROR: ${compact}`;
|
|
101
101
|
}
|
|
102
102
|
function stripThinkTags(text) {
|
|
103
|
-
const
|
|
104
|
-
|
|
103
|
+
const withoutThinking = text.replace(/<think(?:ing)?[\s>][\s\S]*?<\/think(?:ing)?>/gi, "");
|
|
104
|
+
const withoutTrailingWhitespace = withoutThinking.trimEnd();
|
|
105
|
+
const withoutTrailingControlTokens = withoutTrailingWhitespace.replace(
|
|
106
|
+
/(?:\s*(?:<\|(?:im_end|eot_id|end_of_text|eom_id|end)\|>|<\/s>))+$/gi,
|
|
107
|
+
""
|
|
108
|
+
);
|
|
109
|
+
return withoutTrailingControlTokens.trim();
|
|
105
110
|
}
|
|
106
111
|
function hasThinkingContent(response, thinkingField) {
|
|
107
112
|
if (thinkingField && thinkingField.trim().length > 0) return true;
|
|
@@ -182,18 +187,20 @@ function stripTypeAnnotations(code) {
|
|
|
182
187
|
return code;
|
|
183
188
|
} catch {
|
|
184
189
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
190
|
+
if (process.env.METRILLM_ENABLE_EXPERIMENTAL_TS_STRIP === "1") {
|
|
191
|
+
try {
|
|
192
|
+
const mod = __require("module");
|
|
193
|
+
if (typeof mod.stripTypeScriptTypes === "function") {
|
|
194
|
+
const stripped = mod.stripTypeScriptTypes(code);
|
|
195
|
+
try {
|
|
196
|
+
new vm.Script(`${stripped}
|
|
191
197
|
;`);
|
|
192
|
-
|
|
193
|
-
|
|
198
|
+
return stripped;
|
|
199
|
+
} catch {
|
|
200
|
+
}
|
|
194
201
|
}
|
|
202
|
+
} catch {
|
|
195
203
|
}
|
|
196
|
-
} catch {
|
|
197
204
|
}
|
|
198
205
|
let result = code;
|
|
199
206
|
result = stripInterfaceDeclarations(result);
|
|
@@ -506,7 +513,7 @@ function extractCodeBlock(text, preferredFunctionName) {
|
|
|
506
513
|
// ../src/core/ollama-client.ts
|
|
507
514
|
var client = new Ollama();
|
|
508
515
|
var DEFAULT_OLLAMA_HOST = "http://127.0.0.1:11434";
|
|
509
|
-
var OLLAMA_INIT_TIMEOUT_MS =
|
|
516
|
+
var OLLAMA_INIT_TIMEOUT_MS = 12e4;
|
|
510
517
|
var STREAM_STALL_TIMEOUT_MS = 3e4;
|
|
511
518
|
function getOllamaBaseUrl() {
|
|
512
519
|
const configured = process.env.OLLAMA_HOST?.trim();
|
|
@@ -541,7 +548,8 @@ async function listModels() {
|
|
|
541
548
|
size: m.size,
|
|
542
549
|
parameterSize: m.details?.parameter_size,
|
|
543
550
|
quantization: m.details?.quantization_level,
|
|
544
|
-
family: m.details?.family
|
|
551
|
+
family: m.details?.family,
|
|
552
|
+
modelFormat: "gguf"
|
|
545
553
|
}));
|
|
546
554
|
}
|
|
547
555
|
async function listRunningModels() {
|
|
@@ -578,6 +586,7 @@ async function generateStream(model, prompt, callbacks, options) {
|
|
|
578
586
|
let fullResponse = "";
|
|
579
587
|
let fullThinking = "";
|
|
580
588
|
let result = null;
|
|
589
|
+
let firstChunkSeen = false;
|
|
581
590
|
let stallTimer = null;
|
|
582
591
|
const resetStallTimer = () => {
|
|
583
592
|
if (stallTimer) clearTimeout(stallTimer);
|
|
@@ -589,6 +598,10 @@ async function generateStream(model, prompt, callbacks, options) {
|
|
|
589
598
|
resetStallTimer();
|
|
590
599
|
for await (const chunk of stream) {
|
|
591
600
|
resetStallTimer();
|
|
601
|
+
if (!firstChunkSeen) {
|
|
602
|
+
firstChunkSeen = true;
|
|
603
|
+
callbacks?.onFirstChunk?.();
|
|
604
|
+
}
|
|
592
605
|
const chunkAny = chunk;
|
|
593
606
|
if (chunkAny.thinking) {
|
|
594
607
|
fullThinking += String(chunkAny.thinking);
|
|
@@ -639,7 +652,706 @@ function abortOngoingRequests() {
|
|
|
639
652
|
client.abort();
|
|
640
653
|
}
|
|
641
654
|
|
|
655
|
+
// ../src/core/lm-studio-client.ts
|
|
656
|
+
import os from "os";
|
|
657
|
+
import path from "path";
|
|
658
|
+
import { promises as fs } from "fs";
|
|
659
|
+
var DEFAULT_LM_STUDIO_BASE_URL = "http://127.0.0.1:1234";
|
|
660
|
+
var LM_STUDIO_INIT_TIMEOUT_MS = 15e3;
|
|
661
|
+
var LM_STUDIO_METADATA_TIMEOUT_MS = 2e3;
|
|
662
|
+
var DEFAULT_STREAM_STALL_TIMEOUT_MS = 18e4;
|
|
663
|
+
var DEFAULT_LM_STUDIO_HOME_DIR = path.join(os.homedir(), ".lmstudio");
|
|
664
|
+
var DEFAULT_LM_STUDIO_MODELS_DIR = path.join(DEFAULT_LM_STUDIO_HOME_DIR, "models");
|
|
665
|
+
var LM_STUDIO_HOME_DIR_ENV = "LM_STUDIO_HOME_DIR";
|
|
666
|
+
var LM_STUDIO_MODELS_DIR_ENV = "LM_STUDIO_MODELS_DIR";
|
|
667
|
+
var defaultKeepAlive2;
|
|
668
|
+
var activeAbortControllers = /* @__PURE__ */ new Set();
|
|
669
|
+
var directorySizeCache = /* @__PURE__ */ new Map();
|
|
670
|
+
var modelDefinitionCache = /* @__PURE__ */ new Map();
|
|
671
|
+
function buildThinkingConfig(think) {
|
|
672
|
+
if (think === void 0) return {};
|
|
673
|
+
const effort = think ? "high" : "low";
|
|
674
|
+
return {
|
|
675
|
+
include_reasoning: think,
|
|
676
|
+
reasoning_effort: effort,
|
|
677
|
+
reasoning: { effort }
|
|
678
|
+
};
|
|
679
|
+
}
|
|
680
|
+
function parseNonNegativeInt(value) {
|
|
681
|
+
if (!/^\d+$/.test(value)) return null;
|
|
682
|
+
const parsed = Number.parseInt(value, 10);
|
|
683
|
+
if (!Number.isSafeInteger(parsed) || parsed < 0) return null;
|
|
684
|
+
return parsed;
|
|
685
|
+
}
|
|
686
|
+
function resolveStreamStallTimeoutMs(override) {
|
|
687
|
+
if (override !== void 0) {
|
|
688
|
+
if (!Number.isFinite(override) || override < 0) return DEFAULT_STREAM_STALL_TIMEOUT_MS;
|
|
689
|
+
return override === 0 ? void 0 : Math.trunc(override);
|
|
690
|
+
}
|
|
691
|
+
const configured = process.env.LM_STUDIO_STREAM_STALL_TIMEOUT_MS?.trim();
|
|
692
|
+
if (!configured) return DEFAULT_STREAM_STALL_TIMEOUT_MS;
|
|
693
|
+
const parsed = parseNonNegativeInt(configured);
|
|
694
|
+
if (parsed === null) return DEFAULT_STREAM_STALL_TIMEOUT_MS;
|
|
695
|
+
return parsed === 0 ? void 0 : parsed;
|
|
696
|
+
}
|
|
697
|
+
function getLMStudioBaseUrl() {
|
|
698
|
+
const configured = process.env.LM_STUDIO_BASE_URL?.trim();
|
|
699
|
+
if (!configured) return DEFAULT_LM_STUDIO_BASE_URL;
|
|
700
|
+
const candidate = /^https?:\/\//i.test(configured) ? configured : `http://${configured}`;
|
|
701
|
+
try {
|
|
702
|
+
return new URL(candidate).toString();
|
|
703
|
+
} catch {
|
|
704
|
+
return DEFAULT_LM_STUDIO_BASE_URL;
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
function getLMStudioHeaders() {
|
|
708
|
+
const headers = {
|
|
709
|
+
"Content-Type": "application/json"
|
|
710
|
+
};
|
|
711
|
+
const apiKey = process.env.LM_STUDIO_API_KEY?.trim();
|
|
712
|
+
if (apiKey) {
|
|
713
|
+
headers.Authorization = `Bearer ${apiKey}`;
|
|
714
|
+
}
|
|
715
|
+
return headers;
|
|
716
|
+
}
|
|
717
|
+
function extractUsage(payload) {
|
|
718
|
+
if (typeof payload !== "object" || payload === null) return void 0;
|
|
719
|
+
const usage = payload.usage;
|
|
720
|
+
if (!usage) return void 0;
|
|
721
|
+
return usage;
|
|
722
|
+
}
|
|
723
|
+
function extractChoice2(payload) {
|
|
724
|
+
if (typeof payload !== "object" || payload === null) return void 0;
|
|
725
|
+
const choices = payload.choices;
|
|
726
|
+
if (!choices || choices.length === 0) return void 0;
|
|
727
|
+
return choices[0];
|
|
728
|
+
}
|
|
729
|
+
function extractContent(choice) {
|
|
730
|
+
const content = choice?.delta?.content ?? choice?.message?.content;
|
|
731
|
+
return typeof content === "string" ? content : "";
|
|
732
|
+
}
|
|
733
|
+
function extractReasoning(choice) {
|
|
734
|
+
const reasoning = choice?.delta?.reasoning_content ?? choice?.delta?.reasoning ?? choice?.message?.reasoning_content ?? choice?.message?.reasoning;
|
|
735
|
+
return typeof reasoning === "string" ? reasoning : "";
|
|
736
|
+
}
|
|
737
|
+
function asNonEmptyString(value) {
|
|
738
|
+
if (typeof value !== "string") return void 0;
|
|
739
|
+
const trimmed = value.trim();
|
|
740
|
+
return trimmed.length > 0 ? trimmed : void 0;
|
|
741
|
+
}
|
|
742
|
+
function stripOptionalQuotes(value) {
|
|
743
|
+
const trimmed = value.trim();
|
|
744
|
+
return trimmed.replace(/^["']|["']$/g, "").trim();
|
|
745
|
+
}
|
|
746
|
+
function normalizeToken(value) {
|
|
747
|
+
return (value ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "");
|
|
748
|
+
}
|
|
749
|
+
function tokenizeModelName(value) {
|
|
750
|
+
return value.toLowerCase().split(/[^a-z0-9]+/g).map((token) => token.trim()).filter((token) => token.length >= 3 && /[a-z]/.test(token));
|
|
751
|
+
}
|
|
752
|
+
function getLMStudioHomeDir() {
|
|
753
|
+
const fromEnv = process.env[LM_STUDIO_HOME_DIR_ENV]?.trim();
|
|
754
|
+
if (!fromEnv) return DEFAULT_LM_STUDIO_HOME_DIR;
|
|
755
|
+
if (fromEnv.startsWith("~")) {
|
|
756
|
+
return path.join(os.homedir(), fromEnv.slice(1));
|
|
757
|
+
}
|
|
758
|
+
return fromEnv;
|
|
759
|
+
}
|
|
760
|
+
async function pathIsDirectory(targetPath) {
|
|
761
|
+
try {
|
|
762
|
+
const stat = await fs.stat(targetPath);
|
|
763
|
+
return stat.isDirectory();
|
|
764
|
+
} catch {
|
|
765
|
+
return false;
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
async function loadHubModelDefinition(modelId) {
|
|
769
|
+
if (modelDefinitionCache.has(modelId)) {
|
|
770
|
+
return modelDefinitionCache.get(modelId) ?? null;
|
|
771
|
+
}
|
|
772
|
+
const [publisher, modelName] = modelId.split("/", 2);
|
|
773
|
+
if (!publisher || !modelName) {
|
|
774
|
+
modelDefinitionCache.set(modelId, null);
|
|
775
|
+
return null;
|
|
776
|
+
}
|
|
777
|
+
const yamlPath = path.join(getLMStudioHomeDir(), "hub", "models", publisher, modelName, "model.yaml");
|
|
778
|
+
let content = "";
|
|
779
|
+
try {
|
|
780
|
+
content = await fs.readFile(yamlPath, "utf8");
|
|
781
|
+
} catch {
|
|
782
|
+
modelDefinitionCache.set(modelId, null);
|
|
783
|
+
return null;
|
|
784
|
+
}
|
|
785
|
+
const sourcePattern = /user:\s*([^\n#]+?)\s*(?:\r?\n)\s*repo:\s*([^\n#]+?)\s*(?:\r?\n|$)/g;
|
|
786
|
+
const sourceMap = /* @__PURE__ */ new Map();
|
|
787
|
+
let sourceMatch = null;
|
|
788
|
+
while ((sourceMatch = sourcePattern.exec(content)) !== null) {
|
|
789
|
+
const user = stripOptionalQuotes(sourceMatch[1] ?? "");
|
|
790
|
+
const repo = stripOptionalQuotes(sourceMatch[2] ?? "");
|
|
791
|
+
if (!user || !repo) continue;
|
|
792
|
+
sourceMap.set(`${user}/${repo}`, { user, repo });
|
|
793
|
+
}
|
|
794
|
+
const paramsPattern = /paramsStrings:\s*(?:\r?\n)\s*-\s*([^\n#]+)/;
|
|
795
|
+
const paramsMatch = paramsPattern.exec(content);
|
|
796
|
+
const parameterSize = paramsMatch?.[1] ? stripOptionalQuotes(paramsMatch[1]) : void 0;
|
|
797
|
+
const definition = {
|
|
798
|
+
parameterSize: parameterSize || void 0,
|
|
799
|
+
sources: Array.from(sourceMap.values())
|
|
800
|
+
};
|
|
801
|
+
modelDefinitionCache.set(modelId, definition);
|
|
802
|
+
return definition;
|
|
803
|
+
}
|
|
804
|
+
async function readDirectorySizeBytes(dirPath) {
|
|
805
|
+
const cached = directorySizeCache.get(dirPath);
|
|
806
|
+
if (cached !== void 0) return cached;
|
|
807
|
+
let total = 0;
|
|
808
|
+
const queue = [dirPath];
|
|
809
|
+
while (queue.length > 0) {
|
|
810
|
+
const current = queue.pop();
|
|
811
|
+
if (!current) continue;
|
|
812
|
+
let entries = [];
|
|
813
|
+
try {
|
|
814
|
+
entries = await fs.readdir(current, { withFileTypes: true });
|
|
815
|
+
} catch {
|
|
816
|
+
continue;
|
|
817
|
+
}
|
|
818
|
+
for (const entry of entries) {
|
|
819
|
+
if (entry.isSymbolicLink()) continue;
|
|
820
|
+
const fullPath = path.join(current, entry.name);
|
|
821
|
+
if (entry.isDirectory()) {
|
|
822
|
+
queue.push(fullPath);
|
|
823
|
+
continue;
|
|
824
|
+
}
|
|
825
|
+
if (!entry.isFile()) continue;
|
|
826
|
+
try {
|
|
827
|
+
const stat = await fs.stat(fullPath);
|
|
828
|
+
total += stat.size;
|
|
829
|
+
} catch {
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
directorySizeCache.set(dirPath, total);
|
|
834
|
+
return total;
|
|
835
|
+
}
|
|
836
|
+
function scoreModelSourceCandidate(source, modelId, apiModel) {
|
|
837
|
+
const repoToken = normalizeToken(source.repo);
|
|
838
|
+
let score = 0;
|
|
839
|
+
const quantToken = normalizeToken(apiModel?.quantization);
|
|
840
|
+
if (quantToken && repoToken.includes(quantToken)) score += 8;
|
|
841
|
+
const compatibilityToken = normalizeToken(apiModel?.compatibility_type);
|
|
842
|
+
if (compatibilityToken && repoToken.includes(compatibilityToken)) score += 4;
|
|
843
|
+
const [, shortModelName] = modelId.split("/", 2);
|
|
844
|
+
const modelToken = normalizeToken(shortModelName ?? modelId);
|
|
845
|
+
if (modelToken && repoToken.includes(modelToken)) score += 2;
|
|
846
|
+
return score;
|
|
847
|
+
}
|
|
848
|
+
async function resolveModelsRootDir() {
|
|
849
|
+
const fromEnv = process.env[LM_STUDIO_MODELS_DIR_ENV]?.trim();
|
|
850
|
+
if (fromEnv) {
|
|
851
|
+
if (fromEnv.startsWith("~")) {
|
|
852
|
+
return path.join(os.homedir(), fromEnv.slice(1));
|
|
853
|
+
}
|
|
854
|
+
return fromEnv;
|
|
855
|
+
}
|
|
856
|
+
const settingsCandidates = [
|
|
857
|
+
path.join(getLMStudioHomeDir(), "settings.json"),
|
|
858
|
+
path.join(os.homedir(), "Library", "Application Support", "LM Studio", "settings.json")
|
|
859
|
+
];
|
|
860
|
+
for (const settingsPath of settingsCandidates) {
|
|
861
|
+
try {
|
|
862
|
+
const content = await fs.readFile(settingsPath, "utf8");
|
|
863
|
+
const parsed = JSON.parse(content);
|
|
864
|
+
const downloadsFolder = asNonEmptyString(parsed.downloadsFolder);
|
|
865
|
+
if (downloadsFolder) {
|
|
866
|
+
return downloadsFolder.startsWith("~") ? path.join(os.homedir(), downloadsFolder.slice(1)) : downloadsFolder;
|
|
867
|
+
}
|
|
868
|
+
} catch {
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
return DEFAULT_LM_STUDIO_MODELS_DIR;
|
|
872
|
+
}
|
|
873
|
+
async function listImmediateSubdirs(dirPath) {
|
|
874
|
+
let entries = [];
|
|
875
|
+
try {
|
|
876
|
+
entries = await fs.readdir(dirPath, { withFileTypes: true });
|
|
877
|
+
} catch {
|
|
878
|
+
return [];
|
|
879
|
+
}
|
|
880
|
+
return entries.filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()).map((entry) => path.join(dirPath, entry.name));
|
|
881
|
+
}
|
|
882
|
+
async function findPublisherDirs(rootDir, publisher) {
|
|
883
|
+
const normalizedPublisher = normalizeToken(publisher);
|
|
884
|
+
if (!normalizedPublisher) return [];
|
|
885
|
+
const dirs = await listImmediateSubdirs(rootDir);
|
|
886
|
+
return dirs.filter((dir) => normalizeToken(path.basename(dir)) === normalizedPublisher);
|
|
887
|
+
}
|
|
888
|
+
function scorePublisherModelCandidate(modelDirName, modelId, apiModel) {
|
|
889
|
+
const dirToken = normalizeToken(modelDirName);
|
|
890
|
+
if (!dirToken) return 0;
|
|
891
|
+
let score = 0;
|
|
892
|
+
const idToken = normalizeToken(modelId);
|
|
893
|
+
if (idToken && dirToken.includes(idToken)) score += 24;
|
|
894
|
+
const [, shortModelName] = modelId.split("/", 2);
|
|
895
|
+
const shortToken = normalizeToken(shortModelName ?? modelId);
|
|
896
|
+
if (shortToken && dirToken.includes(shortToken)) score += 16;
|
|
897
|
+
for (const token of tokenizeModelName(shortModelName ?? modelId)) {
|
|
898
|
+
if (dirToken.includes(token)) score += 3;
|
|
899
|
+
}
|
|
900
|
+
const quantToken = normalizeToken(apiModel?.quantization);
|
|
901
|
+
if (quantToken && dirToken.includes(quantToken)) score += 6;
|
|
902
|
+
const compatibilityToken = normalizeToken(apiModel?.compatibility_type);
|
|
903
|
+
if (compatibilityToken && dirToken.includes(compatibilityToken)) score += 4;
|
|
904
|
+
return score;
|
|
905
|
+
}
|
|
906
|
+
async function resolvePublisherModelMetadata(modelId, apiModel, modelsRootDir) {
|
|
907
|
+
const explicitPublisher = asNonEmptyString(apiModel?.publisher);
|
|
908
|
+
const [publisherFromId] = modelId.split("/", 1);
|
|
909
|
+
const publisher = explicitPublisher ?? publisherFromId;
|
|
910
|
+
if (!publisher) {
|
|
911
|
+
return { size: 0, parameterSize: inferParameterSizeFromModelId(modelId) };
|
|
912
|
+
}
|
|
913
|
+
const bundledModelsDir = path.join(getLMStudioHomeDir(), ".internal", "bundled-models");
|
|
914
|
+
const roots = Array.from(/* @__PURE__ */ new Set([modelsRootDir, bundledModelsDir]));
|
|
915
|
+
const candidates = [];
|
|
916
|
+
for (const root of roots) {
|
|
917
|
+
const publisherDirs = await findPublisherDirs(root, publisher);
|
|
918
|
+
for (const publisherDir of publisherDirs) {
|
|
919
|
+
const modelDirs = await listImmediateSubdirs(publisherDir);
|
|
920
|
+
if (modelDirs.length === 0) {
|
|
921
|
+
candidates.push({
|
|
922
|
+
fullPath: publisherDir,
|
|
923
|
+
score: scorePublisherModelCandidate(path.basename(publisherDir), modelId, apiModel)
|
|
924
|
+
});
|
|
925
|
+
continue;
|
|
926
|
+
}
|
|
927
|
+
for (const modelDir of modelDirs) {
|
|
928
|
+
candidates.push({
|
|
929
|
+
fullPath: modelDir,
|
|
930
|
+
score: scorePublisherModelCandidate(path.basename(modelDir), modelId, apiModel)
|
|
931
|
+
});
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
if (candidates.length === 0) {
|
|
936
|
+
return { size: 0, parameterSize: inferParameterSizeFromModelId(modelId) };
|
|
937
|
+
}
|
|
938
|
+
candidates.sort((a, b) => {
|
|
939
|
+
const scoreDiff = b.score - a.score;
|
|
940
|
+
if (scoreDiff !== 0) return scoreDiff;
|
|
941
|
+
return a.fullPath.localeCompare(b.fullPath);
|
|
942
|
+
});
|
|
943
|
+
let bestSize = 0;
|
|
944
|
+
for (const candidate of candidates) {
|
|
945
|
+
const size = await readDirectorySizeBytes(candidate.fullPath);
|
|
946
|
+
if (size > bestSize) bestSize = size;
|
|
947
|
+
if (size > 0 && candidate.score > 0) {
|
|
948
|
+
return { size, parameterSize: inferParameterSizeFromModelId(modelId) };
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
return { size: bestSize, parameterSize: inferParameterSizeFromModelId(modelId) };
|
|
952
|
+
}
|
|
953
|
+
async function resolveLocalModelMetadata(modelId, apiModel, modelsRootDir) {
|
|
954
|
+
const definition = await loadHubModelDefinition(modelId);
|
|
955
|
+
if (!definition) {
|
|
956
|
+
return resolvePublisherModelMetadata(modelId, apiModel, modelsRootDir);
|
|
957
|
+
}
|
|
958
|
+
const installedSources = [];
|
|
959
|
+
for (const source of definition.sources) {
|
|
960
|
+
const fullPath = path.join(modelsRootDir, source.user, source.repo);
|
|
961
|
+
if (await pathIsDirectory(fullPath)) {
|
|
962
|
+
installedSources.push({ ...source, fullPath });
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
if (installedSources.length === 0) {
|
|
966
|
+
const fallback2 = await resolvePublisherModelMetadata(modelId, apiModel, modelsRootDir);
|
|
967
|
+
if (fallback2.size > 0) {
|
|
968
|
+
return {
|
|
969
|
+
size: fallback2.size,
|
|
970
|
+
parameterSize: definition.parameterSize ?? fallback2.parameterSize
|
|
971
|
+
};
|
|
972
|
+
}
|
|
973
|
+
return { size: 0, parameterSize: definition.parameterSize };
|
|
974
|
+
}
|
|
975
|
+
installedSources.sort((a, b) => {
|
|
976
|
+
const diff = scoreModelSourceCandidate(b, modelId, apiModel) - scoreModelSourceCandidate(a, modelId, apiModel);
|
|
977
|
+
if (diff !== 0) return diff;
|
|
978
|
+
return a.repo.localeCompare(b.repo);
|
|
979
|
+
});
|
|
980
|
+
let bestSize = 0;
|
|
981
|
+
for (const source of installedSources) {
|
|
982
|
+
const size = await readDirectorySizeBytes(source.fullPath);
|
|
983
|
+
if (size > bestSize) bestSize = size;
|
|
984
|
+
if (size > 0) {
|
|
985
|
+
return { size, parameterSize: definition.parameterSize };
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
if (bestSize > 0) {
|
|
989
|
+
return { size: bestSize, parameterSize: definition.parameterSize };
|
|
990
|
+
}
|
|
991
|
+
const fallback = await resolvePublisherModelMetadata(modelId, apiModel, modelsRootDir);
|
|
992
|
+
if (fallback.size > 0) {
|
|
993
|
+
return {
|
|
994
|
+
size: fallback.size,
|
|
995
|
+
parameterSize: definition.parameterSize ?? fallback.parameterSize
|
|
996
|
+
};
|
|
997
|
+
}
|
|
998
|
+
return { size: 0, parameterSize: definition.parameterSize ?? fallback.parameterSize };
|
|
999
|
+
}
|
|
1000
|
+
function parseSizeBytes(model) {
|
|
1001
|
+
if (!model) return 0;
|
|
1002
|
+
const candidates = [
|
|
1003
|
+
model.size_bytes,
|
|
1004
|
+
model.model_size_bytes,
|
|
1005
|
+
model.file_size_bytes,
|
|
1006
|
+
model.bytes,
|
|
1007
|
+
model.size
|
|
1008
|
+
];
|
|
1009
|
+
for (const candidate of candidates) {
|
|
1010
|
+
if (typeof candidate !== "number" || !Number.isFinite(candidate)) continue;
|
|
1011
|
+
if (candidate > 0) return Math.trunc(candidate);
|
|
1012
|
+
}
|
|
1013
|
+
return 0;
|
|
1014
|
+
}
|
|
1015
|
+
function normalizeModelNumber(value) {
|
|
1016
|
+
return value.replace(/\.0+$/, "");
|
|
1017
|
+
}
|
|
1018
|
+
function inferParameterSizeFromModelId(modelId) {
|
|
1019
|
+
const id = modelId.toLowerCase();
|
|
1020
|
+
const mixtureMatch = id.match(/\b(\d+(?:\.\d+)?)\s*[x*]\s*(\d+(?:\.\d+)?)\s*b\b/);
|
|
1021
|
+
if (mixtureMatch) {
|
|
1022
|
+
const experts = normalizeModelNumber(mixtureMatch[1] ?? "");
|
|
1023
|
+
const perExpert = normalizeModelNumber(mixtureMatch[2] ?? "");
|
|
1024
|
+
if (experts && perExpert) return `${experts}x${perExpert}B`;
|
|
1025
|
+
}
|
|
1026
|
+
const billionMatch = id.match(/\b(\d+(?:\.\d+)?)\s*b\b/);
|
|
1027
|
+
if (billionMatch?.[1]) {
|
|
1028
|
+
return `${normalizeModelNumber(billionMatch[1])}B`;
|
|
1029
|
+
}
|
|
1030
|
+
const millionMatch = id.match(/\b(\d+(?:\.\d+)?)\s*m\b/);
|
|
1031
|
+
if (millionMatch?.[1]) {
|
|
1032
|
+
return `${normalizeModelNumber(millionMatch[1])}M`;
|
|
1033
|
+
}
|
|
1034
|
+
return void 0;
|
|
1035
|
+
}
|
|
1036
|
+
function isLoadedState(state) {
|
|
1037
|
+
if (!state) return false;
|
|
1038
|
+
const normalized = state.trim().toLowerCase();
|
|
1039
|
+
if (!normalized || normalized.includes("not-loaded")) return false;
|
|
1040
|
+
if (normalized === "loaded" || normalized === "ready") return true;
|
|
1041
|
+
return normalized.includes("loaded");
|
|
1042
|
+
}
|
|
1043
|
+
async function fetchApiModels() {
|
|
1044
|
+
try {
|
|
1045
|
+
const resp = await fetchWithTimeout(
|
|
1046
|
+
"/api/v0/models",
|
|
1047
|
+
{ method: "GET", headers: getLMStudioHeaders() },
|
|
1048
|
+
LM_STUDIO_METADATA_TIMEOUT_MS,
|
|
1049
|
+
"LM Studio API metadata"
|
|
1050
|
+
);
|
|
1051
|
+
if (!resp.ok) return null;
|
|
1052
|
+
const payload = await resp.json();
|
|
1053
|
+
return Array.isArray(payload.data) ? payload.data : [];
|
|
1054
|
+
} catch {
|
|
1055
|
+
return null;
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
async function fetchWithTimeout(path2, init, timeoutMs, label) {
|
|
1059
|
+
const controller = new AbortController();
|
|
1060
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
1061
|
+
const baseUrl = getLMStudioBaseUrl();
|
|
1062
|
+
try {
|
|
1063
|
+
const url = new URL(path2, baseUrl);
|
|
1064
|
+
return await fetch(url, { ...init, signal: controller.signal });
|
|
1065
|
+
} catch (err) {
|
|
1066
|
+
if (err instanceof Error && err.name === "AbortError") {
|
|
1067
|
+
throw new Error(`${label} timed out after ${timeoutMs}ms`);
|
|
1068
|
+
}
|
|
1069
|
+
throw err;
|
|
1070
|
+
} finally {
|
|
1071
|
+
clearTimeout(timeout);
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
async function resolveLocalLMStudioVersion() {
|
|
1075
|
+
const historicalVersionPath = path.join(
|
|
1076
|
+
getLMStudioHomeDir(),
|
|
1077
|
+
".internal",
|
|
1078
|
+
"historical-version-info.json"
|
|
1079
|
+
);
|
|
1080
|
+
try {
|
|
1081
|
+
const content = await fs.readFile(historicalVersionPath, "utf8");
|
|
1082
|
+
const parsed = JSON.parse(content);
|
|
1083
|
+
const version = asNonEmptyString(parsed.lastRecordedAppVersion) ?? asNonEmptyString(parsed.lastRecorderdAppVersion);
|
|
1084
|
+
if (!version) return null;
|
|
1085
|
+
const build = asNonEmptyString(parsed.lastRecordedAppBuildVersion);
|
|
1086
|
+
return build ? `${version}+${build}` : version;
|
|
1087
|
+
} catch {
|
|
1088
|
+
return null;
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
async function getLMStudioVersion() {
|
|
1092
|
+
const localVersion = await resolveLocalLMStudioVersion();
|
|
1093
|
+
try {
|
|
1094
|
+
const resp = await fetchWithTimeout(
|
|
1095
|
+
"/v1/models",
|
|
1096
|
+
{ method: "GET", headers: getLMStudioHeaders() },
|
|
1097
|
+
5e3,
|
|
1098
|
+
"LM Studio version check"
|
|
1099
|
+
);
|
|
1100
|
+
if (!resp.ok) return localVersion ?? "unknown";
|
|
1101
|
+
const fromHeader = asNonEmptyString(resp.headers.get("x-lmstudio-version"));
|
|
1102
|
+
if (fromHeader) return fromHeader;
|
|
1103
|
+
return localVersion ?? "unknown";
|
|
1104
|
+
} catch {
|
|
1105
|
+
return localVersion ?? "unknown";
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
async function listModels2() {
|
|
1109
|
+
const resp = await fetchWithTimeout(
|
|
1110
|
+
"/v1/models",
|
|
1111
|
+
{ method: "GET", headers: getLMStudioHeaders() },
|
|
1112
|
+
LM_STUDIO_INIT_TIMEOUT_MS,
|
|
1113
|
+
"LM Studio list models"
|
|
1114
|
+
);
|
|
1115
|
+
if (!resp.ok) {
|
|
1116
|
+
throw new Error(`LM Studio list models failed (${resp.status} ${resp.statusText})`);
|
|
1117
|
+
}
|
|
1118
|
+
const data = await resp.json();
|
|
1119
|
+
const ids = (data.data ?? []).map((m) => m.id?.trim()).filter((id) => Boolean(id));
|
|
1120
|
+
const apiModels = await fetchApiModels();
|
|
1121
|
+
const apiById = /* @__PURE__ */ new Map();
|
|
1122
|
+
for (const model of apiModels ?? []) {
|
|
1123
|
+
const id = asNonEmptyString(model.id);
|
|
1124
|
+
if (!id) continue;
|
|
1125
|
+
apiById.set(id, model);
|
|
1126
|
+
}
|
|
1127
|
+
const modelsRootDir = await resolveModelsRootDir();
|
|
1128
|
+
const localMetadataById = /* @__PURE__ */ new Map();
|
|
1129
|
+
for (const id of ids) {
|
|
1130
|
+
const localMetadata = await resolveLocalModelMetadata(id, apiById.get(id), modelsRootDir);
|
|
1131
|
+
localMetadataById.set(id, localMetadata);
|
|
1132
|
+
}
|
|
1133
|
+
return ids.map((id) => {
|
|
1134
|
+
const apiModel = apiById.get(id);
|
|
1135
|
+
const localMetadata = localMetadataById.get(id);
|
|
1136
|
+
const apiSize = parseSizeBytes(apiModel);
|
|
1137
|
+
return {
|
|
1138
|
+
name: id,
|
|
1139
|
+
size: apiSize > 0 ? apiSize : localMetadata?.size ?? 0,
|
|
1140
|
+
parameterSize: localMetadata?.parameterSize ?? inferParameterSizeFromModelId(id),
|
|
1141
|
+
quantization: asNonEmptyString(apiModel?.quantization),
|
|
1142
|
+
runtimeStatus: asNonEmptyString(apiModel?.state),
|
|
1143
|
+
modelFormat: asNonEmptyString(apiModel?.compatibility_type),
|
|
1144
|
+
family: asNonEmptyString(apiModel?.arch) ?? asNonEmptyString(apiModel?.type) ?? asNonEmptyString(apiModel?.publisher)
|
|
1145
|
+
};
|
|
1146
|
+
});
|
|
1147
|
+
}
|
|
1148
|
+
async function listRunningModels2() {
|
|
1149
|
+
const apiModels = await fetchApiModels();
|
|
1150
|
+
if (!apiModels) return [];
|
|
1151
|
+
return apiModels.filter((model) => isLoadedState(model.state)).map((model) => ({
|
|
1152
|
+
name: model.id ?? "",
|
|
1153
|
+
size: parseSizeBytes(model),
|
|
1154
|
+
vramUsed: 0
|
|
1155
|
+
})).filter((model) => model.name.trim().length > 0);
|
|
1156
|
+
}
|
|
1157
|
+
function setDefaultKeepAlive2(keepAlive) {
|
|
1158
|
+
defaultKeepAlive2 = keepAlive;
|
|
1159
|
+
void defaultKeepAlive2;
|
|
1160
|
+
}
|
|
1161
|
+
async function generate2(model, prompt, options) {
|
|
1162
|
+
const start = Date.now();
|
|
1163
|
+
const controller = new AbortController();
|
|
1164
|
+
activeAbortControllers.add(controller);
|
|
1165
|
+
try {
|
|
1166
|
+
const baseUrl = getLMStudioBaseUrl();
|
|
1167
|
+
const url = new URL("/v1/chat/completions", baseUrl);
|
|
1168
|
+
const resp = await fetch(url, {
|
|
1169
|
+
method: "POST",
|
|
1170
|
+
headers: getLMStudioHeaders(),
|
|
1171
|
+
body: JSON.stringify({
|
|
1172
|
+
model,
|
|
1173
|
+
messages: [{ role: "user", content: prompt }],
|
|
1174
|
+
temperature: options?.temperature ?? 0,
|
|
1175
|
+
max_tokens: options?.num_predict ?? 512,
|
|
1176
|
+
stream: false,
|
|
1177
|
+
...buildThinkingConfig(options?.think)
|
|
1178
|
+
}),
|
|
1179
|
+
signal: controller.signal
|
|
1180
|
+
});
|
|
1181
|
+
if (!resp.ok) {
|
|
1182
|
+
const body = await resp.text().catch(() => "");
|
|
1183
|
+
throw new Error(`LM Studio generate failed (${resp.status} ${resp.statusText}) ${body}`.trim());
|
|
1184
|
+
}
|
|
1185
|
+
const payload = await resp.json();
|
|
1186
|
+
const choice = extractChoice2(payload);
|
|
1187
|
+
const response = extractContent(choice);
|
|
1188
|
+
const reasoning = extractReasoning(choice);
|
|
1189
|
+
const usage = extractUsage(payload);
|
|
1190
|
+
const totalDuration = Math.max(0, Date.now() - start) * 1e6;
|
|
1191
|
+
return {
|
|
1192
|
+
response,
|
|
1193
|
+
...reasoning ? { thinking: reasoning } : {},
|
|
1194
|
+
totalDuration,
|
|
1195
|
+
loadDuration: 0,
|
|
1196
|
+
promptEvalCount: usage?.prompt_tokens ?? 0,
|
|
1197
|
+
promptEvalDuration: 0,
|
|
1198
|
+
evalCount: usage?.completion_tokens ?? 0,
|
|
1199
|
+
evalDuration: totalDuration
|
|
1200
|
+
};
|
|
1201
|
+
} catch (err) {
|
|
1202
|
+
if (err instanceof Error && err.name === "AbortError") {
|
|
1203
|
+
throw new Error("LM Studio generate request aborted");
|
|
1204
|
+
}
|
|
1205
|
+
throw err;
|
|
1206
|
+
} finally {
|
|
1207
|
+
activeAbortControllers.delete(controller);
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
async function generateStream2(model, prompt, callbacks, options) {
|
|
1211
|
+
const start = Date.now();
|
|
1212
|
+
const controller = new AbortController();
|
|
1213
|
+
activeAbortControllers.add(controller);
|
|
1214
|
+
const stallTimeoutMs = resolveStreamStallTimeoutMs(options?.stall_timeout_ms);
|
|
1215
|
+
let abortedByStallTimeout = false;
|
|
1216
|
+
const baseUrl = getLMStudioBaseUrl();
|
|
1217
|
+
const url = new URL("/v1/chat/completions", baseUrl);
|
|
1218
|
+
let stallTimer = null;
|
|
1219
|
+
const resetStallTimer = () => {
|
|
1220
|
+
if (stallTimeoutMs === void 0) return;
|
|
1221
|
+
if (stallTimer) clearTimeout(stallTimer);
|
|
1222
|
+
stallTimer = setTimeout(() => {
|
|
1223
|
+
abortedByStallTimeout = true;
|
|
1224
|
+
controller.abort();
|
|
1225
|
+
}, stallTimeoutMs);
|
|
1226
|
+
};
|
|
1227
|
+
try {
|
|
1228
|
+
resetStallTimer();
|
|
1229
|
+
const resp = await fetch(url, {
|
|
1230
|
+
method: "POST",
|
|
1231
|
+
headers: getLMStudioHeaders(),
|
|
1232
|
+
body: JSON.stringify({
|
|
1233
|
+
model,
|
|
1234
|
+
messages: [{ role: "user", content: prompt }],
|
|
1235
|
+
temperature: options?.temperature ?? 0,
|
|
1236
|
+
max_tokens: options?.num_predict ?? 512,
|
|
1237
|
+
stream: true,
|
|
1238
|
+
stream_options: { include_usage: true },
|
|
1239
|
+
...buildThinkingConfig(options?.think)
|
|
1240
|
+
}),
|
|
1241
|
+
signal: controller.signal
|
|
1242
|
+
});
|
|
1243
|
+
if (!resp.ok) {
|
|
1244
|
+
const body = await resp.text().catch(() => "");
|
|
1245
|
+
throw new Error(`LM Studio stream failed (${resp.status} ${resp.statusText}) ${body}`.trim());
|
|
1246
|
+
}
|
|
1247
|
+
if (!resp.body) {
|
|
1248
|
+
throw new Error("LM Studio stream response body is empty");
|
|
1249
|
+
}
|
|
1250
|
+
const reader = resp.body.getReader();
|
|
1251
|
+
const decoder = new TextDecoder();
|
|
1252
|
+
let buffered = "";
|
|
1253
|
+
let doneReceived = false;
|
|
1254
|
+
let fullResponse = "";
|
|
1255
|
+
let fullThinking = "";
|
|
1256
|
+
let usage;
|
|
1257
|
+
let firstChunkSeen = false;
|
|
1258
|
+
let firstTokenTime = null;
|
|
1259
|
+
let lastTokenTime = null;
|
|
1260
|
+
const processDataLine = (rawLine) => {
|
|
1261
|
+
const line = rawLine.trim();
|
|
1262
|
+
if (!line.startsWith("data:")) return;
|
|
1263
|
+
const dataStr = line.slice(5).trim();
|
|
1264
|
+
if (!dataStr) return;
|
|
1265
|
+
if (dataStr === "[DONE]") {
|
|
1266
|
+
doneReceived = true;
|
|
1267
|
+
return;
|
|
1268
|
+
}
|
|
1269
|
+
let payload;
|
|
1270
|
+
try {
|
|
1271
|
+
payload = JSON.parse(dataStr);
|
|
1272
|
+
} catch {
|
|
1273
|
+
return;
|
|
1274
|
+
}
|
|
1275
|
+
const choice = extractChoice2(payload);
|
|
1276
|
+
const content = extractContent(choice);
|
|
1277
|
+
const reasoning = extractReasoning(choice);
|
|
1278
|
+
const chunkUsage = extractUsage(payload);
|
|
1279
|
+
if (chunkUsage) usage = chunkUsage;
|
|
1280
|
+
if (reasoning) {
|
|
1281
|
+
fullThinking += reasoning;
|
|
1282
|
+
}
|
|
1283
|
+
if (content) {
|
|
1284
|
+
const now = Date.now();
|
|
1285
|
+
if (firstTokenTime === null) firstTokenTime = now;
|
|
1286
|
+
lastTokenTime = now;
|
|
1287
|
+
fullResponse += content;
|
|
1288
|
+
callbacks?.onToken?.(content);
|
|
1289
|
+
}
|
|
1290
|
+
};
|
|
1291
|
+
while (true) {
|
|
1292
|
+
const { value, done } = await reader.read();
|
|
1293
|
+
if (done) break;
|
|
1294
|
+
resetStallTimer();
|
|
1295
|
+
if (!firstChunkSeen) {
|
|
1296
|
+
firstChunkSeen = true;
|
|
1297
|
+
callbacks?.onFirstChunk?.();
|
|
1298
|
+
}
|
|
1299
|
+
buffered += decoder.decode(value, { stream: true });
|
|
1300
|
+
const lines = buffered.split("\n");
|
|
1301
|
+
buffered = lines.pop() ?? "";
|
|
1302
|
+
for (const rawLine of lines) {
|
|
1303
|
+
processDataLine(rawLine);
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
if (buffered.trim().length > 0) {
|
|
1307
|
+
processDataLine(buffered);
|
|
1308
|
+
}
|
|
1309
|
+
if (stallTimer) clearTimeout(stallTimer);
|
|
1310
|
+
if (!doneReceived && !fullResponse && !fullThinking) {
|
|
1311
|
+
throw new Error("LM Studio stream ended without content");
|
|
1312
|
+
}
|
|
1313
|
+
const totalDuration = Math.max(0, Date.now() - start) * 1e6;
|
|
1314
|
+
const evalDurationMs = firstTokenTime !== null && lastTokenTime !== null && lastTokenTime > firstTokenTime ? lastTokenTime - firstTokenTime : Date.now() - start;
|
|
1315
|
+
const result = {
|
|
1316
|
+
response: fullResponse,
|
|
1317
|
+
...fullThinking ? { thinking: fullThinking } : {},
|
|
1318
|
+
totalDuration,
|
|
1319
|
+
loadDuration: 0,
|
|
1320
|
+
promptEvalCount: usage?.prompt_tokens ?? 0,
|
|
1321
|
+
promptEvalDuration: firstTokenTime !== null ? (firstTokenTime - start) * 1e6 : 0,
|
|
1322
|
+
evalCount: usage?.completion_tokens ?? 0,
|
|
1323
|
+
evalDuration: Math.max(1, evalDurationMs) * 1e6
|
|
1324
|
+
};
|
|
1325
|
+
callbacks?.onDone?.(result);
|
|
1326
|
+
return result;
|
|
1327
|
+
} catch (err) {
|
|
1328
|
+
if (err instanceof Error && err.name === "AbortError") {
|
|
1329
|
+
if (abortedByStallTimeout && stallTimeoutMs !== void 0) {
|
|
1330
|
+
throw new Error(`LM Studio stream timed out after ${stallTimeoutMs}ms`);
|
|
1331
|
+
}
|
|
1332
|
+
throw new Error("LM Studio stream request aborted");
|
|
1333
|
+
}
|
|
1334
|
+
throw err;
|
|
1335
|
+
} finally {
|
|
1336
|
+
if (stallTimer) clearTimeout(stallTimer);
|
|
1337
|
+
activeAbortControllers.delete(controller);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
async function unloadModel2(_model) {
|
|
1341
|
+
}
|
|
1342
|
+
function abortOngoingRequests2() {
|
|
1343
|
+
for (const controller of activeAbortControllers) {
|
|
1344
|
+
controller.abort();
|
|
1345
|
+
}
|
|
1346
|
+
activeAbortControllers.clear();
|
|
1347
|
+
}
|
|
1348
|
+
|
|
642
1349
|
// ../src/core/runtime.ts
|
|
1350
|
+
var SUPPORTED_RUNTIME_BACKENDS = ["ollama", "lm-studio"];
|
|
1351
|
+
var RUNTIME_LABELS = {
|
|
1352
|
+
ollama: "Ollama",
|
|
1353
|
+
"lm-studio": "LM Studio"
|
|
1354
|
+
};
|
|
643
1355
|
var OllamaRuntime = class {
|
|
644
1356
|
name = "ollama";
|
|
645
1357
|
modelFormat = "gguf";
|
|
@@ -668,29 +1380,105 @@ var OllamaRuntime = class {
|
|
|
668
1380
|
abortOngoingRequests();
|
|
669
1381
|
}
|
|
670
1382
|
};
|
|
1383
|
+
var LMStudioRuntime = class {
|
|
1384
|
+
name = "lm-studio";
|
|
1385
|
+
modelFormat = "gguf";
|
|
1386
|
+
generate(model, prompt, opts) {
|
|
1387
|
+
return generate2(model, prompt, opts);
|
|
1388
|
+
}
|
|
1389
|
+
generateStream(model, prompt, callbacks, opts) {
|
|
1390
|
+
return generateStream2(model, prompt, callbacks, opts);
|
|
1391
|
+
}
|
|
1392
|
+
listModels() {
|
|
1393
|
+
return listModels2();
|
|
1394
|
+
}
|
|
1395
|
+
listRunningModels() {
|
|
1396
|
+
return listRunningModels2();
|
|
1397
|
+
}
|
|
1398
|
+
getVersion() {
|
|
1399
|
+
return getLMStudioVersion();
|
|
1400
|
+
}
|
|
1401
|
+
unloadModel(model) {
|
|
1402
|
+
return unloadModel2(model);
|
|
1403
|
+
}
|
|
1404
|
+
setKeepAlive(keepAlive) {
|
|
1405
|
+
setDefaultKeepAlive2(keepAlive);
|
|
1406
|
+
}
|
|
1407
|
+
abort() {
|
|
1408
|
+
abortOngoingRequests2();
|
|
1409
|
+
}
|
|
1410
|
+
};
|
|
671
1411
|
var activeRuntime = new OllamaRuntime();
|
|
672
|
-
function
|
|
1412
|
+
function createRuntime(backend) {
|
|
1413
|
+
if (backend === "lm-studio") return new LMStudioRuntime();
|
|
1414
|
+
return new OllamaRuntime();
|
|
1415
|
+
}
|
|
1416
|
+
function normalizeRuntimeBackend(value) {
|
|
1417
|
+
const candidate = (value ?? "ollama").trim().toLowerCase();
|
|
1418
|
+
if (candidate === "ollama" || candidate === "lm-studio") {
|
|
1419
|
+
return candidate;
|
|
1420
|
+
}
|
|
1421
|
+
throw new Error(
|
|
1422
|
+
`Unsupported backend "${value}". Supported backends: ${SUPPORTED_RUNTIME_BACKENDS.join(", ")}`
|
|
1423
|
+
);
|
|
1424
|
+
}
|
|
1425
|
+
function setRuntimeByName(backend) {
|
|
1426
|
+
const normalized = normalizeRuntimeBackend(backend);
|
|
1427
|
+
if (activeRuntime.name === normalized) return normalized;
|
|
1428
|
+
activeRuntime = createRuntime(normalized);
|
|
1429
|
+
return normalized;
|
|
1430
|
+
}
|
|
1431
|
+
function getRuntimeDisplayName(runtimeName = activeRuntime.name) {
|
|
1432
|
+
if (runtimeName === "lm-studio") return RUNTIME_LABELS["lm-studio"];
|
|
1433
|
+
if (runtimeName === "ollama") return RUNTIME_LABELS.ollama;
|
|
1434
|
+
return runtimeName;
|
|
1435
|
+
}
|
|
1436
|
+
function getRuntimeModelInstallHint(runtimeName = activeRuntime.name) {
|
|
1437
|
+
if (runtimeName === "lm-studio") {
|
|
1438
|
+
return "Download/select a model in LM Studio, then load it in the local server.";
|
|
1439
|
+
}
|
|
1440
|
+
if (runtimeName === "ollama") {
|
|
1441
|
+
return "Pull one with: ollama pull <model>";
|
|
1442
|
+
}
|
|
1443
|
+
return "Add at least one model in your selected runtime.";
|
|
1444
|
+
}
|
|
1445
|
+
function getRuntimeSetupHints(runtimeName = activeRuntime.name) {
|
|
1446
|
+
if (runtimeName === "lm-studio") {
|
|
1447
|
+
return [
|
|
1448
|
+
"Start LM Studio local server (Developer tab -> Local Server).",
|
|
1449
|
+
"Optionally set LM_STUDIO_BASE_URL if your server is not on http://127.0.0.1:1234."
|
|
1450
|
+
];
|
|
1451
|
+
}
|
|
1452
|
+
if (runtimeName === "ollama") {
|
|
1453
|
+
return [
|
|
1454
|
+
"Start it with: ollama serve",
|
|
1455
|
+
"Install it at: https://ollama.com"
|
|
1456
|
+
];
|
|
1457
|
+
}
|
|
1458
|
+
return ["Verify the runtime is reachable and configured correctly."];
|
|
1459
|
+
}
|
|
1460
|
+
function generate3(model, prompt, opts) {
|
|
673
1461
|
return activeRuntime.generate(model, prompt, opts);
|
|
674
1462
|
}
|
|
675
|
-
function
|
|
1463
|
+
function generateStream3(model, prompt, callbacks, opts) {
|
|
676
1464
|
return activeRuntime.generateStream(model, prompt, callbacks, opts);
|
|
677
1465
|
}
|
|
678
|
-
function
|
|
1466
|
+
function listModels3() {
|
|
679
1467
|
return activeRuntime.listModels();
|
|
680
1468
|
}
|
|
681
|
-
function
|
|
1469
|
+
function listRunningModels3() {
|
|
682
1470
|
return activeRuntime.listRunningModels();
|
|
683
1471
|
}
|
|
684
1472
|
function getRuntimeVersion() {
|
|
685
1473
|
return activeRuntime.getVersion();
|
|
686
1474
|
}
|
|
687
|
-
function
|
|
1475
|
+
function unloadModel3(model) {
|
|
688
1476
|
return activeRuntime.unloadModel(model);
|
|
689
1477
|
}
|
|
690
1478
|
function setRuntimeKeepAlive(keepAlive) {
|
|
691
1479
|
activeRuntime.setKeepAlive(keepAlive);
|
|
692
1480
|
}
|
|
693
|
-
function
|
|
1481
|
+
function abortOngoingRequests3() {
|
|
694
1482
|
activeRuntime.abort();
|
|
695
1483
|
}
|
|
696
1484
|
function getRuntimeName() {
|
|
@@ -706,7 +1494,7 @@ import chalk8 from "chalk";
|
|
|
706
1494
|
|
|
707
1495
|
// ../src/core/hardware.ts
|
|
708
1496
|
import si from "systeminformation";
|
|
709
|
-
import
|
|
1497
|
+
import os2 from "os";
|
|
710
1498
|
import { execFile as execFile2 } from "child_process";
|
|
711
1499
|
import { readFile } from "fs/promises";
|
|
712
1500
|
function execCommand(cmd, args, timeoutMs = 3e3) {
|
|
@@ -782,6 +1570,56 @@ async function detectPowerMode() {
|
|
|
782
1570
|
return "unknown";
|
|
783
1571
|
}
|
|
784
1572
|
}
|
|
1573
|
+
async function detectThermalPressure() {
|
|
1574
|
+
try {
|
|
1575
|
+
if (process.platform === "darwin") {
|
|
1576
|
+
const output4 = await execCommand("pmset", ["-g", "therm"]);
|
|
1577
|
+
if (!output4) return "unknown";
|
|
1578
|
+
const match = output4.match(/CPU_Speed_Limit\s*=\s*(\d+)/i);
|
|
1579
|
+
if (!match) return "unknown";
|
|
1580
|
+
const limit = parseInt(match[1], 10);
|
|
1581
|
+
if (limit >= 100) return "nominal";
|
|
1582
|
+
if (limit >= 80) return "moderate";
|
|
1583
|
+
if (limit >= 50) return "heavy";
|
|
1584
|
+
return "critical";
|
|
1585
|
+
}
|
|
1586
|
+
const temp = await si.cpuTemperature();
|
|
1587
|
+
const main2 = temp.main;
|
|
1588
|
+
if (main2 == null || main2 <= 0) return "unknown";
|
|
1589
|
+
if (main2 < 80) return "nominal";
|
|
1590
|
+
if (main2 < 90) return "moderate";
|
|
1591
|
+
if (main2 < 100) return "heavy";
|
|
1592
|
+
return "critical";
|
|
1593
|
+
} catch {
|
|
1594
|
+
return "unknown";
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
async function detectBatteryPowered() {
|
|
1598
|
+
try {
|
|
1599
|
+
if (process.platform === "darwin") {
|
|
1600
|
+
const output4 = await execCommand("pmset", ["-g", "ps"]);
|
|
1601
|
+
if (!output4) return void 0;
|
|
1602
|
+
if (output4.includes("Battery Power")) return true;
|
|
1603
|
+
if (output4.includes("AC Power")) return false;
|
|
1604
|
+
return void 0;
|
|
1605
|
+
}
|
|
1606
|
+
const battery = await si.battery();
|
|
1607
|
+
if (!battery.hasBattery) return void 0;
|
|
1608
|
+
if (typeof battery.acConnected === "boolean") return !battery.acConnected;
|
|
1609
|
+
if (typeof battery.isCharging === "boolean") return !battery.isCharging;
|
|
1610
|
+
return void 0;
|
|
1611
|
+
} catch {
|
|
1612
|
+
return void 0;
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
async function getSwapUsedGB() {
|
|
1616
|
+
try {
|
|
1617
|
+
const mem = await si.mem();
|
|
1618
|
+
return +(mem.swapused / 1024 / 1024 / 1024).toFixed(2);
|
|
1619
|
+
} catch {
|
|
1620
|
+
return 0;
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
785
1623
|
async function getHardwareInfo() {
|
|
786
1624
|
const [cpu, mem, graphics, osInfo, memLayout, powerMode, cpuSpeed, machineModel] = await Promise.all([
|
|
787
1625
|
si.cpu(),
|
|
@@ -813,12 +1651,20 @@ async function getHardwareInfo() {
|
|
|
813
1651
|
gpuCores: gpuCores && !isNaN(gpuCores) ? gpuCores : null,
|
|
814
1652
|
gpuVramMB: gpuController?.vram ?? null,
|
|
815
1653
|
os: `${osInfo.distro} ${osInfo.release}`,
|
|
816
|
-
arch:
|
|
1654
|
+
arch: os2.arch(),
|
|
817
1655
|
machineModel: machineModel || null,
|
|
818
1656
|
powerMode,
|
|
819
1657
|
cpuCurrentSpeedGHz: cpuSpeed?.avg ?? null
|
|
820
1658
|
};
|
|
821
1659
|
}
|
|
1660
|
+
async function getCpuLoad() {
|
|
1661
|
+
try {
|
|
1662
|
+
const load = await si.currentLoad();
|
|
1663
|
+
return +load.currentLoad.toFixed(1);
|
|
1664
|
+
} catch {
|
|
1665
|
+
return -1;
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
822
1668
|
async function getMemoryUsage() {
|
|
823
1669
|
const mem = await si.mem();
|
|
824
1670
|
const totalGB = mem.total / 1024 / 1024 / 1024;
|
|
@@ -979,8 +1825,22 @@ var BENCH_PROMPTS = [
|
|
|
979
1825
|
"Write a detailed step-by-step explanation of how a compiler transforms source code into machine code, covering lexing, parsing, semantic analysis, optimization, and code generation.",
|
|
980
1826
|
"Implement a function in pseudocode that checks whether a given string of parentheses, brackets, and braces is balanced. Explain the time and space complexity."
|
|
981
1827
|
];
|
|
982
|
-
var DEFAULT_WARMUP_TIMEOUT_MS =
|
|
983
|
-
var DEFAULT_PROMPT_TIMEOUT_MS =
|
|
1828
|
+
var DEFAULT_WARMUP_TIMEOUT_MS = 3e5;
|
|
1829
|
+
var DEFAULT_PROMPT_TIMEOUT_MS = 12e4;
|
|
1830
|
+
async function optionalProbe(probe, fallback) {
|
|
1831
|
+
try {
|
|
1832
|
+
return await probe();
|
|
1833
|
+
} catch {
|
|
1834
|
+
return fallback;
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
async function optionalProbeWithAvailability(probe, fallback) {
|
|
1838
|
+
try {
|
|
1839
|
+
return { value: await probe(), available: true };
|
|
1840
|
+
} catch {
|
|
1841
|
+
return { value: fallback, available: false };
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
984
1844
|
async function runPerformanceBench(model, options = {}) {
|
|
985
1845
|
const spinner = createSpinner("Warming up model...");
|
|
986
1846
|
spinner.start();
|
|
@@ -989,21 +1849,39 @@ async function runPerformanceBench(model, options = {}) {
|
|
|
989
1849
|
const promptTimeoutMs = options.promptTimeoutMs ?? DEFAULT_PROMPT_TIMEOUT_MS;
|
|
990
1850
|
const minSuccessfulPrompts = options.minSuccessfulPrompts ?? Math.max(1, Math.ceil(BENCH_PROMPTS.length / 2));
|
|
991
1851
|
const failOnPromptError = options.failOnPromptError ?? false;
|
|
992
|
-
const memBefore = await
|
|
1852
|
+
const [memBefore, thermalBefore, swapBeforeResult, batteryPowered] = await Promise.all([
|
|
1853
|
+
getMemoryUsage(),
|
|
1854
|
+
optionalProbe(() => detectThermalPressure(), "unknown"),
|
|
1855
|
+
optionalProbeWithAvailability(() => getSwapUsedGB(), 0),
|
|
1856
|
+
optionalProbe(() => detectBatteryPowered(), void 0)
|
|
1857
|
+
]);
|
|
993
1858
|
const warmup = await withTimeout(
|
|
994
|
-
|
|
1859
|
+
generateStream3(model, WARMUP_PROMPT, void 0, {
|
|
995
1860
|
num_predict: 32,
|
|
996
|
-
think: options.think
|
|
1861
|
+
think: options.think,
|
|
1862
|
+
stall_timeout_ms: options.streamStallTimeoutMs
|
|
997
1863
|
}),
|
|
998
1864
|
warmupTimeoutMs,
|
|
999
1865
|
"Model warmup",
|
|
1000
|
-
|
|
1866
|
+
abortOngoingRequests3
|
|
1001
1867
|
);
|
|
1868
|
+
const runtimeName = getRuntimeName();
|
|
1869
|
+
const loadTimeAvailable = !(runtimeName === "lm-studio" && warmup.loadDuration === 0);
|
|
1002
1870
|
const loadTime = warmup.loadDuration / 1e6;
|
|
1003
|
-
const runningModels = await
|
|
1871
|
+
const runningModels = await listRunningModels3();
|
|
1004
1872
|
const thisModel = runningModels.find((m) => m.name === model);
|
|
1873
|
+
let installedModelSizeBytes = 0;
|
|
1874
|
+
try {
|
|
1875
|
+
const availableModels = await listModels3();
|
|
1876
|
+
const listedModel = availableModels.find((m) => m.name === model);
|
|
1877
|
+
if (listedModel && Number.isFinite(listedModel.size) && listedModel.size > 0) {
|
|
1878
|
+
installedModelSizeBytes = listedModel.size;
|
|
1879
|
+
}
|
|
1880
|
+
} catch {
|
|
1881
|
+
}
|
|
1005
1882
|
spinner.succeed("Model loaded");
|
|
1006
1883
|
const tpsValues = [];
|
|
1884
|
+
const firstChunkValues = [];
|
|
1007
1885
|
const ttftValues = [];
|
|
1008
1886
|
let totalPromptTokens = 0;
|
|
1009
1887
|
let totalCompletionTokens = 0;
|
|
@@ -1013,33 +1891,47 @@ async function runPerformanceBench(model, options = {}) {
|
|
|
1013
1891
|
let failedPrompts = 0;
|
|
1014
1892
|
let thinkingDetected = false;
|
|
1015
1893
|
let totalThinkingTokens = 0;
|
|
1894
|
+
const cpuLoadSamples = [];
|
|
1016
1895
|
for (let i = 0; i < BENCH_PROMPTS.length; i++) {
|
|
1017
1896
|
spinner.start(`Running performance test ${i + 1}/${BENCH_PROMPTS.length}...`);
|
|
1897
|
+
let firstChunkTime = null;
|
|
1018
1898
|
let firstTokenTime = null;
|
|
1019
1899
|
const startTime = Date.now();
|
|
1020
1900
|
try {
|
|
1021
1901
|
const result = await withTimeout(
|
|
1022
|
-
|
|
1902
|
+
generateStream3(
|
|
1023
1903
|
model,
|
|
1024
1904
|
BENCH_PROMPTS[i],
|
|
1025
1905
|
{
|
|
1906
|
+
onFirstChunk: () => {
|
|
1907
|
+
if (firstChunkTime === null) {
|
|
1908
|
+
firstChunkTime = Date.now() - startTime;
|
|
1909
|
+
}
|
|
1910
|
+
},
|
|
1026
1911
|
onToken: () => {
|
|
1027
1912
|
if (firstTokenTime === null) {
|
|
1028
1913
|
firstTokenTime = Date.now() - startTime;
|
|
1029
1914
|
}
|
|
1030
1915
|
}
|
|
1031
1916
|
},
|
|
1032
|
-
{
|
|
1917
|
+
{
|
|
1918
|
+
num_predict: 256,
|
|
1919
|
+
think: options.think,
|
|
1920
|
+
stall_timeout_ms: options.streamStallTimeoutMs
|
|
1921
|
+
}
|
|
1033
1922
|
),
|
|
1034
1923
|
promptTimeoutMs,
|
|
1035
1924
|
"Performance benchmark",
|
|
1036
|
-
|
|
1925
|
+
abortOngoingRequests3
|
|
1037
1926
|
);
|
|
1038
1927
|
const evalDurationSec = result.evalDuration / 1e9;
|
|
1039
1928
|
const tps = evalDurationSec > 0 ? result.evalCount / evalDurationSec : 0;
|
|
1040
1929
|
tpsValues.push(tps);
|
|
1041
1930
|
totalEvalCount += result.evalCount;
|
|
1042
1931
|
totalEvalDurationNs += result.evalDuration;
|
|
1932
|
+
if (firstChunkTime !== null) {
|
|
1933
|
+
firstChunkValues.push(firstChunkTime);
|
|
1934
|
+
}
|
|
1043
1935
|
if (firstTokenTime !== null) {
|
|
1044
1936
|
ttftValues.push(firstTokenTime);
|
|
1045
1937
|
}
|
|
@@ -1052,8 +1944,10 @@ async function runPerformanceBench(model, options = {}) {
|
|
|
1052
1944
|
totalCompletionTokens += result.evalCount;
|
|
1053
1945
|
successfulPrompts++;
|
|
1054
1946
|
subStep(
|
|
1055
|
-
` Prompt ${i + 1}: ${tps.toFixed(1)} tok/s, TTFT ${firstTokenTime ?? "?"}ms`
|
|
1947
|
+
` Prompt ${i + 1}: ${tps.toFixed(1)} tok/s, first chunk ${firstChunkTime ?? "?"}ms, TTFT ${firstTokenTime ?? "?"}ms`
|
|
1056
1948
|
);
|
|
1949
|
+
const cpuSample = await optionalProbe(() => getCpuLoad(), -1);
|
|
1950
|
+
if (cpuSample >= 0) cpuLoadSamples.push(cpuSample);
|
|
1057
1951
|
} catch (err) {
|
|
1058
1952
|
failedPrompts++;
|
|
1059
1953
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -1073,24 +1967,43 @@ async function runPerformanceBench(model, options = {}) {
|
|
|
1073
1967
|
` Completed with ${failedPrompts} failed prompt(s); metrics use ${successfulPrompts} successful prompt(s).`
|
|
1074
1968
|
);
|
|
1075
1969
|
}
|
|
1076
|
-
const memAfter = await
|
|
1970
|
+
const [memAfter, thermalAfter, swapAfterResult] = await Promise.all([
|
|
1971
|
+
getMemoryUsage(),
|
|
1972
|
+
optionalProbe(() => detectThermalPressure(), thermalBefore),
|
|
1973
|
+
optionalProbeWithAvailability(() => getSwapUsedGB(), swapBeforeResult.value)
|
|
1974
|
+
]);
|
|
1077
1975
|
let memoryUsedGB;
|
|
1078
1976
|
let memoryPercent;
|
|
1079
|
-
|
|
1080
|
-
|
|
1977
|
+
const loadedModelSizeBytes = thisModel && thisModel.size > 0 ? thisModel.size : installedModelSizeBytes;
|
|
1978
|
+
if (loadedModelSizeBytes > 0) {
|
|
1979
|
+
memoryUsedGB = loadedModelSizeBytes / 1024 ** 3;
|
|
1081
1980
|
memoryPercent = memoryUsedGB / memAfter.totalGB * 100;
|
|
1082
1981
|
} else {
|
|
1083
1982
|
memoryUsedGB = Math.max(0, memAfter.usedGB - memBefore.usedGB);
|
|
1084
1983
|
memoryPercent = Math.max(0, memAfter.percent - memBefore.percent);
|
|
1085
1984
|
}
|
|
1086
1985
|
spinner.succeed("Performance benchmark complete");
|
|
1986
|
+
const firstChunkMs = firstChunkValues.length > 0 ? avg(firstChunkValues) : void 0;
|
|
1087
1987
|
const ttft = ttftValues.length > 0 ? avg(ttftValues) : -1;
|
|
1988
|
+
const swapDeltaGB = swapBeforeResult.available && swapAfterResult.available ? +(swapAfterResult.value - swapBeforeResult.value).toFixed(2) : void 0;
|
|
1989
|
+
const cpuAvgLoad = cpuLoadSamples.length > 0 ? +(cpuLoadSamples.reduce((a, b) => a + b, 0) / cpuLoadSamples.length).toFixed(1) : void 0;
|
|
1990
|
+
const cpuPeakLoad = cpuLoadSamples.length > 0 ? +Math.max(...cpuLoadSamples).toFixed(1) : void 0;
|
|
1991
|
+
const benchEnvironment = {
|
|
1992
|
+
thermalPressureBefore: thermalBefore,
|
|
1993
|
+
thermalPressureAfter: thermalAfter,
|
|
1994
|
+
...swapDeltaGB !== void 0 && swapDeltaGB > 0 ? { swapDeltaGB } : {},
|
|
1995
|
+
...batteryPowered != null ? { batteryPowered } : {},
|
|
1996
|
+
...cpuAvgLoad !== void 0 ? { cpuAvgLoad } : {},
|
|
1997
|
+
...cpuPeakLoad !== void 0 ? { cpuPeakLoad } : {}
|
|
1998
|
+
};
|
|
1088
1999
|
return {
|
|
1089
2000
|
metrics: {
|
|
1090
2001
|
tokensPerSecond: totalEvalDurationNs > 0 ? totalEvalCount / (totalEvalDurationNs / 1e9) : avg(tpsValues),
|
|
2002
|
+
...firstChunkMs !== void 0 ? { firstChunkMs } : {},
|
|
1091
2003
|
ttft: ttft >= 0 ? ttft : 3e4,
|
|
1092
2004
|
// Fallback: 30s if no TTFT measured
|
|
1093
2005
|
loadTime,
|
|
2006
|
+
loadTimeAvailable,
|
|
1094
2007
|
totalTokens: totalPromptTokens + totalCompletionTokens,
|
|
1095
2008
|
promptTokens: totalPromptTokens,
|
|
1096
2009
|
completionTokens: totalCompletionTokens,
|
|
@@ -1101,7 +2014,8 @@ async function runPerformanceBench(model, options = {}) {
|
|
|
1101
2014
|
tpsStdDev: tpsValues.length >= 2 ? stddev(tpsValues) : void 0,
|
|
1102
2015
|
...totalThinkingTokens > 0 ? { thinkingTokensEstimate: totalThinkingTokens } : {}
|
|
1103
2016
|
},
|
|
1104
|
-
thinkingDetected
|
|
2017
|
+
thinkingDetected,
|
|
2018
|
+
benchEnvironment
|
|
1105
2019
|
};
|
|
1106
2020
|
} catch (err) {
|
|
1107
2021
|
if (spinner.isSpinning) {
|
|
@@ -1467,11 +2381,13 @@ var reasoning_default = [
|
|
|
1467
2381
|
|
|
1468
2382
|
// ../src/benchmarks/reasoning.ts
|
|
1469
2383
|
var questions = reasoning_default;
|
|
2384
|
+
var DEFAULT_REASONING_TIMEOUT_MS = 12e4;
|
|
1470
2385
|
async function runReasoningBench(model, opts) {
|
|
1471
2386
|
const spinner = createSpinner("Running reasoning benchmark...");
|
|
1472
2387
|
spinner.start();
|
|
1473
2388
|
const details = [];
|
|
1474
2389
|
let correct = 0;
|
|
2390
|
+
const timeoutMs = opts?.timeoutMs ?? DEFAULT_REASONING_TIMEOUT_MS;
|
|
1475
2391
|
try {
|
|
1476
2392
|
for (let i = 0; i < questions.length; i++) {
|
|
1477
2393
|
const q = questions[i];
|
|
@@ -1485,10 +2401,10 @@ Answer:`;
|
|
|
1485
2401
|
const startTime = Date.now();
|
|
1486
2402
|
try {
|
|
1487
2403
|
const result = await withTimeout(
|
|
1488
|
-
|
|
1489
|
-
|
|
2404
|
+
generate3(model, prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
|
|
2405
|
+
timeoutMs,
|
|
1490
2406
|
"Reasoning question",
|
|
1491
|
-
|
|
2407
|
+
abortOngoingRequests3
|
|
1492
2408
|
);
|
|
1493
2409
|
const answer = stripThinkTags(result.response);
|
|
1494
2410
|
const actual = extractChoice(answer) ?? answer.trim();
|
|
@@ -1795,11 +2711,13 @@ var math_default = [
|
|
|
1795
2711
|
|
|
1796
2712
|
// ../src/benchmarks/math.ts
|
|
1797
2713
|
var problems = math_default;
|
|
2714
|
+
var DEFAULT_MATH_TIMEOUT_MS = 12e4;
|
|
1798
2715
|
async function runMathBench(model, opts) {
|
|
1799
2716
|
const spinner = createSpinner("Running math benchmark...");
|
|
1800
2717
|
spinner.start();
|
|
1801
2718
|
const details = [];
|
|
1802
2719
|
let correct = 0;
|
|
2720
|
+
const timeoutMs = opts?.timeoutMs ?? DEFAULT_MATH_TIMEOUT_MS;
|
|
1803
2721
|
try {
|
|
1804
2722
|
for (let i = 0; i < problems.length; i++) {
|
|
1805
2723
|
const p = problems[i];
|
|
@@ -1812,10 +2730,10 @@ Answer:`;
|
|
|
1812
2730
|
const startTime = Date.now();
|
|
1813
2731
|
try {
|
|
1814
2732
|
const result = await withTimeout(
|
|
1815
|
-
|
|
1816
|
-
|
|
2733
|
+
generate3(model, prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
|
|
2734
|
+
timeoutMs,
|
|
1817
2735
|
"Math problem",
|
|
1818
|
-
|
|
2736
|
+
abortOngoingRequests3
|
|
1819
2737
|
);
|
|
1820
2738
|
const actual = extractNumber(stripThinkTags(result.response));
|
|
1821
2739
|
const tolerance = p.tolerance ?? 0;
|
|
@@ -5366,6 +6284,7 @@ var DIFFICULTY_WEIGHT = {
|
|
|
5366
6284
|
var SANDBOX_TIMEOUT_MS = 5e3;
|
|
5367
6285
|
var ISOLATED_WALL_TIMEOUT_MIN_MS = 8e3;
|
|
5368
6286
|
var ISOLATED_WALL_TIMEOUT_MAX_MS = 6e4;
|
|
6287
|
+
var DEFAULT_CODING_TIMEOUT_MS = 24e4;
|
|
5369
6288
|
var MAX_SANDBOX_STDOUT_BYTES = 64 * 1024;
|
|
5370
6289
|
var MAX_SANDBOX_STDERR_BYTES = 64 * 1024;
|
|
5371
6290
|
function runTestsInSandbox(code, task, sandboxTimeoutMs, vmModule) {
|
|
@@ -5668,6 +6587,7 @@ async function runCodingBench(model, opts) {
|
|
|
5668
6587
|
const details = [];
|
|
5669
6588
|
let totalPassed = 0;
|
|
5670
6589
|
let totalTests = 0;
|
|
6590
|
+
const timeoutMs = opts?.timeoutMs ?? DEFAULT_CODING_TIMEOUT_MS;
|
|
5671
6591
|
try {
|
|
5672
6592
|
for (let i = 0; i < tasks.length; i++) {
|
|
5673
6593
|
const task = tasks[i];
|
|
@@ -5693,10 +6613,10 @@ Reply with ONLY the function code, no explanation.`;
|
|
|
5693
6613
|
const startTime = Date.now();
|
|
5694
6614
|
try {
|
|
5695
6615
|
const result = await withTimeout(
|
|
5696
|
-
|
|
5697
|
-
|
|
6616
|
+
generate3(model, prompt, { temperature: 0, num_predict: 2048, think: opts?.think }),
|
|
6617
|
+
timeoutMs,
|
|
5698
6618
|
"Coding task",
|
|
5699
|
-
|
|
6619
|
+
abortOngoingRequests3
|
|
5700
6620
|
);
|
|
5701
6621
|
const rawCode = extractCodeBlock(stripThinkTags(result.response), task.functionName);
|
|
5702
6622
|
const code = stripTypeAnnotations(rawCode);
|
|
@@ -5893,6 +6813,7 @@ var instruction_following_default = [
|
|
|
5893
6813
|
|
|
5894
6814
|
// ../src/benchmarks/instruction-following.ts
|
|
5895
6815
|
var questions2 = instruction_following_default;
|
|
6816
|
+
var DEFAULT_INSTRUCTION_FOLLOWING_TIMEOUT_MS = 12e4;
|
|
5896
6817
|
function escapeRegex(value) {
|
|
5897
6818
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
5898
6819
|
}
|
|
@@ -6038,6 +6959,7 @@ async function runInstructionFollowingBench(model, opts) {
|
|
|
6038
6959
|
spinner.start();
|
|
6039
6960
|
const details = [];
|
|
6040
6961
|
let correct = 0;
|
|
6962
|
+
const timeoutMs = opts?.timeoutMs ?? DEFAULT_INSTRUCTION_FOLLOWING_TIMEOUT_MS;
|
|
6041
6963
|
try {
|
|
6042
6964
|
for (let i = 0; i < questions2.length; i++) {
|
|
6043
6965
|
const q = questions2[i];
|
|
@@ -6046,10 +6968,10 @@ async function runInstructionFollowingBench(model, opts) {
|
|
|
6046
6968
|
const startTime = Date.now();
|
|
6047
6969
|
try {
|
|
6048
6970
|
const result = await withTimeout(
|
|
6049
|
-
|
|
6050
|
-
|
|
6971
|
+
generate3(model, prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
|
|
6972
|
+
timeoutMs,
|
|
6051
6973
|
"Instruction following task",
|
|
6052
|
-
|
|
6974
|
+
abortOngoingRequests3
|
|
6053
6975
|
);
|
|
6054
6976
|
const answer = stripThinkTags(result.response);
|
|
6055
6977
|
const isCorrect = validateInstructionFollowingResponse(answer, q);
|
|
@@ -6197,6 +7119,7 @@ var structured_output_default = [
|
|
|
6197
7119
|
|
|
6198
7120
|
// ../src/benchmarks/structured-output.ts
|
|
6199
7121
|
var questions3 = structured_output_default;
|
|
7122
|
+
var DEFAULT_STRUCTURED_OUTPUT_TIMEOUT_MS = 12e4;
|
|
6200
7123
|
function tryParseJson(text) {
|
|
6201
7124
|
try {
|
|
6202
7125
|
return { ok: true, value: JSON.parse(text) };
|
|
@@ -6225,9 +7148,9 @@ function checkType(value, expectedType) {
|
|
|
6225
7148
|
if (expectedType === "object") return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
6226
7149
|
return typeof value === expectedType;
|
|
6227
7150
|
}
|
|
6228
|
-
function getNestedValue(obj,
|
|
7151
|
+
function getNestedValue(obj, path2) {
|
|
6229
7152
|
let current = obj;
|
|
6230
|
-
for (const key of
|
|
7153
|
+
for (const key of path2) {
|
|
6231
7154
|
if (typeof current !== "object" || current === null) return void 0;
|
|
6232
7155
|
current = current[key];
|
|
6233
7156
|
}
|
|
@@ -6303,8 +7226,8 @@ function validateStructuredOutputResponse(response, q) {
|
|
|
6303
7226
|
const parsed = safeParse(response);
|
|
6304
7227
|
if (!parsed) return false;
|
|
6305
7228
|
const paths = ["path1", "path2", "path3", "path4"].filter((k) => k in p).map((k) => p[k]);
|
|
6306
|
-
return paths.every((
|
|
6307
|
-
const val = getNestedValue(parsed,
|
|
7229
|
+
return paths.every((path2) => {
|
|
7230
|
+
const val = getNestedValue(parsed, path2);
|
|
6308
7231
|
return val !== void 0 && val !== null;
|
|
6309
7232
|
});
|
|
6310
7233
|
}
|
|
@@ -6423,6 +7346,7 @@ async function runStructuredOutputBench(model, opts) {
|
|
|
6423
7346
|
spinner.start();
|
|
6424
7347
|
const details = [];
|
|
6425
7348
|
let correct = 0;
|
|
7349
|
+
const timeoutMs = opts?.timeoutMs ?? DEFAULT_STRUCTURED_OUTPUT_TIMEOUT_MS;
|
|
6426
7350
|
try {
|
|
6427
7351
|
for (let i = 0; i < questions3.length; i++) {
|
|
6428
7352
|
const q = questions3[i];
|
|
@@ -6430,10 +7354,10 @@ async function runStructuredOutputBench(model, opts) {
|
|
|
6430
7354
|
const startTime = Date.now();
|
|
6431
7355
|
try {
|
|
6432
7356
|
const result = await withTimeout(
|
|
6433
|
-
|
|
6434
|
-
|
|
7357
|
+
generate3(model, q.prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
|
|
7358
|
+
timeoutMs,
|
|
6435
7359
|
"Structured output task",
|
|
6436
|
-
|
|
7360
|
+
abortOngoingRequests3
|
|
6437
7361
|
);
|
|
6438
7362
|
const answer = stripThinkTags(result.response);
|
|
6439
7363
|
const isCorrect = validateStructuredOutputResponse(answer, q);
|
|
@@ -6616,6 +7540,7 @@ var multilingual_default = [
|
|
|
6616
7540
|
|
|
6617
7541
|
// ../src/benchmarks/multilingual.ts
|
|
6618
7542
|
var questions4 = multilingual_default;
|
|
7543
|
+
var DEFAULT_MULTILINGUAL_TIMEOUT_MS = 12e4;
|
|
6619
7544
|
var NEGATION_PATTERNS = [
|
|
6620
7545
|
/\b(?:not|never|no|non|pas|nicht|kein|keine|ningun|ninguna|nunca|jamas)\b/u,
|
|
6621
7546
|
/\bn'?est\s+pas\b/u,
|
|
@@ -6680,6 +7605,7 @@ async function runMultilingualBench(model, opts) {
|
|
|
6680
7605
|
spinner.start();
|
|
6681
7606
|
const details = [];
|
|
6682
7607
|
let correct = 0;
|
|
7608
|
+
const timeoutMs = opts?.timeoutMs ?? DEFAULT_MULTILINGUAL_TIMEOUT_MS;
|
|
6683
7609
|
try {
|
|
6684
7610
|
for (let i = 0; i < questions4.length; i++) {
|
|
6685
7611
|
const q = questions4[i];
|
|
@@ -6687,10 +7613,10 @@ async function runMultilingualBench(model, opts) {
|
|
|
6687
7613
|
const startTime = Date.now();
|
|
6688
7614
|
try {
|
|
6689
7615
|
const result = await withTimeout(
|
|
6690
|
-
|
|
6691
|
-
|
|
7616
|
+
generate3(model, q.prompt, { temperature: 0, num_predict: 1024, think: opts?.think }),
|
|
7617
|
+
timeoutMs,
|
|
6692
7618
|
"Multilingual task",
|
|
6693
|
-
|
|
7619
|
+
abortOngoingRequests3
|
|
6694
7620
|
);
|
|
6695
7621
|
const answer = stripThinkTags(result.response);
|
|
6696
7622
|
const isCorrect = validateMultilingualResponse(answer, q);
|
|
@@ -6793,24 +7719,24 @@ function deriveHardwareFitTuning(hardware) {
|
|
|
6793
7719
|
};
|
|
6794
7720
|
}
|
|
6795
7721
|
function scoreSpeed(tps, tuning) {
|
|
6796
|
-
if (tps >= tuning.speed.excellent) return
|
|
7722
|
+
if (tps >= tuning.speed.excellent) return 50;
|
|
6797
7723
|
if (tps >= tuning.speed.good) {
|
|
6798
|
-
return lerp(tps, tuning.speed.good, tuning.speed.excellent,
|
|
7724
|
+
return lerp(tps, tuning.speed.good, tuning.speed.excellent, 30, 50);
|
|
6799
7725
|
}
|
|
6800
7726
|
if (tps >= tuning.speed.marginal) {
|
|
6801
|
-
return lerp(tps, tuning.speed.marginal, tuning.speed.good,
|
|
7727
|
+
return lerp(tps, tuning.speed.marginal, tuning.speed.good, 12, 30);
|
|
6802
7728
|
}
|
|
6803
|
-
return lerp(tps, 0, tuning.speed.marginal, 0,
|
|
7729
|
+
return lerp(tps, 0, tuning.speed.marginal, 0, 12);
|
|
6804
7730
|
}
|
|
6805
7731
|
function scoreTTFT(ttft, tuning) {
|
|
6806
|
-
if (ttft <= tuning.ttft.excellentMs) return
|
|
7732
|
+
if (ttft <= tuning.ttft.excellentMs) return 20;
|
|
6807
7733
|
if (ttft <= tuning.ttft.goodMs) {
|
|
6808
|
-
return lerp(ttft, tuning.ttft.excellentMs, tuning.ttft.goodMs,
|
|
7734
|
+
return lerp(ttft, tuning.ttft.excellentMs, tuning.ttft.goodMs, 20, 13);
|
|
6809
7735
|
}
|
|
6810
7736
|
if (ttft <= tuning.ttft.marginalMs) {
|
|
6811
|
-
return lerp(ttft, tuning.ttft.goodMs, tuning.ttft.marginalMs,
|
|
7737
|
+
return lerp(ttft, tuning.ttft.goodMs, tuning.ttft.marginalMs, 13, 6);
|
|
6812
7738
|
}
|
|
6813
|
-
return lerp(ttft, tuning.ttft.marginalMs, tuning.ttft.hardMaxMs,
|
|
7739
|
+
return lerp(ttft, tuning.ttft.marginalMs, tuning.ttft.hardMaxMs, 6, 0);
|
|
6814
7740
|
}
|
|
6815
7741
|
function scoreMemory(memPercent) {
|
|
6816
7742
|
if (memPercent <= 30) return 30;
|
|
@@ -6918,12 +7844,12 @@ function normalizeRawScore(value) {
|
|
|
6918
7844
|
if (!Number.isFinite(value)) return 0;
|
|
6919
7845
|
return clamp(value, 0, 100);
|
|
6920
7846
|
}
|
|
6921
|
-
function computeFitness(perf, quality, hardware) {
|
|
7847
|
+
function computeFitness(perf, quality, hardware, benchEnv) {
|
|
6922
7848
|
const tuning = deriveHardwareFitTuning(hardware);
|
|
6923
7849
|
const performanceScore = computePerformanceScore(perf, hardware);
|
|
6924
7850
|
const qualityScore = quality ? computeQualityScore(quality) : null;
|
|
6925
7851
|
const hardwareFitScore = Math.round(performanceScore.total);
|
|
6926
|
-
const globalScore = qualityScore ? clamp(Math.round(0.
|
|
7852
|
+
const globalScore = qualityScore ? clamp(Math.round(0.3 * hardwareFitScore + 0.7 * qualityScore.total), 0, 100) : null;
|
|
6927
7853
|
const categoryLabels = quality ? [
|
|
6928
7854
|
{ category: "Reasoning", key: "reasoning", result: quality.reasoning },
|
|
6929
7855
|
{ category: "Coding", key: "coding", result: quality.coding },
|
|
@@ -7015,6 +7941,26 @@ function computeFitness(perf, quality, hardware) {
|
|
|
7015
7941
|
`CPU appears throttled (${hardware.cpuCurrentSpeedGHz.toFixed(1)} GHz current vs ${hardware.cpuFreqGHz.toFixed(1)} GHz nominal, ${ratio}%).`
|
|
7016
7942
|
);
|
|
7017
7943
|
}
|
|
7944
|
+
if (benchEnv?.thermalPressureAfter === "heavy" || benchEnv?.thermalPressureAfter === "critical") {
|
|
7945
|
+
warnings.push(
|
|
7946
|
+
`Thermal throttling detected after benchmark (${benchEnv.thermalPressureAfter}). Results may be degraded \u2014 consider cooling or shorter runs.`
|
|
7947
|
+
);
|
|
7948
|
+
}
|
|
7949
|
+
if (benchEnv?.swapDeltaGB != null && benchEnv.swapDeltaGB > 0.5) {
|
|
7950
|
+
warnings.push(
|
|
7951
|
+
`Significant swap activity during benchmark (+${benchEnv.swapDeltaGB.toFixed(1)} GB). Model may exceed available RAM \u2014 results are severely degraded.`
|
|
7952
|
+
);
|
|
7953
|
+
}
|
|
7954
|
+
if (benchEnv?.batteryPowered) {
|
|
7955
|
+
warnings.push(
|
|
7956
|
+
`Running on battery power \u2014 performance may be reduced.`
|
|
7957
|
+
);
|
|
7958
|
+
}
|
|
7959
|
+
if (benchEnv?.cpuAvgLoad != null && benchEnv.cpuAvgLoad > 90) {
|
|
7960
|
+
warnings.push(
|
|
7961
|
+
`High CPU load during inference (avg ${benchEnv.cpuAvgLoad.toFixed(0)}%). System may feel unresponsive \u2014 GPU-accelerated runtimes (MLX) can reduce CPU pressure.`
|
|
7962
|
+
);
|
|
7963
|
+
}
|
|
7018
7964
|
return {
|
|
7019
7965
|
verdict,
|
|
7020
7966
|
globalScore,
|
|
@@ -7101,7 +8047,7 @@ function printHardwareTable(hw) {
|
|
|
7101
8047
|
}
|
|
7102
8048
|
console.log(table.toString());
|
|
7103
8049
|
}
|
|
7104
|
-
function printPerformanceTable(perf) {
|
|
8050
|
+
function printPerformanceTable(perf, benchEnvironment) {
|
|
7105
8051
|
const table = new Table({
|
|
7106
8052
|
head: [chalk3.bold("Metric"), chalk3.bold("Value")],
|
|
7107
8053
|
style: { head: [], border: [] }
|
|
@@ -7111,8 +8057,15 @@ function printPerformanceTable(perf) {
|
|
|
7111
8057
|
const memColor = perf.memoryPercent < 50 ? chalk3.green : perf.memoryPercent < 80 ? chalk3.yellow : chalk3.red;
|
|
7112
8058
|
table.push(
|
|
7113
8059
|
["Tokens/sec", tpsColor(`${perf.tokensPerSecond.toFixed(1)} tok/s`)],
|
|
8060
|
+
[
|
|
8061
|
+
"First Chunk Latency",
|
|
8062
|
+
perf.firstChunkMs !== void 0 ? formatDuration(perf.firstChunkMs) : chalk3.dim("N/A (stream metric unavailable)")
|
|
8063
|
+
],
|
|
7114
8064
|
["Time to First Token", ttftColor(formatDuration(perf.ttft))],
|
|
7115
|
-
[
|
|
8065
|
+
[
|
|
8066
|
+
"Model Load Time",
|
|
8067
|
+
perf.loadTimeAvailable === false ? chalk3.dim("N/A (runtime metric unavailable)") : formatDuration(perf.loadTime)
|
|
8068
|
+
],
|
|
7116
8069
|
["Total Tokens", String(perf.totalTokens)],
|
|
7117
8070
|
["Prompt Tokens", String(perf.promptTokens)],
|
|
7118
8071
|
["Completion Tokens", String(perf.completionTokens)],
|
|
@@ -7133,6 +8086,13 @@ function printPerformanceTable(perf) {
|
|
|
7133
8086
|
chalk3.magenta(`~${perf.thinkingTokensEstimate} tokens`)
|
|
7134
8087
|
]);
|
|
7135
8088
|
}
|
|
8089
|
+
if (benchEnvironment?.cpuAvgLoad != null && benchEnvironment.cpuAvgLoad >= 0) {
|
|
8090
|
+
const cpuColor = benchEnvironment.cpuAvgLoad < 50 ? chalk3.green : benchEnvironment.cpuAvgLoad < 80 ? chalk3.yellow : chalk3.red;
|
|
8091
|
+
table.push([
|
|
8092
|
+
"CPU Load During Bench",
|
|
8093
|
+
cpuColor(`avg ${benchEnvironment.cpuAvgLoad.toFixed(0)}%` + (benchEnvironment.cpuPeakLoad != null ? ` (peak ${benchEnvironment.cpuPeakLoad.toFixed(0)}%)` : ""))
|
|
8094
|
+
]);
|
|
8095
|
+
}
|
|
7136
8096
|
console.log(table.toString());
|
|
7137
8097
|
}
|
|
7138
8098
|
function printQualityTable(quality, timePenalties) {
|
|
@@ -7179,7 +8139,7 @@ function printSummaryTable(results) {
|
|
|
7179
8139
|
const compact = termWidth < 100;
|
|
7180
8140
|
console.log(
|
|
7181
8141
|
chalk3.dim(
|
|
7182
|
-
"Global =
|
|
8142
|
+
"Global = 30% Hardware Fit + 70% Quality. Hardware Fit = host compatibility. Quality = model capability."
|
|
7183
8143
|
)
|
|
7184
8144
|
);
|
|
7185
8145
|
const head = [
|
|
@@ -7362,7 +8322,7 @@ function printVerdict(model, fitness) {
|
|
|
7362
8322
|
);
|
|
7363
8323
|
sectionText(`Active profile: ${fitness.tuning.profile}`, fitBorder, chalk4.white);
|
|
7364
8324
|
sectionText(
|
|
7365
|
-
"Formula: Speed (/
|
|
8325
|
+
"Formula: Speed (/50) + TTFT (/20) + Memory (/30) = Hardware Fit (/100).",
|
|
7366
8326
|
fitBorder
|
|
7367
8327
|
);
|
|
7368
8328
|
sectionText(
|
|
@@ -7380,7 +8340,7 @@ function printVerdict(model, fitness) {
|
|
|
7380
8340
|
sectionText("", fitBorder);
|
|
7381
8341
|
scoreRow("Performance", fitness.performanceScore.total, fitBorder);
|
|
7382
8342
|
sectionText(
|
|
7383
|
-
` Breakdown: Speed ${fitness.performanceScore.speed}/
|
|
8343
|
+
` Breakdown: Speed ${fitness.performanceScore.speed}/50 TTFT ${fitness.performanceScore.ttft}/20 Memory ${fitness.performanceScore.memory}/30`,
|
|
7384
8344
|
fitBorder
|
|
7385
8345
|
);
|
|
7386
8346
|
scoreRow("Hardware Fit", fitness.hardwareFitScore, fitBorder);
|
|
@@ -7449,7 +8409,7 @@ function printVerdict(model, fitness) {
|
|
|
7449
8409
|
sectionStart("C) Global Score", globalBorder);
|
|
7450
8410
|
if (fitness.globalScore !== null) {
|
|
7451
8411
|
sectionText(
|
|
7452
|
-
"Formula: Global =
|
|
8412
|
+
"Formula: Global = 30% Hardware Fit + 70% Quality.",
|
|
7453
8413
|
globalBorder,
|
|
7454
8414
|
chalk4.white
|
|
7455
8415
|
);
|
|
@@ -7519,6 +8479,12 @@ var DEFAULT_CONFIG = {
|
|
|
7519
8479
|
async function ensureDirs() {
|
|
7520
8480
|
await mkdir(RESULTS_DIR, { recursive: true });
|
|
7521
8481
|
}
|
|
8482
|
+
function parseRuntimeBackend(value) {
|
|
8483
|
+
if (value === "ollama" || value === "lm-studio") {
|
|
8484
|
+
return value;
|
|
8485
|
+
}
|
|
8486
|
+
return void 0;
|
|
8487
|
+
}
|
|
7522
8488
|
function resultFilename(result) {
|
|
7523
8489
|
const ts = result.timestamp.replace(/[:.]/g, "-").replace("T", "_").replace(/[^0-9_-]/g, "");
|
|
7524
8490
|
const model = result.model.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
@@ -7540,7 +8506,15 @@ async function loadConfig() {
|
|
|
7540
8506
|
const telemetry = typeof parsed.telemetry === "boolean" ? parsed.telemetry : void 0;
|
|
7541
8507
|
const submitterNickname = typeof parsed.submitterNickname === "string" && isValidNickname(parsed.submitterNickname) ? normalizeNickname(parsed.submitterNickname) : void 0;
|
|
7542
8508
|
const submitterEmail = typeof parsed.submitterEmail === "string" && isValidEmail(parsed.submitterEmail) ? normalizeEmail(parsed.submitterEmail) : void 0;
|
|
7543
|
-
|
|
8509
|
+
const runtimeBackend = parseRuntimeBackend(parsed.runtimeBackend);
|
|
8510
|
+
return {
|
|
8511
|
+
...DEFAULT_CONFIG,
|
|
8512
|
+
autoShare,
|
|
8513
|
+
telemetry,
|
|
8514
|
+
submitterNickname,
|
|
8515
|
+
submitterEmail,
|
|
8516
|
+
runtimeBackend
|
|
8517
|
+
};
|
|
7544
8518
|
} catch {
|
|
7545
8519
|
return { ...DEFAULT_CONFIG };
|
|
7546
8520
|
}
|
|
@@ -7552,24 +8526,40 @@ async function saveConfig(config) {
|
|
|
7552
8526
|
|
|
7553
8527
|
// ../src/core/uploader.ts
|
|
7554
8528
|
import { createClient } from "@supabase/supabase-js";
|
|
8529
|
+
var SUPABASE_URL_DEFAULT = "https://phvvzbgasxobjzjnkewf.supabase.co";
|
|
8530
|
+
var SUPABASE_ANON_KEY_DEFAULT = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InBodnZ6Ymdhc3hvYmp6am5rZXdmIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NzIyOTMxNTksImV4cCI6MjA4Nzg2OTE1OX0.-Wn0V8f-Uv_x_krNUnjmFRcqfAOPQEVc_Qx1kZ2ppgk";
|
|
7555
8531
|
var SUPABASE_URL_PLACEHOLDER = "https://YOUR_SUPABASE_PROJECT.supabase.co";
|
|
8532
|
+
var SUPABASE_URL_PLACEHOLDER_ALT = "https://your-project.supabase.co";
|
|
7556
8533
|
var SUPABASE_ANON_KEY_PLACEHOLDER = "YOUR_SUPABASE_ANON_KEY";
|
|
8534
|
+
var SUPABASE_ANON_KEY_PLACEHOLDER_ALT = "your-supabase-anon-key";
|
|
8535
|
+
var PUBLIC_RESULT_BASE_URL_DEFAULT = "https://metrillm.dev";
|
|
7557
8536
|
var PUBLIC_RESULT_BASE_URL_PLACEHOLDER = "https://YOUR_DASHBOARD_DOMAIN";
|
|
8537
|
+
var PUBLIC_RESULT_BASE_URL_PLACEHOLDER_ALT = "https://your-dashboard-domain";
|
|
7558
8538
|
function resolveUploaderConfig() {
|
|
8539
|
+
const configuredPublicBaseUrl = process.env.METRILLM_PUBLIC_RESULT_BASE_URL?.trim();
|
|
8540
|
+
const publicResultBaseUrl = !configuredPublicBaseUrl || hasPlaceholder(configuredPublicBaseUrl) ? PUBLIC_RESULT_BASE_URL_DEFAULT : configuredPublicBaseUrl;
|
|
7559
8541
|
return {
|
|
7560
|
-
supabaseUrl: process.env.METRILLM_SUPABASE_URL
|
|
7561
|
-
supabaseAnonKey: process.env.METRILLM_SUPABASE_ANON_KEY
|
|
7562
|
-
publicResultBaseUrl
|
|
8542
|
+
supabaseUrl: !process.env.METRILLM_SUPABASE_URL || hasPlaceholder(process.env.METRILLM_SUPABASE_URL) ? SUPABASE_URL_DEFAULT : process.env.METRILLM_SUPABASE_URL,
|
|
8543
|
+
supabaseAnonKey: !process.env.METRILLM_SUPABASE_ANON_KEY || hasPlaceholder(process.env.METRILLM_SUPABASE_ANON_KEY) ? SUPABASE_ANON_KEY_DEFAULT : process.env.METRILLM_SUPABASE_ANON_KEY,
|
|
8544
|
+
publicResultBaseUrl
|
|
7563
8545
|
};
|
|
7564
8546
|
}
|
|
7565
8547
|
function hasPlaceholder(value) {
|
|
7566
|
-
|
|
8548
|
+
const normalized = value.trim();
|
|
8549
|
+
if (normalized.length === 0) return true;
|
|
8550
|
+
return [
|
|
8551
|
+
SUPABASE_URL_PLACEHOLDER,
|
|
8552
|
+
SUPABASE_URL_PLACEHOLDER_ALT,
|
|
8553
|
+
SUPABASE_ANON_KEY_PLACEHOLDER,
|
|
8554
|
+
SUPABASE_ANON_KEY_PLACEHOLDER_ALT,
|
|
8555
|
+
PUBLIC_RESULT_BASE_URL_PLACEHOLDER,
|
|
8556
|
+
PUBLIC_RESULT_BASE_URL_PLACEHOLDER_ALT
|
|
8557
|
+
].some((placeholder) => placeholder.toLowerCase() === normalized.toLowerCase());
|
|
7567
8558
|
}
|
|
7568
8559
|
function assertUploaderConfig(config) {
|
|
7569
8560
|
const missing = [];
|
|
7570
|
-
if (
|
|
7571
|
-
if (
|
|
7572
|
-
if (hasPlaceholder(config.publicResultBaseUrl)) missing.push("METRILLM_PUBLIC_RESULT_BASE_URL");
|
|
8561
|
+
if (!config.supabaseUrl.trim()) missing.push("METRILLM_SUPABASE_URL");
|
|
8562
|
+
if (!config.supabaseAnonKey.trim()) missing.push("METRILLM_SUPABASE_ANON_KEY");
|
|
7573
8563
|
if (missing.length > 0) {
|
|
7574
8564
|
throw new Error(
|
|
7575
8565
|
`Upload is not configured. Set these variables first: ${missing.join(", ")}`
|
|
@@ -8021,6 +9011,13 @@ async function promptThinkingMode() {
|
|
|
8021
9011
|
var BENCHMARK_SPEC_VERSION = "0.2.0";
|
|
8022
9012
|
var PROMPT_PACK_VERSION = "0.1.0";
|
|
8023
9013
|
async function benchCommand(options) {
|
|
9014
|
+
if (options.backend !== void 0) {
|
|
9015
|
+
setRuntimeByName(options.backend);
|
|
9016
|
+
}
|
|
9017
|
+
const runtimeName = getRuntimeName();
|
|
9018
|
+
const runtimeDisplayName = getRuntimeDisplayName(runtimeName);
|
|
9019
|
+
const runtimeModelHint = getRuntimeModelInstallHint(runtimeName);
|
|
9020
|
+
const runtimeSetupHints = getRuntimeSetupHints(runtimeName);
|
|
8024
9021
|
const shouldSetExitCode = options.setExitCode !== false;
|
|
8025
9022
|
const silent = options.json === true;
|
|
8026
9023
|
const shouldUnloadAfterModel = options.unloadAfterBench ?? (!options.model || options.ciNoMenu === true);
|
|
@@ -8053,7 +9050,7 @@ async function benchCommand(options) {
|
|
|
8053
9050
|
runtimeVersion = await getRuntimeVersion();
|
|
8054
9051
|
} catch (err) {
|
|
8055
9052
|
if (!silent) {
|
|
8056
|
-
warnMsg(
|
|
9053
|
+
warnMsg(`Could not detect ${runtimeDisplayName} version (continuing without it).`);
|
|
8057
9054
|
if (err instanceof Error) warnMsg(err.message);
|
|
8058
9055
|
}
|
|
8059
9056
|
}
|
|
@@ -8062,7 +9059,7 @@ async function benchCommand(options) {
|
|
|
8062
9059
|
if (options.model) {
|
|
8063
9060
|
modelNames = [options.model];
|
|
8064
9061
|
try {
|
|
8065
|
-
allModels = await
|
|
9062
|
+
allModels = await listModels3();
|
|
8066
9063
|
} catch {
|
|
8067
9064
|
}
|
|
8068
9065
|
} else {
|
|
@@ -8070,22 +9067,23 @@ async function benchCommand(options) {
|
|
|
8070
9067
|
const spinnerModels = createSpinner("Fetching model list...");
|
|
8071
9068
|
if (!silent) spinnerModels.start();
|
|
8072
9069
|
try {
|
|
8073
|
-
allModels = await
|
|
9070
|
+
allModels = await listModels3();
|
|
8074
9071
|
modelNames = allModels.map((m) => m.name);
|
|
8075
9072
|
if (!silent) spinnerModels.succeed(`Found ${allModels.length} model(s)`);
|
|
8076
9073
|
} catch (err) {
|
|
8077
9074
|
if (!silent) {
|
|
8078
|
-
spinnerModels.fail(
|
|
8079
|
-
errorMsg(
|
|
8080
|
-
|
|
8081
|
-
|
|
9075
|
+
spinnerModels.fail(`Cannot connect to ${runtimeDisplayName}`);
|
|
9076
|
+
errorMsg(`Make sure ${runtimeDisplayName} is installed and running.`);
|
|
9077
|
+
for (const hint of runtimeSetupHints) {
|
|
9078
|
+
errorMsg(` \u2022 ${hint}`);
|
|
9079
|
+
}
|
|
8082
9080
|
if (err instanceof Error) errorMsg(err.message);
|
|
8083
9081
|
}
|
|
8084
9082
|
if (shouldSetExitCode) process.exitCode = 1;
|
|
8085
9083
|
return { results: [], failedModels: [] };
|
|
8086
9084
|
}
|
|
8087
9085
|
if (modelNames.length === 0) {
|
|
8088
|
-
if (!silent) errorMsg(
|
|
9086
|
+
if (!silent) errorMsg(`No models found. ${runtimeModelHint}`);
|
|
8089
9087
|
if (shouldSetExitCode) process.exitCode = 1;
|
|
8090
9088
|
return { results: [], failedModels: [] };
|
|
8091
9089
|
}
|
|
@@ -8097,6 +9095,8 @@ async function benchCommand(options) {
|
|
|
8097
9095
|
thinkEnabled = options.thinking;
|
|
8098
9096
|
} else if (!silent && !options.ciNoMenu) {
|
|
8099
9097
|
thinkEnabled = await promptThinkingMode();
|
|
9098
|
+
} else {
|
|
9099
|
+
thinkEnabled = false;
|
|
8100
9100
|
}
|
|
8101
9101
|
if (!silent && thinkEnabled) {
|
|
8102
9102
|
infoMsg("Thinking mode enabled \u2014 models that support it will use extended reasoning.");
|
|
@@ -8129,29 +9129,38 @@ ${tl}${h.repeat(innerWidth)}${tr}`));
|
|
|
8129
9129
|
promptTimeoutMs: options.perfPromptTimeoutMs,
|
|
8130
9130
|
minSuccessfulPrompts: options.perfMinSuccessfulPrompts,
|
|
8131
9131
|
failOnPromptError: options.perfStrict,
|
|
8132
|
-
think: thinkEnabled
|
|
9132
|
+
think: thinkEnabled,
|
|
9133
|
+
streamStallTimeoutMs: options.lmStudioStreamStallTimeoutMs
|
|
8133
9134
|
});
|
|
8134
9135
|
const perf = perfResult.metrics;
|
|
8135
|
-
const
|
|
8136
|
-
if (!silent) printPerformanceTable(perf);
|
|
9136
|
+
const benchEnvironment = perfResult.benchEnvironment;
|
|
9137
|
+
if (!silent) printPerformanceTable(perf, benchEnvironment);
|
|
8137
9138
|
let quality = null;
|
|
8138
9139
|
if (!options.perfOnly) {
|
|
8139
|
-
const
|
|
9140
|
+
const qualityTimeoutMs = options.qualityTimeoutMs;
|
|
9141
|
+
const codingTimeoutMs = options.codingTimeoutMs ?? options.qualityTimeoutMs;
|
|
9142
|
+
const commonQualityOpts = {
|
|
9143
|
+
think: thinkEnabled,
|
|
9144
|
+
...qualityTimeoutMs !== void 0 ? { timeoutMs: qualityTimeoutMs } : {}
|
|
9145
|
+
};
|
|
8140
9146
|
if (!silent) stepHeader("Quality Benchmark \u2014 Reasoning");
|
|
8141
|
-
const reasoning = await runReasoningBench(modelName,
|
|
9147
|
+
const reasoning = await runReasoningBench(modelName, commonQualityOpts);
|
|
8142
9148
|
if (!silent) stepHeader("Quality Benchmark \u2014 Math");
|
|
8143
|
-
const math = await runMathBench(modelName,
|
|
9149
|
+
const math = await runMathBench(modelName, commonQualityOpts);
|
|
8144
9150
|
if (!silent) stepHeader("Quality Benchmark \u2014 Coding");
|
|
8145
|
-
const coding = await runCodingBench(modelName,
|
|
9151
|
+
const coding = await runCodingBench(modelName, {
|
|
9152
|
+
think: thinkEnabled,
|
|
9153
|
+
...codingTimeoutMs !== void 0 ? { timeoutMs: codingTimeoutMs } : {}
|
|
9154
|
+
});
|
|
8146
9155
|
if (!silent) stepHeader("Quality Benchmark \u2014 Instruction Following");
|
|
8147
|
-
const instructionFollowing = await runInstructionFollowingBench(modelName,
|
|
9156
|
+
const instructionFollowing = await runInstructionFollowingBench(modelName, commonQualityOpts);
|
|
8148
9157
|
if (!silent) stepHeader("Quality Benchmark \u2014 Structured Output");
|
|
8149
|
-
const structuredOutput = await runStructuredOutputBench(modelName,
|
|
9158
|
+
const structuredOutput = await runStructuredOutputBench(modelName, commonQualityOpts);
|
|
8150
9159
|
if (!silent) stepHeader("Quality Benchmark \u2014 Multilingual");
|
|
8151
|
-
const multilingual = await runMultilingualBench(modelName,
|
|
9160
|
+
const multilingual = await runMultilingualBench(modelName, commonQualityOpts);
|
|
8152
9161
|
quality = { reasoning, math, coding, instructionFollowing, structuredOutput, multilingual };
|
|
8153
9162
|
}
|
|
8154
|
-
const fitness = computeFitness(perf, quality, hardware);
|
|
9163
|
+
const fitness = computeFitness(perf, quality, hardware, benchEnvironment);
|
|
8155
9164
|
if (!silent) {
|
|
8156
9165
|
if (quality) {
|
|
8157
9166
|
printQualityTable(quality, fitness.qualityScore?.timePenalties);
|
|
@@ -8163,8 +9172,9 @@ ${tl}${h.repeat(innerWidth)}${tr}`));
|
|
|
8163
9172
|
parameterSize: matchedModel.parameterSize,
|
|
8164
9173
|
quantization: matchedModel.quantization,
|
|
8165
9174
|
family: matchedModel.family,
|
|
8166
|
-
|
|
8167
|
-
|
|
9175
|
+
// Persist the configured benchmark mode (not model auto-detection).
|
|
9176
|
+
thinkingDetected: thinkEnabled
|
|
9177
|
+
} : { thinkingDetected: thinkEnabled };
|
|
8168
9178
|
const partialResult = {
|
|
8169
9179
|
model: modelName,
|
|
8170
9180
|
modelInfo,
|
|
@@ -8172,13 +9182,14 @@ ${tl}${h.repeat(innerWidth)}${tr}`));
|
|
|
8172
9182
|
performance: perf,
|
|
8173
9183
|
quality,
|
|
8174
9184
|
fitness,
|
|
9185
|
+
benchEnvironment,
|
|
8175
9186
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8176
9187
|
metadata: {
|
|
8177
9188
|
benchmarkSpecVersion: BENCHMARK_SPEC_VERSION,
|
|
8178
9189
|
promptPackVersion: PROMPT_PACK_VERSION,
|
|
8179
9190
|
runtimeVersion,
|
|
8180
9191
|
runtimeBackend: getRuntimeName(),
|
|
8181
|
-
modelFormat: getRuntimeModelFormat()
|
|
9192
|
+
modelFormat: matchedModel?.modelFormat ?? getRuntimeModelFormat()
|
|
8182
9193
|
}
|
|
8183
9194
|
};
|
|
8184
9195
|
const rawLogHash = createHash3("sha256").update(JSON.stringify(partialResult)).digest("hex");
|
|
@@ -8233,7 +9244,7 @@ ${tl}${h.repeat(innerWidth)}${tr}`));
|
|
|
8233
9244
|
} finally {
|
|
8234
9245
|
if (shouldUnloadAfterModel) {
|
|
8235
9246
|
try {
|
|
8236
|
-
await
|
|
9247
|
+
await unloadModel3(modelName);
|
|
8237
9248
|
} catch (err) {
|
|
8238
9249
|
if (!silent) {
|
|
8239
9250
|
warnMsg(`Could not unload model ${modelName} after benchmark.`);
|
|
@@ -8330,8 +9341,8 @@ ${tl}${h.repeat(innerWidth)}${tr}`));
|
|
|
8330
9341
|
}
|
|
8331
9342
|
|
|
8332
9343
|
// src/tools.ts
|
|
8333
|
-
var SUPPORTED_RUNTIMES = ["ollama"];
|
|
8334
|
-
var runtimeSchema = z.enum(SUPPORTED_RUNTIMES).optional().default("ollama").describe("Inference runtime to use
|
|
9344
|
+
var SUPPORTED_RUNTIMES = ["ollama", "lm-studio"];
|
|
9345
|
+
var runtimeSchema = z.enum(SUPPORTED_RUNTIMES).optional().default("ollama").describe("Inference runtime to use (ollama | lm-studio).");
|
|
8335
9346
|
var listModelsSchema = z.object({
|
|
8336
9347
|
runtime: runtimeSchema
|
|
8337
9348
|
});
|
|
@@ -8365,18 +9376,24 @@ var toolDefinitions = [
|
|
|
8365
9376
|
},
|
|
8366
9377
|
{
|
|
8367
9378
|
name: "share_result",
|
|
8368
|
-
description: "Upload a benchmark result to the public MetriLLM leaderboard.
|
|
9379
|
+
description: "Upload a benchmark result to the public MetriLLM leaderboard. Uses official upload defaults; METRILLM_* environment variables can override for self-hosted deployments. The resultFile must be an absolute path to a JSON file in ~/.metrillm/results/.",
|
|
8369
9380
|
inputSchema: shareResultSchema
|
|
8370
9381
|
}
|
|
8371
9382
|
];
|
|
8372
9383
|
function assertRuntime(runtime) {
|
|
8373
|
-
if (runtime
|
|
9384
|
+
if (!SUPPORTED_RUNTIMES.includes(runtime)) {
|
|
8374
9385
|
throw new Error(
|
|
8375
9386
|
`Runtime "${runtime}" is not yet supported. Currently supported: ${SUPPORTED_RUNTIMES.join(", ")}. New runtimes will be added as they become available in the MetriLLM CLI.`
|
|
8376
9387
|
);
|
|
8377
9388
|
}
|
|
8378
9389
|
}
|
|
8379
9390
|
var RESULTS_DIR2 = join2(homedir2(), ".metrillm", "results");
|
|
9391
|
+
function normalizeResultRuntimeBackend(result) {
|
|
9392
|
+
const runtimeBackend = result.metadata?.runtimeBackend?.trim().toLowerCase();
|
|
9393
|
+
if (runtimeBackend === "lm-studio") return "lm-studio";
|
|
9394
|
+
if (runtimeBackend === "ollama") return "ollama";
|
|
9395
|
+
return "ollama";
|
|
9396
|
+
}
|
|
8380
9397
|
var benchLock = Promise.resolve();
|
|
8381
9398
|
function withBenchLock(fn) {
|
|
8382
9399
|
const prev = benchLock;
|
|
@@ -8388,29 +9405,37 @@ function withBenchLock(fn) {
|
|
|
8388
9405
|
}
|
|
8389
9406
|
async function handleListModels(args) {
|
|
8390
9407
|
assertRuntime(args.runtime);
|
|
8391
|
-
|
|
8392
|
-
|
|
9408
|
+
return withBenchLock(async () => {
|
|
9409
|
+
setRuntimeByName(args.runtime);
|
|
9410
|
+
const runtimeDisplayName = getRuntimeDisplayName(args.runtime);
|
|
9411
|
+
const runtimeModelHint = getRuntimeModelInstallHint(args.runtime);
|
|
9412
|
+
const models = await listModels3();
|
|
9413
|
+
if (models.length === 0) {
|
|
9414
|
+
return JSON.stringify({
|
|
9415
|
+
models: [],
|
|
9416
|
+
message: `No models found on ${runtimeDisplayName}. ${runtimeModelHint}`
|
|
9417
|
+
});
|
|
9418
|
+
}
|
|
8393
9419
|
return JSON.stringify({
|
|
8394
|
-
models:
|
|
8395
|
-
|
|
9420
|
+
models: models.map((m) => ({
|
|
9421
|
+
name: m.name,
|
|
9422
|
+
size: m.size,
|
|
9423
|
+
parameterSize: m.parameterSize ?? null,
|
|
9424
|
+
quantization: m.quantization ?? null,
|
|
9425
|
+
family: m.family ?? null
|
|
9426
|
+
})),
|
|
9427
|
+
count: models.length
|
|
8396
9428
|
});
|
|
8397
|
-
}
|
|
8398
|
-
return JSON.stringify({
|
|
8399
|
-
models: models.map((m) => ({
|
|
8400
|
-
name: m.name,
|
|
8401
|
-
size: m.size,
|
|
8402
|
-
parameterSize: m.parameterSize ?? null,
|
|
8403
|
-
quantization: m.quantization ?? null,
|
|
8404
|
-
family: m.family ?? null
|
|
8405
|
-
})),
|
|
8406
|
-
count: models.length
|
|
8407
9429
|
});
|
|
8408
9430
|
}
|
|
8409
9431
|
async function handleRunBenchmark(args) {
|
|
8410
9432
|
assertRuntime(args.runtime);
|
|
9433
|
+
const runtimeDisplayName = getRuntimeDisplayName(args.runtime);
|
|
8411
9434
|
return withBenchLock(async () => {
|
|
9435
|
+
setRuntimeByName(args.runtime);
|
|
8412
9436
|
const outcome = await benchCommand({
|
|
8413
9437
|
model: args.model,
|
|
9438
|
+
backend: args.runtime,
|
|
8414
9439
|
perfOnly: args.perfOnly,
|
|
8415
9440
|
json: true,
|
|
8416
9441
|
// suppress all UI output
|
|
@@ -8428,7 +9453,7 @@ async function handleRunBenchmark(args) {
|
|
|
8428
9453
|
if (outcome.results.length === 0) {
|
|
8429
9454
|
return JSON.stringify({
|
|
8430
9455
|
success: false,
|
|
8431
|
-
error:
|
|
9456
|
+
error: `No results produced. Is ${runtimeDisplayName} running?`
|
|
8432
9457
|
});
|
|
8433
9458
|
}
|
|
8434
9459
|
const result = outcome.results[0];
|
|
@@ -8467,6 +9492,10 @@ async function handleGetResults(args) {
|
|
|
8467
9492
|
try {
|
|
8468
9493
|
const content = await readFile3(join2(RESULTS_DIR2, file), "utf8");
|
|
8469
9494
|
const result = JSON.parse(content);
|
|
9495
|
+
const resultRuntime = normalizeResultRuntimeBackend(result);
|
|
9496
|
+
if (resultRuntime !== args.runtime) {
|
|
9497
|
+
continue;
|
|
9498
|
+
}
|
|
8470
9499
|
if (args.model && !result.model.toLowerCase().includes(args.model.toLowerCase())) {
|
|
8471
9500
|
continue;
|
|
8472
9501
|
}
|