@vtstech/pi-model-test 1.0.9 → 1.1.0

package/model-test.js

@@ -12,57 +12,17 @@ import {
   truncate,
   sanitizeForReport
 } from "@vtstech/pi-shared/format";
-import { getOllamaBaseUrl, detectModelFamily, readModelsJson, BUILTIN_PROVIDERS, fetchModelContextLength } from "@vtstech/pi-shared/ollama";
-function detectProvider(ctx) {
-  const model = ctx.model;
-  if (!model) return { kind: "unknown", name: "none" };
-  const providerName = model.provider || "";
-  if (!providerName) return { kind: "unknown", name: "none" };
-  const modelsJson = readModelsJson();
-  const userProviderCfg = (modelsJson.providers || {})[providerName];
-  if (userProviderCfg) {
-    const baseUrl = userProviderCfg.baseUrl || "";
-    const apiMode = userProviderCfg.api || "";
-    const apiKey = userProviderCfg.apiKey || "";
-    const isOllama = /ollama/i.test(providerName) || /localhost:\d+/.test(baseUrl) || /127\.0\.0\.1:\d+/.test(baseUrl) || /0\.0\.0\.0:\d+/.test(baseUrl) || /\/api\/chat/.test(baseUrl) || apiMode === "ollama";
-    if (isOllama) {
-      return { kind: "ollama", name: providerName, apiMode: "ollama", baseUrl, apiKey };
-    }
-    if (/\/api\/chat/.test(baseUrl)) {
-      return { kind: "ollama", name: providerName, apiMode: "ollama", baseUrl, apiKey };
-    }
-    return {
-      kind: "builtin",
-      name: providerName,
-      apiMode: apiMode || userProviderCfg.api || "openai-completions",
-      baseUrl,
-      apiKey
-    };
-  }
-  const builtin = BUILTIN_PROVIDERS[providerName];
-  if (builtin) {
-    const apiKey = process.env[builtin.envKey] || "";
-    return {
-      kind: "builtin",
-      name: providerName,
-      apiMode: builtin.api,
-      baseUrl: builtin.baseUrl,
-      envKey: builtin.envKey,
-      apiKey
-    };
-  }
-  return { kind: "unknown", name: providerName };
-}
+import { getOllamaBaseUrl, detectModelFamily, readModelsJson, writeModelsJson, fetchModelContextLength, EXTENSION_VERSION, detectProvider } from "@vtstech/pi-shared/ollama";
 var CONFIG = {
   // General API settings
   DEFAULT_TIMEOUT_MS: 999999,
-  // 8.3 minutes - default timeout for model responses
+  // ~16.7 minutes — effectively unlimited for slow models
   CONNECT_TIMEOUT_S: 60,
-  // 30 seconds to establish connection
+  // 60 seconds to establish connection
   MAX_RETRIES: 1,
   // Single retry for transient failures
   RETRY_DELAY_MS: 1e4,
-  // 2 seconds between retries
+  // 10 seconds between retries
   EXEC_BUFFER_MS: 8e3,
   // Extra buffer for exec timeout over curl timeout
   // Model generation settings
@@ -74,31 +34,32 @@ var CONFIG = {
   MIN_THINKING_LENGTH: 10,
   // Minimum chars to consider thinking tokens valid
   TOOL_TEST_TIMEOUT_MS: 999999,
-  // 90 seconds for tool usage tests
+  // Effectively unlimited for slow tool usage tests
   TOOL_TEST_MAX_TIME_S: 999999,
   // Max curl time for tool tests (effectively unlimited)
   TOOL_SUPPORT_TIMEOUT_MS: 999999,
-  // 2+ minutes for tool support detection
+  // Effectively unlimited for tool support detection
   TOOL_SUPPORT_MAX_TIME_S: 999999,
   // Max curl time for tool support detection
   // Metadata retrieval
   TAGS_TIMEOUT_MS: 15e3,
   // 15 seconds for /api/tags
   TAGS_CONNECT_TIMEOUT_S: 30,
-  // 10 seconds connection timeout for tags
+  // 30 seconds connection timeout for tags
   MODEL_INFO_TIMEOUT_MS: 3e4,
-  // 10 seconds for model info lookup
+  // 30 seconds for model info lookup
   // Provider API settings
   PROVIDER_TIMEOUT_MS: 999999,
-  // 2 minutes for cloud provider API calls
+  // Effectively unlimited for cloud provider API calls
   PROVIDER_TOOL_TIMEOUT_MS: 12e4,
-  // 60 seconds for tool usage tests on providers
+  // 120 seconds for tool usage tests on providers
   // Rate limiting
   TEST_DELAY_MS: 1e4
-  // 30 seconds between tests to avoid rate limiting
+  // 10 seconds between tests to avoid rate limiting
 };
 var TOOL_SUPPORT_CACHE_DIR = path.join(os.homedir(), ".pi", "agent", "cache");
 var TOOL_SUPPORT_CACHE_PATH = path.join(TOOL_SUPPORT_CACHE_DIR, "tool_support.json");
+var _toolSupportCacheInMemory = null;
 function readToolSupportCache() {
   try {
     if (fs.existsSync(TOOL_SUPPORT_CACHE_PATH)) {
@@ -116,19 +77,21 @@ function writeToolSupportCache(cache) {
   fs.writeFileSync(TOOL_SUPPORT_CACHE_PATH, JSON.stringify(cache, null, 2) + "\n", "utf-8");
 }
 function getCachedToolSupport(model) {
-  const cache = readToolSupportCache();
+  const cache = _toolSupportCacheInMemory || readToolSupportCache();
+  if (!_toolSupportCacheInMemory) _toolSupportCacheInMemory = cache;
   const entry = cache[model];
   if (!entry) return null;
   if (!entry.support || !["native", "react", "none"].includes(entry.support)) return null;
   return entry;
 }
 function cacheToolSupport(model, support, family) {
-  const cache = readToolSupportCache();
+  const cache = _toolSupportCacheInMemory || readToolSupportCache();
   cache[model] = {
     support,
     testedAt: (/* @__PURE__ */ new Date()).toISOString(),
     family
   };
+  _toolSupportCacheInMemory = cache;
   writeToolSupportCache(cache);
 }
 function model_test_temp_default(pi) {
@@ -247,10 +210,8 @@ function model_test_temp_default(pi) {
         { role: "user", content: "Reply with exactly: PONG" }
       ], { maxTokens: 10, timeoutMs: 3e4 });
       const elapsedMs = Date.now() - start;
-      const content = result.content.trim().toUpperCase();
       const reachable = true;
       const authValid = true;
-      const hasPong = content.includes("PONG");
       return {
         pass: reachable && authValid,
         reachable,
@@ -259,7 +220,6 @@ function model_test_temp_default(pi) {
         elapsedMs
       };
     } catch (e) {
-      const start = Date.now();
       let reachable = false;
       let authValid = false;
       const msg = e.message || "";
@@ -290,7 +250,6 @@ function model_test_temp_default(pi) {
     const prompt = `A snail climbs 3 feet up a wall each day, but slides back 2 feet each night. The wall is 10 feet tall. How many days does it take the snail to reach the top? Think step by step and give the final answer on its own line like: ANSWER: <number>`;
     try {
       let response, elapsedMs;
-      let usedThinkingFallback = false;
       try {
         const result = await ollamaChat(model, [
           { role: "user", content: prompt }
@@ -309,7 +268,6 @@ function model_test_temp_default(pi) {
           ], { think: true });
           response = retry.response;
           elapsedMs = retry.elapsedMs;
-          usedThinkingFallback = true;
         } else {
           throw firstErr;
         }
@@ -917,12 +875,30 @@ The JSON object must have exactly these 4 keys:
         parsed = JSON.parse(cleaned);
       } catch {
         const cleaned = msg.replace(/```json?\s*/gi, "").replace(/```/g, "").trim();
-        const openBraces = (cleaned.match(/\{/g) || []).length;
-        const closeBraces = (cleaned.match(/\}/g) || []).length;
-        const openBrackets = (cleaned.match(/\[/g) || []).length;
-        const closeBrackets = (cleaned.match(/\]/g) || []).length;
-        if (openBraces > closeBraces || openBrackets > closeBrackets) {
-          const repaired = cleaned + "}".repeat(Math.max(0, openBraces - closeBraces)) + "]".repeat(Math.max(0, openBrackets - closeBrackets));
+        let braceDepth = 0, bracketDepth = 0;
+        let inString = false, escapeNext = false;
+        for (let i = 0; i < cleaned.length; i++) {
+          const c = cleaned[i];
+          if (escapeNext) {
+            escapeNext = false;
+            continue;
+          }
+          if (c === "\\") {
+            if (inString) escapeNext = true;
+            continue;
+          }
+          if (c === '"') {
+            inString = !inString;
+            continue;
+          }
+          if (inString) continue;
+          if (c === "{") braceDepth++;
+          else if (c === "}") braceDepth = Math.max(0, braceDepth - 1);
+          else if (c === "[") bracketDepth++;
+          else if (c === "]") bracketDepth = Math.max(0, bracketDepth - 1);
+        }
+        if (braceDepth > 0 || bracketDepth > 0) {
+          const repaired = cleaned + "}".repeat(braceDepth) + "]".repeat(bracketDepth);
           try {
             parsed = JSON.parse(repaired);
             repairNote = " (repaired truncated JSON)";
@@ -976,12 +952,30 @@ The JSON object must have exactly these 4 keys:
         parsed = JSON.parse(cleaned);
       } catch {
         const cleaned = msg.replace(/```json?\s*/gi, "").replace(/```/g, "").trim();
-        const openBraces = (cleaned.match(/\{/g) || []).length;
-        const closeBraces = (cleaned.match(/\}/g) || []).length;
-        const openBrackets = (cleaned.match(/\[/g) || []).length;
-        const closeBrackets = (cleaned.match(/\]/g) || []).length;
-        if (openBraces > closeBraces || openBrackets > closeBrackets) {
-          const repaired = cleaned + "}".repeat(Math.max(0, openBraces - closeBraces)) + "]".repeat(Math.max(0, openBrackets - closeBrackets));
+        let braceDepth = 0, bracketDepth = 0;
+        let inString = false, escapeNext = false;
+        for (let i = 0; i < cleaned.length; i++) {
+          const c = cleaned[i];
+          if (escapeNext) {
+            escapeNext = false;
+            continue;
+          }
+          if (c === "\\") {
+            if (inString) escapeNext = true;
+            continue;
+          }
+          if (c === '"') {
+            inString = !inString;
+            continue;
+          }
+          if (inString) continue;
+          if (c === "{") braceDepth++;
+          else if (c === "}") braceDepth = Math.max(0, braceDepth - 1);
+          else if (c === "[") bracketDepth++;
+          else if (c === "]") bracketDepth = Math.max(0, bracketDepth - 1);
+        }
+        if (braceDepth > 0 || bracketDepth > 0) {
+          const repaired = cleaned + "}".repeat(braceDepth) + "]".repeat(bracketDepth);
           try {
             parsed = JSON.parse(repaired);
             repairNote = " (repaired truncated JSON)";
@@ -1187,14 +1181,8 @@ The JSON object must have exactly these 4 keys:
     return ctx.model?.id;
   }
   function updateModelsJsonReasoning(model, hasReasoning) {
-    const agentDir = path.join(os.homedir(), ".pi", "agent");
-    const modelsJsonPath = path.join(agentDir, "models.json");
-    if (!fs.existsSync(modelsJsonPath)) {
-      return { updated: false, message: "models.json not found \u2014 skipped" };
-    }
     try {
-      const raw = fs.readFileSync(modelsJsonPath, "utf-8");
-      const config = JSON.parse(raw);
+      const config = readModelsJson();
       let updated = false;
       for (const provider of Object.values(config.providers || {})) {
         const models = provider.models || [];
@@ -1214,7 +1202,7 @@ The JSON object must have exactly these 4 keys:
       if (!updated) {
         return { updated: false, message: `${model} not found in models.json \u2014 skipped` };
       }
-      fs.writeFileSync(modelsJsonPath, JSON.stringify(config, null, 2) + "\n", "utf-8");
+      writeModelsJson(config);
       const action = hasReasoning ? "set reasoning: true" : "set reasoning: false";
       return { updated: true, message: `\u2705 Updated ${model}: ${action}` };
     } catch (e) {
@@ -1222,7 +1210,7 @@ The JSON object must have exactly these 4 keys:
     }
   }
   const branding = [
-    `  \u26A1 Pi Model Benchmark v1.0.9`,
+    `  \u26A1 Pi Model Benchmark v${EXTENSION_VERSION}`,
     `  Written by VTSTech`,
     `  GitHub: https://github.com/VTSTech`,
     `  Website: www.vts-tech.org`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vtstech/pi-model-test",
-  "version": "1.0.9",
+  "version": "1.1.0",
   "description": "Model benchmark/testing extension for Pi Coding Agent",
   "main": "model-test.js",
   "keywords": ["pi-extensions"],
@@ -14,7 +14,7 @@
     "url": "https://github.com/VTSTech/pi-coding-agent"
   },
   "dependencies": {
-    "@vtstech/pi-shared": "1.0.9"
+    "@vtstech/pi-shared": "1.1.0"
   },
   "peerDependencies": {
     "@mariozechner/pi-coding-agent": ">=0.66"