@vtstech/pi-shared 1.1.6 → 1.1.7

package/README.md

@@ -10,10 +10,10 @@ This is an internal dependency — you don't need to install it directly. It's p
 |--------|-------------|
 | `debug` | Conditional debug logging via `PI_EXTENSIONS_DEBUG=1` env var — `debugLog(module, message, ...args)` |
 | `format` | Section headers, indicators (ok/fail/warn/info), numeric formatters (bytes, ms, percentages), string utilities |
-| `model-test-utils` | Shared test utilities — `ChatFn` abstraction, unified test functions (tool usage, reasoning, instruction following), scoring helpers, tool support cache |
-| `ollama` | Ollama base URL resolution, models.json I/O with TTL cache, model family detection, provider detection, Ollama API helpers |
+| `model-test-utils` | Shared test utilities — `ChatFn` abstraction, unified test functions, scoring helpers, tool support cache, user config (`~/.pi/agent/model-test-config.json`), test history with regression detection (`~/.pi/agent/cache/model-test-history.json`) |
+| `ollama` | Ollama base URL resolution, models.json I/O with TTL cache, async write mutex (`acquireModelsJsonLock`, `readModifyWriteModelsJson`), exponential backoff retry (`withRetry`), model family detection, provider detection, Ollama API helpers |
 | `react-parser` | Multi-dialect ReAct text parser — 4 dialects (react, function, tool, call), `parseReact()`, `detectReactDialect()`, `fuzzyMatchToolName()` |
-| `security` | Security mode toggle (`basic`/`max`), partitioned command blocklist (41 CRITICAL + 25 EXTENDED), mode-aware SSRF (19 + 7 patterns), path validation with symlink dereference, URL validation, command sanitization, audit logging with mode tracking (`AUDIT_LOG_PATH` exported) |
+| `security` | Security mode toggle (`basic`/`max`), partitioned command blocklist (41 CRITICAL + 25 EXTENDED) with full-word scanning, mode-aware SSRF (22 + 7 patterns), path validation with symlink dereference, URL validation, command sanitization, DNS rebinding protection (`resolveAndCheckHostname`), buffered audit logging with mode tracking (`AUDIT_LOG_PATH` exported) |
 | `types` | Type definitions (ToolSupportLevel, AuditEntry, etc.) |
 
 ## Usage

package/model-test-utils.js

@@ -38,6 +38,34 @@ var CONFIG = {
   TEST_DELAY_MS: 1e4
   // 10 seconds between tests to avoid rate limiting
 };
+var TEST_CONFIG_DIR = path.join(os.homedir(), ".pi", "agent");
+var TEST_CONFIG_PATH = path.join(TEST_CONFIG_DIR, "model-test-config.json");
+function readTestConfig() {
+  try {
+    if (fs.existsSync(TEST_CONFIG_PATH)) {
+      const raw = fs.readFileSync(TEST_CONFIG_PATH, "utf-8");
+      return JSON.parse(raw);
+    }
+  } catch {
+  }
+  return {};
+}
+function getEffectiveConfig() {
+  const userConfig = readTestConfig();
+  return {
+    ...CONFIG,
+    DEFAULT_TIMEOUT_MS: userConfig.defaultTimeoutMs ?? CONFIG.DEFAULT_TIMEOUT_MS,
+    CONNECT_TIMEOUT_S: userConfig.connectTimeoutS ?? CONFIG.CONNECT_TIMEOUT_S,
+    MAX_RETRIES: userConfig.maxRetries ?? CONFIG.MAX_RETRIES,
+    RETRY_DELAY_MS: userConfig.retryDelayMs ?? CONFIG.RETRY_DELAY_MS,
+    TEST_DELAY_MS: userConfig.testDelayMs ?? CONFIG.TEST_DELAY_MS,
+    TOOL_TEST_TIMEOUT_MS: userConfig.toolTestTimeoutMs ?? CONFIG.TOOL_TEST_TIMEOUT_MS,
+    PROVIDER_TIMEOUT_MS: userConfig.providerTimeoutMs ?? CONFIG.PROVIDER_TIMEOUT_MS,
+    PROVIDER_TOOL_TIMEOUT_MS: userConfig.providerToolTimeoutMs ?? CONFIG.PROVIDER_TOOL_TIMEOUT_MS,
+    NUM_PREDICT: userConfig.numPredict ?? CONFIG.NUM_PREDICT,
+    TEMPERATURE: userConfig.temperature ?? CONFIG.TEMPERATURE
+  };
+}
 var WEATHER_TOOL_DEFINITION = {
   type: "function",
   function: {
@@ -150,6 +178,91 @@ function cacheToolSupport(model, support, family) {
   _toolSupportCacheInMemory = cache;
   writeToolSupportCache(cache);
 }
+var TEST_HISTORY_DIR = path.join(os.homedir(), ".pi", "agent", "cache");
+var TEST_HISTORY_PATH = path.join(TEST_HISTORY_DIR, "model-test-history.json");
+var MAX_HISTORY_PER_MODEL = 50;
+var MAX_HISTORY_TOTAL = 500;
+function readTestHistory() {
+  try {
+    if (fs.existsSync(TEST_HISTORY_PATH)) {
+      const raw = fs.readFileSync(TEST_HISTORY_PATH, "utf-8");
+      return JSON.parse(raw);
+    }
+  } catch {
+  }
+  return {};
+}
+function writeTestHistory(history) {
+  for (const model of Object.keys(history)) {
+    if (history[model].length > MAX_HISTORY_PER_MODEL) {
+      history[model] = history[model].slice(-MAX_HISTORY_PER_MODEL);
+    }
+  }
+  let totalEntries = 0;
+  const modelsByRecency = Object.entries(history).map(([model, entries]) => ({
+    model,
+    entries,
+    lastEntry: entries[entries.length - 1]?.timestamp || ""
+  })).sort((a, b) => b.lastEntry.localeCompare(a.lastEntry));
+  const trimmedHistory = {};
+  for (const { model, entries } of modelsByRecency) {
+    if (totalEntries + entries.length > MAX_HISTORY_TOTAL) {
+      const remaining = MAX_HISTORY_TOTAL - totalEntries;
+      if (remaining <= 0) break;
+      trimmedHistory[model] = entries.slice(-remaining);
+      totalEntries += remaining;
+    } else {
+      trimmedHistory[model] = entries;
+      totalEntries += entries.length;
+    }
+  }
+  if (!fs.existsSync(TEST_HISTORY_DIR)) {
+    fs.mkdirSync(TEST_HISTORY_DIR, { recursive: true });
+  }
+  fs.writeFileSync(TEST_HISTORY_PATH, JSON.stringify(trimmedHistory, null, 2) + "\n", "utf-8");
+}
+function appendTestHistory(entry) {
+  const history = readTestHistory();
+  if (!history[entry.model]) {
+    history[entry.model] = [];
+  }
+  history[entry.model].push(entry);
+  writeTestHistory(history);
+}
+function getModelHistory(model, limit = 10) {
+  const history = readTestHistory();
+  const entries = history[model] || [];
+  return entries.slice(-limit);
+}
+function detectRegression(model, current) {
+  const history = readTestHistory();
+  const entries = history[model] || [];
+  if (entries.length < 2) return [];
+  const previous = entries[entries.length - 2];
+  const regressions = [];
+  const scoreOrder = ["STRONG", "MODERATE", "WEAK", "FAIL", "ERROR", "NO", "YES"];
+  const scoreRank = (s) => {
+    const idx = scoreOrder.indexOf(s);
+    return idx >= 0 ? idx : 99;
+  };
+  if (scoreRank(current.tests.reasoning.score) > scoreRank(previous.tests.reasoning.score)) {
+    regressions.push({ test: "Reasoning", previous: previous.tests.reasoning.score, current: current.tests.reasoning.score });
+  }
+  if (scoreRank(current.tests.toolUsage.score) > scoreRank(previous.tests.toolUsage.score)) {
+    regressions.push({ test: "Tool Usage", previous: previous.tests.toolUsage.score, current: current.tests.toolUsage.score });
+  }
+  if (scoreRank(current.tests.reactParsing.score) > scoreRank(previous.tests.reactParsing.score)) {
+    regressions.push({ test: "ReAct Parsing", previous: previous.tests.reactParsing.score, current: current.tests.reactParsing.score });
+  }
+  if (scoreRank(current.tests.instructionFollowing.score) > scoreRank(previous.tests.instructionFollowing.score)) {
+    regressions.push({ test: "Instructions", previous: previous.tests.instructionFollowing.score, current: current.tests.instructionFollowing.score });
+  }
+  const supportRank = (s) => s === "native" ? 0 : s === "react" ? 1 : 2;
+  if (supportRank(current.tests.toolSupport.level) > supportRank(previous.tests.toolSupport.level)) {
+    regressions.push({ test: "Tool Support", previous: previous.tests.toolSupport.level, current: current.tests.toolSupport.level });
+  }
+  return regressions;
+}
 var REASONING_PROMPT = `A snail climbs 3 feet up a wall each day, but slides back 2 feet each night. The wall is 10 feet tall. How many days does it take the snail to reach the top? Think step by step and give the final answer on its own line like: ANSWER: <number>`;
 var TOOL_SYSTEM_PROMPT = "You are a helpful assistant. Use the available tools when needed.";
 var TOOL_USER_PROMPT = "What's the weather like in Paris right now?";
@@ -309,11 +422,18 @@ The JSON object must have exactly these 4 keys:
 }
 export {
   CONFIG,
+  TEST_CONFIG_PATH,
   TOOL_SUPPORT_CACHE_PATH,
   WEATHER_TOOL_DEFINITION,
+  appendTestHistory,
   cacheToolSupport,
+  detectRegression,
   getCachedToolSupport,
+  getEffectiveConfig,
+  getModelHistory,
   parseTextToolCall,
+  readTestConfig,
+  readTestHistory,
   readToolSupportCache,
   scoreNativeToolCall,
   scoreReasoning,
@@ -321,5 +441,6 @@ export {
   testInstructionFollowingUnified,
   testReasoningUnified,
   testToolUsageUnified,
+  writeTestHistory,
   writeToolSupportCache
 };

package/ollama.js

@@ -12,7 +12,7 @@ function debugLog(module, message, ...args) {
 }
 
 // shared/ollama.ts
-var EXTENSION_VERSION = "1.1.6";
+var EXTENSION_VERSION = "1.1.7";
 var MODELS_JSON_PATH = path.join(os.homedir(), ".pi", "agent", "models.json");
 var _modelsJsonCache = null;
 var _ollamaBaseUrlCache = null;
@@ -71,35 +71,119 @@ function writeModelsJson(data) {
   _modelsJsonCache = null;
   _ollamaBaseUrlCache = null;
 }
-async function fetchOllamaModels(baseUrl) {
-  const res = await fetch(`${baseUrl}/api/tags`, {
-    signal: AbortSignal.timeout(5e3)
+var _modelsJsonLock = null;
+async function acquireModelsJsonLock() {
+  while (_modelsJsonLock) {
+    await _modelsJsonLock;
+  }
+  let releaseLock;
+  _modelsJsonLock = new Promise((resolve) => {
+    releaseLock = resolve;
   });
-  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
-  const data = await res.json();
-  return data.models ?? [];
+  return {
+    release: () => {
+      releaseLock();
+      _modelsJsonLock = null;
+    }
+  };
 }
-async function fetchModelContextLength(baseUrl, modelName) {
+async function readModifyWriteModelsJson(modifier) {
+  const { release } = await acquireModelsJsonLock();
   try {
-    const res = await fetch(`${baseUrl}/api/show`, {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ name: modelName }),
-      signal: AbortSignal.timeout(3e4)
+    const data = readModelsJson();
+    const modified = modifier(data);
+    if (modified === null) return false;
+    writeModelsJson(modified);
+    return true;
+  } finally {
+    release();
+  }
+}
+var DEFAULT_RETRY_OPTIONS = {
+  maxRetries: 2,
+  baseDelayMs: 1e3,
+  maxDelayMs: 1e4,
+  retryOnTimeout: true,
+  retryOnConnectionError: true
+};
+function backoffDelay(attempt, baseDelayMs, maxDelayMs) {
+  const delay = Math.min(baseDelayMs * Math.pow(2, attempt), maxDelayMs);
+  const jitter = delay * 0.25 * (Math.random() * 2 - 1);
+  return Math.max(0, Math.round(delay + jitter));
+}
+var RETRYABLE_ERROR_PATTERNS = [
+  "ECONNREFUSED",
+  "ECONNRESET",
+  "ENOTFOUND",
+  "ETIMEDOUT",
+  "fetch failed",
+  "network error",
+  "socket hang up",
+  "Empty response"
+];
+function isRetryableError(error, opts) {
+  if (error instanceof Error) {
+    if (error.name === "AbortError" && opts.retryOnTimeout) return true;
+    const msg = error.message;
+    if (opts.retryOnConnectionError && RETRYABLE_ERROR_PATTERNS.some((p) => msg.includes(p))) {
+      return true;
+    }
+  }
+  return false;
+}
+async function withRetry(fn, options) {
+  const opts = { ...DEFAULT_RETRY_OPTIONS, ...options };
+  let lastError;
+  for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      if (attempt < opts.maxRetries && isRetryableError(error, opts)) {
+        const delay = backoffDelay(attempt, opts.baseDelayMs, opts.maxDelayMs);
+        debugLog("ollama", `Retry ${attempt + 1}/${opts.maxRetries} after ${delay}ms: ${error instanceof Error ? error.message : String(error)}`);
+        await new Promise((r) => setTimeout(r, delay));
+        continue;
+      }
+      throw error;
+    }
+  }
+  throw lastError;
+}
+async function fetchOllamaModels(baseUrl) {
+  return withRetry(async () => {
+    const res = await fetch(`${baseUrl}/api/tags`, {
+      signal: AbortSignal.timeout(5e3)
     });
-    if (!res.ok) return void 0;
+    if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
     const data = await res.json();
-    for (const key of Object.keys(data?.model_info ?? {})) {
-      if (key.endsWith(".context_length")) {
-        const val = data.model_info[key];
-        if (typeof val === "number") return val;
+    return data.models ?? [];
+  });
+}
+async function fetchModelContextLength(baseUrl, modelName) {
+  return withRetry(async () => {
+    try {
+      const res = await fetch(`${baseUrl}/api/show`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ name: modelName }),
+        signal: AbortSignal.timeout(3e4)
+      });
+      if (!res.ok) return void 0;
+      const data = await res.json();
+      for (const key of Object.keys(data?.model_info ?? {})) {
+        if (key.endsWith(".context_length")) {
+          const val = data.model_info[key];
+          if (typeof val === "number") return val;
+        }
       }
+      const numCtx = data?.model_info?.["num_ctx"];
+      if (typeof numCtx === "number") return numCtx;
+    } catch {
+      return void 0;
     }
-    const numCtx = data?.model_info?.["num_ctx"];
-    if (typeof numCtx === "number") return numCtx;
-  } catch {
     return void 0;
-  }
+  });
 }
 async function fetchContextLengthsBatched(baseUrl, modelNames, batchSize = 3) {
   const result = /* @__PURE__ */ new Map();
@@ -205,6 +289,7 @@ export {
   BUILTIN_PROVIDERS,
   EXTENSION_VERSION,
   MODELS_JSON_PATH,
+  acquireModelsJsonLock,
   detectModelFamily,
   detectProvider,
   fetchContextLengthsBatched,
@@ -213,5 +298,7 @@ export {
   getOllamaBaseUrl,
   isReasoningModel,
   readModelsJson,
+  readModifyWriteModelsJson,
+  withRetry,
   writeModelsJson
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vtstech/pi-shared",
-  "version": "1.1.6",
+  "version": "1.1.7",
   "description": "Shared utilities for Pi Coding Agent extensions",
   "exports": {
     "./debug": "./debug.js",

package/security.js

@@ -12,6 +12,7 @@ function debugLog(module, message, ...args) {
 }
 
 // shared/security.ts
+import dns from "node:dns";
 var SETTINGS_PATH = path.join(os.homedir(), ".pi", "agent", "settings.json");
 var SECURITY_CONFIG_PATH = path.join(os.homedir(), ".pi", "agent", "security.json");
 function getSecurityMode() {
@@ -248,6 +249,45 @@ function validatePath(filePath, allowedDirs) {
   }
   return { valid: false, error: `Path not in allowed directories: ${filePath}` };
 }
+function isLoopbackIp(ip) {
+  if (ip.startsWith("127.") || ip === "0.0.0.0") return true;
+  if (ip === "::1" || ip === "::ffff:0.0.0.0") return true;
+  if (ip.startsWith("::ffff:127.")) return true;
+  return false;
+}
+function isPrivateIp(ip) {
+  if (ip.startsWith("10.") || ip.startsWith("192.168.")) return true;
+  if (/^172\.(1[6-9]|2\d|3[01])\./.test(ip)) return true;
+  if (ip === "169.254.169.254") return true;
+  if (ip.startsWith("fc") || ip.startsWith("fd")) return true;
+  if (ip.startsWith("fe80:")) return true;
+  return false;
+}
+async function resolveAndCheckHostname(hostname, blockPrivate = true) {
+  try {
+    const addresses = await new Promise((resolve2, reject) => {
+      dns.lookup(hostname, { all: true }, (err, addresses2) => {
+        if (err) reject(err);
+        else resolve2(addresses2);
+      });
+    });
+    if (!addresses || addresses.length === 0) {
+      return { safe: true, error: "" };
+    }
+    for (const addr of addresses) {
+      const ip = addr.address;
+      if (ip === "169.254.169.254") {
+        return { safe: false, error: `SSRF protection: hostname ${hostname} resolves to cloud metadata IP ${ip}` };
+      }
+      if (blockPrivate && (isLoopbackIp(ip) || isPrivateIp(ip))) {
+        return { safe: false, error: `SSRF protection: hostname ${hostname} resolves to private/reserved IP ${ip} (DNS rebinding check)` };
+      }
+    }
+    return { safe: true, error: "" };
+  } catch {
+    return { safe: true, error: "" };
+  }
+}
 function isSafeUrl(url, blockSsrf = true) {
   if (!url) return { safe: false, error: "URL cannot be empty" };
   let parsed;
@@ -312,8 +352,13 @@ function sanitizeCommand(command) {
   let baseCmd = parts[0].toLowerCase();
   if (baseCmd.includes("/")) baseCmd = baseCmd.split("/").pop();
   if (baseCmd.includes("\\")) baseCmd = baseCmd.split("\\").pop();
-  if (CRITICAL_COMMANDS.has(baseCmd)) {
-    return { isSafe: false, error: `Blocked command: ${baseCmd} (critical)`, command: "" };
+  for (const raw of parts) {
+    let word = raw.toLowerCase();
+    if (word.includes("/")) word = word.split("/").pop();
+    if (word.includes("\\")) word = word.split("\\").pop();
+    if (CRITICAL_COMMANDS.has(word)) {
+      return { isSafe: false, error: `Blocked command: ${word} (critical)`, command: "" };
+    }
   }
   const mode = getSecurityMode();
   if (mode === "max" && EXTENDED_COMMANDS.has(baseCmd)) {
@@ -332,16 +377,46 @@ function sanitizeCommand(command) {
 }
 var AUDIT_DIR = path.join(os.homedir(), ".pi", "agent");
 var AUDIT_LOG_PATH = path.join(AUDIT_DIR, "audit.log");
-function appendAuditEntry(entry) {
+var AUDIT_BUFFER_MAX_ENTRIES = 50;
+var AUDIT_FLUSH_INTERVAL_MS = 500;
+var _auditBuffer = [];
+var _auditFlushTimer = null;
+function ensureAuditFlushTimer() {
+  if (_auditFlushTimer) return;
+  _auditFlushTimer = setInterval(() => {
+    if (_auditBuffer.length > 0) {
+      flushAuditBuffer();
+    }
+  }, AUDIT_FLUSH_INTERVAL_MS);
+  const timerRef = _auditFlushTimer;
+  if (timerRef.unref) {
+    timerRef.unref();
+  }
+}
+function flushAuditBuffer() {
+  if (_auditBuffer.length === 0) return;
   try {
     if (!fs.existsSync(AUDIT_DIR)) {
       fs.mkdirSync(AUDIT_DIR, { recursive: true });
     }
+    const batch = _auditBuffer.join("");
+    fs.appendFileSync(AUDIT_LOG_PATH, batch, "utf-8");
+  } catch (err) {
+    debugLog("security", "audit buffer flush failure", err);
+  }
+  _auditBuffer = [];
+}
+function appendAuditEntry(entry) {
+  try {
+    ensureAuditFlushTimer();
     const enriched = { ...entry, securityMode: getSecurityMode() };
     const line = JSON.stringify(enriched) + "\n";
-    fs.appendFileSync(AUDIT_LOG_PATH, line, "utf-8");
+    _auditBuffer.push(line);
+    if (_auditBuffer.length >= AUDIT_BUFFER_MAX_ENTRIES) {
+      flushAuditBuffer();
+    }
   } catch (err) {
-    debugLog("security", "audit log write failure", err);
+    debugLog("security", "audit log entry creation failure", err);
   }
 }
 function readRecentAuditEntries(count = 50) {
@@ -432,9 +507,11 @@ export {
   checkFileToolInput,
   checkHttpToolInput,
   checkInjectionPatterns,
+  flushAuditBuffer,
   getSecurityMode,
   isSafeUrl,
   readRecentAuditEntries,
+  resolveAndCheckHostname,
   sanitizeCommand,
   setSecurityMode,
   validatePath