dsh-tool-folder 0.2.4 → 0.2.5

package/README.md

@@ -149,6 +149,27 @@ were never unregistered, so nothing else changes.
 
 ## Changelog
 
+### v0.2.5 — red-team firewall fixes + retrieval hardening (2026-09-02)
+1. **Exec-tool recognition (red-team F1)** — bare-name matching let MCP-prefixed
+   exec tools (`mcp__windows__Cmd`) bypass the high-risk gate; `isExecTool` now
+   tokenizes the full name and matches every segment (all segments participate —
+   conservative by design, over-recognition is ~free on the gate).
+2. **False-positive roots A/B** — Windows paths are base64-shaped (57-char fake
+   runs) and the bare `&` of `2>&1` looked like an exec verb; a legit gitleaks
+   staged scan was blocked. Path-fingerprint exclusion + `EXEC_VERB` tightening.
+   2/5 measured FPs → 0/5; regression suite `chainguard-fp.test.js` (114 total green).
+3. **Command-substitution download-exec legs** — `$(curl …)` / backtick /
+   `iwr` shapes (no pipe, invisible to the download-pipe pattern).
+4. **BM25 sub-word split** — hyphen/underscore tool names were single index
+   terms (`mcp__codegraph__codegraph_explore`); measured recall 0/10 → 10/10.
+5. **fallbackCatalog round-robin** — pure lexicographic slice hid whole servers
+   (`mcp__*`) from the fallback catalog; now round-robins across servers.
+6. **serverOf unified** — apply-local `split("__")` copy deleted; module-level
+   grouping (native/openhands families) is the single source for search results
+   and sibling closure. `embedTimeoutMs` configurable (default 8000).
+7. Two-axis audit (2026-09-02): comments/tests now match the real conservative
+   design; one-shot repro script removed.
+
 ### v0.2.4 — P1: batch tools_schema + sibling closure (2026-08-31)
 1. **Batch expand** — `tools_schema` accepts `names: [...]` to expand several
    tools in one call (`{ found, results: [...] }`); a single `name` keeps the

package/lib/bm25.js

@@ -1,6 +1,12 @@
 /**
  * bm25.js — zero-dependency BM25 lexical search for tool retrieval.
  *
+ * VENDOR SOURCE (2026-09-02): dsh-skill-folder/lib/bm25.js vendors this file
+ * verbatim (comments may diverge per repo). Modify here, then mirror the code
+ * change into the skill repo. Sync gate: `node E:\DSH-Data\scripts\check-vendor-sync.mjs`.
+ * semantic.js is intentionally DIVERGED (tool has incremental cache/sha1) and
+ * is NOT covered by the gate.
+ *
  * Design (ported from the xiaowan agent's ToolBM25Index idea; the algorithm is
  * the reference, this code is a clean-room JS implementation):
  *   - BM25 with standard k1=1.2, b=0.75 (RAG-MCP / BoR validated the
@@ -32,6 +38,15 @@ function tokenize(text) {
   for (const m of s.matchAll(/[a-z0-9][a-z0-9_+-]{1,}/g)) {
     const t = m[0];
     if (!EN_STOP.has(t)) out.push(t);
+    // Sub-words of hyphen/underscore compounds. Without this, a tool name like
+    // "mcp__open-design__start_run" is ONE term, so querying "design" scores
+    // zero. The whole token is kept too, so exact-name matches and existing
+    // IDF are unchanged; sub-words only ADD recall.
+    if (t.length > 2 && (t.includes("-") || t.includes("_"))) {
+      for (const part of t.split(/[-_]+/)) {
+        if (part.length > 1 && part !== t && !EN_STOP.has(part)) out.push(part);
+      }
+    }
   }
   // CJK bigrams: "开源框架" -> "开源","源框","框架". Bigrams beat single
   // chars for short tool descriptions and are dependency-free.

package/lib/chainguard.js

@@ -24,6 +24,9 @@ const HIGH_RISK_PATTERNS = [
   // download → execute chain
   [/(wget|curl|powershell|pwsh).{0,80}(\||;|&&).{0,30}(sh|bash|powershell|pwsh|cmd|start)\b/i, "download-pipe-execute"],
   [/invoke-expression|iex\s*\(|iwr\s+.*\|.*iex/i, "powershell download-exec"],
+  // command substitution download → execute (bash $() and backticks; no pipe,
+  // so the download-pipe pattern above cannot see it). Added 2026-09-02.
+  [/\$\s*\(\s*(?:curl|wget|iwr)[^)]{0,120}\)|\`(?:curl|wget|iwr)[^\`]{0,120}\`/i, "download-exec via command substitution"],
   // credential / exfil
   // NOTE: `\\+` (one-or-more literal backslashes) instead of `\\` — argv text
   // reaches verdict() both as a raw string (single backslash) and as
@@ -58,6 +61,43 @@ function matchHighRisk(text) {
   return null;
 }
 
+/**
+ * 是否「执行类工具」——参数是真实要执行的命令/脚本，需要高危硬拦。
+ * 内容类工具（write/edit/read/apply_patch 等）的参数是文件内容/文本，
+ * 本身不执行，含危险命令字面量（讨论/测试/文档）不应触发硬拦。
+ *
+ * 2026-08-30: 修复 ChainGuard 对所有工具参数跑高危正则导致的误拦。
+ * 2026-09-02 (red-team F1): bare-name matching let MCP-prefixed exec tools
+ * (`mcp__windows__Cmd`) bypass the gate — tokenize the full name and match
+ * every segment against the verb list. Over-recognition is ~free here: the
+ * gate only decides "run this argv past the firewall", and a non-command
+ * argv has no high-risk literals to trip; missing one tool is a bypass.
+ * @param {string} name - full tool name (mcp__<server>__<tool> or bare).
+ */
+export function isExecTool(name) {
+  if (!name) return false;
+  const n = String(name).toLowerCase();
+  const EXEC_TOKENS = new Set([
+    "cmd", "command", "console", "terminal", "shell", "bash", "pwsh", "powershell",
+    "exec", "execute", "executor", "run", "runner", "sh", "zsh", "script", "ps1",
+  ]);
+  // Legacy fast paths kept for bare names and native prefixes.
+  const EXEC_NAMES = ["pwsh", "shell", "bash", "cmd", "exec", "terminal", "run", "console"];
+  if (EXEC_NAMES.includes(n)) return true;
+  if (n.endsWith("__app")) return true; // mcp__windows__App：launch_executable 模式可启动任意可执行文件
+  if (n.startsWith("run_") || n.startsWith("exec_") || n.startsWith("execute_")) return true;
+  if (n.startsWith("ssh_") || n.startsWith("wsl")) return true;
+  // Segment match: tokenize on mcp__ separator and _ - . (runbook-style names).
+  // All segments except the `mcp` marker participate — a server name carrying
+  // an exec word (e.g. `pwsh` in mcp__pwsh__list_tools) is itself an exec
+  // signal, while `mcp__github__get_file_contents` has none. Over-recognition
+  // is ~free (gate only); under-recognition is a bypass.
+  const tokens = n.split(/__|_|-|\./).filter((t) => t.length > 0);
+  const meaningful = n.startsWith("mcp") && tokens[0] === "mcp" ? tokens.slice(1) : tokens;
+  if (meaningful.length === 0) return false;
+  return meaningful.some((t) => EXEC_TOKENS.has(t));
+}
+
 export function verdict(argvText) {
   if (!argvText) return null;
 

package/lib/index.js

@@ -37,7 +37,7 @@ import { selectionCoverage, isHotTool, prioritizeExact, touchWarm, evictWarm, mi
 import { tierForBudget } from "./budget.js";
 import { readState, writeState } from "./storage.js";
 import { expandQuery, fallbackCatalog, createSearchCache, recordDiscoveries, serializeDiscoveries, deserializeDiscoveries, siblingClosure, buildSchemaResponse } from "./ux.js";
-import { ChainGuard, verdict } from "./chainguard.js";
+import { ChainGuard, verdict, isExecTool } from "./chainguard.js";
 import { matchCategory } from "./category.js";
 import { toonifyValue } from "./toonify.js";
 import { configureSemantic, buildSemanticIndex, searchSemantic, rrfFuse } from "./semantic.js";
@@ -80,6 +80,7 @@ export const Config = z.object({
   semanticEnabled: z.boolean().default(true).description("语义检索腿（本地 Ollama bge-m3）：tools_search 用 BM25+语义 RRF 混合，中英跨语言命中"),
   ollamaBase: z.string().default("http://127.0.0.1:11434").description("Ollama 地址（semanticEnabled 时使用）"),
   embedModel: z.string().default("bge-m3").description("embedding 模型（semanticEnabled 时使用）"),
+  embedTimeoutMs: z.number().min(100).default(8000).description("embedding 请求超时（毫秒；Ollama 慢时可调大）"),
   disclosureBudget: z.number().min(0).max(200000).default(0)
     .description("分级披露预算（字节）：0=关闭。开启后可见工具 schema 估算字节超预算时，动态段按 完整schema→name+desc→仅name→折叠进catalog 逐级降级（core/hot 永不降级）"),
   maxWarmTools: z.number().min(0).max(50).default(0)
@@ -124,6 +125,7 @@ const DEFAULTS = {
   semanticEnabled: true, // P0-1: bge-m3 semantic leg for tools_search (RRF hybrid)
   ollamaBase: "http://127.0.0.1:11434",
   embedModel: "bge-m3",
+  embedTimeoutMs: 8000, // P3-2: embedding 请求超时（毫秒）
   disclosureBudget: 0, // v0.2.2: 分级披露预算（0=关闭）
   maxWarmTools: 0, // v0.2.2: warm LRU 上限（0=关闭，默认关=无配置升级逐字节等价）
   fallbackCatalogSize: 30, // P0: 零匹配回退轻目录条数（0=关闭）
@@ -200,24 +202,6 @@ function serverOf(name) {
   return "native";
 }
 
-/**
- * 是否「执行类工具」——参数是真实要执行的命令/脚本，需要高危硬拦。
- * 内容类工具（write/edit/read/apply_patch 等）的参数是文件内容/文本，
- * 本身不执行，含危险命令字面量（讨论/测试/文档）不应触发硬拦。
- * 2026-08-30：修复 ChainGuard 对所有工具参数跑高危正则导致的误拦。
- */
-function isExecTool(name) {
-  if (!name) return false;
-  const n = String(name).toLowerCase();
-  const EXEC_NAMES = ["pwsh", "shell", "bash", "cmd", "exec", "terminal", "run", "console"];
-  if (EXEC_NAMES.includes(n)) return true;
-  if (n.includes("powershell")) return true; // mcp__windows__PowerShell 等（参数即命令）
-  if (n.endsWith("__app")) return true; // mcp__windows__App：launch_executable 模式可启动任意可执行文件
-  if (n.startsWith("run_") || n.startsWith("exec_") || n.startsWith("execute_")) return true;
-  if (n.startsWith("ssh_") || n.startsWith("wsl")) return true;
-  return false;
-}
-
 function summarizeCaps(tools) {
   // Pull the first few distinct capability keywords from descriptions.
   const seen = new Set();
@@ -279,7 +263,7 @@ export function apply(ctx, config) {
   // Semantic leg (P0-1): point the shared module at our config; the index is
   // built lazily on first tools_search call (doc embeddings cached on disk,
   // so the ~10s cold start happens once, then it's a cache hit).
-  configureSemantic({ ollamaBase: cfg.ollamaBase, embedModel: cfg.embedModel });
+  configureSemantic({ ollamaBase: cfg.ollamaBase, embedModel: cfg.embedModel, embedTimeoutMs: cfg.embedTimeoutMs });
   let semIndexPromise = null;
   const getSemanticIndex = () => {
     if (!semIndexPromise) {
@@ -384,9 +368,12 @@ export function apply(ctx, config) {
   // P0-②: 检索结果缓存（同 query 60s 内稳定；LRU 200 条；进程级共享）。
   const searchCache = createSearchCache(200, 60000);
 
-  // "mcp__viking__remember" → "mcp__viking"; plain tools ("tool-bash") stay
-  // as themselves (single segment survives slice(0,2)).
-  const serverOf = (name) => String(name).split("__").slice(0, 2).join("__");
+  // serverOf is the module-level function defined above (mcp__server__tool →
+  // "mcp__server", openhands_* → "openhands", plain native tools → "native").
+  // P3-1 (2026-09-02): the previous apply-local copy (split("__").slice(0,2))
+  // let plain tools be their own "server", diverging from catalogText grouping
+  // and making sibling closure useless for native/openhands families. One
+  // definition, one口径.
 
   // Pattern matching shared by deny (P0) and include (P2-1): exact name or
   // "prefix*" matches any tool starting with that prefix.

package/lib/obfuscation.js

@@ -65,7 +65,15 @@ const RE_BITSADMIN = /bitsadmin\s+\/transfer\s+\S+\s+\/download/i;
 
 // A long base64 blob co-located with an execution verb in the same command.
 const RE_B64_PAYLOAD = /[A-Za-z0-9+/]{40,}={0,2}/;
-const EXEC_VERB = /(?:iex|invoke-expression|invoke-command|iwr|irm|start-process|start-job|&|cmd\s*\/[cC]|sh\s+-c|bash\s+-c|python\s+-[cC]|node\s+-e|powershell|pwsh|mshta|rundll32|regsvr32|wscript|cscript)\b/i;
+// Path fingerprint: >=3 camel-case segments joined by '/' (C:/Users/L/AppData/...).
+// '/' is a base64 char, so long Windows/URL paths form fake base64 runs; real
+// base64 payloads almost never contain 3+ `/Upper`-word segments. Fixed 2026-09-02
+// (measured: 57-char WinGet path run + `2>&1` was blocked as base64-embedded-command).
+const RE_PATHISH_RUN = /(?:\/[A-Z][A-Za-z]{1,}){3,}/;
+// Exec verb used by the b64/hex legs. NOTE: no bare `&` — `2>&1` redirection
+// and `& ` call operators are not payload-exec evidence by themselves; the
+// verb must be an actual command word (fixed 2026-09-02, fp-repro 2/5).
+const EXEC_VERB = /(?:iex|invoke-expression|invoke-command|iwr|irm|start-process|start-job|&\s*(?:iex|invoke|iwr|irm|cmd|powershell|pwsh|sh|bash|python|node|mshta|rundll32|regsvr32|wscript|cscript)|cmd\s*\/[cC]|sh\s+-c|bash\s+-c|python\s+-[cC]|node\s+-e|powershell|pwsh|mshta|rundll32|regsvr32|wscript|cscript)\b/i;
 
 // Full-width Latin letters (ｃｕｒｌ) — never legit in a command line.
 const RE_FULLWIDTH = /[\uFF21-\uFF3A\uFF41-\uFF5A]/;
@@ -97,7 +105,8 @@ export function detectObfuscation(rawText, cleanText) {
   if (RE_BITSADMIN.test(probe)) {
     return { label: "bitsadmin-download-exec", detail: "bitsadmin /transfer 下载执行" };
   }
-  if (RE_B64_PAYLOAD.test(probe) && EXEC_VERB.test(probe)) {
+  const b64Run = RE_B64_PAYLOAD.exec(probe);
+  if (b64Run && !RE_PATHISH_RUN.test(b64Run[0]) && EXEC_VERB.test(probe)) {
     return { label: "base64-embedded-command", detail: "长 base64 载荷 + 执行动词共存" };
   }
   if (RE_FULLWIDTH.test(raw)) {

package/lib/semantic.js

@@ -86,20 +86,22 @@ function writeCacheFileAt(p, obj) {
 
 let _model = DEFAULT_MODEL;
 let _base = OLLAMA_BASE;
+let _timeout = EMBED_TIMEOUT_MS; // P3-2 (2026-09-02): configurable via configureSemantic
 
 /**
  * Configure the endpoint (called by plugin apply with cfg.ollamaBase/model).
- * @param {{ollamaBase?: string, embedModel?: string}} cfg
+ * @param {{ollamaBase?: string, embedModel?: string, embedTimeoutMs?: number}} cfg
  */
 export function configureSemantic(cfg = {}) {
   if (cfg && typeof cfg === "object") {
     if (typeof cfg.ollamaBase === "string" && cfg.ollamaBase) _base = cfg.ollamaBase;
     if (typeof cfg.embedModel === "string" && cfg.embedModel) _model = cfg.embedModel;
+    if (Number.isFinite(cfg.embedTimeoutMs) && cfg.embedTimeoutMs >= 100) _timeout = cfg.embedTimeoutMs;
   }
 }
 
 /** One embedding vector for a text, or null on any failure. */
-export async function embedText(text, timeoutMs = EMBED_TIMEOUT_MS) {
+export async function embedText(text, timeoutMs = _timeout) {
   const s = String(text || "").trim();
   if (!s) return null;
   try {
@@ -164,7 +166,7 @@ export function cosine(a, b) {
  *                 real ~/.dsh/state unless omitted).
  * @returns {Promise<{vectors: Map<string, number[]>, available: boolean}>}
  */
-export async function buildSemanticIndex(docs, timeoutMs = EMBED_TIMEOUT_MS, opts = {}) {
+export async function buildSemanticIndex(docs, timeoutMs = _timeout, opts = {}) {
   const out = { vectors: new Map(), available: false };
   const list = Array.isArray(docs) ? docs.filter((d) => d && d.id && d.text) : [];
   if (list.length === 0) return out;

package/lib/ux.js

@@ -87,8 +87,14 @@ export function expandQuery(q) {
 /* ------------------------- ② 零匹配回退轻目录 ------------------------- */
 
 /**
- * 轻目录：按名字典序排序取前 limit 个（name + 描述截 80 + server）。
+ * 轻目录：跨 server 轮转配额（name + 描述截 80 + server）。
  * limit<=0 → []（关闭回退，行为与旧版一致）。
+ *
+ * 2026-09-02（审计修复）：纯字典序 slice(0, limit) 会让名字靠后的 server
+ * （mcp__windows 等）在截断后全部不可见 → 模型零匹配回退时误判"没有该能力"。
+ * 改为：按 server 分组（组内字典序）→ round-robin 出队——小 server 必有代表、
+ * 大 server 拿多数、同输入同输出（确定性）。server 函数为每工具独立名时行为
+ * 与旧字典序一致（每组 1 个，轮转 = 字典序）。
  * @param {Array<{name:string,description?:string}>} allTools
  * @param {number} limit
  * @param {(name:string)=>string} [serverOf]
@@ -98,14 +104,28 @@ export function fallbackCatalog(allTools, limit, serverOf) {
   const lim = Math.max(0, Number(limit) || 0);
   if (!lim || !Array.isArray(allTools) || allTools.length === 0) return [];
   const svc = typeof serverOf === "function" ? serverOf : () => "";
-  return [...allTools]
-    .sort((a, b) => String(a.name).localeCompare(String(b.name)))
-    .slice(0, lim)
-    .map((t) => ({
-      name: String(t.name || ""),
-      description: String(t.description || "").slice(0, 80),
-      server: svc(t.name),
-    }));
+  const byServer = new Map();
+  for (const t of allTools) {
+    const s = svc(t.name) || "(none)";
+    if (!byServer.has(s)) byServer.set(s, []);
+    byServer.get(s).push(t);
+  }
+  // Deterministic: server order by name, tools inside each server sorted by name.
+  const queues = [...byServer.entries()]
+    .sort((a, b) => String(a[0]).localeCompare(String(b[0])))
+    .map(([, ts]) => ts.slice().sort((a, b) => String(a.name).localeCompare(String(b.name))));
+  const out = [];
+  while (out.length < lim && queues.some((q) => q.length > 0)) {
+    for (const q of queues) {
+      if (out.length >= lim) break;
+      if (q.length > 0) out.push(q.shift());
+    }
+  }
+  return out.map((t) => ({
+    name: String(t.name || ""),
+    description: String(t.description || "").slice(0, 80),
+    server: svc(t.name),
+  }));
 }
 
 /* ------------------------- ③ 检索结果缓存 ------------------------- */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name":  "dsh-tool-folder",
-    "version":  "0.2.4",
+    "version":  "0.2.5",
     "description":  "Fold the DSH tool surface per request + ChainGuard firewall (high-risk block + exfil-chain detection + anti-obfuscation) + BM25/bge-m3 hybrid tools_search. Shrinks schema tokens 80-90% while keeping selection accuracy. v0.2.0 adds a semantic retrieval leg (local Ollama bge-m3, RRF hybrid) and ChainGuard obfuscation detection (concat-rebuild / -enc / certutil / bitsadmin / homoglyphs).",
     "keywords":  [
                      "dsh-plugin",