dsh-tool-folder 0.2.5 → 0.3.2

package/lib/embed-remote.js

@@ -0,0 +1,193 @@
+/**
+ * embed-remote.js — 远程 OpenAI 兼容 embedding（默认火山方舟 doubao-embedding-vision）。
+ *
+ * 为什么需要（2026-10-17 实测）：v0.2.x 的语义腿只走本地 Ollama bge-m3，
+ * 而本机 `127.0.0.1:11434` 返回 HTTP 000（Ollama 未安装）→ `embedText` 恒为 null
+ * → `tools_search` 实际退化为纯 BM25。中文意图打英文工具描述时经常零命中。
+ *
+ * 设计要点（与 dsh-skill-folder v0.4.0 的同名模块同源）：
+ *   - 批量：OpenAI 兼容端点支持 `input: string[]`，一次 4 条（火山限流下的稳定值）
+ *   - 并发：2 路并发，批间 250ms 节流；429/5xx 退避重试（0.8s / 2s / 5s）
+ *   - 熔断：连续 3 次失败 → 冷却 5 分钟，期间 `remoteEmbedAvailable()` 为 false，
+ *     调用方自动回落到 Ollama / 纯 BM25
+ *   - 任何失败都返回 null/数组中的 null，绝不抛
+ */
+
+const DEFAULT_BASE = "https://ark.cn-beijing.volces.com/api/plan/v3";
+const DEFAULT_MODEL = "doubao-embedding-vision";
+const DEFAULT_TIMEOUT = 15000;
+const FAIL_THRESHOLD = 3;
+const COOLDOWN_MS = 5 * 60 * 1000;
+const DEFAULT_BATCH = 4;
+const DEFAULT_CONCURRENCY = 2;
+const BATCH_GAP_MS = 250;
+const BACKOFF_MS = [800, 2000, 5000];
+
+let _base = process.env.DSH_EMBED_BASE || DEFAULT_BASE;
+let _model = process.env.DSH_EMBED_MODEL || DEFAULT_MODEL;
+let _key = process.env.DSH_EMBED_KEY || process.env.OPENVIKING_ARK_KEY || "";
+let _timeout = DEFAULT_TIMEOUT;
+let _fails = 0;
+let _disabledUntil = 0;
+
+export function configureRemoteEmbed(cfg = {}) {
+  if (!cfg || typeof cfg !== "object") return;
+  if (typeof cfg.apiBase === "string" && cfg.apiBase) _base = cfg.apiBase;
+  if (typeof cfg.model === "string" && cfg.model) _model = cfg.model;
+  if (typeof cfg.apiKey === "string") _key = cfg.apiKey; // 允许显式空串禁用（测试用）
+  if (Number.isFinite(Number(cfg.timeoutMs)) && Number(cfg.timeoutMs) >= 100) _timeout = Number(cfg.timeoutMs);
+}
+
+export function remoteEmbedAvailable() {
+  return Boolean(_key) && Date.now() >= _disabledUntil;
+}
+
+export function remoteEmbedState() {
+  return {
+    available: remoteEmbedAvailable(),
+    base: _base,
+    model: _model,
+    fails: _fails,
+    disabledUntil: _disabledUntil,
+    hasKey: Boolean(_key),
+  };
+}
+
+function trip(reason) {
+  _fails += 1;
+  if (_fails >= FAIL_THRESHOLD && Date.now() >= _disabledUntil) {
+    _disabledUntil = Date.now() + COOLDOWN_MS;
+    try {
+      console.log(`[tool-folder] remote embedding 熔断（${reason}），${COOLDOWN_MS / 1000}s 内只用本地/词法腿`);
+    } catch {
+      /* 日志失败不影响功能 */
+    }
+  }
+}
+
+function reset() {
+  _fails = 0;
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+/** 一次 HTTP 调用，input 可以是单条字符串或字符串数组。 */
+async function callOnce(input, timeoutMs) {
+  const ctrl = new AbortController();
+  const timer = setTimeout(() => ctrl.abort(), timeoutMs);
+  try {
+    const res = await fetch(`${_base.replace(/\/+$/, "")}/embeddings`, {
+      method: "POST",
+      headers: { "content-type": "application/json", authorization: `Bearer ${_key}` },
+      body: JSON.stringify({
+        model: _model,
+        input,
+        encoding_format: "float",
+      }),
+      signal: ctrl.signal,
+    });
+    if (!res.ok) {
+      const body = await res.text().catch(() => "");
+      const err = new Error(`HTTP ${res.status} ${body.slice(0, 120)}`);
+      err.retryable = res.status === 429 || res.status >= 500;
+      err.status = res.status;
+      throw err;
+    }
+    const j = await res.json();
+    const rows = Array.isArray(j?.data) ? j.data : [];
+    if (!rows.length) throw new Error("empty embedding response");
+    // 按 index 对齐（端点不保证顺序）
+    const out = rows
+      .slice()
+      .sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
+      .map((r) => (Array.isArray(r?.embedding) && r.embedding.length ? r.embedding.map(Number) : null));
+    if (!out.some(Boolean)) throw new Error("no usable embedding");
+    return out;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** 单条文本 → 向量；失败返回 null（绝不抛）。 */
+export async function remoteEmbed(text, timeoutMs = _timeout) {
+  const s = String(text || "").trim();
+  if (!s || !remoteEmbedAvailable()) return null;
+  for (let attempt = 0; attempt <= BACKOFF_MS.length; attempt++) {
+    try {
+      const [v] = await callOnce(s.slice(0, 6000) || " ", timeoutMs);
+      if (Array.isArray(v) && v.length) {
+        reset();
+        return v;
+      }
+      trip("empty embedding");
+      return null;
+    } catch (e) {
+      const retryable = e?.retryable || e?.name === "AbortError";
+      if (retryable && attempt < BACKOFF_MS.length) {
+        await sleep(BACKOFF_MS[attempt]);
+        continue;
+      }
+      trip(`${e?.status || e?.name || "error"}`);
+      return null;
+    }
+  }
+  return null;
+}
+
+/**
+ * 批量 embedding。返回与 texts 等长的数组（失败的槽位是 null）。
+ * 批内失败会自动拆成单条重试，尽力保住能拿到的向量。
+ *
+ * @param {string[]} texts
+ * @param {{timeoutMs?: number, batchSize?: number, concurrency?: number, onProgress?: Function}} [opts]
+ * @returns {Promise<Array<number[]|null>>}
+ */
+export async function remoteEmbedBatch(texts, opts = {}) {
+  const list = Array.isArray(texts) ? texts : [];
+  const out = new Array(list.length).fill(null);
+  if (!list.length || !remoteEmbedAvailable()) return out;
+
+  const timeoutMs = Number(opts.timeoutMs) || _timeout;
+  const batchSize = Math.min(Math.max(Number(opts.batchSize) || DEFAULT_BATCH, 1), 16);
+  const concurrency = Math.min(Math.max(Number(opts.concurrency) || DEFAULT_CONCURRENCY, 1), 4);
+
+  const batches = [];
+  for (let i = 0; i < list.length; i += batchSize) {
+    batches.push({ start: i, items: list.slice(i, i + batchSize) });
+  }
+
+  let cursor = 0;
+  const worker = async () => {
+    while (cursor < batches.length) {
+      const b = batches[cursor++];
+      if (!remoteEmbedAvailable()) return; // 熔断后立刻停手
+      const payload = b.items.map((t) => String(t || "").trim().slice(0, 6000) || " ");
+      try {
+        const vecs = await callOnce(payload, timeoutMs);
+        reset();
+        for (let k = 0; k < vecs.length && k < b.items.length; k++) {
+          if (vecs[k]) out[b.start + k] = vecs[k];
+        }
+      } catch (e) {
+        trip(`${e?.status || e?.name || "error"}`);
+        // 批失败拆单条，尽量保住部分向量
+        for (let k = 0; k < payload.length; k++) {
+          if (!remoteEmbedAvailable()) break;
+          const v = await remoteEmbed(payload[k], timeoutMs);
+          if (v) out[b.start + k] = v;
+        }
+      }
+      if (typeof opts.onProgress === "function") {
+        try {
+          opts.onProgress(out.filter(Boolean).length, list.length);
+        } catch {
+          /* 进度回调失败不影响索引 */
+        }
+      }
+      await sleep(BATCH_GAP_MS);
+    }
+  };
+
+  await Promise.all(Array.from({ length: concurrency }, () => worker()));
+  return out;
+}

package/lib/expand.js

@@ -0,0 +1,276 @@
+/**
+ * expand.js — 查询扩展（零依赖、纯本地）
+ *
+ * 解决的核心问题（2026-10-31 评测实测）：
+ *   「上网查一下这个报错怎么解决」→ 期望 web_search，实得 **空**
+ *   「谁在调这个方法」          → 期望 find_references，实得 **空**
+ *   「把这个配置改成新的值」     → 期望 write/edit，实得 **空**
+ *
+ * 根因：纯字面 bigram 匹配没有同义桥接。「上网」和工具描述里的「联网」是两个
+ * 不相交的 bigram；「方法」和「符号」也是。字面不重合 = 零分 = 召回为 0。
+ *
+ * 三条互补的扩展路径：
+ *
+ *   1. 内置开发词表（thesaurus）——覆盖最高频的意图词，中英双向。
+ *      这是唯一能救「首轮零命中」的手段：RM3 需要先有结果才能反馈，
+ *      零命中时它无从下手（鸡生蛋问题）。
+ *
+ *   2. 反向别名扩展（alias 反向）——配置里的 aliases 是
+ *      canonical → [同义词]，只用于「工具能被哪些词命中」。这里做反查：
+ *      查询里出现某个同义词时，把 canonical 也加进查询。
+ *
+ *   3. RM3 伪相关反馈（Lavrenko & Croft 2001，Anserini / Lucene 同款思路）——
+ *      首轮 top-k 结果里抽高权重的词，与原查询加权拼接后二次检索。
+ *      只在首轮有结果时生效，用来扩大召回并纠正漂移。
+ *
+ * 设计约束：不碰 fs / net，纯字符串与数字运算；单次扩展 < 0.1ms。
+ */
+
+/**
+ * 内置中英开发词表。
+ * key 是会被用户在查询里打出来的词；value 是要注入查询的扩展词
+ * （已小写、已分词形态：英文单词 + 中文 bigram）。
+ */
+const THESAURUS = {
+  // ---- 文件读写 ----
+  "看看": ["read", "读取", "内容", "查看"],
+  "查看": ["read", "读取", "内容"],
+  "读取": ["read", "读取"],
+  "打开": ["read", "open", "读取"],
+  "内容": ["read", "读取", "content"],
+  "写入": ["write", "写入", "保存"],
+  "保存": ["write", "保存"],
+  "改成": ["write", "edit", "修改"],
+  "修改": ["edit", "write", "修改", "替换"],
+  "替换": ["edit", "替换"],
+  "删掉": ["edit", "delete", "删除"],
+  "删除": ["delete", "remove", "删除", "移除"],
+  "移除": ["remove", "删除"],
+
+  // ---- 搜索 ----
+  "搜": ["search", "搜索", "查找", "grep"],
+  "搜索": ["search", "搜索", "查找"],
+  "查找": ["search", "find", "查找", "grep"],
+  "找": ["find", "search", "查找"],
+  "上网": ["web", "联网", "搜索", "search"],
+  "联网": ["web", "联网"],
+  "网络": ["web", "网络"],
+  "网页": ["web", "fetch", "抓取"],
+  "抓": ["fetch", "抓取"],
+
+  // ---- 代码符号 ----
+  "方法": ["symbol", "function", "符号", "定义"],
+  "函数": ["symbol", "function", "符号"],
+  "符号": ["symbol", "符号"],
+  "定义": ["symbol", "definition", "定义"],
+  "调用": ["references", "caller", "引用"],
+  "引用": ["references", "引用"],
+  "调": ["references", "call", "引用"],
+  "谁": ["references", "caller", "引用"],
+
+  // ---- Git ----
+  "改动": ["diff", "status", "差异", "改动"],
+  "改了": ["diff", "edit", "改动"],
+  "改": ["edit", "write", "diff", "修改", "改动"],
+  "行": ["line", "diff", "行"],
+  "哪些行": ["diff", "line"],
+  "差异": ["diff", "差异"],
+  "提交": ["commit", "提交"],
+  "历史": ["log", "history", "历史"],
+  "分支": ["branch", "分支"],
+  "工作区": ["status", "工作区"],
+
+  // ---- 安全 ----
+  "安全": ["security", "安全", "semgrep", "审计"],
+  "漏洞": ["vulnerability", "security", "漏洞"],
+  "审计": ["audit", "审计"],
+  "扫描": ["scan", "扫描"],
+
+  // ---- 笔记 / 知识 ----
+  "笔记": ["note", "笔记", "notes"],
+  "文档": ["note", "document", "文档"],
+  "资料": ["document", "资料", "笔记"],
+  "记一条": ["note", "create", "新建"],
+
+  // ---- 交互 / 杂项 ----
+  "报错": ["error", "错误", "异常"],
+  "错误": ["error", "错误"],
+  "异常": ["error", "异常", "exception"],
+  "解决": ["error", "fix", "解决"],
+  "问": ["ask", "question", "提问"],
+  "截屏": ["screenshot", "截图", "screen"],
+  "截图": ["screenshot", "截图"],
+  "截": ["screenshot", "截屏"],
+  "屏": ["screenshot", "screen", "截屏"],
+  "个屏": ["screenshot", "截屏"],
+  "剪贴板": ["clipboard", "剪贴板"],
+  "复制": ["clipboard", "复制"],
+  "讲": ["explain", "讲解", "概念"],
+  "讲讲": ["explain", "讲解"],
+  "概念": ["explain", "concept", "概念"],
+  "浏览器": ["browser", "navigate", "浏览器"],
+  "网址": ["navigate", "url", "网址"],
+  "按钮": ["click", "点击", "按钮"],
+  "点击": ["click", "点击"],
+  "画布": ["canvas", "frame", "画布"],
+  "画框": ["frame", "画框"],
+  "技能": ["skill", "技能"],
+  "交给": ["delegate", "委派"],
+  "委派": ["delegate", "委派"],
+  "拆开": ["subagent", "delegate", "并行"],
+  "并行": ["subagent", "并行"],
+  "跑": ["exec", "run", "执行", "命令"],
+  "执行": ["exec", "执行", "命令"],
+  "命令": ["exec", "command", "命令"],
+  "想想": ["thinking", "reason", "推理"],
+  "一步步": ["thinking", "step", "推理"],
+  "影响": ["impact", "影响", "波及"],
+  "文件": ["file", "文件"],
+};
+
+/** 从中文/英文原文里抽候选词（供词表查表用，不做 IDF 计算） */
+function probeTerms(text) {
+  const out = new Set();
+  const raw = String(text || "");
+  const lower = raw.toLowerCase();
+
+  for (const m of raw.matchAll(/[A-Za-z0-9][A-Za-z0-9_+-]{1,}/g)) {
+    out.add(m[0].toLowerCase());
+  }
+  const cjk = lower.replace(/[^\u4e00-\u9fff]/g, "");
+  // 单字 + bigram 都作为探针，词表的 key 以 1~3 字为主
+  for (let i = 0; i < cjk.length; i++) {
+    out.add(cjk[i]);
+    if (i + 1 < cjk.length) out.add(cjk.slice(i, i + 2));
+    if (i + 2 < cjk.length) out.add(cjk.slice(i, i + 3));
+  }
+  return out;
+}
+
+/**
+ * 用内置词表 + 反向别名扩展查询。
+ * @param {string} query
+ * @param {{aliases?: object, extra?: object}} [opts]
+ * @returns {string[]} 扩展后的 token 数组（可直接喂 bm25.search）
+ */
+function expandQuery(query, opts = {}) {
+  const probes = probeTerms(query);
+  const added = new Set();
+
+  // 1) 内置词表
+  for (const p of probes) {
+    const syn = THESAURUS[p];
+    if (syn) for (const s of syn) added.add(s);
+  }
+
+  // 2) 反向别名：aliases = { canonical: [syn...] }
+  //    查询里出现某个 syn → 把 canonical 加进来
+  const aliases = opts.aliases || {};
+  for (const [canonical, syns] of Object.entries(aliases)) {
+    if (!Array.isArray(syns)) continue;
+    const canon = String(canonical).toLowerCase();
+    for (const s of syns) {
+      const syn = String(s).toLowerCase();
+      if (probes.has(syn)) {
+        added.add(canon);
+        break;
+      }
+    }
+  }
+
+  // 3) 额外词表（调用方可注入）
+  for (const p of probes) {
+    const syn = (opts.extra || {})[p];
+    if (syn) for (const s of syn) added.add(s);
+  }
+
+  return [...added];
+}
+
+/**
+ * RM3 伪相关反馈检索。
+ *
+ * 首轮用原查询取 top-k，从这些文档里按 tf·idf 抽 top-N 词，
+ * 与原查询按 mu 加权融合后二次检索。
+ * 首轮零命中时直接返回空（RM3 无从反馈）。
+ *
+ * @param index    bm25.buildIndex 的产物
+ * @param query    原始查询字符串
+ * @param bm25     bm25 模块（要能拿到 tokenize / score）
+ * @param {{topK?:number, fbDocs?:number, fbTerms?:number, mu?:number, expanded?:string[]}} opts
+ * @returns {number[]} doc 索引数组
+ */
+function searchWithFeedback(index, query, bm25, opts = {}) {
+  const topK = opts.topK ?? 6;
+  const fbDocs = opts.fbDocs ?? 5;
+  const fbTerms = opts.fbTerms ?? 8;
+  const mu = opts.mu ?? 0.65; // 原查询权重
+
+  const { n, docTokens, df, avgdl } = index;
+  if (!n) return [];
+
+  const orig = bm25.tokenize(query);
+  if (!orig.length) return [];
+
+  // 若调用方已给定扩展词，直接并进原查询
+  const baseTerms = orig.concat(opts.expanded || []);
+  const first = bm25.searchScored
+    ? bm25.searchScored(index, baseTerms, topK)
+    : bm25.search(index, baseTerms, topK).map(i => [i, 0]);
+
+  if (!first.length) return [];
+
+  // ---- 从 top 文档抽反馈词 ----
+  const fb = new Map(); // term -> weight
+  const topDocs = first.slice(0, fbDocs);
+  let totalScore = 0;
+  for (const [, s] of topDocs) totalScore += s;
+
+  for (const [di, s] of topDocs) {
+    const docWeight = totalScore > 0 ? s / totalScore : 1 / topDocs.length;
+    const tf = new Map();
+    for (const t of docTokens[di]) tf.set(t, (tf.get(t) || 0) + 1);
+    const seen = new Set(orig);
+    for (const [t, f] of tf) {
+      if (seen.has(t)) continue; // 原查询已有的词不重复加
+      const d = df.get(t) || 0;
+      const idf = Math.log((n - d + 0.5) / (d + 0.5) + 1);
+      fb.set(t, (fb.get(t) || 0) + docWeight * f * idf);
+    }
+  }
+
+  const expansion = [...fb.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, fbTerms)
+    .map(([t]) => t);
+
+  if (!expansion.length) return first.map(([i]) => i);
+
+  // ---- 加权融合：原查询词权重 mu，反馈词权重 (1-mu) 均分 ----
+  // 注意：不能靠「重复词」加权——search 内部用 Map 按词去重，重复无效。
+  // 这里自己构造 queryIdf，把权重直接乘进 idf。
+  const queryIdf = new Map();
+  const origSet = new Set(baseTerms);
+  for (const t of origSet) {
+    const d = df.get(t) || 0;
+    const idf = Math.log((n - d + 0.5) / (d + 0.5) + 1);
+    queryIdf.set(t, idf * mu);
+  }
+  const perTerm = (1 - mu) / Math.max(1, expansion.length);
+  for (const t of expansion) {
+    const d = df.get(t) || 0;
+    const idf = Math.log((n - d + 0.5) / (d + 0.5) + 1);
+    const prev = queryIdf.get(t) || 0;
+    queryIdf.set(t, prev + idf * perTerm);
+  }
+
+  const out = [];
+  for (let i = 0; i < n; i++) {
+    const s = bm25.score(docTokens[i], queryIdf, n, avgdl);
+    if (s > 0) out.push([i, s]);
+  }
+  out.sort((a, b) => b[1] - a[1]);
+  return out.slice(0, topK).map(([i]) => i);
+}
+
+export { expandQuery, searchWithFeedback, probeTerms, THESAURUS };