dsh-tool-folder 0.3.0 → 0.3.2

package/lib/static-embed.js

@@ -0,0 +1,195 @@
+/**
+ * static-embed.js — 离线静态嵌入（零推理 / 零网络 / 零依赖）
+ *
+ * 为什么需要：语义腿原有两条路都不可靠——
+ *   Ark doubao-embedding：订阅过期即 400，熔断 5 分钟
+ *   Ollama bge-m3：本机未安装（11434 返回 000）
+ * 实测（eval/queries-holdout.json，29 条留出查询）纯词法 + 意图词表只到
+ * recall@3 0.448 —— 剩下 55% 是词表覆盖不到的改写，必须有语义。
+ *
+ * 做法（model2vec 思路，MinishLab）：把 sentence transformer 蒸馏成一张
+ * 「token → 向量」查找表，运行期只做 查表 + 均值池化 + 余弦。
+ * 无推理图、无 onnx、无 WASM、无网络。
+ *
+ * 构建：tools/build-static-embed.py（离线，读本地 safetensors）
+ * 产物三件套，写入 lib/static-embed/：
+ *   vocab.json   {"dim":256,"dtype":"int8","count":N,"tokens":{token:rowIndex}}
+ *   vectors.bin  int8 行优先，行已 L2 归一化后再量化
+ *   scales.bin   float32 每行独立 scale
+ *   meta.json    构建信息
+ *
+ * 分词口径必须与 Python 侧 tokenize_mixed() 逐字一致，
+ * 否则构建期建的词和运行期查的词对不上。
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+const CJK_CHAR = /[\u4e00-\u9fff]/;
+const PIECE = /[\u4e00-\u9fff]|[A-Za-z0-9]+/g;
+
+let _cache = null;
+
+/**
+ * 与 Python tokenize_mixed() 完全同构：
+ *  1) camelCase / PascalCase 在 lowercase 之前拆开
+ *  2) CJK 单字成 token
+ *  3) 拉丁/数字按连续段切，lowercase，长度 ≥ 2 才收
+ */
+export function tokenizeMixed(text) {
+  let s = String(text == null ? "" : text);
+  s = s
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
+
+  const out = [];
+  PIECE.lastIndex = 0;
+  let m;
+  while ((m = PIECE.exec(s)) !== null) {
+    const part = m[0];
+    if (part.length === 1 && CJK_CHAR.test(part)) {
+      out.push(part);
+    } else {
+      const w = part.toLowerCase();
+      if (w.length >= 2) out.push(w);
+    }
+  }
+  return out;
+}
+
+/** 读 meta/vocab/vectors/scales，任何一步失败都返回 null（调用方退回词法）。 */
+function loadFrom(dir) {
+  try {
+    const p = (f) => path.join(dir, f);
+    const files = [p("meta.json"), p("vocab.json"), p("vectors.bin")];
+    for (const f of files) if (!fs.existsSync(f)) return null;
+
+    const mtime = fs.statSync(p("vectors.bin")).mtimeMs;
+    if (_cache && _cache.dir === dir && _cache.mtime === mtime) return _cache;
+
+    const meta = JSON.parse(fs.readFileSync(p("meta.json"), "utf8"));
+    const vj = JSON.parse(fs.readFileSync(p("vocab.json"), "utf8"));
+    const dim = Number(meta.dim || vj.dim);
+    const count = Number(meta.count || vj.count);
+    if (!dim || !count) return null;
+
+    const vbuf = fs.readFileSync(p("vectors.bin"));
+    if (vbuf.length < count * dim) return null;
+    const vectors = new Int8Array(vbuf.buffer, vbuf.byteOffset, count * dim);
+
+    // 每行独立 scale；退化为全局 1 时不报错
+    let scales = new Float32Array(count).fill(1);
+    if (fs.existsSync(p("scales.bin"))) {
+      const sbuf = fs.readFileSync(p("scales.bin"));
+      if (sbuf.length >= count * 4) {
+        const dv = new DataView(sbuf.buffer, sbuf.byteOffset, count * 4);
+        for (let i = 0; i < count; i++) scales[i] = dv.getFloat32(i * 4, true);
+      }
+    }
+
+    const tokens = vj.tokens || {};
+    const isArray = Array.isArray(tokens);
+    const vocab = new Map();
+    if (isArray) {
+      for (let i = 0; i < tokens.length; i++) vocab.set(tokens[i], i);
+    } else {
+      for (const [k, v] of Object.entries(tokens)) vocab.set(k, v | 0);
+    }
+
+    _cache = {
+      dir,
+      mtime,
+      vocab,
+      vectors,
+      scales,
+      dim,
+      count,
+      model: meta.source || meta.model || "",
+      bytes: vbuf.length + count * 4,
+    };
+    return _cache;
+  } catch {
+    return null;
+  }
+}
+
+/** token 序列 → 行下标（丢掉词表外的 token）。 */
+export function toIds(text, st) {
+  const ids = [];
+  for (const t of tokenizeMixed(text)) {
+    const id = st.vocab.get(t);
+    if (id !== undefined) ids.push(id);
+  }
+  return ids;
+}
+
+/** 均值池化（含每行反量化）→ L2 归一化 Float32Array。空文本返回 null。 */
+export function pool(ids, st) {
+  if (!ids.length) return null;
+  const { dim, vectors, scales } = st;
+  const v = new Float32Array(dim);
+  for (let k = 0; k < ids.length; k++) {
+    const id = ids[k];
+    const base = id * dim;
+    const sc = scales[id];
+    for (let d = 0; d < dim; d++) v[d] += vectors[base + d] * sc;
+  }
+  let norm = 0;
+  for (let d = 0; d < dim; d++) norm += v[d] * v[d];
+  norm = Math.sqrt(norm);
+  if (norm > 1e-12) {
+    for (let d = 0; d < dim; d++) v[d] /= norm;
+  } else {
+    return null;
+  }
+  return v;
+}
+
+/** 余弦（两边已归一化时等价点积）。 */
+export function cosine(a, b) {
+  if (!a || !b) return 0;
+  const n = Math.min(a.length, b.length);
+  let s = 0;
+  for (let i = 0; i < n; i++) s += a[i] * b[i];
+  return s;
+}
+
+/**
+ * 打开静态嵌入目录。
+ * @param {string} dir
+ * @returns {{available:boolean, dim:number, count:number, model:string, bytes:number,
+ *            embed:(t:string)=>Float32Array|null,
+ *            embedMany:(ts:string[])=>Array<Float32Array|null>}}
+ */
+export function openStaticEmbed(dir) {
+  const st = dir ? loadFrom(dir) : null;
+  if (!st) {
+    return {
+      available: false, dim: 0, count: 0, model: "", bytes: 0,
+      embed: () => null,
+      embedMany: (n) => n.map(() => null),
+    };
+  }
+  return {
+    available: true,
+    dim: st.dim,
+    count: st.count,
+    model: st.model,
+    bytes: st.bytes,
+    embed: (t) => pool(toIds(t, st), st),
+    embedMany: (ts) => ts.map((t) => pool(toIds(t, st), st)),
+  };
+}
+
+/** 在候选向量里按余弦排序，返回 [{index, score}]。 */
+export function rankByVector(queryVec, vectors, topK = 10) {
+  if (!queryVec) return [];
+  const out = [];
+  for (let i = 0; i < vectors.length; i++) {
+    const v = vectors[i];
+    if (!v) continue;
+    out.push([i, cosine(queryVec, v)]);
+  }
+  out.sort((a, b) => b[1] - a[1]);
+  return out.slice(0, topK).map(([i, s]) => ({ index: i, score: s }));
+}

package/package.json

@@ -1,56 +1,56 @@
-{
-  "name": "dsh-tool-folder",
-  "version": "0.3.0",
-  "description": "Fold the DSH tool surface per request + ChainGuard firewall. v0.3.0: remote-first semantic retrieval (Ark doubao-embedding / Ollama fallback), weighted RRF with adaptive BM25 downweight, batched incremental index (partial results cached), body-level rerank (opt-in), duplicate-id fusion fix.",
-  "keywords": [
-    "dsh-plugin",
-    "dsh",
-    "deepseek-harness",
-    "cordis",
-    "tool-selection",
-    "token-optimization",
-    "bm25",
-    "semantic-search",
-    "security",
-    "firewall",
-    "obfuscation"
-  ],
-  "type": "module",
-  "main": "lib/index.js",
-  "files": [
-    "lib",
-    "cordis.patch.yml",
-    "README.md"
-  ],
-  "dsh": {
-    "bundle": {
-      "patch": "./cordis.patch.yml"
-    }
-  },
-  "license": "MIT",
-  "engines": {
-    "node": ">=18"
-  },
-  "dependencies": {
-    "@deepseek-ai/schemastery": "^3.18.1"
-  },
-  "peerDependencies": {
-    "@deepseek-ai/cordis": "^4.0.1",
-    "@deepseek-ai/dsh-tools": "^0.1.0-rc.6"
-  },
-  "scripts": {
-    "test": "node --test \"test/*.test.js\""
-  },
-  "exports": {
-    ".": "./lib/index.js",
-    "./lib/*": "./lib/*"
-  },
-  "peerDependenciesMeta": {
-    "@deepseek-ai/cordis": {
-      "optional": true
-    },
-    "@deepseek-ai/dsh-tools": {
-      "optional": true
-    }
-  }
-}
+{
+  "name": "dsh-tool-folder",
+  "version": "0.3.2",
+  "description": "v0.3.2: offline static embedding leg (7760x256 int8, 2MB, built from potion-multilingual — replaces dead Ark/Ollama paths), staticWeight 0.15 tuned by sweep, empty-query guard. Fold the DSH tool surface per request + ChainGuard firewall. v0.3.1: intent thesaurus expansion (CN/EN bidirectional + reverse aliases), camelCase name splitting, CJK stop-bigram filtering, body-rerank weight tuned 0.35->0.45, RRF fusion option. v0.3.0: remote-first semantic retrieval (Ark doubao-embedding / Ollama fallback), weighted RRF with adaptive BM25 downweight, batched incremental index (partial results cached), body-level rerank (opt-in), duplicate-id fusion fix.",
+  "keywords": [
+    "dsh-plugin",
+    "dsh",
+    "deepseek-harness",
+    "cordis",
+    "tool-selection",
+    "token-optimization",
+    "bm25",
+    "semantic-search",
+    "security",
+    "firewall",
+    "obfuscation"
+  ],
+  "type": "module",
+  "main": "lib/index.js",
+  "files": [
+    "lib",
+    "cordis.patch.yml",
+    "README.md"
+  ],
+  "dsh": {
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    }
+  },
+  "license": "MIT",
+  "engines": {
+    "node": ">=18"
+  },
+  "dependencies": {
+    "@deepseek-ai/schemastery": "^3.18.1"
+  },
+  "peerDependencies": {
+    "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-tools": "^0.1.0-rc.6"
+  },
+  "scripts": {
+    "test": "node --test \"test/*.test.js\""
+  },
+  "exports": {
+    ".": "./lib/index.js",
+    "./lib/*": "./lib/*"
+  },
+  "peerDependenciesMeta": {
+    "@deepseek-ai/cordis": {
+      "optional": true
+    },
+    "@deepseek-ai/dsh-tools": {
+      "optional": true
+    }
+  }
+}