@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0

package/memory-engine/lib/embedder.mjs

@@ -1,242 +1,223 @@
-// 记忆引擎 v2 · 离线层 —— 编码器（可注入）
-//
-// 三种实现，接口一致（encode / encodeBatch / health / describe）：
-//   1. createLocalEmbedder  —— 本地 ONNX（@huggingface/transformers + bge-small-zh-v1.5）
-//   2. createHttpEmbedder   —— 走回环编码桥（serve.mjs，127.0.0.1:8201）
-//   3. createStubEmbedder   —— 零依赖确定性哈希编码器：**仅供测试骨架**，无任何语义质量
-//
-// 设计要点：同步/召回核心一律依赖本接口，不直接 import 模型运行时
-// ⇒ 模型拉不动时，骨架与单测仍可完整跑通（本卡 §二.6 的退路）。
-
-import crypto from 'node:crypto';
+// 编码器层：同一个接口的三种实现，调用方只认 encode / encodeBatch / health / describe。
+//   · stub  —— 确定性哈希，零依赖，只给骨架与单测用（没有半点语义质量）
+//   · http  —— 回环编码桥的客户端（超时默认 250ms）
+//   · local —— 本地 ONNX 推理（bge 小模型，CPU 上跑）
+// 模型运行时只在 local 里动态 import ⇒ 权重拉不下来时，其余实现与测试照样能跑。
+
+import { createHash } from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 
-/** 确定性 stub：char n-gram 哈希到 dim 维 + L2 归一化（测试用，禁止用于生产召回） */
-export function createStubEmbedder({ dim = 512, modelId = 'stub-ngram', sleepMs = 0 } = {}) {
+const STUB_MAX_N = 3;
+const HEALTH_TIMEOUT_MS = 2000;
+const BATCH_TIMEOUT_FLOOR_MS = 30000;
+const LOCAL_PROBE_TEXT = '维度自检';
+
+const errorText = (e) => (e && e.message ? e.message : String(e));
+const sleep = (ms) => new Promise((done) => setTimeout(done, ms));
+
+// 原地单位化；零向量按长度 1 处理，免得除零把整条变成 NaN。
+function unitize(vec) {
+  let squareSum = 0;
+  for (let i = 0; i < vec.length; i += 1) squareSum += vec[i] * vec[i];
+  const length = Math.sqrt(squareSum) || 1;
+  for (let i = 0; i < vec.length; i += 1) vec[i] /= length;
+  return vec;
+}
+
+// 连着的 n 元切片，n = 1..maxN；先统一小写再切。
+function ngramsOf(text, maxN) {
+  const body = text.toLowerCase();
+  const grams = [];
+  for (let n = 1; n <= maxN; n += 1) {
+    for (let at = 0; at + n <= body.length; at += 1) grams.push(body.slice(at, at + n));
+  }
+  return grams;
+}
+
+/** 测试骨架用的确定性编码器：每个 n 元算一次 md5，头两字节定槽位、第三字节定正负。 */
+export function createStubEmbedder({ dim = 512, sleepMs = 0, modelId = 'stub-ngram' } = {}) {
   let encodeCount = 0;
-  const encodeOne = (text) => {
-    const v = new Float32Array(dim);
-    const s = String(text ?? '').toLowerCase();
-    const grams = [];
-    for (let n = 1; n <= 3; n += 1) {
-      for (let i = 0; i + n <= s.length; i += 1) grams.push(s.slice(i, i + n));
-    }
-    for (const g of grams) {
-      const h = crypto.createHash('md5').update(g).digest();
-      const idx = ((h[0] << 8) | h[1]) % dim;
-      const sign = h[2] % 2 === 0 ? 1 : -1;
-      v[idx] += sign;
+
+  const hashToVector = (text) => {
+    const vec = new Float32Array(dim);
+    for (const gram of ngramsOf(String(text ?? ''), STUB_MAX_N)) {
+      const digest = createHash('md5').update(gram).digest();
+      const slot = ((digest[0] << 8) | digest[1]) % dim;
+      vec[slot] += digest[2] % 2 === 0 ? 1 : -1;
     }
-    let norm = 0;
-    for (let i = 0; i < dim; i += 1) norm += v[i] * v[i];
-    norm = Math.sqrt(norm) || 1;
-    for (let i = 0; i < dim; i += 1) v[i] /= norm;
-    return v;
+    return unitize(vec);
   };
+
+  // 每次编码都过这里：模拟耗时 + 计数（两者都是测试要观测的）
+  const encodeOnce = async (text) => {
+    if (sleepMs) await sleep(sleepMs);
+    encodeCount = encodeCount + 1;
+    return hashToVector(text);
+  };
+
   return {
-    kind: 'stub',
-    modelId,
-    dim,
-    describe: () => ({ kind: 'stub', modelId, dim, note: '确定性哈希编码器，仅用于骨架自测' }),
-    async health() {
-      return { ready: true, modelLoading: false, model: modelId, dim };
-    },
-    async encode(text) {
-      if (sleepMs) await new Promise((r) => setTimeout(r, sleepMs));
-      encodeCount += 1;
-      return encodeOne(text);
+    kind: 'stub', modelId, dim,
+    describe: () => ({
+      kind: 'stub',
+      modelId,
+      dim,
+      note: '确定性哈希编码器，仅用于骨架自测',
+    }),
+    health: async () => ({ ready: true, modelLoading: false, model: modelId, dim }),
+    encode: (text) => encodeOnce(text),
+    encodeBatch: async (texts) => {
+      const vectors = [];
+      for (const text of texts) vectors.push(await encodeOnce(text));
+      return vectors;
     },
-    async encodeBatch(texts) {
-      const out = [];
-      for (const t of texts) {
-        if (sleepMs) await new Promise((r) => setTimeout(r, sleepMs));
-        encodeCount += 1;
-        out.push(encodeOne(t));
-      }
-      return out;
+    stats() {
+      return { encodeCount };
     },
-    stats: () => ({ encodeCount }),
   };
 }
 
-/** 回环编码桥客户端（插件侧 / CLI 侧共用；方案 §3.1 编码通道契约：超时 250ms） */
-export function createHttpEmbedder({
-  host = '127.0.0.1',
-  port = 8201,
-  timeoutMs = 250,
-  modelId = null,
-  dim = null,
-} = {}) {
+/** 回环编码桥的客户端：只发 JSON，一切都带超时，桥不在就如实报错而不是挂着。 */
+export function createHttpEmbedder({ host = '127.0.0.1', port = 8201, timeoutMs = 250, modelId = null, dim = null } = {}) {
   const base = `http://${host}:${port}`;
-  let healthCache = null;
-  const fetchJson = async (url, init = {}, ms = timeoutMs) => {
-    const ac = new AbortController();
-    const t = setTimeout(() => ac.abort(), ms);
+  let lastHealth = null;
+
+  // payload 为 null ⇒ 走 GET；否则按 JSON POST 发出去。
+  const callJson = async (url, payload, ms) => {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), ms);
     try {
-      const res = await fetch(url, { ...init, signal: ac.signal });
+      const init = payload
+        ? {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify(payload),
+            signal: controller.signal,
+          }
+        : { signal: controller.signal };
+      const res = await fetch(url, init);
       if (!res.ok) throw new Error(`HTTP ${res.status}`);
       return await res.json();
     } finally {
-      clearTimeout(t);
+      clearTimeout(timer);
     }
   };
+
   return {
     kind: 'http',
-    get modelId() {
-      return modelId || (healthCache && healthCache.model) || 'unknown';
-    },
-    get dim() {
-      return dim || (healthCache && healthCache.dim) || 0;
-    },
+    // 显式给了就用手上的；没给就跟着最近一次探活的结果走（读那一瞬才算）
+    get modelId() { return modelId || (lastHealth && lastHealth.model) || 'unknown'; },
+    get dim() { return dim || (lastHealth && lastHealth.dim) || 0; },
     describe: () => ({ kind: 'http', base, modelId, dim, timeoutMs }),
-    async health() {
+    health: async () => {
       try {
-        healthCache = await fetchJson(`${base}/health`, {}, 2000);
-        return { ...healthCache, bridge: base };
+        lastHealth = await callJson(`${base}/health`, null, HEALTH_TIMEOUT_MS);
+        return { ...lastHealth, bridge: base };
       } catch (e) {
-        return { ready: false, modelLoading: false, error: String(e && e.message ? e.message : e), bridge: base };
+        return { ready: false, modelLoading: false, error: errorText(e), bridge: base };
       }
     },
-    async encode(text) {
-      const out = await fetchJson(
-        `${base}/encode`,
-        {
-          method: 'POST',
-          headers: { 'content-type': 'application/json' },
-          body: JSON.stringify({ texts: [String(text ?? '')] }),
-        },
-        timeoutMs,
-      );
-      return Float32Array.from(out.vectors[0]);
+    encode: async (text) => {
+      const answer = await callJson(`${base}/encode`, { texts: [String(text ?? '')] }, timeoutMs);
+      return Float32Array.from(answer.vectors[0]);
     },
-    async encodeBatch(texts) {
-      if (!texts.length) return [];
-      const out = await fetchJson(
+    encodeBatch: async (texts) => {
+      if (!texts.length) {
+        return [];
+      }
+      const answer = await callJson(
         `${base}/encode`,
-        {
-          method: 'POST',
-          headers: { 'content-type': 'application/json' },
-          body: JSON.stringify({ texts: texts.map((t) => String(t ?? '')) }),
-        },
-        Math.max(timeoutMs, 30000),
+        { texts: texts.map((text) => String(text ?? '')) },
+        Math.max(timeoutMs, BATCH_TIMEOUT_FLOOR_MS),
       );
-      return out.vectors.map((v) => Float32Array.from(v));
+      return answer.vectors.map((row) => Float32Array.from(row));
+    },
+    stats() {
+      return { encodeCount: -1 };
     },
-    stats: () => ({ encodeCount: -1 }),
   };
 }
 
+// 权重目录有两种摆法：要么 config.json 就在这一层，要么下面还套一层以模型名命名的子目录。
+// 推理库要的是"根目录 + 子目录名"，所以先探一下再决定怎么喂。
+function resolveWeightLayout(modelDir, modelId) {
+  if (!fs.existsSync(path.join(modelDir, 'config.json'))) {
+    return { root: modelDir, id: modelId };
+  }
+  return { root: path.dirname(modelDir), id: path.basename(modelDir) };
+}
+
 /**
- * 本地 ONNX 编码器（bge-small-zh-v1.5，512 维，CPU）。
- * 动态 import 模型运行时 ⇒ 未安装依赖时本模块仍可被 import（骨架/单测不受影响）。
+ * 本地 ONNX 编码器：默认取 cls 位做向量，也可以按注意力掩码做均值池化。
+ * 维度以**实测**为准（模型真吐几维就是几维），配置里那个数只作比对。
  */
-export async function createLocalEmbedder({
-  modelDir,
-  modelId = 'bge-small-zh-v1.5',
-  dim = 512,
-  quantized = false,
-  dtype = null,
-  maxLength = 512,
-  pooling = 'cls',
-  queryPrefix = '',
-} = {}) {
-  const mod = await import('@huggingface/transformers');
-  const { env, AutoTokenizer, AutoModel } = mod;
-  // 布局自适应：transformers.js 的 env.localModelPath 是「模型根目录」（其下按 modelId 建子目录），
-  // 而本项目的 modelDir 语义是「直接放 config.json 的权重目录」⇒ 两种布局都支持。
-  let localRoot = modelDir;
-  let localId = modelId;
-  if (fs.existsSync(path.join(modelDir, 'config.json'))) {
-    localRoot = path.dirname(modelDir);
-    localId = path.basename(modelDir);
-  }
-  env.allowRemoteModels = false;
-  env.allowLocalModels = true;
-  env.localModelPath = localRoot;
-  env.useBrowserCache = false;
-
-  const loadOpts = dtype
-    ? { dtype }
-    : quantized
-      ? { dtype: 'q8' }
-      : {};
-  const tokenizer = await AutoTokenizer.from_pretrained(localId, { local_files_only: true });
-  const model = await AutoModel.from_pretrained(localId, { local_files_only: true, ...loadOpts });
-
-  const encodeOne = async (text) => {
-    const input = String(text ?? '');
-    const inputs = await tokenizer(input, { padding: true, truncation: true, max_length: maxLength });
-    const out = await model(inputs);
-    const hidden = out.last_hidden_state ?? out[Object.keys(out)[0]];
-    const [, seq, d] = hidden.dims;
-    const data = hidden.data;
-    const v = new Float32Array(d);
+export async function createLocalEmbedder({ modelDir, modelId = 'bge-small-zh-v1.5', dim = 512, quantized = false, dtype = null, maxLength = 512, pooling = 'cls', queryPrefix = '' } = {}) {
+  const runtime = await import('@huggingface/transformers');
+  const { env, AutoTokenizer, AutoModel } = runtime;
+  const layout = resolveWeightLayout(modelDir, modelId);
+
+  // 全离线：只认本地目录，不碰远端、不占浏览器缓存
+  Object.assign(env, {
+    allowRemoteModels: false,
+    allowLocalModels: true,
+    localModelPath: layout.root,
+    useBrowserCache: false,
+  });
+
+  const loadOptions = { local_files_only: true };
+  if (dtype) loadOptions.dtype = dtype;
+  else if (quantized) loadOptions.dtype = 'q8';
+  const tokenizer = await AutoTokenizer.from_pretrained(layout.id, { local_files_only: true });
+  const model = await AutoModel.from_pretrained(layout.id, loadOptions);
+
+  const embedOnce = async (text) => {
+    const inputs = await tokenizer(String(text ?? ''), {
+      padding: true,
+      truncation: true,
+      max_length: maxLength,
+    });
+    const output = await model(inputs);
+    const hidden = output.last_hidden_state ?? output[Object.keys(output)[0]];
+    const [, tokenCount, width] = hidden.dims;
+    const vec = new Float32Array(width);
     if (pooling === 'cls') {
-      for (let i = 0; i < d; i += 1) v[i] = data[i];
-    } else {
-      // mean pooling（按 attention_mask 加权）
-      const mask = inputs.attention_mask?.data;
-      let n = 0;
-      for (let t = 0; t < seq; t += 1) {
-        const m = mask ? Number(mask[t]) : 1;
-        if (!m) continue;
-        n += 1;
-        for (let i = 0; i < d; i += 1) v[i] += data[t * d + i];
-      }
-      n = n || 1;
-      for (let i = 0; i < d; i += 1) v[i] /= n;
+      vec.set(hidden.data.subarray(0, width));
+      return unitize(vec);
+    }
+    // 均值池化：被掩码盖掉的位不参与平均
+    const mask = inputs.attention_mask?.data;
+    let kept = 0;
+    for (let token = 0; token < tokenCount; token += 1) {
+      if (mask && !Number(mask[token])) continue;
+      kept += 1;
+      for (let i = 0; i < width; i += 1) vec[i] += hidden.data[token * width + i];
     }
-    let norm = 0;
-    for (let i = 0; i < d; i += 1) norm += v[i] * v[i];
-    norm = Math.sqrt(norm) || 1;
-    for (let i = 0; i < d; i += 1) v[i] /= norm;
-    return v;
+    const divisor = kept || 1;
+    for (let i = 0; i < width; i += 1) vec[i] /= divisor;
+    return unitize(vec);
   };
 
-  const actualDim = await (async () => {
-    const probe = await encodeOne('维度自检');
-    return probe.length;
-  })();
+  // 用一句固定探针量真维度（此时不加查询前缀）
+  const measuredDim = (await embedOnce(LOCAL_PROBE_TEXT)).length;
 
   return {
-    kind: 'local',
-    modelId,
-    dim: actualDim,
-    configuredDim: dim,
-    describe: () => ({
-      kind: 'local',
-      modelId,
-      modelDir,
-      dim: actualDim,
-      pooling,
-      quantized,
-      dtype: dtype || (quantized ? 'q8' : 'fp32'),
-      queryPrefix,
-    }),
-    async health() {
-      return {
-        ready: true,
-        modelLoading: false,
-        model: modelId,
-        dim: actualDim,
-        modelDir,
-      };
-    },
-    async encode(text) {
-      return encodeOne(String(queryPrefix || '') + String(text ?? ''));
+    kind: 'local', modelId, dim: measuredDim, configuredDim: dim,
+    describe: () => ({ kind: 'local', modelId, modelDir, dim: measuredDim, pooling, quantized, dtype: dtype || (quantized ? 'q8' : 'fp32'), queryPrefix }),
+    health: async () => ({ ready: true, modelLoading: false, model: modelId, dim: measuredDim, modelDir }),
+    encode: (text) => embedOnce(String(queryPrefix || '') + String(text ?? '')),
+    encodeBatch: async (texts) => {
+      const vectors = [];
+      for (const text of texts) vectors.push(await embedOnce(text));
+      return vectors;
     },
-    async encodeBatch(texts) {
-      const out = [];
-      for (const t of texts) out.push(await encodeOne(t));
-      return out;
+    stats() {
+      return { encodeCount: -1 };
     },
-    stats: () => ({ encodeCount: -1 }),
   };
 }
 
-/** 按配置挑一个可用编码器（桥优先 → 本地 → 报错） */
+/** 按偏好挑一个：要 http 就走桥，其余一律走本地。 */
 export async function createAutoEmbedder({ prefer = 'local', ...opts } = {}) {
-  if (prefer === 'http') return createHttpEmbedder(opts);
-  return createLocalEmbedder(opts);
+  const build = prefer === 'http' ? createHttpEmbedder : createLocalEmbedder;
+  return build(opts);
 }

package/memory-engine/lib/hashes.mjs

@@ -1,12 +1,27 @@
-// 记忆引擎 v2 · 离线层 —— 哈希工具
-import crypto from 'node:crypto';
+// 正文与标识的摘要工具。
+//
+// 全仓只用一种算法：sha256 的十六进制形态。选它的理由是**跨机可复算** ——
+// 日志、台账、去重键都拿这个值对齐，换机器重算必须得到同一串。
+import { createHash } from 'node:crypto';
 
-/** 记忆正文哈希（方案 §3.1：text_hash = sha256(text)） */
+const ALGORITHM = 'sha256';
+const SHORT_LEN = 12;
+
+/**
+ * 正文指纹。
+ * 入参先归一到字符串（null / undefined 折成空串），
+ * 免得同一份正文因为传参形态不同算出两个哈希。
+ */
 export function textHash(text) {
-  return crypto.createHash('sha256').update(String(text ?? ''), 'utf8').digest('hex');
+  const source = text == null ? '' : String(text);
+  return createHash(ALGORITHM).update(source, 'utf8').digest('hex');
 }
 
-/** 稳定摘要（日志/报告用） */
-export function shortHash(h, n = 12) {
-  return String(h || '').slice(0, n);
+/**
+ * 截短显示（日志与报告里用）。
+ * 哈希本身是 ASCII 十六进制，按码元切与按码点切结果相同，这里沿用 String#slice。
+ */
+export function shortHash(h, n = SHORT_LEN) {
+  const value = h || '';
+  return String(value).slice(0, n);
 }