@kira905/dsh-memory-toolkit 0.0.0-stage → 0.1.1

package/dsh-memory-local/src/pointer.mjs

@@ -0,0 +1,152 @@
+/**
+ * FORK-EDIT(2026-09-21): 指针渲染与预算裁剪（方案 §3.4 注入产物 / §8.2 绝对量判据）
+ *
+ * 预算判据（硬）：条数 ≤8、总量 **≤800 字**、**≤600 token**（取更紧的一档实际生效）；
+ * 每条 = `id/路径` + 日期 + tags + **≤80 字摘要首行**（方案 §3.4 Z17 已吸收）。
+ *
+ * token 计法（写死口径，报告里给实测字符数即可复核）：
+ *   · 主口径 = DeepSeek 中文口径：CJK 字符 1 token ≈ 1.5 字，非 CJK 1 token ≈ 3.6 字
+ *     （DeepSeek 官方文档口径为「1 个中文字符 ≈ 0.6 token」，取 1.5 是**偏保守**的一侧）
+ *   · 另有 `estimateTokensWorstCase` = 1 字 = 1 token 的过度保守口径（只用于报告对照）
+ */
+
+/** DeepSeek 中文口径的 token 估算（偏保守：CJK 按 1.5 字/token）。 */
+export function estimateTokens(text) {
+  const s = String(text ?? '');
+  let cjk = 0;
+  let other = 0;
+  for (const ch of s) {
+    if (/[\u3400-\u9fff\uf900-\ufaff\u3000-\u303f\uff00-\uffef]/.test(ch)) cjk += 1;
+    else other += 1;
+  }
+  return Math.ceil(cjk / 1.5 + other / 3.6);
+}
+
+/** 过度保守口径（1 字符 = 1 token），仅供报告对照，不用于裁剪。 */
+export function estimateTokensWorstCase(text) {
+  return String(text ?? '').length;
+}
+
+const POINTER_HEADER =
+  '[相关记忆·指针] 以下可能与当前请求相关的历史记忆索引；需要细节请用 memory_get 展开：';
+
+/** 日期（YYYY-MM-DD，本地时区）。 */
+export function formatDate(ms) {
+  const d = new Date(Number(ms) || 0);
+  if (!Number.isFinite(d.getTime()) || d.getTime() <= 0) return '日期未知';
+  const p = (n) => String(n).padStart(2, '0');
+  return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`;
+}
+
+/**
+ * 剥掉孤立代理项（lone surrogate，U+D800–U+DFFF）。
+ *
+ * ⚠️ 某次事故修复（详见 CHANGELOG）—— 会话历史永久 HTTP 400：
+ * 记忆文本里含 emoji（📦/🐋…，星面字符 = 一对代理项）时，若按 **UTF-16 码元** `slice`
+ * 截断，会正好切在代理对中间，留下孤立高代理项（如 `\ud83d`）。该文本进会话历史后，
+ * 请求体 JSON 变成 `"…\ud83d…"` ⇒ DeepSeek 网关 `Failed to parse the request body as JSON:
+ * unexpected end of hex escape` ⇒ **HTTP 400 / INVALID_REQUEST**；而坏消息**永久留在历史里**，
+ * 该会话此后**每轮必 400、无法自愈**（实测复现：含孤立代理项 → 400，去掉 → 200）。
+ */
+export function stripLoneSurrogate(text) {
+  return String(text ?? '').replace(
+    /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,
+    '',
+  );
+}
+
+/**
+ * 码点安全截断：**不切断代理对**（星面字符算 1 个），并对结果做孤立代理兜底剥离。
+ * @param {string} text
+ * @param {number} maxChars 按码点计的最大长度
+ */
+export function sliceSafe(text, maxChars) {
+  const s = String(text ?? '');
+  const limit = Math.max(0, Number(maxChars) || 0);
+  if (limit === 0) return '';
+  if (s.length <= limit) return stripLoneSurrogate(s);
+  const points = [...s];
+  if (points.length <= limit) return stripLoneSurrogate(s);
+  return stripLoneSurrogate(points.slice(0, limit).join(''));
+}
+
+/**
+ * 摘要首行：取正文第一行，压掉换行/多余空白，截到 maxChars（超出加省略号）。
+ * @param {string} text
+ * @param {number} maxChars 默认 80（方案 §3.4）
+ */
+export function summarize(text, maxChars = 80) {
+  const firstLine = String(text ?? '')
+    .split(/\r?\n/)[0]
+    .replace(/\s+/g, ' ')
+    .trim();
+  if (firstLine.length <= maxChars) return stripLoneSurrogate(firstLine);
+  return `${sliceSafe(firstLine, Math.max(1, maxChars - 1))}…`;
+}
+
+/** tags 压缩：空格分隔取前若干个，总长 ≤ maxChars（码点安全，不切断 emoji）。 */
+export function compactTags(tags, maxChars = 40) {
+  const s = String(tags ?? '').replace(/\s+/g, ' ').trim();
+  if (s.length <= maxChars) return stripLoneSurrogate(s);
+  return `${sliceSafe(s, Math.max(1, maxChars - 1))}…`;
+}
+
+/**
+ * 渲染一条指针行（方案 §3.4 形态）。
+ * @param {{id:number,text:string,tags:string,updated_at:number}} row
+ */
+export function pointerLine(row, { summaryChars = 80, tagChars = 40 } = {}) {
+  const tags = compactTags(row.tags, tagChars);
+  const tagPart = tags ? ` · tags: ${tags}` : '';
+  return `- #${row.id} (${formatDate(row.updated_at)}${tagPart}) ${summarize(row.text, summaryChars)}`;
+}
+
+/**
+ * 按预算裁剪并渲染指针文本。
+ * 逐条累加，任一预算触顶即停（后面的条目丢弃并计数）。
+ * @returns {{text:string, lines:string[], items:Array, chars:number, estTokens:number, dropped:number, overflow:boolean}}
+ */
+export function renderPointer(
+  rows,
+  { header = POINTER_HEADER, maxItems = 8, maxChars = 800, maxTokens = 600, summaryChars = 80, tagChars = 40 } = {},
+) {
+  const lines = [];
+  const items = [];
+  let dropped = 0;
+  let overflow = false;
+  for (const row of rows || []) {
+    if (lines.length >= maxItems) {
+      dropped += 1;
+      overflow = true;
+      continue;
+    }
+    const line = pointerLine(row, { summaryChars, tagChars });
+    const candidate = [header, ...lines, line].join('\n');
+    if (candidate.length > maxChars || estimateTokens(candidate) > maxTokens) {
+      dropped += 1;
+      overflow = true;
+      continue;
+    }
+    lines.push(line);
+    items.push({
+      id: Number(row.id),
+      chars: line.length,
+      summaryChars: summarize(row.text, summaryChars).length,
+      tags: row.tags,
+      updatedAt: Number(row.updated_at),
+    });
+  }
+  const text = lines.length ? [header, ...lines].join('\n') : '';
+  return {
+    text,
+    lines,
+    items,
+    chars: text.length,
+    estTokens: estimateTokens(text),
+    worstCaseTokens: estimateTokensWorstCase(text),
+    dropped,
+    overflow,
+  };
+}
+
+export const POINTER_HEADER_TEXT = POINTER_HEADER;

package/dsh-memory-local/src/recall-log.mjs

@@ -0,0 +1,118 @@
+/**
+ * FORK-EDIT(2026-09-21): 注入可观测日志（方案 §3.4 可观测 / §6.1 ⑤ 失败环 / §8.2 展开率）
+ *
+ * 落点：`<DSH_HOME>/logs/memory-recall.log`（JSON 行：时间/条数/耗时/命中来源/预估字符数/是否被展开）
+ * 轮转：**3 份 × 保留 30 天**（方案 §3.4 Z18）；**不含记忆正文**（只记 id/长度/摘要长度）。
+ * 铁律：**写日志失败不得影响注入**（全部 try/catch 吞掉，返回值只用于自测）。
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+
+export const DEFAULT_LOG_FILE = 'memory-recall.log';
+
+export function createRecallLogger({
+  dir,
+  file = DEFAULT_LOG_FILE,
+  maxBytes = 1024 * 1024,
+  keepFiles = 3,
+  keepDays = 30,
+  fsImpl = fs,
+  now = () => Date.now(),
+  echo = null,
+} = {}) {
+  if (!dir) throw new Error('recall-log: 必须给 dir（日志目录）');
+  const main = path.join(dir, file);
+  const stats = { written: 0, errors: 0, rotations: 0, swept: 0 };
+  let sizeCache = null;
+
+  const stamp = (ms) => {
+    const d = new Date(ms);
+    const p = (n) => String(n).padStart(2, '0');
+    return `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}`;
+  };
+
+  const rotateIfNeeded = () => {
+    let size = 0;
+    try {
+      size = fsImpl.existsSync(main) ? fsImpl.statSync(main).size : 0;
+    } catch {
+      size = 0;
+    }
+    if (size <= maxBytes) return;
+    const target = path.join(dir, `memory-recall.${stamp(now())}.log`);
+    if (fsImpl.existsSync(target)) return; // 同秒重名 → 跳过（不覆盖）
+    fsImpl.renameSync(main, target);
+    sizeCache = 0;
+    stats.rotations += 1;
+  };
+
+  /** 清理：删超过 keepDays 的轮转文件 + 轮转文件数超过 keepFiles 删最旧。 */
+  const sweep = () => {
+    let names = [];
+    try {
+      names = fsImpl
+        .readdirSync(dir)
+        .filter((n) => /^memory-recall\..+\.log$/.test(n))
+        .map((n) => {
+          const full = path.join(dir, n);
+          let mtime = 0;
+          try {
+            mtime = fsImpl.statSync(full).mtimeMs;
+          } catch {
+            mtime = 0;
+          }
+          return { full, n, mtime };
+        })
+        .sort((a, b) => b.mtime - a.mtime);
+    } catch {
+      return;
+    }
+    const cutoff = now() - keepDays * 86400000;
+    const kept = [];
+    for (const f of names) {
+      if (f.mtime && f.mtime < cutoff) {
+        try {
+          fsImpl.rmSync(f.full, { force: true });
+          stats.swept += 1;
+        } catch {
+          /* ignore */
+        }
+        continue;
+      }
+      kept.push(f);
+    }
+    for (const f of kept.slice(keepFiles)) {
+      try {
+        fsImpl.rmSync(f.full, { force: true });
+        stats.swept += 1;
+      } catch {
+        /* ignore */
+      }
+    }
+  };
+
+  /** 写一行 JSON。永不抛错。 */
+  const log = (obj) => {
+    try {
+      const line = JSON.stringify({ t: new Date(now()).toISOString(), ...obj });
+      if (echo) {
+        try {
+          echo(line);
+        } catch {
+          /* ignore */
+        }
+      }
+      fsImpl.mkdirSync(dir, { recursive: true });
+      rotateIfNeeded();
+      fsImpl.appendFileSync(main, `${line}\n`, 'utf8');
+      stats.written += 1;
+      if (stats.written % 200 === 0) sweep();
+      return true;
+    } catch {
+      stats.errors += 1;
+      return false;
+    }
+  };
+
+  return { log, sweep, path: main, stats: () => ({ ...stats }) };
+}

package/dsh-memory-local/src/recall.mjs

@@ -0,0 +1,192 @@
+/**
+ * FORK-EDIT(2026-09-21): 召回主逻辑 —— 从 S1 离线层 `memory-engine/lib/recall.mjs` 移植。
+ *
+ * 移植来源：本仓 `memory-engine/lib/recall.mjs`（记忆引擎 v2 · S1 交付）
+ * 本文件与来源**逐字同语义**（方案 §3.3：向量为主 + FTS 仅补标签/英文/数字 + RRF(k=60)
+ * + 排除 pinned + 非空即触发、纯符号/单字跳过）；**唯一改动**是 import 路径
+ * （`./vectors-store.mjs` → `./vectors-read.mjs`，插件侧只读、不写向量库）。
+ */
+import { loadMatrix } from './vectors-read.mjs';
+
+/** 触发判定（方案 §3.3 Z16 已吸收） */
+export function shouldTrigger(rawQuery) {
+  const q = String(rawQuery ?? '').trim();
+  if (!q) return { trigger: false, reason: 'empty' };
+  const semantic = q.match(/[\p{L}\p{N}]/gu) || [];
+  if (semantic.length === 0) return { trigger: false, reason: 'symbols-only' };
+  if (semantic.length < 2) return { trigger: false, reason: 'single-char' };
+  return { trigger: true, reason: 'non-empty' };
+}
+
+/** 抽 FTS 补位 token：只取英文/数字（A4：中文连续串在 unicode61 下几近不可用） */
+export function extractFtsTokens(rawQuery, max = 8) {
+  const m = String(rawQuery ?? '').match(/[A-Za-z0-9][A-Za-z0-9_.-]*/g) || [];
+  const out = [];
+  for (const t of m) {
+    if (t.length < 2) continue; // 单字母/单数字噪声大
+    if (!out.includes(t)) out.push(t);
+    if (out.length >= max) break;
+  }
+  return out;
+}
+
+/** 构造 FTS5 MATCH 表达式：每个 token 加双引号（防语法注入），OR 连接 */
+export function buildFtsMatch(tokens) {
+  if (!tokens || !tokens.length) return null;
+  return tokens.map((t) => `"${String(t).replace(/"/g, '')}"`).join(' OR ');
+}
+
+/** 载入向量矩阵 + 行范数（余弦召回用；一次构建，多次召回） */
+export function buildVectorIndex(db, { source = 'memory', model = null } = {}) {
+  const matrix = loadMatrix(db, source, model);
+  const norms = new Float32Array(matrix.count);
+  for (let r = 0; r < matrix.count; r += 1) {
+    let n = 0;
+    const off = r * matrix.dim;
+    for (let i = 0; i < matrix.dim; i += 1) n += matrix.data[off + i] * matrix.data[off + i];
+    norms[r] = Math.sqrt(n) || 1;
+  }
+  return { ...matrix, norms, builtAt: Date.now() };
+}
+
+/** 余弦 top-K（纯内存暴力；万级候选毫秒级，方案 §3.3） */
+export function cosineTopK(index, qvec, topK) {
+  const { data, dim, count, norms } = index;
+  if (!dim || !count) return [];
+  let qn = 0;
+  for (let i = 0; i < dim; i += 1) qn += qvec[i] * qvec[i];
+  qn = Math.sqrt(qn) || 1;
+  const scores = new Float32Array(count);
+  for (let r = 0; r < count; r += 1) {
+    let s = 0;
+    const off = r * dim;
+    for (let i = 0; i < dim; i += 1) s += data[off + i] * qvec[i];
+    scores[r] = s / (qn * norms[r]);
+  }
+  const order = Array.from({ length: count }, (_, i) => i);
+  order.sort((a, b) => scores[b] - scores[a] || a - b);
+  const k = Math.min(Number(topK) || 0, count);
+  const out = [];
+  for (let i = 0; i < k; i += 1) {
+    const idx = order[i];
+    out.push({ index: idx, refId: index.refIds[idx], id: index.refIdsNum[idx], score: scores[idx] });
+  }
+  return out;
+}
+
+/**
+ * RRF 融合（k=60）：score(d) = Σ_r 1/(k + rank_r(d))
+ * @param {Array<{source:string, ids:string[]}>} rankLists 各路排名（已按优劣降序，rank 从 1 起）
+ */
+export function rrfFuse(rankLists, k = 60) {
+  const acc = new Map();
+  for (const list of rankLists) {
+    if (!list || !list.ids) continue;
+    list.ids.forEach((id, i) => {
+      const rank = i + 1;
+      const cur = acc.get(String(id)) || { id: String(id), score: 0, sources: [], ranks: {} };
+      cur.score += 1 / (k + rank);
+      if (!cur.sources.includes(list.source)) cur.sources.push(list.source);
+      cur.ranks[list.source] = rank;
+      acc.set(String(id), cur);
+    });
+  }
+  return [...acc.values()].sort(
+    (a, b) => b.score - a.score || Number(a.id) - Number(b.id),
+  );
+}
+
+/**
+ * 召回主入口。
+ * fail-open：任何异常都由调用方兜住（本函数内部对 FTS 异常降级为空路）。
+ * @returns {Promise<object>}
+ */
+export async function recall({
+  query,
+  store,
+  source,
+  embedder,
+  index = null,
+  vectorTopK = 20,
+  ftsTopK = 20,
+  rrfK = 60,
+  resultTopK = 8,
+  log = () => {},
+}) {
+  const t0 = performance.now();
+  const gate = shouldTrigger(query);
+  if (!gate.trigger) {
+    return { triggered: false, reason: gate.reason, query: String(query ?? ''), hits: [], timings: { totalMs: 0 } };
+  }
+
+  const db = store.db;
+  const pinned = new Set((source ? source.pinnedIds() : []).map(String));
+
+  // ① 查询编码
+  const tEnc = performance.now();
+  const qvec = await embedder.encode(query);
+  const encodeMs = performance.now() - tEnc;
+
+  // ② 向量路
+  const tVec = performance.now();
+  const idx = index || buildVectorIndex(db, { source: 'memory' });
+  const rawVec = cosineTopK(idx, qvec, vectorTopK + pinned.size + 5);
+  const vecHits = rawVec.filter((h) => !pinned.has(String(h.refId))).slice(0, vectorTopK);
+  const vectorMs = performance.now() - tVec;
+
+  // ③ FTS 补位路（仅英文/数字；缺失时不参与，不做零分占位）
+  const tFts = performance.now();
+  let ftsHits = [];
+  let ftsMatched = 0;
+  const tokens = extractFtsTokens(query);
+  const matchExpr = buildFtsMatch(tokens);
+  if (matchExpr && source) {
+    const r = source.ftsSearch(matchExpr, ftsTopK + pinned.size + 5);
+    if (Array.isArray(r)) {
+      ftsMatched = r.length;
+      ftsHits = r.filter((h) => !pinned.has(String(h.id))).slice(0, ftsTopK);
+    } else if (r && r.error) {
+      log(`[recall] FTS 补位降级：${r.error}`);
+    }
+  }
+  const ftsMs = performance.now() - tFts;
+
+  // ④ RRF 融合
+  const tFuse = performance.now();
+  const fused = rrfFuse(
+    [
+      { source: 'vector', ids: vecHits.map((h) => String(h.refId)) },
+      { source: 'fts', ids: ftsHits.map((h) => String(h.id)) },
+    ],
+    rrfK,
+  );
+  const hits = fused.slice(0, Number(resultTopK) || 8).map((h) => ({
+    refId: h.id,
+    id: Number(h.id),
+    score: h.score,
+    sources: h.sources,
+    vectorRank: h.ranks.vector ?? null,
+    ftsRank: h.ranks.fts ?? null,
+  }));
+  const fuseMs = performance.now() - tFuse;
+
+  const totalMs = performance.now() - t0;
+  return {
+    triggered: true,
+    reason: gate.reason,
+    query: String(query ?? ''),
+    hits,
+    timings: { encodeMs, vectorMs, ftsMs, fuseMs, totalMs },
+    stats: {
+      indexSize: idx.count,
+      pinnedExcluded: pinned.size,
+      vectorCandidates: rawVec.length,
+      vectorHits: vecHits.length,
+      ftsTokens: tokens,
+      ftsMatchExpr: matchExpr,
+      ftsMatched,
+      ftsHits: ftsHits.length,
+      fused: fused.length,
+    },
+  };
+}