@kira905/dsh-memory-toolkit 0.0.0-stage → 0.1.0

package/memory-engine/tools/fetch-model.mjs

@@ -0,0 +1,128 @@
+// 记忆引擎 v2 · 离线层 —— 模型权重获取器（node 版，不用 curl）
+//
+// 为什么要有这个脚本（Windows + schannel 环境实测坑）：
+//   Windows 自带 curl.exe 走 schannel；HF 的 LFS 大文件会 302 到 Xet CAS 桥
+//   （cas-bridge.xethub.hf.co），部分 Windows 主机上 schannel 对该域名 TLS 握手直接失败
+//   （curl: (35) SEC_E_ILLEGAL_MESSAGE），而小文件（走 hf-mirror 直连）没问题。
+//   node 自带 TLS 栈不受 schannel 影响 ⇒ 用本脚本取权重。
+//
+// 用法：
+//   node tools/fetch-model.mjs --out <模型目录> [--base https://hf-mirror.com]
+//        [--repo Xenova/bge-small-zh-v1.5] [--variant fp32|int8] [--force]
+// 幂等：已存在且大小与远端一致 → 跳过（--force 强制重下）。
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { parseArgs, isMainEntry } from '../lib/cli-util.mjs';
+
+const COMMON = [
+  'config.json',
+  'tokenizer.json',
+  'tokenizer_config.json',
+  'special_tokens_map.json',
+  'vocab.txt',
+];
+const VARIANTS = {
+  fp32: 'onnx/model.onnx',
+  int8: 'onnx/model_quantized.onnx',
+  fp16: 'onnx/model_fp16.onnx',
+};
+
+async function fetchOne({ url, dest, force, log }) {
+  const head = await fetch(url, { method: 'GET', redirect: 'follow' });
+  if (!head.ok) throw new Error(`HTTP ${head.status} ${head.statusText} (${url})`);
+  const total = Number(head.headers.get('content-length') || 0);
+  const name = path.basename(dest);
+  if (!force && fs.existsSync(dest) && total > 0 && fs.statSync(dest).size === total) {
+    await head.body?.cancel?.();
+    return { file: name, status: 'skip', size: total, sha: shortSha(dest) };
+  }
+  fs.mkdirSync(path.dirname(dest), { recursive: true });
+  const part = `${dest}.part`;
+  let got = 0;
+  let lastPct = -1;
+  const src = Readable.fromWeb(head.body);
+  src.on('data', (c) => {
+    got += c.length;
+    if (total) {
+      const pct = Math.floor((got / total) * 100);
+      if (pct >= lastPct + 10) {
+        lastPct = pct;
+        log(`    ${name}: ${pct}%（${(got / 1048576).toFixed(1)}/${(total / 1048576).toFixed(1)} MB）`);
+      }
+    }
+  });
+  await pipeline(src, fs.createWriteStream(part));
+  if (total && fs.statSync(part).size !== total) {
+    throw new Error(`${name} 大小不符：期望 ${total}，实际 ${fs.statSync(part).size}`);
+  }
+  fs.renameSync(part, dest);
+  return { file: name, status: 'fetched', size: fs.statSync(dest).size, sha: shortSha(dest) };
+}
+
+function shortSha(p) {
+  return crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex').slice(0, 32);
+}
+
+export async function fetchModel({ out, base, repo, variant, force, log, retries = 3 }) {
+  const files = [...COMMON, VARIANTS[variant]];
+  const results = [];
+  for (const f of files) {
+    const url = `${base}/${repo}/resolve/main/${f}`;
+    let lastErr = null;
+    for (let attempt = 1; attempt <= retries; attempt += 1) {
+      try {
+        log(`  → ${f}${attempt > 1 ? `（第 ${attempt} 次尝试）` : ''}`);
+        results.push(await fetchOne({ url, dest: path.join(out, f.replace(/\//g, path.sep)), force, log }));
+        lastErr = null;
+        break;
+      } catch (e) {
+        lastErr = e;
+        log(`    ⚠ ${String(e && e.message ? e.message : e)}`);
+        await new Promise((r) => setTimeout(r, 1500 * attempt));
+      }
+    }
+    if (lastErr) {
+      results.push({ file: f, status: 'FAILED', error: String(lastErr.message || lastErr) });
+    }
+  }
+  return results;
+}
+
+async function main() {
+  const { flags } = parseArgs();
+  if (flags.help) {
+    console.log(`用法: node tools/fetch-model.mjs --out <dir> [--base <url>] [--repo <owner/name>] [--variant fp32|int8|fp16] [--force]`);
+    return;
+  }
+  const out = flags.out || process.env.MEMORY_ENGINE_MODEL_DIR;
+  if (!out) throw new Error('必须指定 --out（模型落盘目录）');
+  const opts = {
+    out: path.resolve(out),
+    base: flags.base || 'https://hf-mirror.com',
+    repo: flags.repo || 'Xenova/bge-small-zh-v1.5',
+    variant: flags.variant || 'fp32',
+    force: Boolean(flags.force),
+  };
+  console.log(`模型获取器：${opts.repo}（${opts.variant}）`);
+  console.log(`镜像      ：${opts.base}`);
+  console.log(`目标目录  ：${opts.out}`);
+  if (!VARIANTS[opts.variant]) throw new Error(`未知 variant=${opts.variant}（可选 ${Object.keys(VARIANTS).join('/')}）`);
+  const results = await fetchModel({ ...opts, log: (m) => console.log(m) });
+  console.log('\n=== 结果 ===');
+  let failed = 0;
+  for (const r of results) {
+    if (r.status === 'FAILED') {
+      failed += 1;
+      console.log(`❌ ${r.file}：${r.error}`);
+    } else {
+      console.log(`${r.status === 'skip' ? '⏭️ ' : '✅'} ${r.file.padEnd(28)} ${(r.size / 1048576).toFixed(2)} MB  sha256=${r.sha}`);
+    }
+  }
+  console.log(failed ? `\n❌ ${failed} 个文件失败` : '\n✅ 全部就绪');
+  process.exitCode = failed ? 1 : 0;
+}
+
+if (isMainEntry(import.meta.url)) await main();

package/memory-engine/tools/smoke-recall.mjs

@@ -0,0 +1,76 @@
+// 记忆引擎 v2 · 离线层 —— 真实模型召回冒烟（人读证据）
+//
+// 为什么单独一个脚本：命令行传中文经 cmd/ssh 会乱码，这里把查询集写死在文件里，
+// 在目标机上直接跑，得到「真实模型 + 真实语料」的可读召回证据（含短暗号）。
+import fs from 'node:fs';
+import { parseArgs, isMainEntry } from '../lib/cli-util.mjs';
+import { resolveConfig } from '../lib/config.mjs';
+import { openVectorsStore, countVectors, getMetaAll } from '../lib/vectors-store.mjs';
+import { openMemorySource } from '../lib/memory-source.mjs';
+import { createLocalEmbedder } from '../lib/embedder.mjs';
+import { recall, buildVectorIndex, shouldTrigger } from '../lib/recall.mjs';
+
+const QUERIES = [
+  '退下吧',
+  '整夜干活',
+  '去深水区',
+  '跨机同步挡推',
+  '看板卡怎么验收',
+  '守护鱼崩溃了怎么办',
+  '生图角色一致性',
+  'FTS5 中文分词',
+  'task-board 并回账本',
+  '3080 端口',
+];
+
+async function main() {
+  const { flags } = parseArgs();
+  const config = resolveConfig({
+    memoryDb: flags['memory-db'] || undefined,
+    vectorsDb: flags['vectors-db'] || undefined,
+    modelDir: flags['model-dir'] || undefined,
+    modelId: flags.model || undefined,
+    dim: flags.dim ? Number(flags.dim) : undefined,
+  });
+  if (!fs.existsSync(config.memoryDb)) throw new Error(`记忆库不存在：${config.memoryDb}`);
+
+  const embedder = await createLocalEmbedder({
+    modelDir: config.modelDir,
+    modelId: config.modelId,
+    dim: config.dim,
+  });
+  const store = openVectorsStore(config.vectorsDb);
+  const source = openMemorySource(config.memoryDb);
+  const index = buildVectorIndex(store.db, { source: 'memory' });
+
+  console.log('=== 真实模型召回冒烟 ===');
+  console.log(`主机     : ${process.env.COMPUTERNAME || '?'}`);
+  console.log(`记忆库   : ${config.memoryDb}（${source.count()} 条 / pinned ${source.pinnedIds().length} 条）`);
+  console.log(`向量库   : ${config.vectorsDb}（${countVectors(store.db, 'memory')} 条向量）`);
+  console.log(`meta     : ${JSON.stringify(getMetaAll(store.db))}`);
+  console.log(`模型     : ${config.modelId} dim=${embedder.dim}`);
+  console.log('');
+
+  const pinned = new Set(source.pinnedIds().map(String));
+  let pinnedLeak = 0;
+  for (const q of QUERIES) {
+    const gate = shouldTrigger(q);
+    const t0 = performance.now();
+    const r = await recall({ query: q, store, source, embedder, index, resultTopK: 5 });
+    const ms = performance.now() - t0;
+    console.log(`【${q}】触发=${gate.trigger ? '是' : `否(${gate.reason})`} 命中 ${r.hits.length} 条 · ${ms.toFixed(1)}ms · FTS命中=${r.stats.ftsMatched ?? 0}`);
+    for (const h of r.hits) {
+      if (pinned.has(String(h.refId))) pinnedLeak += 1;
+      const row = source.getById(h.id);
+      const brief = (row ? row.text : '(已删除)').replace(/\s+/g, ' ').slice(0, 58);
+      console.log(`   #${h.refId} score=${h.score.toFixed(5)} [${h.sources.join('+')}] pinned=${row ? row.pinned : '?'} · ${brief}…`);
+    }
+    console.log('');
+  }
+  console.log(`pinned 泄漏检查：${pinnedLeak === 0 ? '✅ 0 条（排除生效）' : `❌ ${pinnedLeak} 条`}`);
+  source.close();
+  store.db.close();
+  process.exitCode = pinnedLeak === 0 ? 0 : 4;
+}
+
+if (isMainEntry(import.meta.url)) await main();

package/package.json

@@ -1,6 +1,55 @@
 {
   "name": "@kira905/dsh-memory-toolkit",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "DSH 记忆面工具集：独立向量记忆引擎（增量同步 / 混合召回 / 文档索引）+ 记忆库只读体检工具链 + 本地记忆插件 fork + persona 记忆体检",
+  "keywords": [
+    "dsh",
+    "deepseek-harness",
+    "memory",
+    "vector-search",
+    "embeddings",
+    "sqlite",
+    "fts5",
+    "plugin"
+  ],
+  "type": "module",
+  "license": "AGPL-3.0-only",
+  "author": "kira905",
+  "homepage": "https://github.com/kira905/dsh-memory-toolkit#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kira905/dsh-memory-toolkit.git"
+  },
+  "bugs": {
+    "url": "https://github.com/kira905/dsh-memory-toolkit/issues"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "bin": {
+    "dsh-memory-audit": "./persona-memory-audit.mjs",
+    "dsh-memory-summary": "./memory-db-tools/list-memory-summary.mjs",
+    "dsh-memory-export": "./memory-db-tools/audit-export.mjs"
+  },
+  "files": [
+    "memory-engine/",
+    "memory-db-tools/",
+    "dsh-memory-local/",
+    "lib/",
+    "persona-memory-audit.mjs",
+    "README.md",
+    "README.en.md",
+    "CHANGELOG.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "help": "node memory-db-tools/audit-export.mjs --help",
+    "audit": "node persona-memory-audit.mjs",
+    "test:engine": "node memory-engine/test/run.mjs",
+    "test:plugin": "node dsh-memory-local/test/regression.mjs"
+  }
+}

package/persona-memory-audit.mjs

@@ -0,0 +1,217 @@
+#!/usr/bin/env node
+// persona ↔ 记忆 防反弹闸门（体检 · 零依赖 · 默认只读）
+//
+// 用途：把「persona 正文体积」与「记忆可见率」两项指标固定下来，供日报/面板/手工调用。
+// 判据来源（只读参考，不改动）：
+//   · persona 正文 = <preset>/agent.cordis.yml 的 `prefix: |-` 块（每会话 100% 注入）
+//   · 记忆注入 = dsh-memory 的 forPrompt + renderPrompt（lib/index.js L124-128 / L185-203）
+//   · 生效预算 = <profile>/cordis.patch.yml 的 `- id: memory` 行
+//
+// 阈值：persona > 20000 字符 ⇒ 🔴；记忆可见率 < 5% ⇒ 🔴
+// 退出码：0 = 全绿/仅黄；3 = 存在 🔴；2 = 关键读数失败
+//
+// 用法：
+//   node persona-memory-audit.mjs [--json] [--quiet]
+//        [--persona-max 20000] [--visible-min 5] [--preset whale-girl] [--profile web]
+//
+// 变更：2026-09-28 首版
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+
+const ARGV = process.argv.slice(2);
+const flag = (name, dflt) => {
+  const i = ARGV.indexOf(name);
+  return i === -1 ? dflt : ARGV[i + 1];
+};
+const has = (name) => ARGV.includes(name);
+
+const PERSONA_MAX = Number(flag('--persona-max', 20000));
+const VISIBLE_MIN = Number(flag('--visible-min', 5));
+const PRESET = flag('--preset', 'whale-girl');
+const PROFILE = flag('--profile', 'web');
+const DH = process.env.DSH_HOME || path.join(process.env.USERPROFILE ?? '.', '.dsh');
+
+function readPersonaBody(file) {
+  const raw = fs.readFileSync(file, 'utf8');
+  const lines = raw.split(/\r?\n/);
+  let start = -1;
+  for (let i = 0; i < lines.length; i++) if (/^ {4}prefix:\s*\|-?\s*$/.test(lines[i])) { start = i; break; }
+  if (start === -1) return null;
+  let end = lines.length;
+  for (let i = start + 1; i < lines.length; i++) {
+    const l = lines[i];
+    if (l.trim() === '') continue;
+    if (l.length - l.trimStart().length < 6) { end = i; break; }
+  }
+  const bodyLines = lines.slice(start + 1, end);
+  // YAML 块标量的 6 空格缩进是语法、不进注入文本 ⇒ 口径以「剥离缩进后」为准
+  // （与 `tools\validate-persona.mjs` 报的 persona text 长度一致）。
+  const stripped = bodyLines.map((l) => (l.startsWith('      ') ? l.slice(6) : l));
+  return {
+    text: stripped.join('\n'),
+    indentedChars: bodyLines.join('\n').length,
+    lines: bodyLines.length,
+    fileChars: raw.length,
+  };
+}
+
+function readBudget(patchFile) {
+  if (!fs.existsSync(patchFile)) return { found: false, promptRecentCount: 10, promptMaxChars: 2000 };
+  const txt = fs.readFileSync(patchFile, 'utf8');
+  const m = /- id: memory\s*\n\s*config:\s*\n([\s\S]*?)(?=\n- |\n# |\s*$)/.exec(txt);
+  const seg = m ? m[1] : '';
+  const num = (k, d) => {
+    const r = new RegExp(`${k}:\\s*(\\d+)`).exec(seg);
+    return r ? Number(r[1]) : d;
+  };
+  return { found: Boolean(m), promptRecentCount: num('promptRecentCount', 10), promptMaxChars: num('promptMaxChars', 2000) };
+}
+
+function safeTags(raw) {
+  try { const v = JSON.parse(raw); return Array.isArray(v) ? v : String(v).split(/[,\s]+/).filter(Boolean); }
+  catch { return String(raw).split(/[,\s]+/).filter(Boolean); }
+}
+
+function measureMemory(dbPath, budget) {
+  if (!fs.existsSync(dbPath)) return { ok: false, error: 'memory.db 不存在' };
+  let db;
+  try { db = new DatabaseSync(dbPath, { readOnly: true }); }
+  catch (e) { return { ok: false, error: `打不开 memory.db: ${e.message}` }; }
+  try {
+    const total = db.prepare('SELECT COUNT(*) n FROM memories').get().n;
+    const pinnedN = db.prepare('SELECT COUNT(*) n FROM memories WHERE pinned = 1').get().n;
+    const totalChars = db.prepare('SELECT SUM(LENGTH(text)) c FROM memories').get().c ?? 0;
+    const records = [
+      ...db.prepare('SELECT * FROM memories WHERE pinned = 1 ORDER BY updated_at DESC, id DESC').all(),
+      ...db.prepare('SELECT * FROM memories WHERE pinned = 0 ORDER BY updated_at DESC, id DESC LIMIT ?').all(budget.promptRecentCount),
+    ].map((r) => ({ id: r.id, pinned: Boolean(r.pinned), tags: r.tags ? safeTags(r.tags) : [], text: r.text }));
+
+    const header = 'Memories you previously stored (use memory_search for anything not listed):\n';
+    let used = 76;
+    let dropped = 0;
+    const kept = [];
+    for (const rec of records) {
+      const tags = rec.tags.length ? ` [${rec.tags.join(',')}]` : '';
+      const line = `- (#${rec.id}${rec.pinned ? ', pinned' : ''})${tags} ${rec.text}`;
+      if (used + line.length + 1 > budget.promptMaxChars) { dropped++; continue; }
+      kept.push({ id: rec.id, pinned: rec.pinned, chars: line.length });
+      used += line.length + 1;
+    }
+    const tail = dropped > 0 ? `\n(${dropped} more memories not shown; use memory_search)` : '';
+    const injectedChars = kept.length ? 76 + kept.reduce((a, k) => a + k.chars, 0) + tail.length : 0;
+    return {
+      ok: true,
+      total, pinnedN, totalChars,
+      candidateCount: records.length,
+      injectedCount: kept.length,
+      injectedChars,
+      dropped,
+      visiblePercent: totalChars > 0 ? injectedChars / totalChars * 100 : 0,
+      kept,
+    };
+  } finally { db.close(); }
+}
+
+function usage() {
+  return `persona ↔ 记忆 体检（只读 · 零依赖）
+
+用法：node persona-memory-audit.mjs [选项]
+
+  --json                机读输出（JSON）
+  --quiet               只输出一行结论
+  --persona-max <n>     persona 正文字符阈值（默认 20000）
+  --visible-min <p>     记忆可见率阈值 %（默认 5）
+  --preset <name>       preset 名（默认 whale-girl；读 <DSH_HOME>/.agent-presets/<name>/agent.cordis.yml）
+  --profile <name>      profile 名（默认 web；读 <DSH_HOME>/profiles/<name>/cordis.patch.yml）
+  -h, --help            本帮助
+
+环境变量：DSH_HOME（缺省 ~/.dsh）
+退出码：0 = 全绿/仅黄 · 3 = 存在 🔴 · 2 = 关键读数失败`;
+}
+
+function main() {
+  if (process.argv.includes('--help') || process.argv.includes('-h')) {
+    console.log(usage());
+    return 0;
+  }
+  const presetFile = path.join(DH, '.agent-presets', PRESET, 'agent.cordis.yml');
+  const patchFile = path.join(DH, 'profiles', PROFILE, 'cordis.patch.yml');
+  const dbPath = path.join(DH, 'memory', 'memory.db');
+
+  const persona = fs.existsSync(presetFile) ? readPersonaBody(presetFile) : null;
+  const budget = readBudget(patchFile);
+  const mem = measureMemory(dbPath, budget);
+
+  const findings = [];
+  if (!persona) {
+    findings.push({ level: 'red', key: 'persona', text: `persona 正文读不到（${presetFile}）` });
+  } else if (persona.text.length > PERSONA_MAX) {
+    findings.push({
+      level: 'red', key: 'persona',
+      text: `persona 正文 ${persona.text.length} 字符 > 阈值 ${PERSONA_MAX} ⇒ 每会话多注入 ${persona.text.length - PERSONA_MAX} 字符`,
+    });
+  }
+  if (!mem.ok) {
+    findings.push({ level: 'red', key: 'memory', text: `记忆库读数失败：${mem.error}` });
+  } else if (mem.visiblePercent < VISIBLE_MIN) {
+    findings.push({
+      level: 'red', key: 'visible',
+      text: `记忆可见率 ${mem.visiblePercent.toFixed(2)}% < ${VISIBLE_MIN}% ⇒ 注入 ${mem.injectedCount}/${mem.total} 条（${mem.injectedChars} 字符）；pinned ${mem.pinnedN} 条/字符占用高是常见主因`,
+    });
+  } else if (mem.visiblePercent < VISIBLE_MIN * 2) {
+    findings.push({ level: 'yellow', key: 'visible', text: `记忆可见率 ${mem.visiblePercent.toFixed(2)}% 偏低（阈值 ${VISIBLE_MIN}%）` });
+  }
+
+  const report = {
+    generatedAt: new Date().toISOString(),
+    host: process.env.COMPUTERNAME ?? 'unknown',
+    dshHome: DH,
+    thresholds: { personaMaxChars: PERSONA_MAX, visibleMinPercent: VISIBLE_MIN },
+    persona: persona ? { file: presetFile, bodyChars: persona.text.length, bodyCharsIndented: persona.indentedChars, bodyLines: persona.lines, fileChars: persona.fileChars } : { file: presetFile, error: '未找到 prefix 块' },
+    memoryBudget: budget,
+    memory: mem.ok ? {
+      db: dbPath,
+      total: mem.total, pinned: mem.pinnedN, totalChars: mem.totalChars,
+      candidateCount: mem.candidateCount,
+      injectedCount: mem.injectedCount, injectedChars: mem.injectedChars, dropped: mem.dropped,
+      visiblePercent: Number(mem.visiblePercent.toFixed(2)),
+      kept: mem.kept,
+    } : { error: mem.error },
+    findings,
+    worst: findings.some((f) => f.level === 'red') ? 'red' : findings.length ? 'yellow' : 'green',
+  };
+
+  if (has('--json')) {
+    process.stdout.write(JSON.stringify(report, null, 2) + '\n');
+    return report.worst === 'red' ? 3 : 0;
+  }
+
+  const icon = { red: '🔴', yellow: '🟡', green: '🟢' }[report.worst];
+  if (!has('--quiet')) {
+    console.log(`${icon} persona ↔ 记忆 体检（${report.host}）`);
+    if (persona) console.log(`  persona 正文: ${persona.text.length} 字符 / ${persona.lines} 行（阈值 ${PERSONA_MAX}；含 YAML 缩进 ${persona.indentedChars}）`);
+    else console.log('  persona 正文: 读不到');
+    if (mem.ok) {
+      console.log(`  记忆库: ${mem.total} 条 / ${mem.totalChars} 字符（pinned ${mem.pinnedN}）`);
+      console.log(`  注入预算: promptRecentCount=${budget.promptRecentCount} / promptMaxChars=${budget.promptMaxChars}（profile 覆盖 ${budget.found ? '在位' : '未找到'}）`);
+      console.log(`  本轮注入: ${mem.injectedCount} 条 / ${mem.injectedChars} 字符（丢弃 ${mem.dropped}）`);
+      console.log(`  记忆可见率: ${mem.visiblePercent.toFixed(2)}%（阈值 ${VISIBLE_MIN}%）`);
+    } else {
+      console.log(`  记忆库: ${mem.error}`);
+    }
+    for (const f of findings) console.log(`  ${icon === '🟢' ? '·' : (f.level === 'red' ? '🔴' : '🟡')} ${f.text}`);
+    if (!findings.length) console.log('  · 全绿：persona 未超阈值，记忆可见率达标');
+  } else {
+    console.log(`${icon} persona ${persona ? persona.text.length : '?'} 字符 | 记忆可见率 ${mem.ok ? mem.visiblePercent.toFixed(2) + '%' : '?'} | 注入 ${mem.ok ? mem.injectedCount : '?'} 条`);
+  }
+  return report.worst === 'red' ? 3 : 0;
+}
+
+try {
+  process.exitCode = main();
+} catch (e) {
+  console.error('体检失败:', e.message);
+  process.exitCode = 2;
+}