@kira905/dsh-memory-toolkit 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/LICENSE +661 -0
- package/README.en.md +110 -0
- package/README.md +102 -2
- package/dsh-memory-local/FORK-BASE.json +24 -0
- package/dsh-memory-local/README.md +113 -0
- package/dsh-memory-local/build.mjs +101 -0
- package/dsh-memory-local/cordis.patch.yml +20 -0
- package/dsh-memory-local/deploy.mjs +306 -0
- package/dsh-memory-local/entity-status.mjs +75 -0
- package/dsh-memory-local/lib/bridge.mjs +108 -0
- package/dsh-memory-local/lib/host-adapter.js +1171 -0
- package/dsh-memory-local/lib/index.d.ts +128 -0
- package/dsh-memory-local/lib/index.js +944 -0
- package/dsh-memory-local/lib/injector.mjs +400 -0
- package/dsh-memory-local/lib/memory-read.mjs +116 -0
- package/dsh-memory-local/lib/module-gate.mjs +68 -0
- package/dsh-memory-local/lib/pointer.mjs +152 -0
- package/dsh-memory-local/lib/prod-gate.mjs +121 -0
- package/dsh-memory-local/lib/recall-log.mjs +118 -0
- package/dsh-memory-local/lib/recall.mjs +192 -0
- package/dsh-memory-local/lib/schema-migrate.mjs +365 -0
- package/dsh-memory-local/lib/user-message.mjs +32 -0
- package/dsh-memory-local/lib/vectors-read.mjs +110 -0
- package/dsh-memory-local/package.json +51 -0
- package/dsh-memory-local/restore.mjs +259 -0
- package/dsh-memory-local/rollback-injector-fix.mjs +166 -0
- package/dsh-memory-local/src/bridge.mjs +108 -0
- package/dsh-memory-local/src/host-adapter.js +1171 -0
- package/dsh-memory-local/src/index.d.ts +128 -0
- package/dsh-memory-local/src/index.js +944 -0
- package/dsh-memory-local/src/injector.mjs +400 -0
- package/dsh-memory-local/src/memory-read.mjs +116 -0
- package/dsh-memory-local/src/module-gate.mjs +68 -0
- package/dsh-memory-local/src/pointer.mjs +152 -0
- package/dsh-memory-local/src/recall-log.mjs +118 -0
- package/dsh-memory-local/src/recall.mjs +192 -0
- package/dsh-memory-local/src/schema-migrate.mjs +365 -0
- package/dsh-memory-local/src/user-message.mjs +32 -0
- package/dsh-memory-local/src/vectors-read.mjs +110 -0
- package/dsh-memory-local/test/fixtures-s2b.mjs +216 -0
- package/dsh-memory-local/test/probe-entity.mjs +170 -0
- package/dsh-memory-local/test/regression.mjs +270 -0
- package/dsh-memory-local/test/s2b.test.mjs +568 -0
- package/dsh-memory-local/test/schema-migrate.test.mjs +334 -0
- package/dsh-memory-local/test/test-adapter-converge.mjs +127 -0
- package/dsh-memory-local/test/visible-injections.test.mjs +239 -0
- package/dsh-memory-local/verify-after-restart.mjs +181 -0
- package/lib/deploy-guard.mjs +391 -0
- package/lib/guard-http.mjs +93 -0
- package/lib/pitfall-form-map.mjs +130 -0
- package/lib/pitfall-score.mjs +185 -0
- package/lib/session-file-name.mjs +160 -0
- package/lib/zstd-frames.mjs +78 -0
- package/memory-db-tools/README.md +68 -0
- package/memory-db-tools/_test/test-memory-scope.mjs +302 -0
- package/memory-db-tools/audit-analyze.mjs +352 -0
- package/memory-db-tools/audit-dump-ids.mjs +245 -0
- package/memory-db-tools/audit-export.mjs +270 -0
- package/memory-db-tools/audit-hit-probe.mjs +227 -0
- package/memory-db-tools/audit-ledger-probe.mjs +236 -0
- package/memory-db-tools/audit-persona-overlap.mjs +277 -0
- package/memory-db-tools/audit-struct-probe.mjs +213 -0
- package/memory-db-tools/audit-verify-unchanged.mjs +276 -0
- package/memory-db-tools/check-memory-db.mjs +8 -0
- package/memory-db-tools/dump-memory-db.mjs +12 -0
- package/memory-db-tools/lib/memory-scope.mjs +446 -0
- package/memory-db-tools/list-memory-summary.mjs +477 -0
- package/memory-db-tools/memory-identity-check.mjs +101 -0
- package/memory-db-tools/memory-schema-migrate.mjs +290 -0
- package/memory-db-tools/memory-scope-fixture.mjs +197 -0
- package/memory-db-tools/read-memories.mjs +26 -0
- package/memory-db-tools/read-memory-db.mjs +20 -0
- package/memory-db-tools/refine-memories.mjs +247 -0
- package/memory-db-tools/show-mem-times.mjs +22 -0
- package/memory-engine/README.md +70 -0
- package/memory-engine/bench/bench.mjs +221 -0
- package/memory-engine/cli/distill-rollback.mjs +86 -0
- package/memory-engine/cli/distill.mjs +156 -0
- package/memory-engine/cli/doc-index-rebuild.mjs +155 -0
- package/memory-engine/cli/doc-index.mjs +48 -0
- package/memory-engine/cli/domain-tags.mjs +251 -0
- package/memory-engine/cli/pitfall-hook.mjs +91 -0
- package/memory-engine/cli/recall.mjs +132 -0
- package/memory-engine/cli/selfcheck.mjs +57 -0
- package/memory-engine/cli/serve-start.mjs +247 -0
- package/memory-engine/cli/sweep.mjs +58 -0
- package/memory-engine/cli/sync.mjs +69 -0
- package/memory-engine/lib/cli-util.mjs +197 -0
- package/memory-engine/lib/config.mjs +135 -0
- package/memory-engine/lib/dirty-ledger.mjs +174 -0
- package/memory-engine/lib/distill-rollback.mjs +178 -0
- package/memory-engine/lib/distill.mjs +898 -0
- package/memory-engine/lib/doc-index-store.mjs +325 -0
- package/memory-engine/lib/doc-index.mjs +529 -0
- package/memory-engine/lib/doc-scan.mjs +271 -0
- package/memory-engine/lib/doc-summarize.mjs +305 -0
- package/memory-engine/lib/domain-tags.mjs +429 -0
- package/memory-engine/lib/embedder.mjs +242 -0
- package/memory-engine/lib/hashes.mjs +12 -0
- package/memory-engine/lib/memory-source.mjs +127 -0
- package/memory-engine/lib/pitfall-source.mjs +170 -0
- package/memory-engine/lib/policy-source.mjs +371 -0
- package/memory-engine/lib/recall.mjs +331 -0
- package/memory-engine/lib/selfcheck.mjs +131 -0
- package/memory-engine/lib/sweep.mjs +159 -0
- package/memory-engine/lib/sync.mjs +180 -0
- package/memory-engine/lib/tick-hook.mjs +218 -0
- package/memory-engine/lib/vectors-store.mjs +292 -0
- package/memory-engine/package-lock.json +1086 -0
- package/memory-engine/package.json +22 -0
- package/memory-engine/serve.mjs +171 -0
- package/memory-engine/test/01-store.test.mjs +127 -0
- package/memory-engine/test/02-sync.test.mjs +196 -0
- package/memory-engine/test/03-sweep.test.mjs +145 -0
- package/memory-engine/test/04-recall.test.mjs +214 -0
- package/memory-engine/test/05-selfcheck.test.mjs +175 -0
- package/memory-engine/test/06-doc-index.test.mjs +928 -0
- package/memory-engine/test/07-domain-tags.test.mjs +298 -0
- package/memory-engine/test/08-distill.test.mjs +686 -0
- package/memory-engine/test/09-pitfall-source.test.mjs +365 -0
- package/memory-engine/test/10-tick-hook.test.mjs +252 -0
- package/memory-engine/test/11-policy-source.test.mjs +224 -0
- package/memory-engine/test/_tickhook-worker.mjs +56 -0
- package/memory-engine/test/fixtures.mjs +205 -0
- package/memory-engine/test/harness.mjs +73 -0
- package/memory-engine/test/run.mjs +35 -0
- package/memory-engine/tools/bridge-smoke.mjs +80 -0
- package/memory-engine/tools/fetch-model.mjs +128 -0
- package/memory-engine/tools/smoke-recall.mjs +76 -0
- package/package.json +53 -4
- package/persona-memory-audit.mjs +217 -0
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FORK-EDIT(2026-09-21): 指针渲染与预算裁剪(方案 §3.4 注入产物 / §8.2 绝对量判据)
|
|
3
|
+
*
|
|
4
|
+
* 预算判据(硬):条数 ≤8、总量 **≤800 字**、**≤600 token**(取更紧的一档实际生效);
|
|
5
|
+
* 每条 = `id/路径` + 日期 + tags + **≤80 字摘要首行**(方案 §3.4 Z17 已吸收)。
|
|
6
|
+
*
|
|
7
|
+
* token 计法(写死口径,报告里给实测字符数即可复核):
|
|
8
|
+
* · 主口径 = DeepSeek 中文口径:CJK 字符 1 token ≈ 1.5 字,非 CJK 1 token ≈ 3.6 字
|
|
9
|
+
* (DeepSeek 官方文档口径为「1 个中文字符 ≈ 0.6 token」,取 1.5 是**偏保守**的一侧)
|
|
10
|
+
* · 另有 `estimateTokensWorstCase` = 1 字 = 1 token 的过度保守口径(只用于报告对照)
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/** DeepSeek 中文口径的 token 估算(偏保守:CJK 按 1.5 字/token)。 */
|
|
14
|
+
export function estimateTokens(text) {
|
|
15
|
+
const s = String(text ?? '');
|
|
16
|
+
let cjk = 0;
|
|
17
|
+
let other = 0;
|
|
18
|
+
for (const ch of s) {
|
|
19
|
+
if (/[\u3400-\u9fff\uf900-\ufaff\u3000-\u303f\uff00-\uffef]/.test(ch)) cjk += 1;
|
|
20
|
+
else other += 1;
|
|
21
|
+
}
|
|
22
|
+
return Math.ceil(cjk / 1.5 + other / 3.6);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** 过度保守口径(1 字符 = 1 token),仅供报告对照,不用于裁剪。 */
|
|
26
|
+
export function estimateTokensWorstCase(text) {
|
|
27
|
+
return String(text ?? '').length;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const POINTER_HEADER =
|
|
31
|
+
'[相关记忆·指针] 以下可能与当前请求相关的历史记忆索引;需要细节请用 memory_get 展开:';
|
|
32
|
+
|
|
33
|
+
/** 日期(YYYY-MM-DD,本地时区)。 */
|
|
34
|
+
export function formatDate(ms) {
|
|
35
|
+
const d = new Date(Number(ms) || 0);
|
|
36
|
+
if (!Number.isFinite(d.getTime()) || d.getTime() <= 0) return '日期未知';
|
|
37
|
+
const p = (n) => String(n).padStart(2, '0');
|
|
38
|
+
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* 剥掉孤立代理项(lone surrogate,U+D800–U+DFFF)。
|
|
43
|
+
*
|
|
44
|
+
* ⚠️ 某次事故修复(详见 CHANGELOG)—— 会话历史永久 HTTP 400:
|
|
45
|
+
* 记忆文本里含 emoji(📦/🐋…,星面字符 = 一对代理项)时,若按 **UTF-16 码元** `slice`
|
|
46
|
+
* 截断,会正好切在代理对中间,留下孤立高代理项(如 `\ud83d`)。该文本进会话历史后,
|
|
47
|
+
* 请求体 JSON 变成 `"…\ud83d…"` ⇒ DeepSeek 网关 `Failed to parse the request body as JSON:
|
|
48
|
+
* unexpected end of hex escape` ⇒ **HTTP 400 / INVALID_REQUEST**;而坏消息**永久留在历史里**,
|
|
49
|
+
* 该会话此后**每轮必 400、无法自愈**(实测复现:含孤立代理项 → 400,去掉 → 200)。
|
|
50
|
+
*/
|
|
51
|
+
export function stripLoneSurrogate(text) {
|
|
52
|
+
return String(text ?? '').replace(
|
|
53
|
+
/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,
|
|
54
|
+
'',
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* 码点安全截断:**不切断代理对**(星面字符算 1 个),并对结果做孤立代理兜底剥离。
|
|
60
|
+
* @param {string} text
|
|
61
|
+
* @param {number} maxChars 按码点计的最大长度
|
|
62
|
+
*/
|
|
63
|
+
export function sliceSafe(text, maxChars) {
|
|
64
|
+
const s = String(text ?? '');
|
|
65
|
+
const limit = Math.max(0, Number(maxChars) || 0);
|
|
66
|
+
if (limit === 0) return '';
|
|
67
|
+
if (s.length <= limit) return stripLoneSurrogate(s);
|
|
68
|
+
const points = [...s];
|
|
69
|
+
if (points.length <= limit) return stripLoneSurrogate(s);
|
|
70
|
+
return stripLoneSurrogate(points.slice(0, limit).join(''));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* 摘要首行:取正文第一行,压掉换行/多余空白,截到 maxChars(超出加省略号)。
|
|
75
|
+
* @param {string} text
|
|
76
|
+
* @param {number} maxChars 默认 80(方案 §3.4)
|
|
77
|
+
*/
|
|
78
|
+
export function summarize(text, maxChars = 80) {
|
|
79
|
+
const firstLine = String(text ?? '')
|
|
80
|
+
.split(/\r?\n/)[0]
|
|
81
|
+
.replace(/\s+/g, ' ')
|
|
82
|
+
.trim();
|
|
83
|
+
if (firstLine.length <= maxChars) return stripLoneSurrogate(firstLine);
|
|
84
|
+
return `${sliceSafe(firstLine, Math.max(1, maxChars - 1))}…`;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** tags 压缩:空格分隔取前若干个,总长 ≤ maxChars(码点安全,不切断 emoji)。 */
|
|
88
|
+
export function compactTags(tags, maxChars = 40) {
|
|
89
|
+
const s = String(tags ?? '').replace(/\s+/g, ' ').trim();
|
|
90
|
+
if (s.length <= maxChars) return stripLoneSurrogate(s);
|
|
91
|
+
return `${sliceSafe(s, Math.max(1, maxChars - 1))}…`;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* 渲染一条指针行(方案 §3.4 形态)。
|
|
96
|
+
* @param {{id:number,text:string,tags:string,updated_at:number}} row
|
|
97
|
+
*/
|
|
98
|
+
export function pointerLine(row, { summaryChars = 80, tagChars = 40 } = {}) {
|
|
99
|
+
const tags = compactTags(row.tags, tagChars);
|
|
100
|
+
const tagPart = tags ? ` · tags: ${tags}` : '';
|
|
101
|
+
return `- #${row.id} (${formatDate(row.updated_at)}${tagPart}) ${summarize(row.text, summaryChars)}`;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* 按预算裁剪并渲染指针文本。
|
|
106
|
+
* 逐条累加,任一预算触顶即停(后面的条目丢弃并计数)。
|
|
107
|
+
* @returns {{text:string, lines:string[], items:Array, chars:number, estTokens:number, dropped:number, overflow:boolean}}
|
|
108
|
+
*/
|
|
109
|
+
export function renderPointer(
|
|
110
|
+
rows,
|
|
111
|
+
{ header = POINTER_HEADER, maxItems = 8, maxChars = 800, maxTokens = 600, summaryChars = 80, tagChars = 40 } = {},
|
|
112
|
+
) {
|
|
113
|
+
const lines = [];
|
|
114
|
+
const items = [];
|
|
115
|
+
let dropped = 0;
|
|
116
|
+
let overflow = false;
|
|
117
|
+
for (const row of rows || []) {
|
|
118
|
+
if (lines.length >= maxItems) {
|
|
119
|
+
dropped += 1;
|
|
120
|
+
overflow = true;
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const line = pointerLine(row, { summaryChars, tagChars });
|
|
124
|
+
const candidate = [header, ...lines, line].join('\n');
|
|
125
|
+
if (candidate.length > maxChars || estimateTokens(candidate) > maxTokens) {
|
|
126
|
+
dropped += 1;
|
|
127
|
+
overflow = true;
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
lines.push(line);
|
|
131
|
+
items.push({
|
|
132
|
+
id: Number(row.id),
|
|
133
|
+
chars: line.length,
|
|
134
|
+
summaryChars: summarize(row.text, summaryChars).length,
|
|
135
|
+
tags: row.tags,
|
|
136
|
+
updatedAt: Number(row.updated_at),
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
const text = lines.length ? [header, ...lines].join('\n') : '';
|
|
140
|
+
return {
|
|
141
|
+
text,
|
|
142
|
+
lines,
|
|
143
|
+
items,
|
|
144
|
+
chars: text.length,
|
|
145
|
+
estTokens: estimateTokens(text),
|
|
146
|
+
worstCaseTokens: estimateTokensWorstCase(text),
|
|
147
|
+
dropped,
|
|
148
|
+
overflow,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export const POINTER_HEADER_TEXT = POINTER_HEADER;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// lib/prod-gate.mjs —— **生产路径闸**(写入落在生产树内时,必须显式放行)
|
|
2
|
+
//
|
|
3
|
+
// 一句话:还原 / 回滚 / 清理类脚本动**生产树**里的东西之前,必须在**机器层**被拦住 ——
|
|
4
|
+
// 没显式 `--allow-production` ⇒ 拒跑(不是"警告后继续");dry-run 是第一道门(只读档零写入,不该拦计划)。
|
|
5
|
+
//
|
|
6
|
+
// 为什么需要它(本仓的来历):上游的实现把「生产树」写死成某一台机器的具体路径,
|
|
7
|
+
// 结果两个洞 ——
|
|
8
|
+
// ① 只比"**等于**"某个默认 home ⇒ `--target <生产树内的子目录>` 一律放行;
|
|
9
|
+
// ② 只比**逻辑**字符串、不走 realpath ⇒ 目录 junction / 符号链接跨面可绕。
|
|
10
|
+
// ⇒ 本文件是它的**可移植等价实现**:生产树由环境变量给出,判定一律 realpath + 逻辑路径**双比**。
|
|
11
|
+
//
|
|
12
|
+
// 判定口径:
|
|
13
|
+
// · 生产树集合 = `$DSH_HOME`(缺省 `~/.dsh`)+ `$DSH_PROD_ROOTS`(多个用 `;` 或 `:` 分隔)
|
|
14
|
+
// · 目标落在任一根之内(任一形态命中即算)⇒ 命中
|
|
15
|
+
// · `dryRun: true` ⇒ 直接放行(第一道门,函数内判,不依赖调用方)
|
|
16
|
+
// · `allowProduction: true` ⇒ 放行,但把命中项回传给调用方(便于留痕/打印)
|
|
17
|
+
//
|
|
18
|
+
// 退出码口径(与各 restore/rollback 脚本一致):
|
|
19
|
+
// ok:true = 通过 · ok:false + exitCode:2 = 缺放行(用法错)
|
|
20
|
+
|
|
21
|
+
import fs from 'node:fs';
|
|
22
|
+
import os from 'node:os';
|
|
23
|
+
import path from 'node:path';
|
|
24
|
+
|
|
25
|
+
/** 归一:绝对路径 + 反斜杠形态 + 小写(Windows 比较用)。 */
|
|
26
|
+
function normalize(p) {
|
|
27
|
+
return path.resolve(String(p)).replace(/\//g, path.sep).toLowerCase();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** 尽头真身(不存在或解析失败则回落逻辑路径 —— 不能因为读不到就当成"不在生产树")。 */
|
|
31
|
+
function realish(p) {
|
|
32
|
+
try {
|
|
33
|
+
return fs.realpathSync.native ? fs.realpathSync.native(p) : fs.realpathSync(p);
|
|
34
|
+
} catch {
|
|
35
|
+
return p;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** 生产树根集合。 */
|
|
40
|
+
export function productionRoots(opts = {}) {
|
|
41
|
+
const env = opts.env ?? process.env;
|
|
42
|
+
const extra = String(env.DSH_PROD_ROOTS ?? '')
|
|
43
|
+
.split(/[;:]/)
|
|
44
|
+
.map((s) => s.trim())
|
|
45
|
+
.filter(Boolean);
|
|
46
|
+
const home = opts.dshHome ?? env.DSH_HOME ?? path.join(os.homedir(), '.dsh');
|
|
47
|
+
return [home, ...extra];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** 目标是否落在生产树内(双比:逻辑路径 + realpath)。 */
|
|
51
|
+
export function isProductionPath(target, opts = {}) {
|
|
52
|
+
const roots = productionRoots(opts);
|
|
53
|
+
const cands = [normalize(target), normalize(realish(target))];
|
|
54
|
+
const hits = [];
|
|
55
|
+
for (const root of roots) {
|
|
56
|
+
for (const r of [normalize(root), normalize(realish(root))]) {
|
|
57
|
+
for (const c of cands) {
|
|
58
|
+
if (c === r || c.startsWith(r.endsWith(path.sep) ? r : r + path.sep)) hits.push({ target, root });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return hits;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* 生产写闸。
|
|
67
|
+
* @param {{targets?:Array<{path:string,label?:string}>, allowProduction?:boolean, dryRun?:boolean, op?:string, reason?:string}} args
|
|
68
|
+
* @returns {{ok:boolean, exitCode:number, dryRun:boolean, hits:Array, message:string}}
|
|
69
|
+
*/
|
|
70
|
+
export function gateProductionWrite(args = {}) {
|
|
71
|
+
const targets = Array.isArray(args.targets) ? args.targets : [];
|
|
72
|
+
const dryRun = args.dryRun === true;
|
|
73
|
+
const allowProduction = args.allowProduction === true;
|
|
74
|
+
const op = args.op || '写入';
|
|
75
|
+
|
|
76
|
+
// ── 第一道门:dry-run 一律放行(只读档不产生任何写) ──────────────────────
|
|
77
|
+
if (dryRun) {
|
|
78
|
+
return { ok: true, exitCode: 0, dryRun: true, hits: [], allowProduction, message: `dry-run:${op}计划放行(零写入)` };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const hits = [];
|
|
82
|
+
for (const t of targets) {
|
|
83
|
+
if (!t || !t.path) continue;
|
|
84
|
+
for (const h of isProductionPath(t.path, args)) hits.push({ ...h, label: t.label || '目标' });
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (hits.length === 0) {
|
|
88
|
+
return { ok: true, exitCode: 0, dryRun: false, hits: [], allowProduction, message: `${op}目标不在生产树内` };
|
|
89
|
+
}
|
|
90
|
+
if (allowProduction) {
|
|
91
|
+
return {
|
|
92
|
+
ok: true,
|
|
93
|
+
exitCode: 0,
|
|
94
|
+
dryRun: false,
|
|
95
|
+
hits,
|
|
96
|
+
allowProduction: true,
|
|
97
|
+
bypassed: true,
|
|
98
|
+
message: `显式放行:${op}目标在生产树内(${hits.map((h) => h.label).join('、')})`,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
return {
|
|
102
|
+
ok: false,
|
|
103
|
+
exitCode: 2,
|
|
104
|
+
dryRun: false,
|
|
105
|
+
hits,
|
|
106
|
+
allowProduction: false,
|
|
107
|
+
message: `拒绝:${op}目标落在生产树内(${hits.map((h) => `${h.label}=${h.target}`).join(',')})`,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** 人类可读的拒绝行(供 `console.error`)。 */
|
|
112
|
+
export function refuseLine(gate) {
|
|
113
|
+
const lines = [gate.message || '拒绝'];
|
|
114
|
+
if (gate.hits?.length) {
|
|
115
|
+
for (const h of gate.hits) lines.push(` 命中:${h.label} → ${h.target}(生产根 ${h.root})`);
|
|
116
|
+
}
|
|
117
|
+
lines.push(' 怎么给:确认这是有意为之后加 `--allow-production`;只读检查请用 `--dry-run`。');
|
|
118
|
+
return lines.join('\n');
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export const EXIT = { OK: 0, GATE: 1, USAGE: 2 };
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FORK-EDIT(2026-09-21): 注入可观测日志(方案 §3.4 可观测 / §6.1 ⑤ 失败环 / §8.2 展开率)
|
|
3
|
+
*
|
|
4
|
+
* 落点:`<DSH_HOME>/logs/memory-recall.log`(JSON 行:时间/条数/耗时/命中来源/预估字符数/是否被展开)
|
|
5
|
+
* 轮转:**3 份 × 保留 30 天**(方案 §3.4 Z18);**不含记忆正文**(只记 id/长度/摘要长度)。
|
|
6
|
+
* 铁律:**写日志失败不得影响注入**(全部 try/catch 吞掉,返回值只用于自测)。
|
|
7
|
+
*/
|
|
8
|
+
import fs from 'node:fs';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
|
|
11
|
+
export const DEFAULT_LOG_FILE = 'memory-recall.log';
|
|
12
|
+
|
|
13
|
+
export function createRecallLogger({
|
|
14
|
+
dir,
|
|
15
|
+
file = DEFAULT_LOG_FILE,
|
|
16
|
+
maxBytes = 1024 * 1024,
|
|
17
|
+
keepFiles = 3,
|
|
18
|
+
keepDays = 30,
|
|
19
|
+
fsImpl = fs,
|
|
20
|
+
now = () => Date.now(),
|
|
21
|
+
echo = null,
|
|
22
|
+
} = {}) {
|
|
23
|
+
if (!dir) throw new Error('recall-log: 必须给 dir(日志目录)');
|
|
24
|
+
const main = path.join(dir, file);
|
|
25
|
+
const stats = { written: 0, errors: 0, rotations: 0, swept: 0 };
|
|
26
|
+
let sizeCache = null;
|
|
27
|
+
|
|
28
|
+
const stamp = (ms) => {
|
|
29
|
+
const d = new Date(ms);
|
|
30
|
+
const p = (n) => String(n).padStart(2, '0');
|
|
31
|
+
return `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}`;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
const rotateIfNeeded = () => {
|
|
35
|
+
let size = 0;
|
|
36
|
+
try {
|
|
37
|
+
size = fsImpl.existsSync(main) ? fsImpl.statSync(main).size : 0;
|
|
38
|
+
} catch {
|
|
39
|
+
size = 0;
|
|
40
|
+
}
|
|
41
|
+
if (size <= maxBytes) return;
|
|
42
|
+
const target = path.join(dir, `memory-recall.${stamp(now())}.log`);
|
|
43
|
+
if (fsImpl.existsSync(target)) return; // 同秒重名 → 跳过(不覆盖)
|
|
44
|
+
fsImpl.renameSync(main, target);
|
|
45
|
+
sizeCache = 0;
|
|
46
|
+
stats.rotations += 1;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/** 清理:删超过 keepDays 的轮转文件 + 轮转文件数超过 keepFiles 删最旧。 */
|
|
50
|
+
const sweep = () => {
|
|
51
|
+
let names = [];
|
|
52
|
+
try {
|
|
53
|
+
names = fsImpl
|
|
54
|
+
.readdirSync(dir)
|
|
55
|
+
.filter((n) => /^memory-recall\..+\.log$/.test(n))
|
|
56
|
+
.map((n) => {
|
|
57
|
+
const full = path.join(dir, n);
|
|
58
|
+
let mtime = 0;
|
|
59
|
+
try {
|
|
60
|
+
mtime = fsImpl.statSync(full).mtimeMs;
|
|
61
|
+
} catch {
|
|
62
|
+
mtime = 0;
|
|
63
|
+
}
|
|
64
|
+
return { full, n, mtime };
|
|
65
|
+
})
|
|
66
|
+
.sort((a, b) => b.mtime - a.mtime);
|
|
67
|
+
} catch {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
const cutoff = now() - keepDays * 86400000;
|
|
71
|
+
const kept = [];
|
|
72
|
+
for (const f of names) {
|
|
73
|
+
if (f.mtime && f.mtime < cutoff) {
|
|
74
|
+
try {
|
|
75
|
+
fsImpl.rmSync(f.full, { force: true });
|
|
76
|
+
stats.swept += 1;
|
|
77
|
+
} catch {
|
|
78
|
+
/* ignore */
|
|
79
|
+
}
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
kept.push(f);
|
|
83
|
+
}
|
|
84
|
+
for (const f of kept.slice(keepFiles)) {
|
|
85
|
+
try {
|
|
86
|
+
fsImpl.rmSync(f.full, { force: true });
|
|
87
|
+
stats.swept += 1;
|
|
88
|
+
} catch {
|
|
89
|
+
/* ignore */
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
/** 写一行 JSON。永不抛错。 */
|
|
95
|
+
const log = (obj) => {
|
|
96
|
+
try {
|
|
97
|
+
const line = JSON.stringify({ t: new Date(now()).toISOString(), ...obj });
|
|
98
|
+
if (echo) {
|
|
99
|
+
try {
|
|
100
|
+
echo(line);
|
|
101
|
+
} catch {
|
|
102
|
+
/* ignore */
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
fsImpl.mkdirSync(dir, { recursive: true });
|
|
106
|
+
rotateIfNeeded();
|
|
107
|
+
fsImpl.appendFileSync(main, `${line}\n`, 'utf8');
|
|
108
|
+
stats.written += 1;
|
|
109
|
+
if (stats.written % 200 === 0) sweep();
|
|
110
|
+
return true;
|
|
111
|
+
} catch {
|
|
112
|
+
stats.errors += 1;
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
return { log, sweep, path: main, stats: () => ({ ...stats }) };
|
|
118
|
+
}
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FORK-EDIT(2026-09-21): 召回主逻辑 —— 从 S1 离线层 `memory-engine/lib/recall.mjs` 移植。
|
|
3
|
+
*
|
|
4
|
+
* 移植来源:本仓 `memory-engine/lib/recall.mjs`(记忆引擎 v2 · S1 交付)
|
|
5
|
+
* 本文件与来源**逐字同语义**(方案 §3.3:向量为主 + FTS 仅补标签/英文/数字 + RRF(k=60)
|
|
6
|
+
* + 排除 pinned + 非空即触发、纯符号/单字跳过);**唯一改动**是 import 路径
|
|
7
|
+
* (`./vectors-store.mjs` → `./vectors-read.mjs`,插件侧只读、不写向量库)。
|
|
8
|
+
*/
|
|
9
|
+
import { loadMatrix } from './vectors-read.mjs';
|
|
10
|
+
|
|
11
|
+
/** 触发判定(方案 §3.3 Z16 已吸收) */
|
|
12
|
+
export function shouldTrigger(rawQuery) {
|
|
13
|
+
const q = String(rawQuery ?? '').trim();
|
|
14
|
+
if (!q) return { trigger: false, reason: 'empty' };
|
|
15
|
+
const semantic = q.match(/[\p{L}\p{N}]/gu) || [];
|
|
16
|
+
if (semantic.length === 0) return { trigger: false, reason: 'symbols-only' };
|
|
17
|
+
if (semantic.length < 2) return { trigger: false, reason: 'single-char' };
|
|
18
|
+
return { trigger: true, reason: 'non-empty' };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** 抽 FTS 补位 token:只取英文/数字(A4:中文连续串在 unicode61 下几近不可用) */
|
|
22
|
+
export function extractFtsTokens(rawQuery, max = 8) {
|
|
23
|
+
const m = String(rawQuery ?? '').match(/[A-Za-z0-9][A-Za-z0-9_.-]*/g) || [];
|
|
24
|
+
const out = [];
|
|
25
|
+
for (const t of m) {
|
|
26
|
+
if (t.length < 2) continue; // 单字母/单数字噪声大
|
|
27
|
+
if (!out.includes(t)) out.push(t);
|
|
28
|
+
if (out.length >= max) break;
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** 构造 FTS5 MATCH 表达式:每个 token 加双引号(防语法注入),OR 连接 */
|
|
34
|
+
export function buildFtsMatch(tokens) {
|
|
35
|
+
if (!tokens || !tokens.length) return null;
|
|
36
|
+
return tokens.map((t) => `"${String(t).replace(/"/g, '')}"`).join(' OR ');
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** 载入向量矩阵 + 行范数(余弦召回用;一次构建,多次召回) */
|
|
40
|
+
export function buildVectorIndex(db, { source = 'memory', model = null } = {}) {
|
|
41
|
+
const matrix = loadMatrix(db, source, model);
|
|
42
|
+
const norms = new Float32Array(matrix.count);
|
|
43
|
+
for (let r = 0; r < matrix.count; r += 1) {
|
|
44
|
+
let n = 0;
|
|
45
|
+
const off = r * matrix.dim;
|
|
46
|
+
for (let i = 0; i < matrix.dim; i += 1) n += matrix.data[off + i] * matrix.data[off + i];
|
|
47
|
+
norms[r] = Math.sqrt(n) || 1;
|
|
48
|
+
}
|
|
49
|
+
return { ...matrix, norms, builtAt: Date.now() };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** 余弦 top-K(纯内存暴力;万级候选毫秒级,方案 §3.3) */
|
|
53
|
+
export function cosineTopK(index, qvec, topK) {
|
|
54
|
+
const { data, dim, count, norms } = index;
|
|
55
|
+
if (!dim || !count) return [];
|
|
56
|
+
let qn = 0;
|
|
57
|
+
for (let i = 0; i < dim; i += 1) qn += qvec[i] * qvec[i];
|
|
58
|
+
qn = Math.sqrt(qn) || 1;
|
|
59
|
+
const scores = new Float32Array(count);
|
|
60
|
+
for (let r = 0; r < count; r += 1) {
|
|
61
|
+
let s = 0;
|
|
62
|
+
const off = r * dim;
|
|
63
|
+
for (let i = 0; i < dim; i += 1) s += data[off + i] * qvec[i];
|
|
64
|
+
scores[r] = s / (qn * norms[r]);
|
|
65
|
+
}
|
|
66
|
+
const order = Array.from({ length: count }, (_, i) => i);
|
|
67
|
+
order.sort((a, b) => scores[b] - scores[a] || a - b);
|
|
68
|
+
const k = Math.min(Number(topK) || 0, count);
|
|
69
|
+
const out = [];
|
|
70
|
+
for (let i = 0; i < k; i += 1) {
|
|
71
|
+
const idx = order[i];
|
|
72
|
+
out.push({ index: idx, refId: index.refIds[idx], id: index.refIdsNum[idx], score: scores[idx] });
|
|
73
|
+
}
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* RRF 融合(k=60):score(d) = Σ_r 1/(k + rank_r(d))
|
|
79
|
+
* @param {Array<{source:string, ids:string[]}>} rankLists 各路排名(已按优劣降序,rank 从 1 起)
|
|
80
|
+
*/
|
|
81
|
+
export function rrfFuse(rankLists, k = 60) {
|
|
82
|
+
const acc = new Map();
|
|
83
|
+
for (const list of rankLists) {
|
|
84
|
+
if (!list || !list.ids) continue;
|
|
85
|
+
list.ids.forEach((id, i) => {
|
|
86
|
+
const rank = i + 1;
|
|
87
|
+
const cur = acc.get(String(id)) || { id: String(id), score: 0, sources: [], ranks: {} };
|
|
88
|
+
cur.score += 1 / (k + rank);
|
|
89
|
+
if (!cur.sources.includes(list.source)) cur.sources.push(list.source);
|
|
90
|
+
cur.ranks[list.source] = rank;
|
|
91
|
+
acc.set(String(id), cur);
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
return [...acc.values()].sort(
|
|
95
|
+
(a, b) => b.score - a.score || Number(a.id) - Number(b.id),
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* 召回主入口。
|
|
101
|
+
* fail-open:任何异常都由调用方兜住(本函数内部对 FTS 异常降级为空路)。
|
|
102
|
+
* @returns {Promise<object>}
|
|
103
|
+
*/
|
|
104
|
+
export async function recall({
|
|
105
|
+
query,
|
|
106
|
+
store,
|
|
107
|
+
source,
|
|
108
|
+
embedder,
|
|
109
|
+
index = null,
|
|
110
|
+
vectorTopK = 20,
|
|
111
|
+
ftsTopK = 20,
|
|
112
|
+
rrfK = 60,
|
|
113
|
+
resultTopK = 8,
|
|
114
|
+
log = () => {},
|
|
115
|
+
}) {
|
|
116
|
+
const t0 = performance.now();
|
|
117
|
+
const gate = shouldTrigger(query);
|
|
118
|
+
if (!gate.trigger) {
|
|
119
|
+
return { triggered: false, reason: gate.reason, query: String(query ?? ''), hits: [], timings: { totalMs: 0 } };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const db = store.db;
|
|
123
|
+
const pinned = new Set((source ? source.pinnedIds() : []).map(String));
|
|
124
|
+
|
|
125
|
+
// ① 查询编码
|
|
126
|
+
const tEnc = performance.now();
|
|
127
|
+
const qvec = await embedder.encode(query);
|
|
128
|
+
const encodeMs = performance.now() - tEnc;
|
|
129
|
+
|
|
130
|
+
// ② 向量路
|
|
131
|
+
const tVec = performance.now();
|
|
132
|
+
const idx = index || buildVectorIndex(db, { source: 'memory' });
|
|
133
|
+
const rawVec = cosineTopK(idx, qvec, vectorTopK + pinned.size + 5);
|
|
134
|
+
const vecHits = rawVec.filter((h) => !pinned.has(String(h.refId))).slice(0, vectorTopK);
|
|
135
|
+
const vectorMs = performance.now() - tVec;
|
|
136
|
+
|
|
137
|
+
// ③ FTS 补位路(仅英文/数字;缺失时不参与,不做零分占位)
|
|
138
|
+
const tFts = performance.now();
|
|
139
|
+
let ftsHits = [];
|
|
140
|
+
let ftsMatched = 0;
|
|
141
|
+
const tokens = extractFtsTokens(query);
|
|
142
|
+
const matchExpr = buildFtsMatch(tokens);
|
|
143
|
+
if (matchExpr && source) {
|
|
144
|
+
const r = source.ftsSearch(matchExpr, ftsTopK + pinned.size + 5);
|
|
145
|
+
if (Array.isArray(r)) {
|
|
146
|
+
ftsMatched = r.length;
|
|
147
|
+
ftsHits = r.filter((h) => !pinned.has(String(h.id))).slice(0, ftsTopK);
|
|
148
|
+
} else if (r && r.error) {
|
|
149
|
+
log(`[recall] FTS 补位降级:${r.error}`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
const ftsMs = performance.now() - tFts;
|
|
153
|
+
|
|
154
|
+
// ④ RRF 融合
|
|
155
|
+
const tFuse = performance.now();
|
|
156
|
+
const fused = rrfFuse(
|
|
157
|
+
[
|
|
158
|
+
{ source: 'vector', ids: vecHits.map((h) => String(h.refId)) },
|
|
159
|
+
{ source: 'fts', ids: ftsHits.map((h) => String(h.id)) },
|
|
160
|
+
],
|
|
161
|
+
rrfK,
|
|
162
|
+
);
|
|
163
|
+
const hits = fused.slice(0, Number(resultTopK) || 8).map((h) => ({
|
|
164
|
+
refId: h.id,
|
|
165
|
+
id: Number(h.id),
|
|
166
|
+
score: h.score,
|
|
167
|
+
sources: h.sources,
|
|
168
|
+
vectorRank: h.ranks.vector ?? null,
|
|
169
|
+
ftsRank: h.ranks.fts ?? null,
|
|
170
|
+
}));
|
|
171
|
+
const fuseMs = performance.now() - tFuse;
|
|
172
|
+
|
|
173
|
+
const totalMs = performance.now() - t0;
|
|
174
|
+
return {
|
|
175
|
+
triggered: true,
|
|
176
|
+
reason: gate.reason,
|
|
177
|
+
query: String(query ?? ''),
|
|
178
|
+
hits,
|
|
179
|
+
timings: { encodeMs, vectorMs, ftsMs, fuseMs, totalMs },
|
|
180
|
+
stats: {
|
|
181
|
+
indexSize: idx.count,
|
|
182
|
+
pinnedExcluded: pinned.size,
|
|
183
|
+
vectorCandidates: rawVec.length,
|
|
184
|
+
vectorHits: vecHits.length,
|
|
185
|
+
ftsTokens: tokens,
|
|
186
|
+
ftsMatchExpr: matchExpr,
|
|
187
|
+
ftsMatched,
|
|
188
|
+
ftsHits: ftsHits.length,
|
|
189
|
+
fused: fused.length,
|
|
190
|
+
},
|
|
191
|
+
};
|
|
192
|
+
}
|