feiyu-memory 0.0.0-stage → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +559 -0
- package/LICENSE +36 -0
- package/README.md +536 -3
- package/SPEC.md +1380 -0
- package/cordis.patch.yml +62 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +62 -4
- package/src/client/index.js +484 -0
- package/src/core/archive.mjs +572 -0
- package/src/core/config.mjs +313 -0
- package/src/core/consolidate.mjs +759 -0
- package/src/core/gate.mjs +42 -0
- package/src/core/handnote.mjs +117 -0
- package/src/core/judge.mjs +116 -0
- package/src/core/keywords.mjs +121 -0
- package/src/core/persona.default.md +34 -0
- package/src/core/persona.mjs +594 -0
- package/src/core/privacy.mjs +252 -0
- package/src/core/recall.mjs +280 -0
- package/src/core/render-diary.mjs +171 -0
- package/src/core/timeline.mjs +127 -0
- package/src/core/tokenize.mjs +70 -0
- package/src/core/zip.mjs +239 -0
- package/src/host/index.mjs +2039 -0
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 隐私层:落盘前的最后一道闸。
|
|
3
|
+
*
|
|
4
|
+
* 分三类,处理方式完全不同:
|
|
5
|
+
*
|
|
6
|
+
* 1. format —— 强确定性的可识别格式(手机号/证件/卡号/邮箱)
|
|
7
|
+
* 只要开了第三方泛化就一律处理。误伤概率极低。
|
|
8
|
+
*
|
|
9
|
+
* 2. secret —— 密钥。**硬阻断**,永远处理,没有开关。
|
|
10
|
+
* 命中即替换,绝不入档、绝不进日志。
|
|
11
|
+
*
|
|
12
|
+
* 3. name / amount —— 弱信号。**只认高置信标记**。
|
|
13
|
+
* 人名只认称谓(王总/李老师)、小X/老X、"叫X"三种明确写法;
|
|
14
|
+
* 金额只处理超过门槛的数(保住"3 块钱的包子"这种生活细节)。
|
|
15
|
+
*
|
|
16
|
+
* 设计原则:**宁可少泛化,不要过度泛化**。
|
|
17
|
+
* 糊成一团"某人和某人在某地发生了某事"的日记,既没有隐私价值也没有记忆价值,
|
|
18
|
+
* 用户会直接关掉这个功能 —— 那就等于没有隐私保护。
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/** 永远处理:密钥。命中的文本块直接丢弃。 */
|
|
22
|
+
const SECRET_RULES = [
|
|
23
|
+
[/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, '私钥'],
|
|
24
|
+
[/\bsk-[A-Za-z0-9_-]{16,}\b/g, 'API Key'],
|
|
25
|
+
[/\bgh[pousr]_[A-Za-z0-9]{20,}\b/g, 'GitHub Token'],
|
|
26
|
+
[/\bgithub_pat_[A-Za-z0-9_]{20,}\b/g, 'GitHub Token'],
|
|
27
|
+
[/\bAKIA[0-9A-Z]{16}\b/g, 'AWS Key'],
|
|
28
|
+
[/\bAIza[0-9A-Za-z_-]{35}\b/g, 'Google Key'],
|
|
29
|
+
[/\bxox[baprs]-[0-9A-Za-z-]{10,}\b/g, 'Slack Token'],
|
|
30
|
+
[/\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, 'JWT'],
|
|
31
|
+
[/\bBearer\s+[A-Za-z0-9._~+/-]{20,}=*/g, 'Bearer Token'],
|
|
32
|
+
[
|
|
33
|
+
/\b(?:pass(?:wo?r?d)?|pwd|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key)\b\s*(?::|=|\bis\b|是|为)\s*["']?[^\s"',;]{6,}["']?/gi,
|
|
34
|
+
'凭据',
|
|
35
|
+
],
|
|
36
|
+
// ── 中文键名 ──
|
|
37
|
+
//
|
|
38
|
+
// 审计报告(F2,P2):上面那条只认英文键名,而这是个中文插件 ——
|
|
39
|
+
// README 承诺"密码……被替换成 [已拦截]",可是这些写法全都放行:
|
|
40
|
+
//
|
|
41
|
+
// 服务器 root 密码是 Zx9!kq2#Lm7$Pw
|
|
42
|
+
// WiFi 密码:hunter2pass2026
|
|
43
|
+
// 数据库口令 Hunter2pass
|
|
44
|
+
// 阿里云 AccessKey Secret 是 wJalrXUtn...
|
|
45
|
+
// my password is Hunter2pass
|
|
46
|
+
//
|
|
47
|
+
// 中文没有 \b 词边界,所以单独一条规则,分隔符接受 : : = 是 为 is。
|
|
48
|
+
[/(?:密码|口令|密钥|秘钥|验证码)\s*(?::|:|=|是|为|\bis\b)?\s*["']?[^\s"',;,。]{6,}["']?/g, '凭据'],
|
|
49
|
+
// 只写冒号、没有连接词的那种("密码:xxx" 上面已覆盖,这条兜 Union 场景)
|
|
50
|
+
[/(?:密码|口令|密钥|秘钥)\s*[::]\s*\S{6,}/g, '凭据'],
|
|
51
|
+
// ── URL 里内嵌的凭据 ──
|
|
52
|
+
//
|
|
53
|
+
// postgres://admin:Hunter2pass@db.internal:5432/app
|
|
54
|
+
//
|
|
55
|
+
// 原来这条"碰巧"被邮箱规则吃掉了(Hunter2pass@db.internal 像个邮箱),
|
|
56
|
+
// 换个 scheme 或者密码里没 @ 就漏。单独成规则才稳。
|
|
57
|
+
[/\/\/[^\/\s:@]+:[^@\s/]{4,}@/g, '凭据'],
|
|
58
|
+
// ── 长数字 ──
|
|
59
|
+
//
|
|
60
|
+
// 社区报告(2026-10-24):一串 13 位数字 `123456789012` 原样进了
|
|
61
|
+
// relationship.json 的 evidence 字段。手机号规则只认 11 位大陆号,
|
|
62
|
+
// 11 位以上的一律放行。
|
|
63
|
+
//
|
|
64
|
+
// **门槛是 12 位,不是 11** —— 11 位是手机号,那个由上面 `RULES` 里的手机号
|
|
65
|
+
// 而且用户明确表示过"露着吧"。11 位以上基本只有电话号码和身份证号。
|
|
66
|
+
[/\b\d{12,}\b/g, '长数字'],
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
const SECRET_PLACEHOLDER = '[已拦截]';
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* @typedef {'format'|'name'|'amount'} RuleGroup
|
|
73
|
+
* @typedef {object} Rule
|
|
74
|
+
* @property {RegExp} re
|
|
75
|
+
* @property {string | ((...m: string[]) => string)} replace
|
|
76
|
+
* @property {string} label
|
|
77
|
+
* @property {RuleGroup} group
|
|
78
|
+
*/
|
|
79
|
+
|
|
80
|
+
/** @type {Rule[]} */
|
|
81
|
+
const RULES = [
|
|
82
|
+
{ group: 'format', label: '手机号', re: /(?<!\d)1[3-9]\d{9}(?!\d)/g, replace: '[手机号]' },
|
|
83
|
+
{ group: 'format', label: '证件号', re: /(?<!\d)\d{17}[\dXx](?!\d)/g, replace: '[证件号]' },
|
|
84
|
+
{ group: 'format', label: '卡号', re: /(?<!\d)\d{16,19}(?!\d)/g, replace: '[卡号]' },
|
|
85
|
+
{
|
|
86
|
+
group: 'format',
|
|
87
|
+
label: '邮箱',
|
|
88
|
+
re: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g,
|
|
89
|
+
replace: '[邮箱]',
|
|
90
|
+
},
|
|
91
|
+
|
|
92
|
+
// 称谓式人名:姓氏表 + 称谓。高置信,误伤极小。
|
|
93
|
+
{
|
|
94
|
+
group: 'name',
|
|
95
|
+
label: '称谓式人名',
|
|
96
|
+
re: /[\u5f20\u738b\u674e\u8d75\u5218\u9648\u6768\u9ec4\u5468\u5434\u5f90\u5b59\u9a6c\u6731\u80e1\u6797\u90ed\u4f55\u9ad8\u7f57\u90d1\u6881\u8c22\u5b8b\u5510\u8bb8\u97e9\u51af\u9093\u66f9\u5f6d\u66fe\u8096\u7530\u8463\u6f58\u8881\u8521\u848b\u4f59\u4e8e\u675c\u53f6\u7a0b\u82cf\u9b4f\u5415\u4e01\u4efb\u6c88\u59da\u5362\u59dc\u5d14\u949f\u8c2d\u9646\u6c6a\u8303\u91d1\u77f3\u5ed6\u8d3e\u590f\u97e6\u4ed8\u65b9\u767d\u90b9\u5b5f\u718a\u79e6\u90b1\u6c5f\u5c39\u859b\u95eb\u6bb5\u96f7\u4faf\u9f99\u53f2\u9676\u9ece\u8d3a\u987e\u6bdb\u90dd\u9f9a\u90b5\u4e07\u94b1\u4e25\u8983\u6b66\u6234\u83ab\u5b54\u5411\u6c64][\u603b\u8001\u5e08\u533b\u751f\u7ecf\u7406\u4e3b\u4efb\u54e5\u59d0\u53d4\u59e8\u5de5]/g,
|
|
97
|
+
replace: '[某人]',
|
|
98
|
+
},
|
|
99
|
+
// 小X / 老X
|
|
100
|
+
{
|
|
101
|
+
group: 'name',
|
|
102
|
+
label: '小X/老X',
|
|
103
|
+
re: /[\u5c0f\u8001][\u5f20\u738b\u674e\u8d75\u5218\u9648\u6768\u9ec4\u5468\u5434\u5f90\u5b59\u9a6c\u6731\u80e1\u6797\u90ed\u4f55\u9ad8\u7f57\u90d1\u6881\u8c22\u5b8b\u5510\u8bb8\u97e9\u51af\u9093\u66f9]/g,
|
|
104
|
+
replace: '[某人]',
|
|
105
|
+
},
|
|
106
|
+
// 明确引入的名字:"叫X" / "名叫X" / "名字是X"
|
|
107
|
+
{
|
|
108
|
+
group: 'name',
|
|
109
|
+
label: '明确指名的名字',
|
|
110
|
+
re: /(叫|名叫|名字是)([\u4e00-\u9fa5]{2,4})(?=[\s\uff0c\u3002\uff01\uff1f,]|$)/g,
|
|
111
|
+
replace: (_m, lead) => `${lead}[某人]`,
|
|
112
|
+
},
|
|
113
|
+
];
|
|
114
|
+
|
|
115
|
+
/** 金额门槛:低于此值的金额不动 */
|
|
116
|
+
export const DEFAULT_AMOUNT_THRESHOLD = 100;
|
|
117
|
+
|
|
118
|
+
/** 带单位的金额(元/块/万/亿…),要读数字,所以单独处理 */
|
|
119
|
+
const AMOUNT_UNIT_RE = /(?<![\d.])(\d[\d,]*(?:\.\d+)?)\s*(万元|万块|亿元|万|亿|元|块钱|块)(?![\d])/g;
|
|
120
|
+
/** 带币种符号的金额 */
|
|
121
|
+
const AMOUNT_SYMBOL_RE = /(?:¥|¥|\$)\s?(\d[\d,]*(?:\.\d+)?)/g;
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* @typedef {object} PrivacyOptions
|
|
125
|
+
* @property {boolean} [thirdParty] 总开关:泛化第三方信息(默认 true)
|
|
126
|
+
* @property {boolean} [redactNames] 处理人名的弱信号规则(默认随 thirdParty)
|
|
127
|
+
* @property {boolean} [amounts] 泛化大额金额(默认随 thirdParty)
|
|
128
|
+
* @property {number} [amountThreshold] 金额门槛(默认 100)
|
|
129
|
+
*/
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* 清洗一段文本。幂等 —— 占位符不会再次命中规则,重复调用结果稳定。
|
|
133
|
+
*
|
|
134
|
+
* @param {string} text
|
|
135
|
+
* @param {PrivacyOptions} [options]
|
|
136
|
+
* @returns {{text: string, hits: Array<{kind: string, label: string, count: number}>, blocked: boolean}}
|
|
137
|
+
*/
|
|
138
|
+
export function scrub(text, options = {}) {
|
|
139
|
+
if (typeof text !== 'string' || text.length === 0) {
|
|
140
|
+
return { text: typeof text === 'string' ? text : '', hits: [], blocked: false };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const thirdParty = options.thirdParty !== false;
|
|
144
|
+
const redactNames = options.redactNames !== false && thirdParty;
|
|
145
|
+
const doAmounts = options.amounts !== false && thirdParty;
|
|
146
|
+
const threshold = Number.isFinite(options.amountThreshold)
|
|
147
|
+
? options.amountThreshold
|
|
148
|
+
: DEFAULT_AMOUNT_THRESHOLD;
|
|
149
|
+
|
|
150
|
+
const tally = new Map();
|
|
151
|
+
const bump = (kind, label, n = 1) => {
|
|
152
|
+
const key = `${kind}\u0000${label}`;
|
|
153
|
+
const cur = tally.get(key) ?? { kind, label, count: 0 };
|
|
154
|
+
cur.count += n;
|
|
155
|
+
tally.set(key, cur);
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
let out = text;
|
|
159
|
+
|
|
160
|
+
// 1) 密钥先做,且无条件做。后面任何一步出错,密钥都已经没了。
|
|
161
|
+
let blocked = false;
|
|
162
|
+
for (const [re, label] of SECRET_RULES) {
|
|
163
|
+
if (!re.test(out)) continue;
|
|
164
|
+
re.lastIndex = 0;
|
|
165
|
+
out = out.replace(re, () => {
|
|
166
|
+
blocked = true;
|
|
167
|
+
bump('secret', label);
|
|
168
|
+
return SECRET_PLACEHOLDER;
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// 2) 格式类 / 人名类
|
|
173
|
+
if (thirdParty) {
|
|
174
|
+
for (const rule of RULES) {
|
|
175
|
+
if (rule.group === 'name' && !redactNames) continue;
|
|
176
|
+
out = out.replace(rule.re, (...args) => {
|
|
177
|
+
bump('thirdParty', rule.label);
|
|
178
|
+
return typeof rule.replace === 'function' ? rule.replace(...args) : rule.replace;
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// 3) 金额(带门槛)
|
|
184
|
+
if (doAmounts) {
|
|
185
|
+
const overThreshold = (digits) => {
|
|
186
|
+
const v = Number(String(digits).replace(/,/g, ''));
|
|
187
|
+
return Number.isFinite(v) && v >= threshold;
|
|
188
|
+
};
|
|
189
|
+
out = out.replace(AMOUNT_UNIT_RE, (match, digits) => {
|
|
190
|
+
if (!overThreshold(digits)) return match;
|
|
191
|
+
bump('thirdParty', '金额');
|
|
192
|
+
return '一笔钱';
|
|
193
|
+
});
|
|
194
|
+
out = out.replace(AMOUNT_SYMBOL_RE, (match, digits) => {
|
|
195
|
+
if (!overThreshold(digits)) return match;
|
|
196
|
+
bump('thirdParty', '金额');
|
|
197
|
+
return '一笔钱';
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
return { text: out, hits: [...tally.values()], blocked };
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* 批量清洗,合并命中统计。
|
|
206
|
+
* @param {string[]} texts
|
|
207
|
+
* @param {PrivacyOptions} [options]
|
|
208
|
+
*/
|
|
209
|
+
export function scrubAll(texts, options = {}) {
|
|
210
|
+
const merged = new Map();
|
|
211
|
+
let blocked = false;
|
|
212
|
+
const out = texts.map((t) => {
|
|
213
|
+
const r = scrub(t, options);
|
|
214
|
+
blocked ||= r.blocked;
|
|
215
|
+
for (const h of r.hits) {
|
|
216
|
+
const key = `${h.kind}\u0000${h.label}`;
|
|
217
|
+
const cur = merged.get(key) ?? { kind: h.kind, label: h.label, count: 0 };
|
|
218
|
+
cur.count += h.count;
|
|
219
|
+
merged.set(key, cur);
|
|
220
|
+
}
|
|
221
|
+
return r.text;
|
|
222
|
+
});
|
|
223
|
+
return { texts: out, hits: [...merged.values()], blocked };
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* 给用户看的一句提示。刻意短,不弹窗、不说教 —— 像鱼,不像系统。
|
|
228
|
+
* @param {Array<{kind: string, label: string, count: number}>} hits
|
|
229
|
+
* @returns {string} 无命中时返回空串
|
|
230
|
+
*/
|
|
231
|
+
export function noticeFor(hits) {
|
|
232
|
+
if (!hits || hits.length === 0) return '';
|
|
233
|
+
const sum = (kind) =>
|
|
234
|
+
hits.filter((h) => (kind === 'secret' ? h.kind === 'secret' : h.kind !== 'secret'))
|
|
235
|
+
.reduce((s, h) => s + h.count, 0);
|
|
236
|
+
const parts = [];
|
|
237
|
+
const secrets = sum('secret');
|
|
238
|
+
const others = sum('other');
|
|
239
|
+
if (secrets > 0) parts.push(`拦下 ${secrets} 处密钥`);
|
|
240
|
+
if (others > 0) parts.push(`模糊了 ${others} 处别人家的信息`);
|
|
241
|
+
return parts.length > 0 ? parts.join(',') + '。' : '';
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* 预设:给"我就想让它记住我女朋友名字"的用户。
|
|
246
|
+
* 注意密钥与格式类仍然生效 —— 那两类没有关闭的理由。
|
|
247
|
+
*/
|
|
248
|
+
export const PRESET_REMEMBER_NAMES = Object.freeze({
|
|
249
|
+
thirdParty: true,
|
|
250
|
+
redactNames: false,
|
|
251
|
+
amounts: false,
|
|
252
|
+
});
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 记忆召回 —— 完全本地、零模型 token。
|
|
3
|
+
*
|
|
4
|
+
* 为什么不用 SQLite / 向量库:
|
|
5
|
+
* - 向量库要下载模型 + 占几百 MB,还得开 GPU 之外的线程池,与"轻到不像插件"背道而驰
|
|
6
|
+
* - SQLite 要么引原生依赖(插件装不上),要么赌运行时有 `node:sqlite`
|
|
7
|
+
* - 倒排索引对我们这个量级(十年日记 ≈ 几千条)是**毫秒级**的,
|
|
8
|
+
* 而且索引是纯派生的 —— 每次加载重建,不需要存盘,文件坏了也不影响档案本体
|
|
9
|
+
*
|
|
10
|
+
* 打分 = a·文本相似度 + b·salience + c·新鲜度 − d·最近刚提过的惩罚
|
|
11
|
+
*
|
|
12
|
+
* 分流刻意保留随机性:人本来就是"联想到什么说什么",
|
|
13
|
+
* 每次都精准翻出最相关的那条,是数据库,不是鱼。
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { tokenize } from './keywords.mjs';
|
|
17
|
+
|
|
18
|
+
/** IDF 平滑,避免 df=0 时除零 */
|
|
19
|
+
const IDF_SMOOTH = 1;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* 建倒排索引。纯派生数据,随用随建。
|
|
23
|
+
*
|
|
24
|
+
* @param {Array<{id: string, keywords?: string[], text?: string}>} entries
|
|
25
|
+
* @returns {{postings: Map<string, Map<string, number>>, df: Map<string, number>, lengths: Map<string, number>, N: number}}
|
|
26
|
+
*/
|
|
27
|
+
export function buildIndex(entries) {
|
|
28
|
+
const postings = new Map();
|
|
29
|
+
const df = new Map();
|
|
30
|
+
const lengths = new Map();
|
|
31
|
+
const list = Array.isArray(entries) ? entries : [];
|
|
32
|
+
|
|
33
|
+
for (const entry of list) {
|
|
34
|
+
// 以正文为主索引;已存的关键词额外加权(重复计入),因为它们是刻意提炼过的
|
|
35
|
+
const tokens = tokenize(entry.text ?? '');
|
|
36
|
+
if (Array.isArray(entry.keywords)) {
|
|
37
|
+
for (const k of entry.keywords) {
|
|
38
|
+
const kt = tokenize(k);
|
|
39
|
+
tokens.push(...kt, ...kt);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if (tokens.length === 0) continue;
|
|
43
|
+
|
|
44
|
+
const tf = new Map();
|
|
45
|
+
for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
|
|
46
|
+
|
|
47
|
+
for (const [t, n] of tf) {
|
|
48
|
+
let bucket = postings.get(t);
|
|
49
|
+
if (bucket === undefined) {
|
|
50
|
+
bucket = new Map();
|
|
51
|
+
postings.set(t, bucket);
|
|
52
|
+
}
|
|
53
|
+
bucket.set(entry.id, n);
|
|
54
|
+
df.set(t, (df.get(t) ?? 0) + 1);
|
|
55
|
+
}
|
|
56
|
+
lengths.set(entry.id, tokens.length);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return { postings, df, lengths, N: lengths.size };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* BM25-lite 检索,分数映射到 0~1(便于设阈值)。
|
|
64
|
+
*
|
|
65
|
+
* 归一化方式经过一次返工:最初用"命中证据 / 查询全部 token 的 idf 之和",
|
|
66
|
+
* 结果**永远匹配不上** —— 中文 bigram 会产生大量噪声 token("序算""法还"),
|
|
67
|
+
* 它们把分母顶得很大,一条明显相关的日记只能拿到 0.10 分。
|
|
68
|
+
*
|
|
69
|
+
* 现在改用**饱和函数**:只看命中了多少有区分度的证据,与查询长度解耦。
|
|
70
|
+
*
|
|
71
|
+
* sim = 1 − exp(−Σgain / saturation)
|
|
72
|
+
*
|
|
73
|
+
* 好处:分数含义稳定 —— 命中 1 个实词约 0.4,命中 2 个约 0.67,完全无关是 0。
|
|
74
|
+
* 阈值因此可以说人话,也不会因为用户多打几个字就掉到阈值以下。
|
|
75
|
+
*
|
|
76
|
+
* @param {ReturnType<typeof buildIndex>} index
|
|
77
|
+
* @param {string} query
|
|
78
|
+
* @param {{limit?: number, saturation?: number}} [options]
|
|
79
|
+
* @returns {Map<string, number>} id → 相似度 0~1(未命中的条目不会出现在 map 里)
|
|
80
|
+
*/
|
|
81
|
+
export function searchScores(index, query, options = {}) {
|
|
82
|
+
const limit = options.limit ?? 50;
|
|
83
|
+
const saturation = Number.isFinite(options.saturation) && options.saturation > 0
|
|
84
|
+
? options.saturation
|
|
85
|
+
: 2;
|
|
86
|
+
const scores = new Map();
|
|
87
|
+
if (index.N === 0) return scores;
|
|
88
|
+
|
|
89
|
+
// 查询侧去重:bigram 噪声里重复 token 不该重复计分
|
|
90
|
+
const qTokens = [...new Set(tokenize(query))];
|
|
91
|
+
if (qTokens.length === 0) return scores;
|
|
92
|
+
|
|
93
|
+
const idf = (t) => Math.log(1 + (index.N + IDF_SMOOTH) / ((index.df.get(t) ?? 0) + IDF_SMOOTH));
|
|
94
|
+
|
|
95
|
+
const k1 = 1.2;
|
|
96
|
+
const b = 0.75;
|
|
97
|
+
const avgLen =
|
|
98
|
+
[...index.lengths.values()].reduce((s, n) => s + n, 0) / Math.max(1, index.lengths.size);
|
|
99
|
+
|
|
100
|
+
const raw = new Map();
|
|
101
|
+
for (const t of qTokens) {
|
|
102
|
+
const bucket = index.postings.get(t);
|
|
103
|
+
if (bucket === undefined) continue;
|
|
104
|
+
const w = idf(t);
|
|
105
|
+
for (const [id, tf] of bucket) {
|
|
106
|
+
const len = index.lengths.get(id) ?? avgLen;
|
|
107
|
+
const norm = 1 - b + (b * len) / Math.max(1, avgLen);
|
|
108
|
+
const gain = (w * (tf * (k1 + 1))) / (tf + k1 * norm);
|
|
109
|
+
raw.set(id, (raw.get(id) ?? 0) + gain);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
for (const [id, r] of raw) scores.set(id, Math.min(1, 1 - Math.exp(-r / saturation)));
|
|
114
|
+
|
|
115
|
+
if (scores.size > limit) {
|
|
116
|
+
const kept = [...scores.entries()].sort((a, b2) => b2[1] - a[1]).slice(0, limit);
|
|
117
|
+
return new Map(kept);
|
|
118
|
+
}
|
|
119
|
+
return scores;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** 新鲜度:指数衰减 */
|
|
123
|
+
function recencyFactor(createdAt, now, halfLifeDays) {
|
|
124
|
+
if (!Number.isFinite(createdAt)) return 0.5;
|
|
125
|
+
const ageDays = Math.max(0, (now - createdAt) / 86400000);
|
|
126
|
+
return Math.pow(0.5, ageDays / Math.max(0.5, halfLifeDays));
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* @typedef {object} RecallOptions
|
|
131
|
+
* @property {number} [highSim] 高相似必触发(默认 0.5 ≈ 共享 2 个以上实词)
|
|
132
|
+
* @property {number} [midSim] 中相似进随机池(默认 0.25 ≈ 共享 1 个实词)
|
|
133
|
+
* @property {number} [minSalience] 进入随机池的最低 salience(默认 0.5)
|
|
134
|
+
* @property {number} [saturation] 相似度饱和尺度(默认 2)
|
|
135
|
+
* @property {number} [a] 相似度权重(默认 1)
|
|
136
|
+
* @property {number} [b] salience 权重(默认 0.45)
|
|
137
|
+
* @property {number} [c] 新鲜度权重(默认 0.2)
|
|
138
|
+
* @property {number} [d] 最近刚提过的惩罚(默认 0.8)
|
|
139
|
+
* @property {number} [temperature] softmax 温度(默认 0.15)
|
|
140
|
+
* @property {number} [cooldownTurns] 同一条的冷却轮数(默认 20)
|
|
141
|
+
* @property {number} [maxShow] 每轮最多露几条(默认 1)
|
|
142
|
+
* @property {number} [wanderChance] 没有强匹配时"走神"随机想起一条的概率(默认 0.08)
|
|
143
|
+
* @property {number} [halfLifeDays] 新鲜度半衰期(默认 45)
|
|
144
|
+
* @property {number} [now] 当前时间戳
|
|
145
|
+
* @property {() => number} [rng] 注入随机源(测试用)
|
|
146
|
+
*/
|
|
147
|
+
|
|
148
|
+
export const DEFAULT_RECALL_OPTIONS = Object.freeze({
|
|
149
|
+
highSim: 0.5,
|
|
150
|
+
midSim: 0.25,
|
|
151
|
+
minSalience: 0.5,
|
|
152
|
+
saturation: 2,
|
|
153
|
+
a: 1,
|
|
154
|
+
b: 0.45,
|
|
155
|
+
c: 0.2,
|
|
156
|
+
d: 0.8,
|
|
157
|
+
temperature: 0.15,
|
|
158
|
+
cooldownTurns: 20,
|
|
159
|
+
maxShow: 1,
|
|
160
|
+
wanderChance: 0.08,
|
|
161
|
+
halfLifeDays: 45,
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* @typedef {object} RecallState
|
|
166
|
+
* @property {Map<string, number>} [lastShownTurn] 条目 → 上次露出的轮次
|
|
167
|
+
*/
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* 挑出这一轮要提起的记忆。
|
|
171
|
+
*
|
|
172
|
+
* @param {Array<object>} entries 全部日记条目
|
|
173
|
+
* @param {ReturnType<typeof buildIndex>} index
|
|
174
|
+
* @param {string} query 本轮用户消息
|
|
175
|
+
* @param {RecallState} state
|
|
176
|
+
* @param {number} turn 当前轮次
|
|
177
|
+
* @param {RecallOptions} [options]
|
|
178
|
+
* @returns {{chosen: Array<{entry: object, score: number, sim: number, why: string}>, considered: number}}
|
|
179
|
+
*/
|
|
180
|
+
export function selectMemories(entries, index, query, state, turn, options = {}) {
|
|
181
|
+
const opts = { ...DEFAULT_RECALL_OPTIONS, ...options };
|
|
182
|
+
const now = Number.isFinite(opts.now) ? opts.now : Date.now();
|
|
183
|
+
const rng = typeof opts.rng === 'function' ? opts.rng : Math.random;
|
|
184
|
+
const lastShownTurn = state?.lastShownTurn ?? new Map();
|
|
185
|
+
|
|
186
|
+
const sims = searchScores(index, query, { saturation: opts.saturation });
|
|
187
|
+
/** @type {Array<{entry: object, score: number, sim: number, why: string}>} */
|
|
188
|
+
const scored = [];
|
|
189
|
+
|
|
190
|
+
for (const entry of entries) {
|
|
191
|
+
const sim = sims.get(entry.id) ?? 0;
|
|
192
|
+
const salience = Number.isFinite(entry.salience) ? entry.salience : 0.5;
|
|
193
|
+
|
|
194
|
+
// 冷却:刚提过就别再提
|
|
195
|
+
const last = lastShownTurn.get(entry.id);
|
|
196
|
+
if (Number.isFinite(last) && turn - last < opts.cooldownTurns) continue;
|
|
197
|
+
|
|
198
|
+
const fresh = recencyFactor(entry.created_at, now, opts.halfLifeDays);
|
|
199
|
+
const score =
|
|
200
|
+
opts.a * sim + opts.b * salience + opts.c * fresh - (Number.isFinite(last) ? opts.d : 0);
|
|
201
|
+
|
|
202
|
+
if (sim < opts.midSim && salience < opts.minSalience) continue;
|
|
203
|
+
scored.push({ entry, score, sim, why: '' });
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// 未解决的失败记忆:稍微加权,因为它们最该被主动提起
|
|
207
|
+
const addWhy = (c, reason) => {
|
|
208
|
+
c.why = c.why ? `${c.why} + ${reason}` : reason;
|
|
209
|
+
};
|
|
210
|
+
for (const c of scored) {
|
|
211
|
+
if (c.entry.kind === 'failure' && c.entry.resolved !== true) {
|
|
212
|
+
c.score += 0.15;
|
|
213
|
+
addWhy(c, '未解决');
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
scored.sort((x, y) => y.score - x.score);
|
|
218
|
+
|
|
219
|
+
const strong = scored.filter((c) => c.sim >= opts.highSim);
|
|
220
|
+
const chosen = [];
|
|
221
|
+
|
|
222
|
+
// 1) 强匹配:必触发
|
|
223
|
+
for (const c of strong.slice(0, opts.maxShow)) {
|
|
224
|
+
addWhy(c, '强匹配');
|
|
225
|
+
chosen.push(c);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// 2) 还不够,且允许走神:从中等候选里按 softmax 随机抽
|
|
229
|
+
if (chosen.length < opts.maxShow) {
|
|
230
|
+
const pool = scored.filter(
|
|
231
|
+
(c) => !chosen.includes(c) && c.sim >= opts.midSim && (c.entry.salience ?? 0) >= opts.minSalience,
|
|
232
|
+
);
|
|
233
|
+
if (pool.length > 0) {
|
|
234
|
+
const picked = softmaxPick(pool, opts.temperature, rng);
|
|
235
|
+
if (picked !== undefined) {
|
|
236
|
+
addWhy(picked, '想起来一点');
|
|
237
|
+
chosen.push(picked);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// 3) 走神:完全没匹配上时,小概率随机想起一条旧事。这条纯粹为了像人。
|
|
243
|
+
if (chosen.length === 0 && rng() < opts.wanderChance) {
|
|
244
|
+
const pool = scored.filter((c) => (c.entry.salience ?? 0) >= opts.minSalience);
|
|
245
|
+
if (pool.length > 0) {
|
|
246
|
+
const picked = pool[Math.floor(rng() * pool.length)];
|
|
247
|
+
addWhy(picked, '走神想起来的');
|
|
248
|
+
chosen.push(picked);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// 记下露出时间,供冷却使用
|
|
253
|
+
for (const c of chosen) lastShownTurn.set(c.entry.id, turn);
|
|
254
|
+
|
|
255
|
+
return { chosen, considered: scored.length };
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* 按 softmax 概率抽一个。温度越低越接近"取最高分"。
|
|
260
|
+
* @template T
|
|
261
|
+
* @param {T[]} items 必须已按 score 降序,且每项有 .score
|
|
262
|
+
* @param {number} temperature
|
|
263
|
+
* @param {() => number} rng
|
|
264
|
+
* @returns {T | undefined}
|
|
265
|
+
*/
|
|
266
|
+
export function softmaxPick(items, temperature, rng) {
|
|
267
|
+
if (items.length === 0) return undefined;
|
|
268
|
+
if (items.length === 1) return items[0];
|
|
269
|
+
const t = Math.max(1e-6, temperature);
|
|
270
|
+
const max = items[0].score;
|
|
271
|
+
const exps = items.map((it) => Math.exp((it.score - max) / t));
|
|
272
|
+
const sum = exps.reduce((s, e) => s + e, 0);
|
|
273
|
+
if (!(sum > 0)) return items[0];
|
|
274
|
+
let r = rng() * sum;
|
|
275
|
+
for (let i = 0; i < items.length; i++) {
|
|
276
|
+
r -= exps[i];
|
|
277
|
+
if (r <= 0) return items[i];
|
|
278
|
+
}
|
|
279
|
+
return items[items.length - 1];
|
|
280
|
+
}
|