dsh-plain-memory 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.en.md +76 -72
- package/README.md +155 -147
- package/cards.js +69 -69
- package/cordis.patch.yml +5 -5
- package/git.js +119 -119
- package/index.js +149 -20
- package/locale/en.json +6 -6
- package/locale/zh.json +6 -6
- package/memory.js +417 -396
- package/package.json +49 -48
- package/prompts/daily-consolidate.md +29 -28
- package/template/README.md +40 -39
- package/template/self.md +15 -15
package/memory.js
CHANGED
|
@@ -1,396 +1,417 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Markdown 记忆仓的内核:解析、BM25 召回、追加写入。零依赖。
|
|
3
|
-
*
|
|
4
|
-
* 目录约定(全部是纯 Markdown):
|
|
5
|
-
* self.md 自我层(身份、偏好、长期约束;需用户确认)
|
|
6
|
-
* episodic/YYYY-MM.md 情景层(做过什么、结论是什么)
|
|
7
|
-
* semantic/<主题>.md 语义层(跨会话可复用的事实与方法)
|
|
8
|
-
* .state/ 运行时状态(状态卡、待确认提案),已被 .gitignore
|
|
9
|
-
*
|
|
10
|
-
* 每条记忆是一个 `### 标题` 小节,元数据行固定为:
|
|
11
|
-
* - id / date / session / cues / related
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from 'node:fs'
|
|
15
|
-
import { basename, dirname, join, relative } from 'node:path'
|
|
16
|
-
|
|
17
|
-
const K1 = 1.2
|
|
18
|
-
const B = 0.75
|
|
19
|
-
const MAX_BODY = 600
|
|
20
|
-
/** 记忆召回至少要两个不同的匹配词才算想起来;状态卡因为条数少、且 24 小时就过期,只要求 1 个。 */
|
|
21
|
-
export const MIN_MATCHED_TERMS = 2
|
|
22
|
-
|
|
23
|
-
/** 本地日期,YYYY-MM-DD。 */
|
|
24
|
-
export function today(now = new Date()) {
|
|
25
|
-
const p = (n) => String(n).padStart(2, '0')
|
|
26
|
-
return `${now.getFullYear()}-${p(now.getMonth() + 1)}-${p(now.getDate())}`
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
/** 文件名安全的短标识;保留中文,去掉路径与空白。 */
|
|
30
|
-
export function slug(text) {
|
|
31
|
-
const s = String(text || '')
|
|
32
|
-
.toLowerCase()
|
|
33
|
-
.replace(/[^\p{L}\p{N}]+/gu, '-')
|
|
34
|
-
.replace(/^-+|-+$/g, '')
|
|
35
|
-
.slice(0, 40)
|
|
36
|
-
return s || 'memory'
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/** 稳定条目 id:层-日期-内容短哈希。 */
|
|
40
|
-
export function entryId(layer, title, date) {
|
|
41
|
-
let h = 0
|
|
42
|
-
const s = `${layer}:${title}:${date}`
|
|
43
|
-
for (let i = 0; i < s.length; i += 1) h = (Math.imul(h, 31) + s.charCodeAt(i)) >>> 0
|
|
44
|
-
return `${layer}-${date.replace(/-/g, '')}-${h.toString(36)}`
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
function metadata(entry) {
|
|
48
|
-
const lines = [
|
|
49
|
-
`- id: ${entry.id}`,
|
|
50
|
-
`- date: ${entry.date}`,
|
|
51
|
-
`- session: ${entry.session}`,
|
|
52
|
-
]
|
|
53
|
-
if (entry.cues && entry.cues.length) lines.push(`- cues: ${entry.cues.join(', ')}`)
|
|
54
|
-
if (entry.related && entry.related.length) lines.push(`- related: ${entry.related.join(', ')}`)
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
*
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
let
|
|
88
|
-
let
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
line = line.slice(0, start)
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
if (!bodyStarted &&
|
|
111
|
-
bodyStarted = true
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
.
|
|
134
|
-
.
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
return '
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
if (d.
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
'
|
|
172
|
-
'
|
|
173
|
-
'
|
|
174
|
-
'
|
|
175
|
-
'
|
|
176
|
-
'
|
|
177
|
-
'
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
*
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
['
|
|
193
|
-
['
|
|
194
|
-
['
|
|
195
|
-
['
|
|
196
|
-
['
|
|
197
|
-
['
|
|
198
|
-
['
|
|
199
|
-
['
|
|
200
|
-
['
|
|
201
|
-
['
|
|
202
|
-
['
|
|
203
|
-
['
|
|
204
|
-
['
|
|
205
|
-
['
|
|
206
|
-
['
|
|
207
|
-
['
|
|
208
|
-
['
|
|
209
|
-
['
|
|
210
|
-
['
|
|
211
|
-
['
|
|
212
|
-
['
|
|
213
|
-
]
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
const
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
const
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
*
|
|
253
|
-
*
|
|
254
|
-
*
|
|
255
|
-
*
|
|
256
|
-
*
|
|
257
|
-
*
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
const
|
|
273
|
-
const
|
|
274
|
-
|
|
275
|
-
const
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
const
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
const
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
return
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
const
|
|
368
|
-
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
return
|
|
396
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Markdown 记忆仓的内核:解析、BM25 召回、追加写入。零依赖。
|
|
3
|
+
*
|
|
4
|
+
* 目录约定(全部是纯 Markdown):
|
|
5
|
+
* self.md 自我层(身份、偏好、长期约束;需用户确认)
|
|
6
|
+
* episodic/YYYY-MM.md 情景层(做过什么、结论是什么)
|
|
7
|
+
* semantic/<主题>.md 语义层(跨会话可复用的事实与方法)
|
|
8
|
+
* .state/ 运行时状态(状态卡、待确认提案),已被 .gitignore
|
|
9
|
+
*
|
|
10
|
+
* 每条记忆是一个 `### 标题` 小节,元数据行固定为:
|
|
11
|
+
* - id / date / session / cues / related
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from 'node:fs'
|
|
15
|
+
import { basename, dirname, join, relative } from 'node:path'
|
|
16
|
+
|
|
17
|
+
const K1 = 1.2
|
|
18
|
+
const B = 0.75
|
|
19
|
+
const MAX_BODY = 600
|
|
20
|
+
/** 记忆召回至少要两个不同的匹配词才算想起来;状态卡因为条数少、且 24 小时就过期,只要求 1 个。 */
|
|
21
|
+
export const MIN_MATCHED_TERMS = 2
|
|
22
|
+
|
|
23
|
+
/** 本地日期,YYYY-MM-DD。 */
|
|
24
|
+
export function today(now = new Date()) {
|
|
25
|
+
const p = (n) => String(n).padStart(2, '0')
|
|
26
|
+
return `${now.getFullYear()}-${p(now.getMonth() + 1)}-${p(now.getDate())}`
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** 文件名安全的短标识;保留中文,去掉路径与空白。 */
|
|
30
|
+
export function slug(text) {
|
|
31
|
+
const s = String(text || '')
|
|
32
|
+
.toLowerCase()
|
|
33
|
+
.replace(/[^\p{L}\p{N}]+/gu, '-')
|
|
34
|
+
.replace(/^-+|-+$/g, '')
|
|
35
|
+
.slice(0, 40)
|
|
36
|
+
return s || 'memory'
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** 稳定条目 id:层-日期-内容短哈希。 */
|
|
40
|
+
export function entryId(layer, title, date) {
|
|
41
|
+
let h = 0
|
|
42
|
+
const s = `${layer}:${title}:${date}`
|
|
43
|
+
for (let i = 0; i < s.length; i += 1) h = (Math.imul(h, 31) + s.charCodeAt(i)) >>> 0
|
|
44
|
+
return `${layer}-${date.replace(/-/g, '')}-${h.toString(36)}`
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function metadata(entry) {
|
|
48
|
+
const lines = [
|
|
49
|
+
`- id: ${entry.id}`,
|
|
50
|
+
`- date: ${entry.date}`,
|
|
51
|
+
`- session: ${entry.session}`,
|
|
52
|
+
]
|
|
53
|
+
if (entry.cues && entry.cues.length) lines.push(`- cues: ${entry.cues.join(', ')}`)
|
|
54
|
+
if (entry.related && entry.related.length) lines.push(`- related: ${entry.related.join(', ')}`)
|
|
55
|
+
if (entry.expires) lines.push(`- expires: ${entry.expires}`)
|
|
56
|
+
return lines.join('\n')
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** 一条记忆的 Markdown 文本。 */
|
|
60
|
+
export function formatEntry(entry) {
|
|
61
|
+
const body = String(entry.body || '').trim().slice(0, MAX_BODY)
|
|
62
|
+
return `${metadata(entry)}\n\n${body}\n`
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* 一条记忆一个文件:并发写不同记忆时不会互相覆盖。
|
|
67
|
+
* 同名同日重复写入落到同一个文件(幂等覆盖),不会堆出重复条目。
|
|
68
|
+
*/
|
|
69
|
+
function targetFile(entry) {
|
|
70
|
+
if (entry.layer === 'self') return 'self.md'
|
|
71
|
+
const dir = entry.layer === 'episodic' ? 'episodic' : 'semantic'
|
|
72
|
+
const suffix = String(entry.id).slice(-4)
|
|
73
|
+
return `${dir}/${entry.date}-${slug(entry.title)}-${suffix}.md`
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function headerFor(layer) {
|
|
77
|
+
return layer === 'self'
|
|
78
|
+
? '# 自我\n\n长期有效的身份、偏好与约束。只有用户明确确认过的内容才能写在这里。\n'
|
|
79
|
+
: ''
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const META = /^- (id|date|session|cues|related|expires):\s*(.*)$/
|
|
83
|
+
|
|
84
|
+
/** 解析一个 Markdown 文件里的全部条目。HTML 注释整段跳过,避免把示例当成记忆。 */
|
|
85
|
+
export function parseMarkdown(text, file, layer) {
|
|
86
|
+
const entries = []
|
|
87
|
+
let cur = null
|
|
88
|
+
let bodyStarted = false
|
|
89
|
+
let inComment = false
|
|
90
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
91
|
+
let line = raw
|
|
92
|
+
if (inComment) {
|
|
93
|
+
if (line.includes('-->')) { inComment = false; line = line.slice(line.indexOf('-->') + 3) }
|
|
94
|
+
else continue
|
|
95
|
+
}
|
|
96
|
+
while (line.includes('<!--')) {
|
|
97
|
+
const start = line.indexOf('<!--')
|
|
98
|
+
const end = line.indexOf('-->', start)
|
|
99
|
+
if (end === -1) { line = line.slice(0, start); inComment = true; break }
|
|
100
|
+
line = line.slice(0, start) + line.slice(end + 3)
|
|
101
|
+
}
|
|
102
|
+
if (line.startsWith('### ')) {
|
|
103
|
+
if (cur) entries.push(cur)
|
|
104
|
+
cur = { title: line.slice(4).trim(), meta: {}, bodyLines: [], file, layer }
|
|
105
|
+
bodyStarted = false
|
|
106
|
+
continue
|
|
107
|
+
}
|
|
108
|
+
if (!cur) continue
|
|
109
|
+
const m = META.exec(line)
|
|
110
|
+
if (!bodyStarted && m) { cur.meta[m[1]] = m[2].trim(); continue }
|
|
111
|
+
if (!bodyStarted && line.trim() === '') { bodyStarted = true; continue }
|
|
112
|
+
bodyStarted = true
|
|
113
|
+
cur.bodyLines.push(line)
|
|
114
|
+
}
|
|
115
|
+
if (cur) entries.push(cur)
|
|
116
|
+
return entries.map((e) => ({
|
|
117
|
+
id: e.meta.id || entryId(e.layer, e.title, e.meta.date || today()),
|
|
118
|
+
layer: e.layer,
|
|
119
|
+
title: e.title,
|
|
120
|
+
date: e.meta.date || '',
|
|
121
|
+
session: e.meta.session || '',
|
|
122
|
+
cues: splitCues(e.meta.cues || ''),
|
|
123
|
+
related: splitCues(e.meta.related || ''),
|
|
124
|
+
expires: e.meta.expires || '',
|
|
125
|
+
body: e.bodyLines.join('\n').trim(),
|
|
126
|
+
file: e.file,
|
|
127
|
+
}))
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** 逗号、顿号、空白都算线索词分隔符。 */
|
|
131
|
+
export function splitCues(text) {
|
|
132
|
+
return String(text || '')
|
|
133
|
+
.split(/[,,、;;\s]+/)
|
|
134
|
+
.map((s) => s.trim())
|
|
135
|
+
.filter((s) => s.length > 0)
|
|
136
|
+
.slice(0, 12)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function layerOf(rel) {
|
|
140
|
+
if (basename(rel) === 'self.md') return 'self'
|
|
141
|
+
if (rel.split(/[\\/]/)[0] === 'episodic') return 'episodic'
|
|
142
|
+
return 'semantic'
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** 读取整个记忆仓的条目(跳过 README 与 .state)。 */
|
|
146
|
+
export function readRepo(repo) {
|
|
147
|
+
const out = []
|
|
148
|
+
const walk = (dir) => {
|
|
149
|
+
let names
|
|
150
|
+
try { names = readdirSync(dir, { withFileTypes: true }) } catch { return }
|
|
151
|
+
for (const d of names) {
|
|
152
|
+
if (d.name.startsWith('.') || d.name === 'node_modules') continue
|
|
153
|
+
const full = join(dir, d.name)
|
|
154
|
+
if (d.isDirectory()) { walk(full); continue }
|
|
155
|
+
if (!d.name.endsWith('.md') || d.name === 'README.md') continue
|
|
156
|
+
const rel = relative(repo, full).replaceAll('\\', '/')
|
|
157
|
+
let text = ''
|
|
158
|
+
try { text = readFileSync(full, 'utf8') } catch { continue }
|
|
159
|
+
out.push(...parseMarkdown(text, rel, layerOf(rel)))
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
walk(repo)
|
|
163
|
+
return out
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* 功能词与套话:只出现在提问里、不指向任何记忆的词。不滤掉它们,
|
|
168
|
+
* 「帮我写一个快速排序」会因为记忆里有一句「一个文件」而误召回。
|
|
169
|
+
*/
|
|
170
|
+
const STOPWORDS = new Set([
|
|
171
|
+
'一个', '这个', '那个', '这些', '那些', '我们', '你们', '他们', '她们', '它们', '自己',
|
|
172
|
+
'什么', '怎么', '为什', '什幺', '可以', '可能', '应该', '需要', '因为', '所以', '但是',
|
|
173
|
+
'如果', '虽然', '然后', '现在', '已经', '还是', '或者', '而且', '不过', '就是', '不是',
|
|
174
|
+
'没有', '有没', '时候', '一些', '这样', '那样', '这种', '那种', '东西', '事情', '地方',
|
|
175
|
+
'帮我', '我的', '你的', '他的', '她的', '一下', '一点', '非常', '特别', '其实', '只是',
|
|
176
|
+
'还有', '以及', '对于', '关于', '通过', '进行', '正在', '一直', '有时', '怎样', '如何',
|
|
177
|
+
'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'on', 'for', 'with', 'is', 'are',
|
|
178
|
+
'was', 'were', 'be', 'it', 'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she',
|
|
179
|
+
'we', 'they', 'my', 'your', 'his', 'her', 'our', 'their',
|
|
180
|
+
])
|
|
181
|
+
|
|
182
|
+
function informative(token) {
|
|
183
|
+
return token.startsWith('+') || !STOPWORDS.has(token)
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* 概念桥:把同一个意思的不同说法折到组首那个词上。这是纯词表,不是语义模型,
|
|
188
|
+
* 所以只在「用户换了个说法」时起作用,不能理解没收录的说法——这一点写进 README。
|
|
189
|
+
* 命中后追加带 `+` 前缀的概念 token,打分按 0.6 计权。
|
|
190
|
+
*/
|
|
191
|
+
const CONCEPTS = [
|
|
192
|
+
['卡住', ['卡住', '转圈', '无响应', '冻结', 'hang']],
|
|
193
|
+
['超时', ['超时', 'timeout', '等了很久']],
|
|
194
|
+
['迁移', ['换电脑', '迁移', '搬家', 'migrate']],
|
|
195
|
+
['并发', ['同时', '并发', '并行', '一起', 'concurrent']],
|
|
196
|
+
['失败', ['坏了', '失败', '故障', '挂了', '失败', 'error']],
|
|
197
|
+
['告警', ['发现', '察觉', '监控', '告警', '报警', 'alert']],
|
|
198
|
+
['评审', ['评审', '评委', '审阅', 'review', 'reviewer']],
|
|
199
|
+
['复现', ['复现', '重现', '复刻', 'reproduce']],
|
|
200
|
+
['信任', ['信任', '可信', 'trust', 'credit']],
|
|
201
|
+
['成本', ['成本', '代价', '开销', 'cost']],
|
|
202
|
+
['验证', ['验证', '检验', '核对', 'verify']],
|
|
203
|
+
['缓存', ['缓存', 'cache']],
|
|
204
|
+
['文档', ['文档', '说明', '注释', 'docs', 'documentation']],
|
|
205
|
+
['性能', ['性能', '速度', '延迟', 'latency', 'performance']],
|
|
206
|
+
['依赖', ['依赖', 'dependency']],
|
|
207
|
+
['配置', ['配置', 'config']],
|
|
208
|
+
['记忆', ['记忆', 'memory']],
|
|
209
|
+
['会话', ['窗口', '会话', 'session']],
|
|
210
|
+
['测试', ['测试', 'test']],
|
|
211
|
+
['发布', ['部署', '发布', '上线', 'deploy', 'release']],
|
|
212
|
+
['权限', ['权限', '授权', 'permission']],
|
|
213
|
+
['脚本', ['脚本', '批量', '自动', 'script']],
|
|
214
|
+
['锁', ['锁', 'lock', '占用']],
|
|
215
|
+
]
|
|
216
|
+
|
|
217
|
+
/** 分词:ASCII 词 + 中文二元组 + 概念桥展开(带 `+` 前缀)。 */
|
|
218
|
+
export function tokenize(text) {
|
|
219
|
+
const s = String(text || '').toLowerCase()
|
|
220
|
+
const tokens = s.match(/[a-z0-9][a-z0-9_+.#/-]*/g) || []
|
|
221
|
+
const cjk = s.match(/[\u3400-\u4dbf\u4e00-\u9fff]+/g) || []
|
|
222
|
+
for (const run of cjk) {
|
|
223
|
+
if (run.length === 1) { tokens.push(run); continue }
|
|
224
|
+
for (let i = 0; i < run.length - 1; i += 1) tokens.push(run.slice(i, i + 2))
|
|
225
|
+
}
|
|
226
|
+
for (const [head, terms] of CONCEPTS) {
|
|
227
|
+
if (terms.some((term) => s.includes(term))) tokens.push(`+${head}`)
|
|
228
|
+
}
|
|
229
|
+
return tokens
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** 建立 BM25 索引。 */
|
|
233
|
+
export function buildIndex(entries) {
|
|
234
|
+
const docs = entries.map((e) => {
|
|
235
|
+
const tf = new Map()
|
|
236
|
+
const tokens = tokenize(`${e.title} ${e.cues.join(' ')} ${e.body}`).filter(informative)
|
|
237
|
+
for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1)
|
|
238
|
+
return { entry: e, tf, len: tokens.length }
|
|
239
|
+
})
|
|
240
|
+
const df = new Map()
|
|
241
|
+
for (const d of docs) for (const t of d.tf.keys()) df.set(t, (df.get(t) || 0) + 1)
|
|
242
|
+
const avgdl = docs.length === 0 ? 0 : docs.reduce((n, d) => n + d.len, 0) / docs.length
|
|
243
|
+
return { docs, df, avgdl: avgdl || 1, size: docs.length, byId: new Map(entries.map((e) => [e.id, e])) }
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function idf(index, term) {
|
|
247
|
+
const df = index.df.get(term) || 0
|
|
248
|
+
return Math.log(1 + (index.size - df + 0.5) / (df + 0.5))
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* 召回分数是"这条查询里落在文档上的信息量占比"(0–1),因此不随语料规模漂移:
|
|
253
|
+
* 同一个阈值在 1 条记忆和 500 条记忆的仓库里含义一致。
|
|
254
|
+
* 在 4 条以上的语料里,出现在 60% 以上文档中的词按停用词处理。
|
|
255
|
+
*
|
|
256
|
+
* 「两个独立线索」:一条记忆至少要有 2 个不同的匹配词才算想起来。
|
|
257
|
+
* 记忆少的时候 IDF 分不开常用词和关键词,单个泛词(记忆/工具/下周)也能过阈值;
|
|
258
|
+
* 人回忆时也是两三个线索同时对上才确定。只命中 1 个词 → 不注入,模型仍可主动 memory_recall。
|
|
259
|
+
*
|
|
260
|
+
* 概念桥 token(`+xxx`)只做同义词折算、参与打分,**不算独立线索**:
|
|
261
|
+
* 否则「窗口/会话/测试」这类词出现在任何一条记忆里,就把整条拉进阈值。
|
|
262
|
+
*
|
|
263
|
+
* 带 `expires` 且已过期的记忆整条跳过:过期的日期型记忆再准也没用,还会反复冒出来。
|
|
264
|
+
* @returns {{entry: object, score: number, hop: boolean, matched: number, terms: string[]}[]}
|
|
265
|
+
*/
|
|
266
|
+
export function expired(entry, now = today()) {
|
|
267
|
+
return Boolean(entry?.expires) && String(entry.expires) < now
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
export function recall(index, text, limit, minScore = 0.3, minMatched = MIN_MATCHED_TERMS, withHop = true, countConcept = false) {
|
|
271
|
+
if (!index || index.size === 0) return []
|
|
272
|
+
const now = today()
|
|
273
|
+
const terms = [...new Set(tokenize(text))].filter((t) => {
|
|
274
|
+
if (!informative(t)) return false
|
|
275
|
+
const df = index.df.get(t)
|
|
276
|
+
if (!df) return false
|
|
277
|
+
return !(index.size >= 4 && df >= index.size * 0.6)
|
|
278
|
+
})
|
|
279
|
+
if (terms.length === 0) return []
|
|
280
|
+
const weightOf = (t) => (t.startsWith('+') ? 0.6 : 1)
|
|
281
|
+
const total = terms.reduce((n, t) => n + idf(index, t) * weightOf(t), 0)
|
|
282
|
+
if (total <= 0) return []
|
|
283
|
+
// 每个匹配词在查询里的位置;用来把重叠的二元组折成一个线索(「报价/价说/说明」算 1 个)
|
|
284
|
+
const lower = String(text || '').toLowerCase()
|
|
285
|
+
const at = new Map(terms.map((t) => [t, t.startsWith('+') ? -1 : lower.indexOf(t)]))
|
|
286
|
+
const independent = (matchedTerms) => {
|
|
287
|
+
const sorted = [...matchedTerms]
|
|
288
|
+
.filter((t) => countConcept || !t.startsWith('+'))
|
|
289
|
+
.sort((a, b) => (at.get(a) ?? -1) - (at.get(b) ?? -1))
|
|
290
|
+
let n = 0
|
|
291
|
+
let end = -1
|
|
292
|
+
for (const t of sorted) {
|
|
293
|
+
const p = at.get(t) ?? -1
|
|
294
|
+
if (p < 0) { n += 1; continue }
|
|
295
|
+
if (p <= end) continue
|
|
296
|
+
n += 1
|
|
297
|
+
end = p + t.length - 1
|
|
298
|
+
}
|
|
299
|
+
return n
|
|
300
|
+
}
|
|
301
|
+
const scored = index.docs
|
|
302
|
+
.filter((doc) => !expired(doc.entry, now))
|
|
303
|
+
.map((doc) => {
|
|
304
|
+
let acc = 0
|
|
305
|
+
const hits = []
|
|
306
|
+
for (const t of terms) {
|
|
307
|
+
const tf = doc.tf.get(t)
|
|
308
|
+
if (!tf) continue
|
|
309
|
+
hits.push(t)
|
|
310
|
+
acc += idf(index, t) * weightOf(t) * (tf / (tf + K1 * (1 - B + (B * doc.len) / index.avgdl)))
|
|
311
|
+
}
|
|
312
|
+
return { entry: doc.entry, score: acc / total, matched: independent(hits), terms: hits }
|
|
313
|
+
})
|
|
314
|
+
.filter((r) => r.score >= minScore && r.matched >= minMatched)
|
|
315
|
+
.sort((a, b) => b.score - a.score)
|
|
316
|
+
.slice(0, limit)
|
|
317
|
+
const out = scored.map((r) => ({ ...r, hop: false }))
|
|
318
|
+
if (!withHop) return out
|
|
319
|
+
const seen = new Set(out.map((r) => r.entry.id))
|
|
320
|
+
for (const hit of scored) {
|
|
321
|
+
for (const rel of hit.entry.related) {
|
|
322
|
+
const linked = index.byId.get(rel)
|
|
323
|
+
if (!linked || seen.has(linked.id)) continue
|
|
324
|
+
seen.add(linked.id)
|
|
325
|
+
out.push({ entry: linked, score: hit.score * 0.5, hop: true, matched: 0, terms: [] })
|
|
326
|
+
break
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return out.slice(0, limit + 1)
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* 长消息按句子切开分别召回,再按条目合并取最高分。
|
|
334
|
+
* 十个问题挤在一条消息里时,整段当查询会被稀释到阈值以下;切开就恢复成十个短查询。
|
|
335
|
+
* 整段也保留一次,避免跨句的主题被漏掉。
|
|
336
|
+
*
|
|
337
|
+
* **只有主动查询(memory_recall)才切开。** 自动注入不切:
|
|
338
|
+
* 切句会让「三个字的一段刚好碰上一个词」拿到接近 1.0 的分,是生产日志里噪声的主要来源。
|
|
339
|
+
* @param {{split?: boolean, hop?: boolean}} [options]
|
|
340
|
+
* @returns {{entry: object, score: number, hop: boolean, matched: number, terms: string[]}[]}
|
|
341
|
+
*/
|
|
342
|
+
export function recallAll(index, text, limit, minScore = 0.3, minMatched = MIN_MATCHED_TERMS, options = {}) {
|
|
343
|
+
if (!index || index.size === 0) return []
|
|
344
|
+
const { split = true, hop = true, countConcept = false } = options
|
|
345
|
+
const raw = String(text || '')
|
|
346
|
+
const segments = split ? raw.split(/[。!?!?;;\n]+/).map((s) => s.trim()).filter((s) => s.length >= 2) : []
|
|
347
|
+
const queries = split ? [...segments.slice(0, 12), raw] : [raw]
|
|
348
|
+
const best = new Map()
|
|
349
|
+
for (const query of queries) {
|
|
350
|
+
for (const hit of recall(index, query, limit, minScore, minMatched, hop, countConcept)) {
|
|
351
|
+
const prev = best.get(hit.entry.id)
|
|
352
|
+
if (!prev || hit.score > prev.score) best.set(hit.entry.id, hit)
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
return [...best.values()].sort((a, b) => b.score - a.score).slice(0, limit)
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
function writeAtomic(file, text) {
|
|
359
|
+
mkdirSync(dirname(file), { recursive: true })
|
|
360
|
+
const tmp = `${file}.${process.pid}.tmp`
|
|
361
|
+
writeFileSync(tmp, text, 'utf8')
|
|
362
|
+
try { renameSync(tmp, file) } catch { writeFileSync(file, text, 'utf8') }
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** 写入一条记忆。非自我层一条一个文件,并发写不同记忆互不覆盖。 */
|
|
366
|
+
export function appendEntry(repo, entry) {
|
|
367
|
+
const rel = targetFile(entry)
|
|
368
|
+
const file = join(repo, rel)
|
|
369
|
+
mkdirSync(dirname(file), { recursive: true })
|
|
370
|
+
if (entry.layer !== 'self') {
|
|
371
|
+
writeAtomic(file, `### ${entry.title}\n${formatEntry(entry)}`)
|
|
372
|
+
return rel
|
|
373
|
+
}
|
|
374
|
+
const head = existsSync(file) ? readFileSync(file, 'utf8') : headerFor(entry.layer)
|
|
375
|
+
writeAtomic(file, `${head.replace(/\s*$/, '')}\n\n### ${entry.title}\n${formatEntry(entry)}`)
|
|
376
|
+
return rel
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* 条目的一句话展示文本。截断时优先停在句号/分号处,保住前面的要点,
|
|
381
|
+
* 并在末尾标明全文还能用 memory_recall 读到——不让最有用的一条被腰斩。
|
|
382
|
+
*/
|
|
383
|
+
export function summarize(entry, maxChars = 160) {
|
|
384
|
+
const body = entry.body.replace(/\s+/g, ' ').trim()
|
|
385
|
+
if (body.length <= maxChars) return body
|
|
386
|
+
const cut = body.slice(0, maxChars)
|
|
387
|
+
const stop = Math.max(cut.lastIndexOf('。'), cut.lastIndexOf(';'), cut.lastIndexOf(';'), cut.lastIndexOf('. '))
|
|
388
|
+
const kept = stop > maxChars * 0.45 ? cut.slice(0, stop + 1) : cut
|
|
389
|
+
return `${kept.trim()}…(全文可用 memory_recall 读取)`
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** 粗略 token 估算:中文按 1 token/字,ASCII 按 4 字符/token。 */
|
|
393
|
+
export function estimateTokens(text) {
|
|
394
|
+
const cjk = (text.match(/[\u3400-\u4dbf\u4e00-\u9fff]/g) || []).length
|
|
395
|
+
return Math.ceil(cjk + (text.length - cjk) / 4)
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/** 疑似凭据的形态。命中即拒写;宁可误拒,也不让密钥进记忆仓。 */
|
|
399
|
+
const SECRET_PATTERNS = [
|
|
400
|
+
['OpenAI/DeepSeek 风格的 sk- 密钥', /\bsk-[A-Za-z0-9_-]{20,}/],
|
|
401
|
+
['GitHub 令牌', /\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})/],
|
|
402
|
+
['JWT / 账号令牌', /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/],
|
|
403
|
+
['Bearer 令牌', /\bBearer\s+[A-Za-z0-9._~+\/-]{20,}/],
|
|
404
|
+
['密钥/令牌赋值', /(?:ANTHROPIC_AUTH_TOKEN|ANTHROPIC_API_KEY|DEEPSEEK_API_KEY|OPENAI_API_KEY|api[_-]?key|access[_-]?token|client[_-]?secret|password)["']?\s*[:=]\s*["']?[A-Za-z0-9._~+\/-]{16,}/i],
|
|
405
|
+
['AWS 访问密钥', /\bAKIA[0-9A-Z]{16}\b/],
|
|
406
|
+
['私钥块', /-----BEGIN (?:RSA |OPENSSH |EC )?PRIVATE KEY-----/],
|
|
407
|
+
]
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* 检查一段文本里有没有疑似凭据。
|
|
411
|
+
* @returns {string|null} 命中的类型名,没有则 null。
|
|
412
|
+
*/
|
|
413
|
+
export function findSecret(text) {
|
|
414
|
+
const s = String(text || '')
|
|
415
|
+
for (const [kind, re] of SECRET_PATTERNS) if (re.test(s)) return kind
|
|
416
|
+
return null
|
|
417
|
+
}
|