@ilaomianyang/dsh-memory-graph 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +259 -0
- package/LICENSE +21 -0
- package/README.en.md +466 -0
- package/README.md +459 -0
- package/client.js +1953 -0
- package/cordis.patch.yml +13 -0
- package/docs/auto-settings.png +0 -0
- package/docs/distill-button.png +0 -0
- package/docs/graph-dark.png +0 -0
- package/docs/graph-light.png +0 -0
- package/docs/session-tab-cards.png +0 -0
- package/docs/session-tab-open.png +0 -0
- package/docs/session-tab.png +0 -0
- package/docs/vault-panel.png +0 -0
- package/eval/REPORT.md +437 -0
- package/eval/questions.json +277 -0
- package/eval/run.mjs +460 -0
- package/eval/seed.mjs +489 -0
- package/icon.svg +26 -0
- package/index.js +243 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +77 -0
- package/src/auto.js +194 -0
- package/src/capture.js +237 -0
- package/src/config.js +280 -0
- package/src/distillapi.js +30 -0
- package/src/distillrun.js +106 -0
- package/src/extract.js +660 -0
- package/src/graph.js +1582 -0
- package/src/graphdata.js +475 -0
- package/src/http.js +32 -0
- package/src/pageapi.js +40 -0
- package/src/privacy.js +78 -0
- package/src/schema.js +323 -0
- package/src/search.js +317 -0
- package/src/tools.js +534 -0
- package/src/vault.js +888 -0
- package/test/auto.mjs +295 -0
- package/test/capture.mjs +264 -0
- package/test/client.mjs +1035 -0
- package/test/distillapi.mjs +212 -0
- package/test/extract.mjs +660 -0
- package/test/graphdata.mjs +295 -0
- package/test/live-supersede.mjs +78 -0
- package/test/live.mjs +156 -0
- package/test/page.mjs +116 -0
- package/test/schema.mjs +256 -0
- package/test/smoke.mjs +428 -0
- package/test/tools.mjs +238 -0
package/src/schema.js
ADDED
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The plugin's `Config` schema.
|
|
3
|
+
*
|
|
4
|
+
* The harness draws this plugin's page in 设置 (Settings) from the schema
|
|
5
|
+
* exported here: `@deepseek-ai/dsh-settings` reads the `Config` of an active
|
|
6
|
+
* profile entry, keeps only the fields marked `.volatile()`, and serves them to
|
|
7
|
+
* the page as one editable form. A field that is *not* volatile never appears in
|
|
8
|
+
* Settings — it is ordinary Cordis configuration, editable only by hand in
|
|
9
|
+
* `cordis.patch.yml`, and changing it remounts the plugin.
|
|
10
|
+
*
|
|
11
|
+
* So the split below is a contract, not a style choice:
|
|
12
|
+
*
|
|
13
|
+
* * `live(...)` leaves — knobs read at call time. A write reaches the running
|
|
14
|
+
* plugin through `loader/volatile-update` with no restart, which is what you
|
|
15
|
+
* want for thresholds, weights and patterns.
|
|
16
|
+
* * plain leaves — `enabled`, `vaultDir`, `defaultProject`, `crossProject`.
|
|
17
|
+
* They are consumed once while the plugin activates (register the tools,
|
|
18
|
+
* `mkdir` the vault, tag the session), so a live write could not take effect
|
|
19
|
+
* and would silently do nothing. Editing them is a re-activation, and the
|
|
20
|
+
* Loader performs one when an ordinary field changes.
|
|
21
|
+
*
|
|
22
|
+
* Every field here has a default, so an empty `config:` block in the profile
|
|
23
|
+
* patch is valid. `src/config.js` still normalises whatever arrives: the form is
|
|
24
|
+
* a convenience, not the contract.
|
|
25
|
+
*/
|
|
26
|
+
import z from '@deepseek-ai/schemastery'
|
|
27
|
+
import { DEFAULT_SKIP_PATTERNS } from './config.js'
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Mark a leaf editable in Settings without remounting the plugin.
|
|
31
|
+
*
|
|
32
|
+
* A volatile leaf validates to a frozen `{ get() }` reference rather than a
|
|
33
|
+
* value, so anything reading it must go through `resolveConfig` (which unwraps
|
|
34
|
+
* refs — see `src/config.js`). Volatile is only allowed on a fixed object path,
|
|
35
|
+
* which is why whole arrays (`skipPatterns`, `denyPatterns`) are marked instead
|
|
36
|
+
* of their items.
|
|
37
|
+
*/
|
|
38
|
+
const live = (schema) => schema.volatile()
|
|
39
|
+
|
|
40
|
+
const capture = z.object({
|
|
41
|
+
enabled: live(
|
|
42
|
+
z
|
|
43
|
+
.boolean()
|
|
44
|
+
.default(true)
|
|
45
|
+
.description('记录原始对话片段 · record raw episodes (off = nothing new is stored)'),
|
|
46
|
+
),
|
|
47
|
+
userChars: live(
|
|
48
|
+
z
|
|
49
|
+
.number()
|
|
50
|
+
.min(200)
|
|
51
|
+
.max(20000)
|
|
52
|
+
.step(100)
|
|
53
|
+
.default(1500)
|
|
54
|
+
.description('每条用户消息最多保留多少字符 · characters kept per user message'),
|
|
55
|
+
),
|
|
56
|
+
assistantChars: live(
|
|
57
|
+
z
|
|
58
|
+
.number()
|
|
59
|
+
.min(200)
|
|
60
|
+
.max(20000)
|
|
61
|
+
.step(100)
|
|
62
|
+
.default(1500)
|
|
63
|
+
.description('每条助手回复最多保留多少字符 · characters kept per assistant reply'),
|
|
64
|
+
),
|
|
65
|
+
includeToolOutput: live(
|
|
66
|
+
z
|
|
67
|
+
.boolean()
|
|
68
|
+
.default(false)
|
|
69
|
+
.description('是否连工具输出一起记 · also store tool output (usually noise)'),
|
|
70
|
+
),
|
|
71
|
+
minChars: live(
|
|
72
|
+
z
|
|
73
|
+
.number()
|
|
74
|
+
.min(0)
|
|
75
|
+
.max(500)
|
|
76
|
+
.step(1)
|
|
77
|
+
.default(12)
|
|
78
|
+
.description('短于这个长度的消息不记 · ignore messages shorter than this'),
|
|
79
|
+
),
|
|
80
|
+
skipPatterns: live(
|
|
81
|
+
z
|
|
82
|
+
.array(z.string())
|
|
83
|
+
.default([...DEFAULT_SKIP_PATTERNS])
|
|
84
|
+
.description('不记录的文本正则(整段替换默认值)· regexes whose matching messages are skipped; replaces the defaults'),
|
|
85
|
+
),
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
const privacy = z.object({
|
|
89
|
+
useDefaultPatterns: live(
|
|
90
|
+
z
|
|
91
|
+
.boolean()
|
|
92
|
+
.default(true)
|
|
93
|
+
.description('启用内置密钥/令牌规则 · enable the built-in secret patterns'),
|
|
94
|
+
),
|
|
95
|
+
denyPatterns: live(
|
|
96
|
+
z
|
|
97
|
+
.array(z.string())
|
|
98
|
+
.default([])
|
|
99
|
+
.description('额外强制打码的正则 · extra regexes redacted before anything is written'),
|
|
100
|
+
),
|
|
101
|
+
redactEmails: live(
|
|
102
|
+
z.boolean().default(true).description('邮箱打码 · redact email addresses'),
|
|
103
|
+
),
|
|
104
|
+
redactPaths: live(
|
|
105
|
+
z.boolean().default(true).description('本地路径打码 · redact local absolute paths'),
|
|
106
|
+
),
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
const extractor = z.object({
|
|
110
|
+
enabled: live(
|
|
111
|
+
z
|
|
112
|
+
.boolean()
|
|
113
|
+
.default(false)
|
|
114
|
+
.description('用模型把片段蒸馏成记忆卡 · distil episodes into cards with a model'),
|
|
115
|
+
),
|
|
116
|
+
transport: live(
|
|
117
|
+
z
|
|
118
|
+
.union([z.const('harness'), z.const('http')])
|
|
119
|
+
.default('harness')
|
|
120
|
+
.description(
|
|
121
|
+
'从哪里取模型:harness = 复用设置里已经配好的模型和密钥;http = 自己填地址和密钥 · where the model comes from: harness borrows a route configured in Settings, http posts to your own endpoint',
|
|
122
|
+
),
|
|
123
|
+
),
|
|
124
|
+
useCurrentModel: live(
|
|
125
|
+
z
|
|
126
|
+
.boolean()
|
|
127
|
+
.default(true)
|
|
128
|
+
.description(
|
|
129
|
+
'harness 模式下跟随「设置 → 模型」当前选中的模型 · harness: use the model currently selected in Settings → Models',
|
|
130
|
+
),
|
|
131
|
+
),
|
|
132
|
+
provider: live(
|
|
133
|
+
z
|
|
134
|
+
.string()
|
|
135
|
+
.default('')
|
|
136
|
+
.description(
|
|
137
|
+
'harness 模式下的 provider(如 deepseek、wb、agy-local);只有关掉 useCurrentModel 才需要 · harness provider id, only when useCurrentModel is off',
|
|
138
|
+
),
|
|
139
|
+
),
|
|
140
|
+
reasoningEffort: live(
|
|
141
|
+
z
|
|
142
|
+
.string()
|
|
143
|
+
.default('')
|
|
144
|
+
.description(
|
|
145
|
+
'思考强度(如 low/medium/high,取决于模型);留空跟随该模型的默认 · thinking level for distillation; empty keeps the route default',
|
|
146
|
+
),
|
|
147
|
+
),
|
|
148
|
+
baseUrl: live(
|
|
149
|
+
z
|
|
150
|
+
.string()
|
|
151
|
+
.default('https://api.deepseek.com/v1')
|
|
152
|
+
.description('仅 http 模式:OpenAI 兼容接口地址 · http transport only: OpenAI-compatible base URL'),
|
|
153
|
+
),
|
|
154
|
+
path: live(
|
|
155
|
+
z.string().default('/chat/completions').description('仅 http 模式:补全路径 · http transport only: completion path'),
|
|
156
|
+
),
|
|
157
|
+
model: live(
|
|
158
|
+
z
|
|
159
|
+
.string()
|
|
160
|
+
.default('')
|
|
161
|
+
.description(
|
|
162
|
+
'模型名;http 模式留空即 deepseek-chat,harness 模式下只有关掉 useCurrentModel 才用到 · model name; http mode falls back to deepseek-chat',
|
|
163
|
+
),
|
|
164
|
+
),
|
|
165
|
+
apiKeyEnv: live(
|
|
166
|
+
z
|
|
167
|
+
.string()
|
|
168
|
+
.default('MEMORY_LLM_API_KEY')
|
|
169
|
+
.description(
|
|
170
|
+
'仅 http 模式:放密钥的环境变量或 ~/.dsh/.credentials.yaml 里的 ref 名 · http transport only: env var or credentials ref holding the key',
|
|
171
|
+
),
|
|
172
|
+
),
|
|
173
|
+
apiKey: live(
|
|
174
|
+
z
|
|
175
|
+
.string()
|
|
176
|
+
.role('secret')
|
|
177
|
+
.default('')
|
|
178
|
+
.description(
|
|
179
|
+
'仅 http 模式:直接写密钥(不建议,会存进 profile 配置)· http transport only: inline key (not recommended)',
|
|
180
|
+
),
|
|
181
|
+
),
|
|
182
|
+
maxOutputTokens: live(
|
|
183
|
+
z
|
|
184
|
+
.number()
|
|
185
|
+
.min(256)
|
|
186
|
+
.max(32000)
|
|
187
|
+
.step(1)
|
|
188
|
+
.default(3000)
|
|
189
|
+
.description('单次蒸馏输出上限 · output token cap per run'),
|
|
190
|
+
),
|
|
191
|
+
temperature: live(
|
|
192
|
+
z
|
|
193
|
+
.number()
|
|
194
|
+
.min(0)
|
|
195
|
+
.max(2)
|
|
196
|
+
.step(0.05)
|
|
197
|
+
.default(0.1)
|
|
198
|
+
.description('仅 http 模式:采样温度(harness 模式沿用模型设置)· http transport only: sampling temperature'),
|
|
199
|
+
),
|
|
200
|
+
timeoutMs: live(
|
|
201
|
+
z
|
|
202
|
+
.number()
|
|
203
|
+
.min(5000)
|
|
204
|
+
.max(600000)
|
|
205
|
+
.step(1000)
|
|
206
|
+
.default(90000)
|
|
207
|
+
.description('请求超时(毫秒)· request timeout in ms'),
|
|
208
|
+
),
|
|
209
|
+
maxEpisodesPerRun: live(
|
|
210
|
+
z
|
|
211
|
+
.number()
|
|
212
|
+
.min(1)
|
|
213
|
+
.max(400)
|
|
214
|
+
.step(1)
|
|
215
|
+
.default(40)
|
|
216
|
+
.description('一次最多蒸馏多少片段 · episodes handled per run'),
|
|
217
|
+
),
|
|
218
|
+
})
|
|
219
|
+
|
|
220
|
+
const retrieval = z.object({
|
|
221
|
+
limit: live(z.number().min(1).max(40).step(1).default(8).description('默认返回几条 · default result count')),
|
|
222
|
+
hops: live(z.number().min(0).max(2).step(1).default(1).description('沿实体扩展几跳 · co-mention expansion hops')),
|
|
223
|
+
snippetChars: live(
|
|
224
|
+
z
|
|
225
|
+
.number()
|
|
226
|
+
.min(60)
|
|
227
|
+
.max(2000)
|
|
228
|
+
.step(20)
|
|
229
|
+
.default(220)
|
|
230
|
+
.description('每条摘要截多长 · snippet length'),
|
|
231
|
+
),
|
|
232
|
+
timelineLimit: live(
|
|
233
|
+
z
|
|
234
|
+
.number()
|
|
235
|
+
.min(1)
|
|
236
|
+
.max(200)
|
|
237
|
+
.step(1)
|
|
238
|
+
.default(30)
|
|
239
|
+
.description('时间线默认条数 · timeline default size'),
|
|
240
|
+
),
|
|
241
|
+
})
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* The automatic distiller. Two independent rules over one minute-long tick:
|
|
245
|
+
* "this many episodes are waiting" and "this many minutes have passed". Both are
|
|
246
|
+
* live, so switching one on takes effect without remounting the plugin — and both
|
|
247
|
+
* default to zero/off, because a background run spends the user's model budget.
|
|
248
|
+
*/
|
|
249
|
+
const auto = z.object({
|
|
250
|
+
enabled: live(
|
|
251
|
+
z
|
|
252
|
+
.boolean()
|
|
253
|
+
.default(false)
|
|
254
|
+
.description('自动蒸馏总开关 · run distillations without being asked (spends model budget)'),
|
|
255
|
+
),
|
|
256
|
+
minEpisodes: live(
|
|
257
|
+
z
|
|
258
|
+
.number()
|
|
259
|
+
.min(0)
|
|
260
|
+
.max(400)
|
|
261
|
+
.step(1)
|
|
262
|
+
.default(0)
|
|
263
|
+
.description('攒够多少条未蒸馏情节就蒸一次;0 = 关闭这条规则 · distill once this many episodes are waiting; 0 turns the rule off'),
|
|
264
|
+
),
|
|
265
|
+
everyMinutes: live(
|
|
266
|
+
z
|
|
267
|
+
.number()
|
|
268
|
+
.min(0)
|
|
269
|
+
.max(10080)
|
|
270
|
+
.step(5)
|
|
271
|
+
.default(0)
|
|
272
|
+
.description('每隔多少分钟蒸一次;0 = 关闭这条规则 · distill at most every N minutes; 0 turns the rule off'),
|
|
273
|
+
),
|
|
274
|
+
minGapMinutes: live(
|
|
275
|
+
z
|
|
276
|
+
.number()
|
|
277
|
+
.min(0)
|
|
278
|
+
.max(1440)
|
|
279
|
+
.step(5)
|
|
280
|
+
.default(30)
|
|
281
|
+
.description('两次自动蒸馏之间至少隔多久 · minimum gap between two automatic runs'),
|
|
282
|
+
),
|
|
283
|
+
quietHours: live(
|
|
284
|
+
z
|
|
285
|
+
.string()
|
|
286
|
+
.default('')
|
|
287
|
+
.description('不跑的时间段,如 23:00-08:00;空 = 全天 · quiet window like 23:00-08:00; empty = always on'),
|
|
288
|
+
),
|
|
289
|
+
project: live(
|
|
290
|
+
z
|
|
291
|
+
.string()
|
|
292
|
+
.default('')
|
|
293
|
+
.description('自动蒸馏记到哪个项目;空 = 最近情节的项目 · project for automatic cards; empty = the newest episode\'s project'),
|
|
294
|
+
),
|
|
295
|
+
})
|
|
296
|
+
|
|
297
|
+
export const Config = z.object({
|
|
298
|
+
// Structural: read once while activating. Editing one remounts the plugin,
|
|
299
|
+
// which is exactly the semantics it needs, so they stay ordinary.
|
|
300
|
+
enabled: z
|
|
301
|
+
.boolean()
|
|
302
|
+
.default(true)
|
|
303
|
+
.description('总开关;关掉后工具和记录都不注册 · master switch; off registers neither tools nor capture'),
|
|
304
|
+
vaultDir: z
|
|
305
|
+
.string()
|
|
306
|
+
.default('')
|
|
307
|
+
.description('记忆库目录,留空是 ~/dsh-memory · vault directory; empty means ~/dsh-memory'),
|
|
308
|
+
defaultProject: z
|
|
309
|
+
.string()
|
|
310
|
+
.default('shared')
|
|
311
|
+
.description('会话没有可用目录时用的项目名 · project tag when a session has no usable cwd'),
|
|
312
|
+
crossProject: z
|
|
313
|
+
.boolean()
|
|
314
|
+
.default(true)
|
|
315
|
+
.description('把新卡片连到项目实体上,跨项目可召回 · link cards into project hubs so recall crosses projects'),
|
|
316
|
+
capture,
|
|
317
|
+
privacy,
|
|
318
|
+
extractor,
|
|
319
|
+
auto,
|
|
320
|
+
retrieval,
|
|
321
|
+
})
|
|
322
|
+
|
|
323
|
+
export default Config
|
package/src/search.js
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retrieval: keyword scoring (BM25 over cards) + entity matching + one-hop
|
|
3
|
+
* associative expansion + recency weighting.
|
|
4
|
+
*
|
|
5
|
+
* The point of the graph is here, not in the storage layout: a card is a hub
|
|
6
|
+
* for the entities it mentions, so "what else do I know that is connected to
|
|
7
|
+
* this" is an entity lookup, not a similarity search that may drift.
|
|
8
|
+
*/
|
|
9
|
+
const STOPWORDS = new Set([
|
|
10
|
+
'the', 'a', 'an', 'and', 'or', 'but', 'if', 'then', 'of', 'to', 'in', 'on', 'for', 'with',
|
|
11
|
+
'is', 'are', 'was', 'were', 'be', 'been', 'it', 'this', 'that', 'these', 'those', 'as', 'at',
|
|
12
|
+
'by', 'from', 'we', 'you', 'i', 'he', 'she', 'they', 'do', 'does', 'did', 'can', 'could',
|
|
13
|
+
'should', 'would', 'will', 'not', 'no', 'yes', 'so', 'than', 'too', 'very', 'just', 'about',
|
|
14
|
+
'still', 'now', 'currently', 'actually', 'really', 'what', 'why', 'how', 'when', 'where',
|
|
15
|
+
'which', 'who', 'whose', 'ever', 'any', 'some', 'there', 'here',
|
|
16
|
+
// Single characters.
|
|
17
|
+
'的', '了', '是', '在', '和', '与', '就', '都', '也', '很', '要', '会', '把', '被', '让',
|
|
18
|
+
'我', '你', '他', '她', '它', '这', '那', '个', '们', '有', '没', '不', '吗', '呢', '啊',
|
|
19
|
+
// Multi-character function words. Chinese is bigram-tokenised, so a question word
|
|
20
|
+
// becomes tokens that look rare to BM25 (`什么` may appear in two cards out of forty)
|
|
21
|
+
// and would otherwise outrank the actual subject of the question. These carry no
|
|
22
|
+
// topic: they are how the question is asked, not what it is about.
|
|
23
|
+
'什么', '为什', '怎么', '怎样', '如何', '是否', '能否', '多少', '哪些', '哪个', '哪里',
|
|
24
|
+
'这个', '那个', '这些', '那些', '这样', '那样', '一个', '一下', '一样', '一点', '一些',
|
|
25
|
+
'是不', '不是', '要不', '不要', '不用', '没有', '可以', '可能', '应该', '需要', '现在', '已经',
|
|
26
|
+
'还是', '就是', '也是', '都是', '只是', '但是', '因为', '所以', '如果', '然后', '而且',
|
|
27
|
+
'出来', '起来', '下来', '过来', '进去', '上来', '还在', '在用', '有没', '有什', '么样',
|
|
28
|
+
'我们', '你们', '他们', '她们', '它们', '自己', '这套', '那套', '东西', '时候', '地方',
|
|
29
|
+
'啥样', '咋样', '一眼', '确定', '知道', '觉得', '看看', '说说',
|
|
30
|
+
])
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Characters that carry grammar rather than subject matter.
|
|
34
|
+
*
|
|
35
|
+
* Enumerating whole function *words* does not generalise: every new way of asking
|
|
36
|
+
* a question produces new fragments — "到底是不是" splits into `底是`, `是不`,
|
|
37
|
+
* `不是`; "做了没有" into `做了`, `了没`, `没有`. Each fragment is rare in the
|
|
38
|
+
* corpus, so BM25 gives it a high idf and it behaves like a rare topic word. The
|
|
39
|
+
* filter therefore has to be a rule about the characters: a bigram touching a
|
|
40
|
+
* function character at either end is a piece of a phrase, not a term.
|
|
41
|
+
*
|
|
42
|
+
* Only characters that are *almost always* grammatical belong here. `有`, `不`,
|
|
43
|
+
* `要` and `得` are deliberately absent: they are equally common inside content
|
|
44
|
+
* words (`所有`, `不同`, `需要`, `得到`), and a first version that included them
|
|
45
|
+
* did real damage — the question "所有项目都要零依赖吗" lost `所有` and `要零`,
|
|
46
|
+
* which is exactly what it was asking about.
|
|
47
|
+
*/
|
|
48
|
+
const CJK_FUNCTION_CHARS = new Set(
|
|
49
|
+
[...'的了吗呢吧啊呀嘛么什怎哪谁几是在和与就都也很把被让我你他她它们这那没为个么要'],
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Words that ask about the *history* of something rather than its current state.
|
|
54
|
+
*
|
|
55
|
+
* `原来` is not in the list on purpose: as often as not it means "so that's why",
|
|
56
|
+
* which is a discovery about the present, and treating it as history lifted a
|
|
57
|
+
* retired card by 6.7× in probing.
|
|
58
|
+
*/
|
|
59
|
+
const HISTORY_INTENT =
|
|
60
|
+
/(最早|最初|起初|一开始|先用|起初那|以前|之前|先前|原先|原本|原来那|当初|当年|从前|那会儿|那阵子|当时|上一版|上一套|上一代|老版本|老做法|旧版|旧方案|旧的做法|旧的|那版|历史|曾经|改过|变过|换过|放弃了|不用了|what did we (?:originally|first)|originally|initially|used to|previously|earlier|before that|first version)/i
|
|
61
|
+
|
|
62
|
+
const CJK = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]/
|
|
63
|
+
|
|
64
|
+
export function tokenize(text) {
|
|
65
|
+
const out = []
|
|
66
|
+
const value = String(text ?? '').toLowerCase()
|
|
67
|
+
let latin = ''
|
|
68
|
+
let cjk = ''
|
|
69
|
+
|
|
70
|
+
const flushLatin = () => {
|
|
71
|
+
if (latin.length >= 2 && !STOPWORDS.has(latin)) out.push(latin)
|
|
72
|
+
latin = ''
|
|
73
|
+
}
|
|
74
|
+
const flushCjk = () => {
|
|
75
|
+
if (cjk.length === 1) {
|
|
76
|
+
if (!STOPWORDS.has(cjk) && !CJK_FUNCTION_CHARS.has(cjk)) out.push(cjk)
|
|
77
|
+
} else if (cjk.length > 1) {
|
|
78
|
+
for (let i = 0; i < cjk.length - 1; i++) {
|
|
79
|
+
const bigram = cjk.slice(i, i + 2)
|
|
80
|
+
if (STOPWORDS.has(bigram)) continue
|
|
81
|
+
// "开着吗" must not yield `着吗`, and "到底是不是" must not yield `底是`.
|
|
82
|
+
if (CJK_FUNCTION_CHARS.has(bigram[0]) || CJK_FUNCTION_CHARS.has(bigram[1])) continue
|
|
83
|
+
out.push(bigram)
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
cjk = ''
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
for (const char of value) {
|
|
90
|
+
if (CJK.test(char)) {
|
|
91
|
+
flushLatin()
|
|
92
|
+
cjk += char
|
|
93
|
+
} else if (/[a-z0-9_]/.test(char)) {
|
|
94
|
+
flushCjk()
|
|
95
|
+
latin += char
|
|
96
|
+
} else {
|
|
97
|
+
flushLatin()
|
|
98
|
+
flushCjk()
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
flushLatin()
|
|
102
|
+
flushCjk()
|
|
103
|
+
|
|
104
|
+
return out
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function termFrequency(tokens) {
|
|
108
|
+
const tf = new Map()
|
|
109
|
+
for (const token of tokens) tf.set(token, (tf.get(token) ?? 0) + 1)
|
|
110
|
+
return tf
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function docTokens(card) {
|
|
114
|
+
const tokens = []
|
|
115
|
+
for (let i = 0; i < 3; i++) tokens.push(...tokenize(card.title))
|
|
116
|
+
for (const entity of card.entities ?? []) tokens.push(...tokenize(entity))
|
|
117
|
+
tokens.push(...tokenize(card.kind))
|
|
118
|
+
tokens.push(...tokenize(card.body))
|
|
119
|
+
return tokens
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function ageInDays(iso) {
|
|
123
|
+
const time = Date.parse(String(iso ?? ''))
|
|
124
|
+
if (!Number.isFinite(time)) return 365
|
|
125
|
+
return Math.max(0, (Date.now() - time) / 86_400_000)
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function recencyWeight(iso) {
|
|
129
|
+
return 0.55 + 0.45 * Math.exp(-ageInDays(iso) / 120)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function statusWeight(status, history = false) {
|
|
133
|
+
if (status === 'retracted') return 0
|
|
134
|
+
// A retired memory keeps only a trace of its weight: it is still worth finding
|
|
135
|
+
// when nothing else matches, but it must never outrank the card that replaced it
|
|
136
|
+
// — and a question asked in the old wording is exactly when that would happen.
|
|
137
|
+
// Unless the question is asking about the past: "最早那版是什么" is a normal thing
|
|
138
|
+
// to ask a memory, and demoting the old card there makes the answer unreachable.
|
|
139
|
+
if (status === 'superseded') return history ? 1 : 0.15
|
|
140
|
+
return 1
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function makeSnippet(text, queryTokens, maxChars) {
|
|
144
|
+
const body = String(text ?? '').replace(/\s+/g, ' ').trim()
|
|
145
|
+
if (body.length <= maxChars) return body
|
|
146
|
+
const lower = body.toLowerCase()
|
|
147
|
+
let best = -1
|
|
148
|
+
for (const token of queryTokens) {
|
|
149
|
+
if (token.length < 2) continue
|
|
150
|
+
const at = lower.indexOf(token)
|
|
151
|
+
if (at !== -1 && (best === -1 || at < best)) best = at
|
|
152
|
+
}
|
|
153
|
+
if (best === -1) return `${body.slice(0, maxChars)}…`
|
|
154
|
+
const start = Math.max(0, best - Math.floor(maxChars / 3))
|
|
155
|
+
return `${start > 0 ? '…' : ''}${body.slice(start, start + maxChars)}…`
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* @returns {{ card: object, score: number, snippet: string, via: string }[]}
|
|
160
|
+
*/
|
|
161
|
+
export function searchCards(cards, query, options = {}) {
|
|
162
|
+
const limit = options.limit ?? 8
|
|
163
|
+
const hops = options.hops ?? 1
|
|
164
|
+
const snippetChars = options.snippetChars ?? 220
|
|
165
|
+
const project = options.project ?? null
|
|
166
|
+
const kinds = options.kinds ?? null
|
|
167
|
+
|
|
168
|
+
const pool = cards.filter((card) => {
|
|
169
|
+
if (project && card.project !== project) return false
|
|
170
|
+
if (kinds && kinds.length > 0 && !kinds.includes(card.kind)) return false
|
|
171
|
+
return true
|
|
172
|
+
})
|
|
173
|
+
if (pool.length === 0) return []
|
|
174
|
+
|
|
175
|
+
const queryTokens = tokenize(query)
|
|
176
|
+
// Asking about the past is a different question from asking about the present:
|
|
177
|
+
// the same words should surface the superseded card, not only its successor.
|
|
178
|
+
const history = HISTORY_INTENT.test(String(query ?? ''))
|
|
179
|
+
const uniqueQuery = [...new Set(queryTokens)]
|
|
180
|
+
|
|
181
|
+
// ---- BM25
|
|
182
|
+
const docs = pool.map((card) => {
|
|
183
|
+
const tokens = docTokens(card)
|
|
184
|
+
return { card, tokens, tf: termFrequency(tokens), len: tokens.length }
|
|
185
|
+
})
|
|
186
|
+
const avgLen = docs.reduce((sum, d) => sum + d.len, 0) / docs.length || 1
|
|
187
|
+
const df = new Map()
|
|
188
|
+
for (const token of uniqueQuery) {
|
|
189
|
+
let count = 0
|
|
190
|
+
for (const doc of docs) if (doc.tf.has(token)) count++
|
|
191
|
+
df.set(token, count)
|
|
192
|
+
}
|
|
193
|
+
const N = docs.length
|
|
194
|
+
const k1 = 1.2
|
|
195
|
+
const b = 0.75
|
|
196
|
+
|
|
197
|
+
const scored = new Map()
|
|
198
|
+
for (const doc of docs) {
|
|
199
|
+
let score = 0
|
|
200
|
+
for (const token of uniqueQuery) {
|
|
201
|
+
const tf = doc.tf.get(token)
|
|
202
|
+
if (!tf) continue
|
|
203
|
+
const n = df.get(token) ?? 0
|
|
204
|
+
const idf = Math.log(1 + (N - n + 0.5) / (n + 0.5))
|
|
205
|
+
score += idf * ((tf * (k1 + 1)) / (tf + k1 * (1 - b + (b * doc.len) / avgLen)))
|
|
206
|
+
}
|
|
207
|
+
if (score <= 0) continue
|
|
208
|
+
scored.set(doc.card.id, {
|
|
209
|
+
card: doc.card,
|
|
210
|
+
raw: score,
|
|
211
|
+
score: score * recencyWeight(doc.card.created) * statusWeight(doc.card.status, history),
|
|
212
|
+
via: 'keyword',
|
|
213
|
+
})
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// ---- entity exact / substring matching (the cross-project bridge)
|
|
217
|
+
const needle = String(query ?? '').toLowerCase()
|
|
218
|
+
for (const card of pool) {
|
|
219
|
+
let boost = 0
|
|
220
|
+
for (const slug of card.entities ?? []) {
|
|
221
|
+
if (slug === '' || slug.length < 2) continue
|
|
222
|
+
const flat = slug.replace(/-/g, ' ')
|
|
223
|
+
if (needle.includes(flat) || needle.includes(slug)) boost += 0.8
|
|
224
|
+
}
|
|
225
|
+
if (boost === 0) continue
|
|
226
|
+
const entry = scored.get(card.id)
|
|
227
|
+
const value = (entry?.score ?? 0) + boost * recencyWeight(card.created) * statusWeight(card.status, history)
|
|
228
|
+
scored.set(card.id, { card, raw: (entry?.raw ?? 0) + boost, score: value, via: entry ? 'keyword+entity' : 'entity' })
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// ---- follow the superseded edge forward.
|
|
232
|
+
// Matching a retired memory is a signal, not a hit: the question was phrased in
|
|
233
|
+
// the old words and the answer is now its successor. Without this the freshest
|
|
234
|
+
// state of a topic loses to the stale card that happens to share its wording.
|
|
235
|
+
//
|
|
236
|
+
// Not when the question is about the past, though. "告警最早是发邮件的吗" wants
|
|
237
|
+
// the old card, and pushing its own score onto the successor lifts the successor
|
|
238
|
+
// above the very card that was asked for.
|
|
239
|
+
const byId = new Map(pool.map((card) => [card.id, card]))
|
|
240
|
+
const staleHits = history ? [] : [...scored.values()].filter((e) => e.card.status === 'superseded')
|
|
241
|
+
for (const entry of staleHits) {
|
|
242
|
+
if (!entry.card.supersededBy) continue
|
|
243
|
+
const successor = byId.get(entry.card.supersededBy)
|
|
244
|
+
if (!successor || successor.status !== 'active') continue
|
|
245
|
+
const pull = (entry.raw ?? 0) * 0.9 * recencyWeight(successor.created) * statusWeight(successor.status, history)
|
|
246
|
+
const existing = scored.get(successor.id)
|
|
247
|
+
if (!existing) {
|
|
248
|
+
scored.set(successor.id, { card: successor, raw: (entry.raw ?? 0) * 0.9, score: pull, via: `supersedes:${entry.card.id}` })
|
|
249
|
+
} else if (existing.score < pull) {
|
|
250
|
+
existing.score = pull
|
|
251
|
+
existing.raw = Math.max(existing.raw ?? 0, (entry.raw ?? 0) * 0.9)
|
|
252
|
+
existing.via = `${existing.via}+supersedes`
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
let ranked = [...scored.values()].sort((a, b2) => b2.score - a.score)
|
|
257
|
+
if (ranked.length === 0) {
|
|
258
|
+
// Nothing matched lexically: fall back to the most recent durable memories.
|
|
259
|
+
ranked = pool
|
|
260
|
+
.slice()
|
|
261
|
+
.sort((a, b2) => String(b2.created).localeCompare(String(a.created)))
|
|
262
|
+
.slice(0, limit)
|
|
263
|
+
.map((card) => ({ card, score: 0.05 * recencyWeight(card.created), via: 'recency' }))
|
|
264
|
+
return ranked
|
|
265
|
+
.filter((entry) => entry.card.status !== 'retracted')
|
|
266
|
+
.slice(0, limit)
|
|
267
|
+
.map((entry) => ({ ...entry, snippet: makeSnippet(entry.card.body, uniqueQuery, snippetChars) }))
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// ---- associative expansion: co-mentioned entities pull in neighbours
|
|
271
|
+
if (hops > 0 && ranked.length > 0) {
|
|
272
|
+
const seeds = ranked.slice(0, 3)
|
|
273
|
+
const seedEntities = new Set(seeds.flatMap((entry) => entry.card.entities ?? []))
|
|
274
|
+
for (const card of pool) {
|
|
275
|
+
if (scored.has(card.id)) continue
|
|
276
|
+
if (statusWeight(card.status, history) === 0) continue
|
|
277
|
+
const shared = (card.entities ?? []).filter((slug) => seedEntities.has(slug))
|
|
278
|
+
if (shared.length === 0) continue
|
|
279
|
+
const seedScore = seeds[0]?.score ?? 1
|
|
280
|
+
scored.set(card.id, {
|
|
281
|
+
card,
|
|
282
|
+
score: seedScore * 0.25 * Math.min(1, shared.length / 2) * recencyWeight(card.created) * statusWeight(card.status, history),
|
|
283
|
+
via: `linked:${shared.slice(0, 2).join(',')}`,
|
|
284
|
+
})
|
|
285
|
+
}
|
|
286
|
+
ranked = [...scored.values()].sort((a, b2) => b2.score - a.score)
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
return ranked.slice(0, limit).map((entry) => ({
|
|
290
|
+
...entry,
|
|
291
|
+
snippet: makeSnippet(entry.card.body, uniqueQuery, snippetChars),
|
|
292
|
+
}))
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
export function timelineCards(cards, options = {}) {
|
|
296
|
+
const limit = options.limit ?? 30
|
|
297
|
+
const project = options.project ?? null
|
|
298
|
+
const from = options.from ? String(options.from) : null
|
|
299
|
+
const to = options.to ? String(options.to) : null
|
|
300
|
+
const entity = options.entity ? String(options.entity).toLowerCase() : null
|
|
301
|
+
|
|
302
|
+
return cards
|
|
303
|
+
.filter((card) => {
|
|
304
|
+
if (card.status === 'retracted') return false
|
|
305
|
+
if (project && card.project !== project) return false
|
|
306
|
+
const day = String(card.created).slice(0, 10)
|
|
307
|
+
if (from && day < from) return false
|
|
308
|
+
if (to && day > to) return false
|
|
309
|
+
if (entity) {
|
|
310
|
+
const haystack = `${(card.entities ?? []).join(' ')} ${card.title} ${card.body}`.toLowerCase()
|
|
311
|
+
if (!haystack.includes(entity.replace(/-/g, ' ')) && !haystack.includes(entity)) return false
|
|
312
|
+
}
|
|
313
|
+
return true
|
|
314
|
+
})
|
|
315
|
+
.sort((a, b) => String(b.created).localeCompare(String(a.created)))
|
|
316
|
+
.slice(0, limit)
|
|
317
|
+
}
|