@restregular/dsh-memory 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -58
- package/cordis.patch.yml +6 -8
- package/icon.svg +1 -0
- package/lib/client.js +859 -0
- package/lib/index.js +72 -0
- package/lib/inject/apply.js +80 -80
- package/lib/inject/inject.js +127 -127
- package/lib/panel.js +353 -0
- package/lib/provider/apply.js +133 -128
- package/lib/provider/embed-install.js +1144 -1002
- package/lib/provider/embed-server.js +308 -294
- package/lib/provider/embed.js +103 -103
- package/lib/provider/hardware.js +225 -0
- package/lib/provider/index.js +1271 -903
- package/lib/provider/models.js +253 -0
- package/lib/provider/recommend.js +187 -0
- package/lib/provider/rerank-cache.js +62 -62
- package/lib/provider/rerank.js +169 -169
- package/lib/provider/scoring.js +143 -143
- package/lib/provider/secrets.js +55 -55
- package/lib/provider/store.js +491 -485
- package/lib/tools/apply.js +104 -104
- package/lib/tools/index.js +224 -224
- package/package.json +17 -6
package/lib/provider/index.js
CHANGED
|
@@ -1,905 +1,1273 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* dsh-memory-local — local JSONL provider for the DSH memory service.
|
|
3
|
-
*
|
|
4
|
-
* This entry is intentionally cordis-free so unit tests and standalone tools
|
|
5
|
-
* run without the harness. The Cordis wiring (Service subclass, effects,
|
|
6
|
-
* plugin apply) lives in ./apply.ts and is only loaded inside DSH.
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
import { MemoryStore, } from './store.js'
|
|
18
|
-
import { buildStats, lexicalScore, recordScore, tokenize, cosine, normalizeInPlace, hasStrongHit } from './scoring.js'
|
|
19
|
-
import { EmbedClient } from './embed.js'
|
|
20
|
-
import { DailyBudget, blendScores, rerank, } from './rerank.js'
|
|
21
|
-
import { RerankCache } from './rerank-cache.js'
|
|
22
|
-
import { scanSecrets, redactSecrets } from './secrets.js'
|
|
23
|
-
import { EmbedServerManager, } from './embed-server.js'
|
|
24
|
-
import { VectorInstaller, detectInstalled, installPaths, } from './embed-install.js'
|
|
25
|
-
import { existsSync, readdirSync, rmSync, statSync } from 'node:fs'
|
|
26
|
-
import { dirname, isAbsolute, join, relative, resolve } from 'node:path'
|
|
27
|
-
import { homedir } from 'node:os'
|
|
1
|
+
/**
|
|
2
|
+
* dsh-memory-local — local JSONL provider for the DSH memory service.
|
|
3
|
+
*
|
|
4
|
+
* This entry is intentionally cordis-free so unit tests and standalone tools
|
|
5
|
+
* run without the harness. The Cordis wiring (Service subclass, effects,
|
|
6
|
+
* plugin apply) lives in ./apply.ts and is only loaded inside DSH.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
import { MemoryStore, } from './store.js'
|
|
18
|
+
import { buildStats, lexicalScore, recordScore, tokenize, cosine, normalizeInPlace, hasStrongHit } from './scoring.js'
|
|
19
|
+
import { EmbedClient } from './embed.js'
|
|
20
|
+
import { DailyBudget, blendScores, rerank, } from './rerank.js'
|
|
21
|
+
import { RerankCache } from './rerank-cache.js'
|
|
22
|
+
import { scanSecrets, redactSecrets } from './secrets.js'
|
|
23
|
+
import { EmbedServerManager, } from './embed-server.js'
|
|
24
|
+
import { VectorInstaller, detectInstalled, installPaths, } from './embed-install.js'
|
|
25
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
26
|
+
import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'
|
|
27
|
+
import { homedir } from 'node:os'
|
|
28
|
+
import {
|
|
29
|
+
DEFAULT_MODEL_ID,
|
|
30
|
+
EMBEDDING_MODELS,
|
|
31
|
+
findModel,
|
|
32
|
+
findVariant,
|
|
33
|
+
|
|
34
|
+
} from './models.js'
|
|
35
|
+
import { probeHardware, } from './hardware.js'
|
|
36
|
+
import { pickFlavor, recommendModel } from './recommend.js'
|
|
37
|
+
|
|
38
|
+
/** Recursive byte count; 0 for anything unreadable. */
|
|
39
|
+
function pathSize(path ) {
|
|
40
|
+
try {
|
|
41
|
+
const st = statSync(path)
|
|
42
|
+
if (!st.isDirectory()) return st.size
|
|
43
|
+
let total = 0
|
|
44
|
+
for (const entry of readdirSync(path)) total += pathSize(join(path, entry))
|
|
45
|
+
return total
|
|
46
|
+
} catch {
|
|
47
|
+
return 0
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** True when `child` is `parent` or lives under it (both resolved). */
|
|
52
|
+
function isInside(parent , child ) {
|
|
53
|
+
const rel = relative(resolve(parent), resolve(child))
|
|
54
|
+
return rel === '' || (!rel.startsWith('..') && !isAbsolute(rel))
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
/** vector 配置节的类型别名,给下面的模型解析helpers 用。 */
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
/** 面板可以在安装时临时指定的模型选择。 */
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* 「面板选过什么」的记忆,落在安装目录里(<installRoot>/selected-model.json)。
|
|
199
|
+
*
|
|
200
|
+
* 为什么需要它:面板没法改写 profile 配置,而"选了模型 → 重启后配置又指回老模型"
|
|
201
|
+
* 会让装好的模型认不出来。这里是**低于配置**的一层记忆:配置里写了 preset/customModel
|
|
202
|
+
* 就听配置的,没写才用这份。
|
|
203
|
+
*/
|
|
204
|
+
export const MODEL_STATE_FILE = 'selected-model.json'
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
function readModelState(root ) {
|
|
214
|
+
try {
|
|
215
|
+
const text = readFileSync(join(root, MODEL_STATE_FILE), 'utf8')
|
|
216
|
+
const parsed = JSON.parse(text)
|
|
217
|
+
return parsed !== null && typeof parsed === 'object' ? parsed : null
|
|
218
|
+
} catch {
|
|
219
|
+
return null
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function writeModelState(root , state ) {
|
|
224
|
+
try {
|
|
225
|
+
mkdirSync(root, { recursive: true })
|
|
226
|
+
writeFileSync(join(root, MODEL_STATE_FILE), `${JSON.stringify(state, null, 2)}\n`, 'utf8')
|
|
227
|
+
} catch {
|
|
228
|
+
// 记不住不影响本次安装;下次面板仍会显示配置文件里的选择。
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** 系统语言(用来判断"中文优先");拿不到就当英文环境。 */
|
|
233
|
+
function safeLocale() {
|
|
234
|
+
try {
|
|
235
|
+
return Intl.DateTimeFormat().resolvedOptions().locale || 'en'
|
|
236
|
+
} catch {
|
|
237
|
+
return 'en'
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** 该模型某个档位的体积(不指定档位时用默认档),用于清单展示与磁盘校验。 */
|
|
242
|
+
function variantBytes(model , variantId ) {
|
|
243
|
+
const variant = variantId !== undefined ? model.variants.find((v) => v.id === variantId) : undefined
|
|
244
|
+
return (variant ?? model.variants.find((v) => v.isDefault) ?? model.variants[0] ).bytes
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* 把配置里的模型选择解析成安装器要的形态(文件名 / 体积 / 源 / 维度 / pooling / 前缀)。
|
|
265
|
+
*
|
|
266
|
+
* 优先级:customModel(完全自定义) → preset(清单 id) → 用 `model` 字符串去清单里找
|
|
267
|
+
* → 内置默认(bge-m3 Q4_K_M,与老行为逐字一致)。
|
|
268
|
+
* 认不出的 `model` 字符串**不改身份**(请求与边车仍用它),只是安装器退回默认模型 ——
|
|
269
|
+
* 用户指向云端/自建服务时本来就该只写 endpoint + model + dims。
|
|
270
|
+
*/
|
|
271
|
+
function resolveModelSelection(v , root ) {
|
|
272
|
+
const state = root !== undefined ? readModelState(root) : null
|
|
273
|
+
const custom = v.customModel ?? state?.customModel
|
|
274
|
+
if (custom !== undefined) {
|
|
275
|
+
let file = custom.file
|
|
276
|
+
if (file === undefined || file === '') {
|
|
277
|
+
try {
|
|
278
|
+
file = basename(new URL(custom.url).pathname) || 'custom-model.gguf'
|
|
279
|
+
} catch {
|
|
280
|
+
file = 'custom-model.gguf'
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return {
|
|
284
|
+
presetId: 'custom',
|
|
285
|
+
variantId: 'custom',
|
|
286
|
+
label: file,
|
|
287
|
+
file,
|
|
288
|
+
...(custom.bytes !== undefined ? { bytes: custom.bytes } : {}),
|
|
289
|
+
sources: custom.sources ?? [custom.url],
|
|
290
|
+
dims: custom.dims,
|
|
291
|
+
pooling: custom.pooling ?? 'auto',
|
|
292
|
+
modelName: v.model ?? file.replace(/\.gguf$/i, ''),
|
|
293
|
+
...(v.flavor ?? state?.flavor ? { flavor: v.flavor ?? state?.flavor } : {}),
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
// 配置 > 面板的记忆 > 用 model 字符串去清单里找 > 内置默认。
|
|
297
|
+
const presetId = v.preset
|
|
298
|
+
?? state?.presetId
|
|
299
|
+
?? (findModel(v.model) !== undefined ? v.model : DEFAULT_MODEL_ID)
|
|
300
|
+
const model = findModel(presetId) ?? findModel(DEFAULT_MODEL_ID)
|
|
301
|
+
const variant = findVariant(model, v.variant ?? state?.variantId)
|
|
302
|
+
return {
|
|
303
|
+
presetId: model.id,
|
|
304
|
+
variantId: variant.id,
|
|
305
|
+
label: `${model.label} · ${variant.label}`,
|
|
306
|
+
file: variant.file,
|
|
307
|
+
bytes: variant.bytes,
|
|
308
|
+
sources: [...variant.sources],
|
|
309
|
+
dims: model.dims,
|
|
310
|
+
pooling: model.pooling,
|
|
311
|
+
modelName: v.model ?? model.id,
|
|
312
|
+
...(model.queryPrefix !== undefined ? { queryPrefix: model.queryPrefix } : {}),
|
|
313
|
+
...(model.documentPrefix !== undefined ? { documentPrefix: model.documentPrefix } : {}),
|
|
314
|
+
...(v.flavor ?? state?.flavor ? { flavor: v.flavor ?? state?.flavor } : {}),
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* 接管已装好的后端时,spawn 该用哪个模型文件。
|
|
320
|
+
*
|
|
321
|
+
* 坑:换模型时 `spawn.model` 可能还留着**上一个**模型的路径(显式配置或上次 adopt
|
|
322
|
+
* 写进去的),而 dims/pooling 已经跟着新模型变了 —— 那会启动一个"模型与参数不匹配"
|
|
323
|
+
* 的服务,而且不报错,只是检索质量悄悄变差。所以只有"配置的模型就是当前选中的那个"
|
|
324
|
+
* 才沿用它,否则一律用刚落盘的那份。
|
|
325
|
+
*/
|
|
326
|
+
export function pickSpawnModel(currentFile , configuredModel , installedModel ) {
|
|
327
|
+
if (configuredModel !== undefined && basename(configuredModel) === currentFile) return configuredModel
|
|
328
|
+
return installedModel
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
export function resolveConfig(config = {}) {
|
|
332
|
+
const v = config.retrieval?.vector ?? {}
|
|
333
|
+
// Do not touch an embedding backend the user never asked for: pointing at the
|
|
334
|
+
// stock localhost endpoint on a machine that has nothing listening means
|
|
335
|
+
// failed embeds, a tripped breaker and scary log lines for a feature that is
|
|
336
|
+
// optional. Vector retrieval is therefore opt-in by CONFIGURATION — writing
|
|
337
|
+
// `endpoint` (any OpenAI-compatible service), configuring `spawn`, or having
|
|
338
|
+
// the panel's installer put a backend on disk implies intent; an explicit
|
|
339
|
+
// `enabled` always wins.
|
|
340
|
+
const storeDir = config.storeDir ?? '.dsh-memory'
|
|
341
|
+
// Installer target: explicit config → that directory; explicit storeDir →
|
|
342
|
+
// its parent (so a hand-made `<root>/llamacpp` + `<root>/models` layout is
|
|
343
|
+
// recognised as-is); otherwise a stable per-user directory under ~/.dsh.
|
|
344
|
+
const installRoot = v.autoInstall?.dir
|
|
345
|
+
?? (config.storeDir !== undefined ? dirname(resolve(storeDir)) : join(homedir(), '.dsh', 'memory-vector'))
|
|
346
|
+
// 模型选择:自定义 → 显式 preset → 用 model 字符串去清单里找 → 内置默认。
|
|
347
|
+
// 这一步必须在 detectInstalled 之前:安装判据是「exe + **所选模型的**文件」。
|
|
348
|
+
const selection = resolveModelSelection(v, installRoot)
|
|
349
|
+
const installed = detectInstalled(installRoot, selection.file)
|
|
350
|
+
const endpointConfigured = v.endpoint !== undefined
|
|
351
|
+
// A configured spawn path that no longer exists (plugin uninstalled the
|
|
352
|
+
// backend, or the user moved it) must not count as configured: that would
|
|
353
|
+
// leave the panel showing a managed server that can never start, instead of
|
|
354
|
+
// offering the download button again.
|
|
355
|
+
const spawnExe = v.spawn?.exe
|
|
356
|
+
const spawnExeUsable = spawnExe === undefined || spawnExe === '' || existsSync(spawnExe)
|
|
357
|
+
const spawnConfigured = ((v.spawn?.enabled === true || (spawnExe ?? '') !== '') && spawnExeUsable) || installed !== null
|
|
358
|
+
const configured = endpointConfigured || spawnConfigured
|
|
359
|
+
const vectorEnabled = v.enabled ?? configured
|
|
360
|
+
const flavor = selection.flavor ?? 'cpu'
|
|
361
|
+
return {
|
|
362
|
+
storeDir,
|
|
363
|
+
globalEnabled: config.scopes?.global ?? true,
|
|
364
|
+
projectEnabled: config.scopes?.project ?? true,
|
|
365
|
+
topK: config.retrieval?.topK ?? 5,
|
|
366
|
+
recallN: config.retrieval?.recallN ?? 30,
|
|
367
|
+
globalThreshold: config.retrieval?.globalThreshold ?? 0.12,
|
|
368
|
+
projectThreshold: config.retrieval?.projectThreshold ?? 0.1,
|
|
369
|
+
halfLifeDays: config.retrieval?.recencyHalfLifeDays ?? 30,
|
|
370
|
+
vector: {
|
|
371
|
+
enabled: vectorEnabled,
|
|
372
|
+
configured,
|
|
373
|
+
endpointConfigured,
|
|
374
|
+
spawnConfigured,
|
|
375
|
+
installRoot,
|
|
376
|
+
...(v.autoInstall?.modelSources ? { installModelSources: v.autoInstall.modelSources } : {}),
|
|
377
|
+
installModel: {
|
|
378
|
+
file: selection.file,
|
|
379
|
+
...(selection.bytes !== undefined ? { bytes: selection.bytes } : {}),
|
|
380
|
+
...(selection.sources.length > 0 ? { sources: selection.sources } : {}),
|
|
381
|
+
},
|
|
382
|
+
presetId: selection.presetId,
|
|
383
|
+
variantId: selection.variantId,
|
|
384
|
+
modelLabel: selection.label,
|
|
385
|
+
pooling: v.pooling ?? selection.pooling,
|
|
386
|
+
flavor,
|
|
387
|
+
gpuLayers: v.gpuLayers ?? (flavor === 'cpu' || flavor === 'auto' ? 0 : 99),
|
|
388
|
+
...(selection.queryPrefix !== undefined ? { queryPrefix: selection.queryPrefix } : {}),
|
|
389
|
+
...(selection.documentPrefix !== undefined ? { documentPrefix: selection.documentPrefix } : {}),
|
|
390
|
+
endpoint: v.endpoint ?? 'http://127.0.0.1:8078/v1/embeddings',
|
|
391
|
+
model: v.model ?? selection.modelName,
|
|
392
|
+
dims: v.dims ?? selection.dims,
|
|
393
|
+
timeoutMs: v.timeoutMs ?? 300,
|
|
394
|
+
// Full contents are orders of magnitude slower than a short query on CPU
|
|
395
|
+
// (measured 3143ms for three ~600-char records), so write/backfill paths
|
|
396
|
+
// get their own generous budgets instead of the interactive 300ms.
|
|
397
|
+
writeTimeoutMs: v.writeTimeoutMs ?? 10_000,
|
|
398
|
+
backfillTimeoutMs: v.backfillTimeoutMs ?? 30_000,
|
|
399
|
+
backfillMaxChars: v.backfillMaxChars ?? 8_000,
|
|
400
|
+
// BGE CLS baseline noise floor measured ~0.42 on unrelated pairs
|
|
401
|
+
minScore: v.minScore ?? 0.5,
|
|
402
|
+
backfillBatchSize: v.backfillBatchSize ?? 32,
|
|
403
|
+
// Port defaults to the one in the embeddings endpoint so the two can never
|
|
404
|
+
// drift apart; opt-in (enabled:false) keeps dev/test runs from spawning.
|
|
405
|
+
// A backend the panel installed (or a hand-made one at installRoot) is
|
|
406
|
+
// adopted as the spawn target when the user configured no explicit paths.
|
|
407
|
+
spawn: {
|
|
408
|
+
...(v.spawn ?? {}),
|
|
409
|
+
enabled: v.spawn?.enabled ?? installed !== null,
|
|
410
|
+
...(installed
|
|
411
|
+
? {
|
|
412
|
+
exe: v.spawn?.exe ?? installed.exe,
|
|
413
|
+
model: v.spawn?.model ?? installed.model,
|
|
414
|
+
cwd: v.spawn?.cwd ?? installed.cwd,
|
|
415
|
+
logFile: v.spawn?.logFile ?? installed.logFile,
|
|
416
|
+
}
|
|
417
|
+
: {}),
|
|
418
|
+
// 模型自带 pooling 与 GPU 层数:llama-server 拿不到就是错的检索结果,
|
|
419
|
+
// 所以跟着所选模型一起传下去(extraArgs 仍在最后,可覆盖)。
|
|
420
|
+
pooling: v.pooling ?? selection.pooling,
|
|
421
|
+
gpuLayers: v.gpuLayers ?? (flavor === 'cpu' || flavor === 'auto' ? 0 : 99),
|
|
422
|
+
port: v.spawn?.port ?? portOf(v.endpoint ?? 'http://127.0.0.1:8078/v1/embeddings') ?? 8078,
|
|
423
|
+
},
|
|
424
|
+
},
|
|
425
|
+
llm: {
|
|
426
|
+
enabled: config.retrieval?.llm?.enabled ?? false,
|
|
427
|
+
provider: config.retrieval?.llm?.provider ?? '',
|
|
428
|
+
model: config.retrieval?.llm?.model ?? '',
|
|
429
|
+
timeoutMs: config.retrieval?.llm?.timeoutMs ?? 1500,
|
|
430
|
+
dailyBudgetTokens: config.retrieval?.llm?.dailyBudgetTokens ?? 200_000,
|
|
431
|
+
weight: config.retrieval?.llm?.weight ?? 0.6,
|
|
432
|
+
topN: config.retrieval?.llm?.topN ?? 10,
|
|
433
|
+
},
|
|
434
|
+
maxRecordsPerScope: config.limits?.maxRecordsPerScope ?? 2000,
|
|
435
|
+
maxRecordChars: config.limits?.maxRecordChars ?? 4000,
|
|
436
|
+
secrets: {
|
|
437
|
+
enabled: config.secrets?.enabled ?? true,
|
|
438
|
+
mode: config.secrets?.mode ?? 'refuse',
|
|
439
|
+
},
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
export class LocalMemoryService {
|
|
444
|
+
cfg
|
|
445
|
+
store
|
|
446
|
+
embed
|
|
447
|
+
rerankBudget
|
|
448
|
+
rerankCache = new RerankCache()
|
|
449
|
+
/** Host-provided one-shot LLM call; null = rerank disabled (recall order). */
|
|
450
|
+
llmInvoke = null
|
|
451
|
+
/** Where saves land when the caller does not pin a scope. */
|
|
452
|
+
defaultScope = 'global'
|
|
453
|
+
|
|
454
|
+
constructor(config = {}) {
|
|
455
|
+
this.cfg = resolveConfig(config)
|
|
456
|
+
this.rerankBudget = new DailyBudget(this.cfg.llm.dailyBudgetTokens)
|
|
457
|
+
this.store = new MemoryStore(this.cfg.storeDir, { onWarn: (m) => console.warn(`[dsh-memory-local] ${m}`) })
|
|
458
|
+
// Identity BEFORE the eager load (issue #1): ScopeStore skips its sidecar
|
|
459
|
+
// validation entirely while the identity is unset (`if (this.vecIdentity && …)`),
|
|
460
|
+
// so loading first would accept vectors from a previous embedding model
|
|
461
|
+
// whenever the dimensions matched — and the next flush would then rewrite the
|
|
462
|
+
// header as the new model, making the mismatch permanent.
|
|
463
|
+
this.store.setVecIdentity(this.cfg.vector.model, this.cfg.vector.dims)
|
|
464
|
+
this.store.loadAll(this.cfg.projectEnabled && this.cfg.globalEnabled ? [this.defaultScope, 'global'] : [this.defaultScope])
|
|
465
|
+
this.embed = new EmbedClient({
|
|
466
|
+
endpoint: this.cfg.vector.endpoint,
|
|
467
|
+
model: this.cfg.vector.model,
|
|
468
|
+
dims: this.cfg.vector.dims,
|
|
469
|
+
timeoutMs: this.cfg.vector.timeoutMs,
|
|
470
|
+
})
|
|
471
|
+
this.startMaintenance()
|
|
472
|
+
this.startEmbedServer()
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/** Own the embedding backend's lifetime: spawn on activation, stop on dispose. */
|
|
476
|
+
startEmbedServer() {
|
|
477
|
+
// Nothing configured and retrieval switched off → do not even build the
|
|
478
|
+
// manager, so no probe, no spawn attempt, no misleading panel state.
|
|
479
|
+
if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return
|
|
480
|
+
this.embedServer = new EmbedServerManager(this.cfg.vector.spawn)
|
|
481
|
+
// fire-and-forget: activation must never block on a model load
|
|
482
|
+
void this.ensureEmbedServer(60_000).catch(() => { /* lifecycle never breaks activation */ })
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* Start (or adopt) the embedding backend and arm the exit safety net.
|
|
487
|
+
* Shared by the activation path and the panel's manual start button.
|
|
488
|
+
*/
|
|
489
|
+
async ensureEmbedServer(waitMs ) {
|
|
490
|
+
const mgr = this.embedServer
|
|
491
|
+
if (!mgr) return 'disabled'
|
|
492
|
+
const state = await mgr.ensureRunning()
|
|
493
|
+
if (state !== 'spawned' && state !== 'adopted-managed') return state
|
|
494
|
+
// hard-exit safety net: cordis disposal covers a clean quit; this covers
|
|
495
|
+
// the host dying without disposing us (the child must not be orphaned)
|
|
496
|
+
if (!this.exitHook) {
|
|
497
|
+
this.exitHook = () => { this.embedServer?.stop() }
|
|
498
|
+
process.once('exit', this.exitHook)
|
|
499
|
+
}
|
|
500
|
+
const ready = await mgr.waitUntilReady(waitMs)
|
|
501
|
+
if (!ready) console.warn('[dsh-memory-local] embed server: 启动超时,检索继续走词法降级')
|
|
502
|
+
return state
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
/** Manual start from the memory panel (same semantics as activation). */
|
|
506
|
+
startEmbedServerManually() {
|
|
507
|
+
return this.ensureEmbedServer(25_000)
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/**
|
|
511
|
+
* Manual stop from the memory panel. An instance we do not own is left alone
|
|
512
|
+
* (it was started outside the app), so the UI can explain the refusal.
|
|
513
|
+
*/
|
|
514
|
+
stopEmbedServerManually() {
|
|
515
|
+
const mgr = this.embedServer
|
|
516
|
+
if (!mgr) return { stopped: false, reason: 'NOT_RUNNING' }
|
|
517
|
+
if (!mgr.isOwned) return { stopped: false, reason: 'EXTERNAL_INSTANCE' }
|
|
518
|
+
return { stopped: mgr.stop(), reason: mgr.isOwned ? 'NOT_RUNNING' : 'STOPPED' }
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
/**
|
|
522
|
+
* Lifecycle snapshot for the memory panel.
|
|
523
|
+
*
|
|
524
|
+
* `mode` is what the panel needs to decide whether a start/stop switch even
|
|
525
|
+
* makes sense:
|
|
526
|
+
* - 'off' nothing configured → no switch, show how to opt in
|
|
527
|
+
* - 'external' we only know an endpoint (cloud or a server the user runs)
|
|
528
|
+
* → no switch, we must not claim ownership of that process
|
|
529
|
+
* - 'managed' spawn is configured → the switch toggles OUR child process
|
|
530
|
+
* `enabled` is the retrieval switch from config; when it is false while a
|
|
531
|
+
* backend IS configured the panel explains the config instead of showing a
|
|
532
|
+
* switch that cannot turn retrieval on.
|
|
533
|
+
*/
|
|
534
|
+
async embedServerStatus()
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
{
|
|
559
|
+
const mgr = this.embedServer
|
|
560
|
+
const cfg = this.cfg.vector
|
|
561
|
+
const pendingEmbeds = this.pendingEmbedCount
|
|
562
|
+
const mode = !cfg.configured ? 'off' : cfg.spawnConfigured ? 'managed' : 'external'
|
|
563
|
+
const usage = this.vectorInstallUsage()
|
|
564
|
+
const base = {
|
|
565
|
+
enabled: cfg.enabled,
|
|
566
|
+
configured: cfg.configured,
|
|
567
|
+
mode,
|
|
568
|
+
endpoint: cfg.endpoint,
|
|
569
|
+
model: cfg.model,
|
|
570
|
+
modelLabel: cfg.modelLabel,
|
|
571
|
+
presetId: cfg.presetId,
|
|
572
|
+
variantId: cfg.variantId,
|
|
573
|
+
modelFile: cfg.installModel.file,
|
|
574
|
+
dims: cfg.dims,
|
|
575
|
+
pooling: cfg.pooling,
|
|
576
|
+
flavor: cfg.flavor,
|
|
577
|
+
spawnConfigured: cfg.spawnConfigured,
|
|
578
|
+
pendingEmbeds,
|
|
579
|
+
install: this.installSnapshot(),
|
|
580
|
+
maintenance: {
|
|
581
|
+
usageBytes: usage.bytes,
|
|
582
|
+
paths: usage.paths,
|
|
583
|
+
// NOTE: `cfg` is the VECTOR config here, not the whole config — reading
|
|
584
|
+
// cfg.storeDir would silently yield undefined (and JSON drops it).
|
|
585
|
+
storeDir: this.cfg.storeDir,
|
|
586
|
+
records: this.store.allScopes().reduce((n, s) => n + s.records.size, 0),
|
|
587
|
+
},
|
|
588
|
+
}
|
|
589
|
+
if (!mgr) return { ...base, owned: false, pid: null, baseUrl: '', reachable: false }
|
|
590
|
+
const reachable = await mgr.healthCheck()
|
|
591
|
+
return { ...base, ...mgr.info(), reachable, pendingEmbeds }
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
/** Installer state for the panel (idle when nothing was ever requested). */
|
|
595
|
+
installSnapshot() {
|
|
596
|
+
const root = this.cfg.vector.installRoot
|
|
597
|
+
const modelFile = this.cfg.vector.installModel.file
|
|
598
|
+
const installed = detectInstalled(root, modelFile) !== null
|
|
599
|
+
const progress = this.installer?.progress ?? {
|
|
600
|
+
phase: 'idle' ,
|
|
601
|
+
message: '',
|
|
602
|
+
receivedBytes: 0,
|
|
603
|
+
totalBytes: null,
|
|
604
|
+
error: null,
|
|
605
|
+
paths: installed ? installPaths(root, modelFile) : null,
|
|
606
|
+
}
|
|
607
|
+
return { ...progress, installed, root }
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/**
|
|
611
|
+
* 面板的"模型与硬件"报告:清单 + 当前选择 + 硬件 + 推荐。
|
|
612
|
+
*
|
|
613
|
+
* 硬件探测会 spawn 外部命令(Windows 上还有 PowerShell),所以**只在面板问的时候**跑,
|
|
614
|
+
* 并且结果缓存 10 分钟 —— 绝不在插件激活路径上。
|
|
615
|
+
*/
|
|
616
|
+
async modelsReport(force = false)
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
{
|
|
628
|
+
const root = this.cfg.vector.installRoot
|
|
629
|
+
const catalog = EMBEDDING_MODELS.map((model) => ({
|
|
630
|
+
id: model.id,
|
|
631
|
+
label: model.label,
|
|
632
|
+
note: model.note,
|
|
633
|
+
dims: model.dims,
|
|
634
|
+
pooling: model.pooling,
|
|
635
|
+
languages: model.languages,
|
|
636
|
+
contextTokens: model.contextTokens,
|
|
637
|
+
sizeBytes: variantBytes(model, this.cfg.vector.presetId === model.id ? this.cfg.vector.variantId : undefined),
|
|
638
|
+
installed: model.variants.some((variant) => detectInstalled(root, variant.file) !== null),
|
|
639
|
+
// 逐档判断"装没装":bge-m3 装了 Q4_K_M 不等于 Q8_0 也装了。
|
|
640
|
+
// 判据与 detectInstalled 一致 —— 运行时 exe + 该档模型文件都在。
|
|
641
|
+
variants: model.variants.map((variant) => ({
|
|
642
|
+
id: variant.id, label: variant.label, bytes: variant.bytes, file: variant.file,
|
|
643
|
+
installed: detectInstalled(root, variant.file) !== null,
|
|
644
|
+
})),
|
|
645
|
+
}))
|
|
646
|
+
const records = this.store.allScopes().reduce((n, s) => n + s.records.size, 0)
|
|
647
|
+
const locale = safeLocale()
|
|
648
|
+
let hardware = null
|
|
649
|
+
try {
|
|
650
|
+
hardware = await this.hardwareSnapshot(force)
|
|
651
|
+
} catch {
|
|
652
|
+
hardware = null // 探测失败就只显示清单,不给推荐 —— 不猜
|
|
653
|
+
}
|
|
654
|
+
const recommendation = hardware === null
|
|
655
|
+
? null
|
|
656
|
+
: recommendModel({ hardware, locale, recordCount: records, currentModelId: this.cfg.vector.presetId })
|
|
657
|
+
return {
|
|
658
|
+
catalog,
|
|
659
|
+
current: {
|
|
660
|
+
presetId: this.cfg.vector.presetId,
|
|
661
|
+
variantId: this.cfg.vector.variantId,
|
|
662
|
+
modelLabel: this.cfg.vector.modelLabel,
|
|
663
|
+
file: this.cfg.vector.installModel.file,
|
|
664
|
+
dims: this.cfg.vector.dims,
|
|
665
|
+
pooling: this.cfg.vector.pooling,
|
|
666
|
+
flavor: this.cfg.vector.flavor,
|
|
667
|
+
model: this.cfg.vector.model,
|
|
668
|
+
},
|
|
669
|
+
hardware,
|
|
670
|
+
recommendation,
|
|
671
|
+
locale,
|
|
672
|
+
records,
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
/** 硬件探测结果缓存(默认 10 分钟),'auto' 风味与推荐共用它。 */
|
|
677
|
+
async hardwareSnapshot(force = false) {
|
|
678
|
+
const now = Date.now()
|
|
679
|
+
if (!force && this.hardwareCache !== null && now - this.hardwareAt < 600_000) return this.hardwareCache
|
|
680
|
+
const hw = await probeHardware({ dir: this.cfg.vector.installRoot })
|
|
681
|
+
this.hardwareCache = hw
|
|
682
|
+
this.hardwareAt = now
|
|
683
|
+
return hw
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
/** 把 'auto' 风味解析成具体风味(需要探测;失败则退回 CPU)。 */
|
|
687
|
+
async resolveFlavor() {
|
|
688
|
+
const configured = this.cfg.vector.flavor
|
|
689
|
+
if (configured !== 'auto') return configured
|
|
690
|
+
try {
|
|
691
|
+
return pickFlavor(await this.hardwareSnapshot())
|
|
692
|
+
} catch {
|
|
693
|
+
return 'cpu'
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
/** 面板选了模型/风味之后,把选择写进安装目录的 state 文件并更新运行中的配置。 */
|
|
698
|
+
applyModelSelection(override ) {
|
|
699
|
+
const synthetic = override.custom !== undefined
|
|
700
|
+
? { customModel: override.custom }
|
|
701
|
+
: { ...(override.preset !== undefined ? { preset: override.preset } : {}), ...(override.variant !== undefined ? { variant: override.variant } : {}) }
|
|
702
|
+
const sel = resolveModelSelection(synthetic)
|
|
703
|
+
const cfg = this.cfg.vector
|
|
704
|
+
cfg.installModel = {
|
|
705
|
+
file: sel.file,
|
|
706
|
+
...(sel.bytes !== undefined ? { bytes: sel.bytes } : {}),
|
|
707
|
+
...(sel.sources.length > 0 ? { sources: sel.sources } : {}),
|
|
708
|
+
}
|
|
709
|
+
cfg.presetId = sel.presetId
|
|
710
|
+
cfg.variantId = sel.variantId
|
|
711
|
+
cfg.modelLabel = sel.label
|
|
712
|
+
cfg.pooling = sel.pooling
|
|
713
|
+
cfg.dims = sel.dims
|
|
714
|
+
cfg.model = sel.modelName
|
|
715
|
+
cfg.spawn.pooling = sel.pooling
|
|
716
|
+
if (override.flavor !== undefined) cfg.flavor = override.flavor
|
|
717
|
+
cfg.spawn.gpuLayers = cfg.flavor === 'cpu' || cfg.flavor === 'auto' ? 0 : (cfg.spawn.gpuLayers || 99)
|
|
718
|
+
writeModelState(cfg.installRoot, {
|
|
719
|
+
presetId: sel.presetId === 'custom' ? undefined : sel.presetId,
|
|
720
|
+
variantId: sel.variantId === 'custom' ? undefined : sel.variantId,
|
|
721
|
+
...(override.custom !== undefined ? { customModel: override.custom } : {}),
|
|
722
|
+
...(override.flavor !== undefined ? { flavor: override.flavor } : {}),
|
|
723
|
+
})
|
|
724
|
+
return cfg.flavor
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
/**
|
|
728
|
+
* Panel action: download llama.cpp + the GGML model into installRoot, then
|
|
729
|
+
* adopt them (start the managed server). Never runs on its own — a fresh
|
|
730
|
+
* install downloads nothing until the user asks.
|
|
731
|
+
*
|
|
732
|
+
* `override` 是面板的选择(预设/档位/自定义 URL/风味):它会先落进安装目录的
|
|
733
|
+
* state 文件(下次启动仍然生效),再按新选择下载。配置里写了 preset/customModel
|
|
734
|
+
* 的话,配置永远优先 —— state 只是"面板选过什么"的记忆。
|
|
735
|
+
*/
|
|
736
|
+
installVectorBackend(override = {}) {
|
|
737
|
+
const flavor = this.applyModelSelection(override)
|
|
738
|
+
const model = this.cfg.vector.installModel
|
|
739
|
+
if (detectInstalled(this.cfg.vector.installRoot, model.file)) {
|
|
740
|
+
this.adoptInstalledBackend()
|
|
741
|
+
return
|
|
742
|
+
}
|
|
743
|
+
// 换模型时把上一个安装器的进度丢掉:它描述的是另一个文件。
|
|
744
|
+
this.installer?.cancel()
|
|
745
|
+
this.installer = new VectorInstaller({
|
|
746
|
+
root: this.cfg.vector.installRoot,
|
|
747
|
+
model: { file: model.file, ...(model.bytes !== undefined ? { bytes: model.bytes } : {}), ...(model.sources !== undefined ? { sources: model.sources } : {}) },
|
|
748
|
+
...(flavor === 'auto' ? {} : { flavor }),
|
|
749
|
+
...(this.cfg.vector.installModelSources && model.sources === undefined ? { modelSources: this.cfg.vector.installModelSources } : {}),
|
|
750
|
+
onProgress: (progress) => {
|
|
751
|
+
if (progress.phase === 'done') this.adoptInstalledBackend()
|
|
752
|
+
},
|
|
753
|
+
})
|
|
754
|
+
this.installer.start()
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
/** Panel action: abort an in-flight download. */
|
|
758
|
+
cancelVectorInstall() {
|
|
759
|
+
this.installer?.cancel()
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/**
|
|
763
|
+
* What the installer put on disk, so the panel can say "释放 xxx MB" before
|
|
764
|
+
* offering to remove it. Counts only our own layout (runtime dir, models dir,
|
|
765
|
+
* log + pid), never the memory store.
|
|
766
|
+
*/
|
|
767
|
+
vectorInstallUsage() {
|
|
768
|
+
const paths = installPaths(this.cfg.vector.installRoot, this.cfg.vector.installModel.file)
|
|
769
|
+
const roots = [paths.dir, join(this.cfg.vector.installRoot, 'models')]
|
|
770
|
+
const files = [paths.logFile, `${paths.logFile}.pid`, `${paths.logFile}.err`]
|
|
771
|
+
let bytes = 0
|
|
772
|
+
const seen = []
|
|
773
|
+
for (const p of [...roots, ...files]) {
|
|
774
|
+
if (!existsSync(p)) continue
|
|
775
|
+
seen.push(p)
|
|
776
|
+
bytes += pathSize(p)
|
|
777
|
+
}
|
|
778
|
+
return { bytes, paths: seen }
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Remove the vector backend we installed (runtime + model + logs).
|
|
783
|
+
*
|
|
784
|
+
* Refuses when the configured backend lives outside installRoot: that is a
|
|
785
|
+
* server the user installed and manages, and deleting it would be destroying
|
|
786
|
+
* something we never created. The memory store is never touched — it lives
|
|
787
|
+
* beside the runtime, not inside it.
|
|
788
|
+
*/
|
|
789
|
+
uninstallVectorBackend()
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
{
|
|
795
|
+
const root = this.cfg.vector.installRoot
|
|
796
|
+
const configuredExe = this.cfg.vector.spawn.exe
|
|
797
|
+
if (configuredExe && !isInside(root, configuredExe)) {
|
|
798
|
+
return { removed: false, reason: 'EXTERNAL_BACKEND', freedBytes: 0, paths: [] }
|
|
799
|
+
}
|
|
800
|
+
const usage = this.vectorInstallUsage()
|
|
801
|
+
if (usage.paths.length === 0) {
|
|
802
|
+
return { removed: false, reason: 'NOT_INSTALLED', freedBytes: 0, paths: [] }
|
|
803
|
+
}
|
|
804
|
+
// Stop our child first: on Windows the executable and its DLLs are locked
|
|
805
|
+
// while it runs, so removal would fail with EBUSY/EPERM.
|
|
806
|
+
try { this.embedServer?.stop() } catch { /* best effort */ }
|
|
807
|
+
const removedPaths = []
|
|
808
|
+
for (const p of usage.paths) {
|
|
809
|
+
// Belt and braces: never step outside the install root, and never delete
|
|
810
|
+
// the memory store even if someone pointed storeDir inside it.
|
|
811
|
+
if (!isInside(root, p)) continue
|
|
812
|
+
if (isInside(p, this.cfg.storeDir) || resolve(p) === resolve(this.cfg.storeDir)) continue
|
|
813
|
+
try {
|
|
814
|
+
rmSync(p, { recursive: true, force: true })
|
|
815
|
+
removedPaths.push(p)
|
|
816
|
+
} catch { /* leave it; the panel reports what went */ }
|
|
817
|
+
}
|
|
818
|
+
this.cfg.vector.configured = false
|
|
819
|
+
this.cfg.vector.spawnConfigured = false
|
|
820
|
+
this.cfg.vector.enabled = false
|
|
821
|
+
this.embedServer = null
|
|
822
|
+
return { removed: true, reason: 'REMOVED', freedBytes: usage.bytes, paths: removedPaths }
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/**
|
|
826
|
+
* Delete every memory record in every scope (the panel's "delete all data"
|
|
827
|
+
* action, used when someone wants the plugin gone without leaving notes
|
|
828
|
+
* behind). Goes through the store so the JSONL + vector sidecars stay
|
|
829
|
+
* consistent instead of ripping files out from under a live store.
|
|
830
|
+
*/
|
|
831
|
+
wipeAllRecords() {
|
|
832
|
+
let removed = 0
|
|
833
|
+
const scopes = []
|
|
834
|
+
for (const scope of this.store.allScopes()) {
|
|
835
|
+
const ids = [...scope.records.keys()]
|
|
836
|
+
if (ids.length === 0) continue
|
|
837
|
+
let local = 0
|
|
838
|
+
for (const id of ids) if (scope.remove(id)) local += 1
|
|
839
|
+
removed += local
|
|
840
|
+
scopes.push(scope.scope)
|
|
841
|
+
}
|
|
842
|
+
try { this.store.flushSync() } catch { /* scheduled flush covers it */ }
|
|
843
|
+
return { removed, scopes }
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
/**
|
|
847
|
+
* Switch the provider over to a backend that is now on disk: fill the spawn
|
|
848
|
+
* config from the installed layout and start it. Idempotent.
|
|
849
|
+
*/
|
|
850
|
+
adoptInstalledBackend() {
|
|
851
|
+
const paths = detectInstalled(this.cfg.vector.installRoot, this.cfg.vector.installModel.file)
|
|
852
|
+
if (!paths) return
|
|
853
|
+
this.cfg.vector.configured = true
|
|
854
|
+
this.cfg.vector.spawnConfigured = true
|
|
855
|
+
this.cfg.vector.enabled = true
|
|
856
|
+
this.cfg.vector.spawn = {
|
|
857
|
+
...this.cfg.vector.spawn,
|
|
858
|
+
enabled: true,
|
|
859
|
+
exe: this.cfg.vector.spawn.exe ?? paths.exe,
|
|
860
|
+
// 模型路径必须跟着**当前选择的模型**:换了模型就不能继续用上一个的文件(见 pickSpawnModel)。
|
|
861
|
+
model: pickSpawnModel(this.cfg.vector.installModel.file, this.cfg.vector.spawn.model, paths.model),
|
|
862
|
+
cwd: this.cfg.vector.spawn.cwd ?? paths.cwd,
|
|
863
|
+
logFile: this.cfg.vector.spawn.logFile ?? paths.logFile,
|
|
864
|
+
pooling: this.cfg.vector.pooling,
|
|
865
|
+
gpuLayers: this.cfg.vector.flavor === 'cpu' || this.cfg.vector.flavor === 'auto' ? 0 : (this.cfg.vector.spawn.gpuLayers || 99),
|
|
866
|
+
port: this.cfg.vector.spawn.port,
|
|
867
|
+
}
|
|
868
|
+
if (!this.embedServer) {
|
|
869
|
+
this.startEmbedServer()
|
|
870
|
+
return
|
|
871
|
+
}
|
|
872
|
+
void this.ensureEmbedServer(60_000).catch(() => { /* panel shows the outcome */ })
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
/**
|
|
876
|
+
* Hourly expired-record purge + capacity eviction. M4 housekeeping; the
|
|
877
|
+
* timer is unref'd so it never holds the process open, and cleared by
|
|
878
|
+
* flushSync()/dispose.
|
|
879
|
+
*/
|
|
880
|
+
maintenanceTimer = null
|
|
881
|
+
|
|
882
|
+
startMaintenance() {
|
|
883
|
+
this.maintenanceTimer = setInterval(() => {
|
|
884
|
+
this.store.purgeExpired()
|
|
885
|
+
this.store.enforceLimit(this.cfg.maxRecordsPerScope)
|
|
886
|
+
this.store.scheduleFlush(0)
|
|
887
|
+
}, 3_600_000)
|
|
888
|
+
this.maintenanceTimer.unref?.()
|
|
889
|
+
// Pending-embed drain: the store queues ids whose embed failed (backend
|
|
890
|
+
// down at save time, or vectors restored from a jsonl-only backup). Until
|
|
891
|
+
// this timer existed nothing ever drained that queue — a single outage
|
|
892
|
+
// stranded them forever (observable as 待嵌入>0 in the memory panel).
|
|
893
|
+
this.embedRetryTimer = setInterval(() => {
|
|
894
|
+
void this.retryPendingEmbeds().catch(() => {})
|
|
895
|
+
}, 60_000)
|
|
896
|
+
this.embedRetryTimer.unref?.()
|
|
897
|
+
// First drain shortly after boot, when the backend is usually already up.
|
|
898
|
+
const initial = setTimeout(() => {
|
|
899
|
+
this.initialDrainTimer = null
|
|
900
|
+
void this.retryPendingEmbeds().catch(() => {})
|
|
901
|
+
}, 5_000)
|
|
902
|
+
initial.unref?.()
|
|
903
|
+
this.initialDrainTimer = initial
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
embedRetryTimer = null
|
|
907
|
+
initialDrainTimer = null
|
|
908
|
+
/** Owns the llama-server child process when vector.spawn.enabled. */
|
|
909
|
+
embedServer = null
|
|
910
|
+
/** Lazily created by the panel's "download and configure" action. */
|
|
911
|
+
installer = null
|
|
912
|
+
/** 硬件探测缓存(面板问了才探测,10 分钟内复用)。 */
|
|
913
|
+
hardwareCache = null
|
|
914
|
+
hardwareAt = 0
|
|
915
|
+
exitHook = null
|
|
916
|
+
|
|
917
|
+
/**
|
|
918
|
+
* Drain the pending-embed queue, probing through the circuit breaker first so
|
|
919
|
+
* a backend that came back online is noticed (backfillEmbeddings itself
|
|
920
|
+
* refuses to touch the network while marked unhealthy, and embed() is the
|
|
921
|
+
* only path that can flip that flag back to healthy).
|
|
922
|
+
*/
|
|
923
|
+
async retryPendingEmbeds() {
|
|
924
|
+
// Vector retrieval opt-in: with no configured backend there is nothing to
|
|
925
|
+
// retry, and probing would only produce failed requests in the log.
|
|
926
|
+
if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return 0
|
|
927
|
+
// Scopes load lazily, so without discovery the queue of an untouched scope
|
|
928
|
+
// (e.g. a project store written by a previous session) is invisible here and
|
|
929
|
+
// would never drain — exactly the 待嵌入>0 the panel showed on 2026-10-10.
|
|
930
|
+
try { this.store.loadDiscovered() } catch { /* discovery is best-effort */ }
|
|
931
|
+
if (this.pendingEmbedCount === 0) return 0
|
|
932
|
+
if (!this.embed.isHealthy) {
|
|
933
|
+
const probe = await this.embed.embed(['ping'], undefined, { timeoutMs: this.cfg.vector.writeTimeoutMs }).catch(() => null)
|
|
934
|
+
if (!probe) return 0
|
|
935
|
+
}
|
|
936
|
+
return this.backfillAll({ throttleMs: 100 })
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
setDefaultScope(scope ) {
|
|
940
|
+
this.defaultScope = scope
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/**
|
|
944
|
+
* Provide the one-shot LLM caller used by the rerank stage. Passing null
|
|
945
|
+
* (or never calling) keeps retrieval on recall-order only. The cordis
|
|
946
|
+
* wiring adapts `ctx.llm` into this; it is optional by design — the service
|
|
947
|
+
* must never sit PENDING just because no LLM adapter is loaded.
|
|
948
|
+
*/
|
|
949
|
+
setLlmInvoke(invoke ) {
|
|
950
|
+
this.llmInvoke = invoke
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
get activeDefaultScope() {
|
|
954
|
+
return this.defaultScope
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
assertScopeAllowed(scope ) {
|
|
958
|
+
if (scope === 'global' && !this.cfg.globalEnabled) {
|
|
959
|
+
throw memoryError('SCOPE_DISABLED', 'global scope is disabled by configuration')
|
|
960
|
+
}
|
|
961
|
+
if (scope.startsWith('project:') && !this.cfg.projectEnabled) {
|
|
962
|
+
throw memoryError('SCOPE_DISABLED', 'project scope is disabled by configuration')
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
async save(input ) {
|
|
967
|
+
const scope = input.scope ?? this.defaultScope
|
|
968
|
+
this.assertScopeAllowed(scope)
|
|
969
|
+
let content = input.content?.trim()
|
|
970
|
+
if (!content) throw memoryError('CONTENT_EMPTY', 'content must be a non-empty string')
|
|
971
|
+
if (content.length > this.cfg.maxRecordChars) {
|
|
972
|
+
throw memoryError('CONTENT_TOO_LONG', `content exceeds ${this.cfg.maxRecordChars} chars`)
|
|
973
|
+
}
|
|
974
|
+
if (input.supersedes && !this.store.scope(scope).records.get(input.supersedes)) {
|
|
975
|
+
throw memoryError('SUPERSEDES_UNKNOWN', `supersedes target ${input.supersedes} not found in ${scope}`)
|
|
976
|
+
}
|
|
977
|
+
// design §11: keep credentials out of the plaintext store and the embedder
|
|
978
|
+
if (this.cfg.secrets.enabled) {
|
|
979
|
+
const findings = scanSecrets(content)
|
|
980
|
+
if (findings.length > 0) {
|
|
981
|
+
if (this.cfg.secrets.mode === 'refuse') {
|
|
982
|
+
throw memoryError('SECRET_DETECTED', `content matches credential denylist: ${[...new Set(findings.map((f) => f.kind))].join(', ')}`)
|
|
983
|
+
}
|
|
984
|
+
content = redactSecrets(content)
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
const { record } = this.store.save(
|
|
988
|
+
{ ...input, scope, content },
|
|
989
|
+
{ importance: input.source?.origin === 'auto' ? 0.4 : 0.8, source: { origin: 'explicit' } },
|
|
990
|
+
)
|
|
991
|
+
void this.embedRecord(record).catch(() => {})
|
|
992
|
+
this.store.scheduleFlush()
|
|
993
|
+
return record
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
async embedRecord(record ) {
|
|
997
|
+
if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return
|
|
998
|
+
const s = this.store.scope(record.scope)
|
|
999
|
+
// save-time budget: one full record, much slower than a search query
|
|
1000
|
+
// 记录按"文档"嵌入:有些模型(nomic/e5)要求文档与查询带不同前缀,不带质量会掉。
|
|
1001
|
+
const vec = await this.embed.embed([`${this.cfg.vector.documentPrefix ?? ''}${record.content}`], undefined, { timeoutMs: this.cfg.vector.writeTimeoutMs }).catch(() => null)
|
|
1002
|
+
if (vec?.[0]) s.appendVector(record.id, normalizeInPlace(vec[0]))
|
|
1003
|
+
else s.embedPending.add(record.id) // backfill queue; the retry timer drains it
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
/**
|
|
1007
|
+
* Retry pending embeddings in batches; returns number embedded.
|
|
1008
|
+
* Batches are bounded by count AND total characters, and use the backfill
|
|
1009
|
+
* budget: full record contents on CPU take seconds per batch, so the
|
|
1010
|
+
* interactive timeout would abort every attempt.
|
|
1011
|
+
*/
|
|
1012
|
+
async backfillEmbeddings() {
|
|
1013
|
+
if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return 0
|
|
1014
|
+
if (!this.embed.isHealthy) return 0
|
|
1015
|
+
let done = 0
|
|
1016
|
+
for (const s of this.store.allScopes()) {
|
|
1017
|
+
if (s.embedPending.size === 0) continue
|
|
1018
|
+
const texts = []
|
|
1019
|
+
const valid = []
|
|
1020
|
+
let chars = 0
|
|
1021
|
+
for (const id of s.embedPending) {
|
|
1022
|
+
if (valid.length >= this.cfg.vector.backfillBatchSize) break
|
|
1023
|
+
const t = s.records.get(id)?.content
|
|
1024
|
+
if (!t) continue
|
|
1025
|
+
if (valid.length > 0 && chars + t.length > this.cfg.vector.backfillMaxChars) break
|
|
1026
|
+
texts.push(`${this.cfg.vector.documentPrefix ?? ''}${t}`)
|
|
1027
|
+
valid.push(id)
|
|
1028
|
+
chars += t.length
|
|
1029
|
+
}
|
|
1030
|
+
if (texts.length === 0) continue
|
|
1031
|
+
const vecs = await this.embed.embed(texts, undefined, { timeoutMs: this.cfg.vector.backfillTimeoutMs }).catch(() => null)
|
|
1032
|
+
if (!vecs) return done
|
|
1033
|
+
for (let i = 0; i < valid.length; i++) {
|
|
1034
|
+
if (s.records.has(valid[i] ) && vecs[i]) s.appendVector(valid[i] , normalizeInPlace(vecs[i] ))
|
|
1035
|
+
s.embedPending.delete(valid[i] )
|
|
1036
|
+
done++
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
return done
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
/**
|
|
1043
|
+
* First-fill: embed the ENTIRE backlog (unlike backfillEmbeddings' single
|
|
1044
|
+
* batch per scope), in throttled batches so a big import does not hammer
|
|
1045
|
+
* llama-server or block the caller. Yields to the event loop between
|
|
1046
|
+
* batches and stops early on abort/unhealthy embedder.
|
|
1047
|
+
*/
|
|
1048
|
+
async backfillAll(opts = {}) {
|
|
1049
|
+
const throttle = opts.throttleMs ?? 50
|
|
1050
|
+
let done = 0
|
|
1051
|
+
while (!opts.signal?.aborted) {
|
|
1052
|
+
const n = await this.backfillEmbeddings()
|
|
1053
|
+
done += n
|
|
1054
|
+
if (n === 0) break // per-scope batch was empty → backlog drained
|
|
1055
|
+
if (!this.embed.isHealthy) break
|
|
1056
|
+
await new Promise((r) => setTimeout(r, throttle))
|
|
1057
|
+
}
|
|
1058
|
+
return done
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
async search(query ) {
|
|
1062
|
+
const topK = query.topK ?? this.cfg.topK
|
|
1063
|
+
const recallN = query.recallN ?? this.cfg.recallN
|
|
1064
|
+
const now = Date.now()
|
|
1065
|
+
const queryTokens = [...new Set(tokenize(query.text))]
|
|
1066
|
+
|
|
1067
|
+
// candidate scopes: pinned, else project + global
|
|
1068
|
+
const stores = []
|
|
1069
|
+
if (query.scope) {
|
|
1070
|
+
stores.push({ store: this.store.scope(query.scope), isGlobal: query.scope === 'global' })
|
|
1071
|
+
} else {
|
|
1072
|
+
if (this.cfg.projectEnabled) stores.push({ store: this.store.scope(this.defaultScope), isGlobal: this.defaultScope === 'global' })
|
|
1073
|
+
if (this.cfg.globalEnabled) stores.push({ store: this.store.scope('global'), isGlobal: true })
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
// --- channel 1: lexical recall, scored per scope with local IDF ---
|
|
1077
|
+
|
|
1078
|
+
const candidates = new Map ()
|
|
1079
|
+
for (const { store, isGlobal } of stores) {
|
|
1080
|
+
const all = [...store.records.values()]
|
|
1081
|
+
const tokenSets = all.map((r) => new Set(tokenize(r.content)))
|
|
1082
|
+
const stats = buildStats(tokenSets)
|
|
1083
|
+
for (let i = 0; i < all.length; i++) {
|
|
1084
|
+
const rec = all[i]
|
|
1085
|
+
if (rec.expiredAt && rec.expiredAt < now) continue
|
|
1086
|
+
const lex = lexicalScore(queryTokens, tokenSets[i] , stats)
|
|
1087
|
+
if (lex <= 0) continue
|
|
1088
|
+
// Weak-only overlap (no shared word/bigram) must clear the "real hit"
|
|
1089
|
+
// floor, not the permissive threshold — kills single-char collisions.
|
|
1090
|
+
const floor = isGlobal ? this.cfg.globalThreshold : this.cfg.projectThreshold
|
|
1091
|
+
const effective = hasStrongHit(queryTokens, tokenSets[i] ) ? floor : Math.max(floor, REAL_HIT_FLOOR)
|
|
1092
|
+
if (lex < effective) continue
|
|
1093
|
+
candidates.set(rec.id, { record: rec, recallScore: lex, isGlobal })
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
// --- channel 2: vector recall over embedded records ---
|
|
1098
|
+
// Skipped entirely when no backend is configured (lexical recall above
|
|
1099
|
+
// already produced the candidates; a request here would just fail).
|
|
1100
|
+
const qvec = this.cfg.vector.enabled && this.cfg.vector.configured
|
|
1101
|
+
? await this.embed.embed([`${this.cfg.vector.queryPrefix ?? ''}${query.text}`], query.signal).catch(() => null)
|
|
1102
|
+
: null
|
|
1103
|
+
if (qvec?.[0]) {
|
|
1104
|
+
const vectorScores = new Map ()
|
|
1105
|
+
for (const { store } of stores) {
|
|
1106
|
+
for (const [id, vec] of store.vectors) {
|
|
1107
|
+
if (!store.records.has(id)) continue
|
|
1108
|
+
const sim = cosine(qvec[0] , vec)
|
|
1109
|
+
if (sim >= this.cfg.vector.minScore) vectorScores.set(id, sim)
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
for (const [id, sim] of vectorScores) {
|
|
1113
|
+
const existing = candidates.get(id)
|
|
1114
|
+
if (existing) {
|
|
1115
|
+
existing.recallScore = Math.max(existing.recallScore, sim) // best channel wins
|
|
1116
|
+
} else {
|
|
1117
|
+
const rec = this.findById(id)
|
|
1118
|
+
if (rec && !(rec.expiredAt && rec.expiredAt < now)) {
|
|
1119
|
+
if (!(rec.scope === 'global' && sim < this.cfg.globalThreshold)) {
|
|
1120
|
+
candidates.set(id, { record: rec, recallScore: sim, isGlobal: rec.scope === 'global' })
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
// shape with recency/importance, sort, cut
|
|
1128
|
+
const supersededIds = new Set ()
|
|
1129
|
+
for (const { store } of stores) {
|
|
1130
|
+
for (const r of store.records.values()) if (r.supersedes) supersededIds.add(r.supersedes)
|
|
1131
|
+
}
|
|
1132
|
+
const shape = (c ) =>
|
|
1133
|
+
recordScore(c.recallScore, c.record.importance, c.record.accessCount, c.record.lastAccessedAt, {
|
|
1134
|
+
now,
|
|
1135
|
+
halfLifeDays: this.cfg.halfLifeDays,
|
|
1136
|
+
superseded: supersededIds.has(c.record.id),
|
|
1137
|
+
})
|
|
1138
|
+
|
|
1139
|
+
const ranked = [...candidates.values()].sort((a, b) => shape(b) - shape(a)).slice(0, recallN)
|
|
1140
|
+
|
|
1141
|
+
// --- channel 3: LLM rerank (optional; any skip/failure keeps recall order) ---
|
|
1142
|
+
const finalScores = new Map (ranked.map((c) => [c.id, shape(c)]))
|
|
1143
|
+
if (this.cfg.llm.enabled && this.llmInvoke && ranked.length >= 2) {
|
|
1144
|
+
const asCands = ranked.map((c) => ({ id: c.id, content: c.record.content, recallScore: finalScores.get(c.id) }))
|
|
1145
|
+
const cacheKey = RerankCache.key(query.text, ranked.map((c) => c.id), this.cfg.llm.model)
|
|
1146
|
+
let outcome = this.rerankCache.get(cacheKey)
|
|
1147
|
+
if (!outcome) {
|
|
1148
|
+
outcome = await rerank({
|
|
1149
|
+
invoke: this.llmInvoke,
|
|
1150
|
+
cfg: this.cfg.llm,
|
|
1151
|
+
query: query.text,
|
|
1152
|
+
candidates: asCands,
|
|
1153
|
+
budget: this.rerankBudget,
|
|
1154
|
+
signal: query.signal,
|
|
1155
|
+
})
|
|
1156
|
+
if (outcome) this.rerankCache.set(cacheKey, outcome)
|
|
1157
|
+
}
|
|
1158
|
+
if (outcome) {
|
|
1159
|
+
for (const { id, score } of blendScores(asCands, outcome, this.cfg.llm.weight)) {
|
|
1160
|
+
finalScores.set(id, score)
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
const hits = ranked.map((c) => ({ record: touch(c.record), score: round4(finalScores.get(c.id) ) }))
|
|
1166
|
+
|
|
1167
|
+
this.store.scheduleFlush()
|
|
1168
|
+
return hits.slice(0, topK)
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
findById(id ) {
|
|
1172
|
+
for (const s of this.store.allScopes()) {
|
|
1173
|
+
const r = s.records.get(id)
|
|
1174
|
+
if (r) return r
|
|
1175
|
+
}
|
|
1176
|
+
return undefined
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
async forget(input ) {
|
|
1180
|
+
const ids = []
|
|
1181
|
+
if (input.id) {
|
|
1182
|
+
for (const s of this.store.allScopes()) {
|
|
1183
|
+
if (s.remove(input.id)) ids.push(input.id)
|
|
1184
|
+
}
|
|
1185
|
+
if (ids.length === 0) throw memoryError('RECORD_NOT_FOUND', `id ${input.id} not found`)
|
|
1186
|
+
} else if (input.query) {
|
|
1187
|
+
const scope = input.scope ?? this.defaultScope
|
|
1188
|
+
ids.push(...this.store.findIdsByQuery(scope, input.query))
|
|
1189
|
+
const s = this.store.scope(scope)
|
|
1190
|
+
for (const id of ids) s.remove(id)
|
|
1191
|
+
if (ids.length === 0) throw memoryError('RECORD_NOT_FOUND', `no record matching query in ${scope}`)
|
|
1192
|
+
} else {
|
|
1193
|
+
throw memoryError('RECORD_NOT_FOUND', 'forget requires id or query')
|
|
1194
|
+
}
|
|
1195
|
+
this.store.scheduleFlush()
|
|
1196
|
+
return { removed: ids.length, ids }
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
async list(input = {}) {
|
|
1200
|
+
const scope = input.scope ?? this.defaultScope
|
|
1201
|
+
let records = [...this.store.scope(scope).records.values()]
|
|
1202
|
+
if (input.kind) records = records.filter((r) => r.kind === input.kind)
|
|
1203
|
+
records.sort((a, b) => b.updatedAt - a.updatedAt)
|
|
1204
|
+
return records.slice(0, input.limit ?? 50)
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
/** Force a synchronous flush (dispose/HMR path). */
|
|
1208
|
+
flushSync() {
|
|
1209
|
+
if (this.maintenanceTimer) {
|
|
1210
|
+
clearInterval(this.maintenanceTimer)
|
|
1211
|
+
this.maintenanceTimer = null
|
|
1212
|
+
}
|
|
1213
|
+
if (this.embedRetryTimer) {
|
|
1214
|
+
clearInterval(this.embedRetryTimer)
|
|
1215
|
+
this.embedRetryTimer = null
|
|
1216
|
+
}
|
|
1217
|
+
if (this.initialDrainTimer) {
|
|
1218
|
+
clearTimeout(this.initialDrainTimer)
|
|
1219
|
+
this.initialDrainTimer = null
|
|
1220
|
+
}
|
|
1221
|
+
if (this.embedServer) this.embedServer.stop()
|
|
1222
|
+
if (this.exitHook) {
|
|
1223
|
+
process.removeListener('exit', this.exitHook)
|
|
1224
|
+
this.exitHook = null
|
|
1225
|
+
}
|
|
1226
|
+
this.store.purgeExpired()
|
|
1227
|
+
this.store.enforceLimit(this.cfg.maxRecordsPerScope)
|
|
1228
|
+
this.store.close()
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
get pendingEmbedCount() {
|
|
1232
|
+
return this.store.allScopes().reduce((n, s) => n + s.embedPending.size, 0)
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
function touch(record ) {
|
|
1237
|
+
record.lastAccessedAt = Date.now()
|
|
1238
|
+
record.accessCount += 1
|
|
1239
|
+
return record
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
function round4(n ) {
|
|
1243
|
+
return Math.round(n * 10_000) / 10_000
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
/** Port of an http(s) endpoint, or null when it cannot be parsed. */
|
|
1247
|
+
function portOf(endpoint ) {
|
|
1248
|
+
try {
|
|
1249
|
+
const port = new URL(endpoint).port
|
|
1250
|
+
return port ? Number.parseInt(port, 10) : null
|
|
1251
|
+
} catch {
|
|
1252
|
+
return null
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
/**
|
|
1257
|
+
* Measured score floor of genuine lexical hits; below this only single-char
|
|
1258
|
+
* coincidences live. Used to gate weak-only overlaps regardless of config.
|
|
1259
|
+
*/
|
|
1260
|
+
const REAL_HIT_FLOOR = 0.25
|
|
1261
|
+
|
|
1262
|
+
function memoryError(code , message ) {
|
|
1263
|
+
const err = new Error(`[memory:${code}] ${message}`)
|
|
1264
|
+
err.code = code
|
|
1265
|
+
return err
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
export function createMemoryService(config = {}) {
|
|
1269
|
+
return new LocalMemoryService(config)
|
|
1270
|
+
}
|
|
28
1271
|
|
|
29
|
-
/** Recursive byte count; 0 for anything unreadable. */
|
|
30
|
-
function pathSize(path ) {
|
|
31
|
-
try {
|
|
32
|
-
const st = statSync(path)
|
|
33
|
-
if (!st.isDirectory()) return st.size
|
|
34
|
-
let total = 0
|
|
35
|
-
for (const entry of readdirSync(path)) total += pathSize(join(path, entry))
|
|
36
|
-
return total
|
|
37
|
-
} catch {
|
|
38
|
-
return 0
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
1272
|
|
|
42
|
-
|
|
43
|
-
function isInside(parent , child ) {
|
|
44
|
-
const rel = relative(resolve(parent), resolve(child))
|
|
45
|
-
return rel === '' || (!rel.startsWith('..') && !isAbsolute(rel))
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
export function resolveConfig(config = {}) {
|
|
138
|
-
const v = config.retrieval?.vector ?? {}
|
|
139
|
-
// Do not touch an embedding backend the user never asked for: pointing at the
|
|
140
|
-
// stock localhost endpoint on a machine that has nothing listening means
|
|
141
|
-
// failed embeds, a tripped breaker and scary log lines for a feature that is
|
|
142
|
-
// optional. Vector retrieval is therefore opt-in by CONFIGURATION — writing
|
|
143
|
-
// `endpoint` (any OpenAI-compatible service), configuring `spawn`, or having
|
|
144
|
-
// the panel's installer put a backend on disk implies intent; an explicit
|
|
145
|
-
// `enabled` always wins.
|
|
146
|
-
const storeDir = config.storeDir ?? '.dsh-memory'
|
|
147
|
-
// Installer target: explicit config → that directory; explicit storeDir →
|
|
148
|
-
// its parent (so a hand-made `<root>/llamacpp` + `<root>/models` layout is
|
|
149
|
-
// recognised as-is); otherwise a stable per-user directory under ~/.dsh.
|
|
150
|
-
const installRoot = v.autoInstall?.dir
|
|
151
|
-
?? (config.storeDir !== undefined ? dirname(resolve(storeDir)) : join(homedir(), '.dsh', 'memory-vector'))
|
|
152
|
-
const installed = detectInstalled(installRoot)
|
|
153
|
-
const endpointConfigured = v.endpoint !== undefined
|
|
154
|
-
// A configured spawn path that no longer exists (plugin uninstalled the
|
|
155
|
-
// backend, or the user moved it) must not count as configured: that would
|
|
156
|
-
// leave the panel showing a managed server that can never start, instead of
|
|
157
|
-
// offering the download button again.
|
|
158
|
-
const spawnExe = v.spawn?.exe
|
|
159
|
-
const spawnExeUsable = spawnExe === undefined || spawnExe === '' || existsSync(spawnExe)
|
|
160
|
-
const spawnConfigured = ((v.spawn?.enabled === true || (spawnExe ?? '') !== '') && spawnExeUsable) || installed !== null
|
|
161
|
-
const configured = endpointConfigured || spawnConfigured
|
|
162
|
-
const vectorEnabled = v.enabled ?? configured
|
|
163
|
-
return {
|
|
164
|
-
storeDir,
|
|
165
|
-
globalEnabled: config.scopes?.global ?? true,
|
|
166
|
-
projectEnabled: config.scopes?.project ?? true,
|
|
167
|
-
topK: config.retrieval?.topK ?? 5,
|
|
168
|
-
recallN: config.retrieval?.recallN ?? 30,
|
|
169
|
-
globalThreshold: config.retrieval?.globalThreshold ?? 0.12,
|
|
170
|
-
projectThreshold: config.retrieval?.projectThreshold ?? 0.1,
|
|
171
|
-
halfLifeDays: config.retrieval?.recencyHalfLifeDays ?? 30,
|
|
172
|
-
vector: {
|
|
173
|
-
enabled: vectorEnabled,
|
|
174
|
-
configured,
|
|
175
|
-
endpointConfigured,
|
|
176
|
-
spawnConfigured,
|
|
177
|
-
installRoot,
|
|
178
|
-
...(v.autoInstall?.modelSources ? { installModelSources: v.autoInstall.modelSources } : {}),
|
|
179
|
-
endpoint: v.endpoint ?? 'http://127.0.0.1:8078/v1/embeddings',
|
|
180
|
-
model: v.model ?? 'bge-m3',
|
|
181
|
-
dims: v.dims ?? 1024,
|
|
182
|
-
timeoutMs: v.timeoutMs ?? 300,
|
|
183
|
-
// Full contents are orders of magnitude slower than a short query on CPU
|
|
184
|
-
// (measured 3143ms for three ~600-char records), so write/backfill paths
|
|
185
|
-
// get their own generous budgets instead of the interactive 300ms.
|
|
186
|
-
writeTimeoutMs: v.writeTimeoutMs ?? 10_000,
|
|
187
|
-
backfillTimeoutMs: v.backfillTimeoutMs ?? 30_000,
|
|
188
|
-
backfillMaxChars: v.backfillMaxChars ?? 8_000,
|
|
189
|
-
// BGE CLS baseline noise floor measured ~0.42 on unrelated pairs
|
|
190
|
-
minScore: v.minScore ?? 0.5,
|
|
191
|
-
backfillBatchSize: v.backfillBatchSize ?? 32,
|
|
192
|
-
// Port defaults to the one in the embeddings endpoint so the two can never
|
|
193
|
-
// drift apart; opt-in (enabled:false) keeps dev/test runs from spawning.
|
|
194
|
-
// A backend the panel installed (or a hand-made one at installRoot) is
|
|
195
|
-
// adopted as the spawn target when the user configured no explicit paths.
|
|
196
|
-
spawn: {
|
|
197
|
-
...(v.spawn ?? {}),
|
|
198
|
-
enabled: v.spawn?.enabled ?? installed !== null,
|
|
199
|
-
...(installed
|
|
200
|
-
? {
|
|
201
|
-
exe: v.spawn?.exe ?? installed.exe,
|
|
202
|
-
model: v.spawn?.model ?? installed.model,
|
|
203
|
-
cwd: v.spawn?.cwd ?? installed.cwd,
|
|
204
|
-
logFile: v.spawn?.logFile ?? installed.logFile,
|
|
205
|
-
}
|
|
206
|
-
: {}),
|
|
207
|
-
port: v.spawn?.port ?? portOf(v.endpoint ?? 'http://127.0.0.1:8078/v1/embeddings') ?? 8078,
|
|
208
|
-
},
|
|
209
|
-
},
|
|
210
|
-
llm: {
|
|
211
|
-
enabled: config.retrieval?.llm?.enabled ?? false,
|
|
212
|
-
provider: config.retrieval?.llm?.provider ?? '',
|
|
213
|
-
model: config.retrieval?.llm?.model ?? '',
|
|
214
|
-
timeoutMs: config.retrieval?.llm?.timeoutMs ?? 1500,
|
|
215
|
-
dailyBudgetTokens: config.retrieval?.llm?.dailyBudgetTokens ?? 200_000,
|
|
216
|
-
weight: config.retrieval?.llm?.weight ?? 0.6,
|
|
217
|
-
topN: config.retrieval?.llm?.topN ?? 10,
|
|
218
|
-
},
|
|
219
|
-
maxRecordsPerScope: config.limits?.maxRecordsPerScope ?? 2000,
|
|
220
|
-
maxRecordChars: config.limits?.maxRecordChars ?? 4000,
|
|
221
|
-
secrets: {
|
|
222
|
-
enabled: config.secrets?.enabled ?? true,
|
|
223
|
-
mode: config.secrets?.mode ?? 'refuse',
|
|
224
|
-
},
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
export class LocalMemoryService {
|
|
229
|
-
cfg
|
|
230
|
-
store
|
|
231
|
-
embed
|
|
232
|
-
rerankBudget
|
|
233
|
-
rerankCache = new RerankCache()
|
|
234
|
-
/** Host-provided one-shot LLM call; null = rerank disabled (recall order). */
|
|
235
|
-
llmInvoke = null
|
|
236
|
-
/** Where saves land when the caller does not pin a scope. */
|
|
237
|
-
defaultScope = 'global'
|
|
238
|
-
|
|
239
|
-
constructor(config = {}) {
|
|
240
|
-
this.cfg = resolveConfig(config)
|
|
241
|
-
this.rerankBudget = new DailyBudget(this.cfg.llm.dailyBudgetTokens)
|
|
242
|
-
this.store = new MemoryStore(this.cfg.storeDir, { onWarn: (m) => console.warn(`[dsh-memory-local] ${m}`) })
|
|
243
|
-
this.store.loadAll(this.cfg.projectEnabled && this.cfg.globalEnabled ? [this.defaultScope, 'global'] : [this.defaultScope])
|
|
244
|
-
this.embed = new EmbedClient({
|
|
245
|
-
endpoint: this.cfg.vector.endpoint,
|
|
246
|
-
model: this.cfg.vector.model,
|
|
247
|
-
dims: this.cfg.vector.dims,
|
|
248
|
-
timeoutMs: this.cfg.vector.timeoutMs,
|
|
249
|
-
})
|
|
250
|
-
// sidecar validation: model/dims change discards old vectors on load
|
|
251
|
-
this.store.setVecIdentity(this.cfg.vector.model, this.cfg.vector.dims)
|
|
252
|
-
this.startMaintenance()
|
|
253
|
-
this.startEmbedServer()
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
/** Own the embedding backend's lifetime: spawn on activation, stop on dispose. */
|
|
257
|
-
startEmbedServer() {
|
|
258
|
-
// Nothing configured and retrieval switched off → do not even build the
|
|
259
|
-
// manager, so no probe, no spawn attempt, no misleading panel state.
|
|
260
|
-
if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return
|
|
261
|
-
this.embedServer = new EmbedServerManager(this.cfg.vector.spawn)
|
|
262
|
-
// fire-and-forget: activation must never block on a model load
|
|
263
|
-
void this.ensureEmbedServer(60_000).catch(() => { /* lifecycle never breaks activation */ })
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
/**
|
|
267
|
-
* Start (or adopt) the embedding backend and arm the exit safety net.
|
|
268
|
-
* Shared by the activation path and the panel's manual start button.
|
|
269
|
-
*/
|
|
270
|
-
async ensureEmbedServer(waitMs ) {
|
|
271
|
-
const mgr = this.embedServer
|
|
272
|
-
if (!mgr) return 'disabled'
|
|
273
|
-
const state = await mgr.ensureRunning()
|
|
274
|
-
if (state !== 'spawned' && state !== 'adopted-managed') return state
|
|
275
|
-
// hard-exit safety net: cordis disposal covers a clean quit; this covers
|
|
276
|
-
// the host dying without disposing us (the child must not be orphaned)
|
|
277
|
-
if (!this.exitHook) {
|
|
278
|
-
this.exitHook = () => { this.embedServer?.stop() }
|
|
279
|
-
process.once('exit', this.exitHook)
|
|
280
|
-
}
|
|
281
|
-
const ready = await mgr.waitUntilReady(waitMs)
|
|
282
|
-
if (!ready) console.warn('[dsh-memory-local] embed server: 启动超时,检索继续走词法降级')
|
|
283
|
-
return state
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
/** Manual start from the memory panel (same semantics as activation). */
|
|
287
|
-
startEmbedServerManually() {
|
|
288
|
-
return this.ensureEmbedServer(25_000)
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
/**
|
|
292
|
-
* Manual stop from the memory panel. An instance we do not own is left alone
|
|
293
|
-
* (it was started outside the app), so the UI can explain the refusal.
|
|
294
|
-
*/
|
|
295
|
-
stopEmbedServerManually() {
|
|
296
|
-
const mgr = this.embedServer
|
|
297
|
-
if (!mgr) return { stopped: false, reason: 'NOT_RUNNING' }
|
|
298
|
-
if (!mgr.isOwned) return { stopped: false, reason: 'EXTERNAL_INSTANCE' }
|
|
299
|
-
return { stopped: mgr.stop(), reason: mgr.isOwned ? 'NOT_RUNNING' : 'STOPPED' }
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
/**
|
|
303
|
-
* Lifecycle snapshot for the memory panel.
|
|
304
|
-
*
|
|
305
|
-
* `mode` is what the panel needs to decide whether a start/stop switch even
|
|
306
|
-
* makes sense:
|
|
307
|
-
* - 'off' nothing configured → no switch, show how to opt in
|
|
308
|
-
* - 'external' we only know an endpoint (cloud or a server the user runs)
|
|
309
|
-
* → no switch, we must not claim ownership of that process
|
|
310
|
-
* - 'managed' spawn is configured → the switch toggles OUR child process
|
|
311
|
-
* `enabled` is the retrieval switch from config; when it is false while a
|
|
312
|
-
* backend IS configured the panel explains the config instead of showing a
|
|
313
|
-
* switch that cannot turn retrieval on.
|
|
314
|
-
*/
|
|
315
|
-
async embedServerStatus()
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
{
|
|
331
|
-
const mgr = this.embedServer
|
|
332
|
-
const cfg = this.cfg.vector
|
|
333
|
-
const pendingEmbeds = this.pendingEmbedCount
|
|
334
|
-
const mode = !cfg.configured ? 'off' : cfg.spawnConfigured ? 'managed' : 'external'
|
|
335
|
-
const usage = this.vectorInstallUsage()
|
|
336
|
-
const base = {
|
|
337
|
-
enabled: cfg.enabled,
|
|
338
|
-
configured: cfg.configured,
|
|
339
|
-
mode,
|
|
340
|
-
endpoint: cfg.endpoint,
|
|
341
|
-
model: cfg.model,
|
|
342
|
-
spawnConfigured: cfg.spawnConfigured,
|
|
343
|
-
pendingEmbeds,
|
|
344
|
-
install: this.installSnapshot(),
|
|
345
|
-
maintenance: {
|
|
346
|
-
usageBytes: usage.bytes,
|
|
347
|
-
paths: usage.paths,
|
|
348
|
-
// NOTE: `cfg` is the VECTOR config here, not the whole config — reading
|
|
349
|
-
// cfg.storeDir would silently yield undefined (and JSON drops it).
|
|
350
|
-
storeDir: this.cfg.storeDir,
|
|
351
|
-
records: this.store.allScopes().reduce((n, s) => n + s.records.size, 0),
|
|
352
|
-
},
|
|
353
|
-
}
|
|
354
|
-
if (!mgr) return { ...base, owned: false, pid: null, baseUrl: '', reachable: false }
|
|
355
|
-
const reachable = await mgr.healthCheck()
|
|
356
|
-
return { ...base, ...mgr.info(), reachable, pendingEmbeds }
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
/** Installer state for the panel (idle when nothing was ever requested). */
|
|
360
|
-
installSnapshot() {
|
|
361
|
-
const root = this.cfg.vector.installRoot
|
|
362
|
-
const installed = detectInstalled(root) !== null
|
|
363
|
-
const progress = this.installer?.progress ?? {
|
|
364
|
-
phase: 'idle' ,
|
|
365
|
-
message: '',
|
|
366
|
-
receivedBytes: 0,
|
|
367
|
-
totalBytes: null,
|
|
368
|
-
error: null,
|
|
369
|
-
paths: installed ? installPaths(root) : null,
|
|
370
|
-
}
|
|
371
|
-
return { ...progress, installed, root }
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
/**
|
|
375
|
-
* Panel action: download llama.cpp + the GGML model into installRoot, then
|
|
376
|
-
* adopt them (start the managed server). Never runs on its own — a fresh
|
|
377
|
-
* install downloads nothing until the user asks.
|
|
378
|
-
*/
|
|
379
|
-
installVectorBackend() {
|
|
380
|
-
if (detectInstalled(this.cfg.vector.installRoot)) {
|
|
381
|
-
this.adoptInstalledBackend()
|
|
382
|
-
return
|
|
383
|
-
}
|
|
384
|
-
if (!this.installer) {
|
|
385
|
-
this.installer = new VectorInstaller({
|
|
386
|
-
root: this.cfg.vector.installRoot,
|
|
387
|
-
...(this.cfg.vector.installModelSources ? { modelSources: this.cfg.vector.installModelSources } : {}),
|
|
388
|
-
onProgress: (progress) => {
|
|
389
|
-
if (progress.phase === 'done') this.adoptInstalledBackend()
|
|
390
|
-
},
|
|
391
|
-
})
|
|
392
|
-
}
|
|
393
|
-
this.installer.start()
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
/** Panel action: abort an in-flight download. */
|
|
397
|
-
cancelVectorInstall() {
|
|
398
|
-
this.installer?.cancel()
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
/**
|
|
402
|
-
* What the installer put on disk, so the panel can say "释放 xxx MB" before
|
|
403
|
-
* offering to remove it. Counts only our own layout (runtime dir, models dir,
|
|
404
|
-
* log + pid), never the memory store.
|
|
405
|
-
*/
|
|
406
|
-
vectorInstallUsage() {
|
|
407
|
-
const paths = installPaths(this.cfg.vector.installRoot)
|
|
408
|
-
const roots = [paths.dir, join(this.cfg.vector.installRoot, 'models')]
|
|
409
|
-
const files = [paths.logFile, `${paths.logFile}.pid`, `${paths.logFile}.err`]
|
|
410
|
-
let bytes = 0
|
|
411
|
-
const seen = []
|
|
412
|
-
for (const p of [...roots, ...files]) {
|
|
413
|
-
if (!existsSync(p)) continue
|
|
414
|
-
seen.push(p)
|
|
415
|
-
bytes += pathSize(p)
|
|
416
|
-
}
|
|
417
|
-
return { bytes, paths: seen }
|
|
418
|
-
}
|
|
419
|
-
|
|
420
|
-
/**
|
|
421
|
-
* Remove the vector backend we installed (runtime + model + logs).
|
|
422
|
-
*
|
|
423
|
-
* Refuses when the configured backend lives outside installRoot: that is a
|
|
424
|
-
* server the user installed and manages, and deleting it would be destroying
|
|
425
|
-
* something we never created. The memory store is never touched — it lives
|
|
426
|
-
* beside the runtime, not inside it.
|
|
427
|
-
*/
|
|
428
|
-
uninstallVectorBackend()
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
{
|
|
434
|
-
const root = this.cfg.vector.installRoot
|
|
435
|
-
const configuredExe = this.cfg.vector.spawn.exe
|
|
436
|
-
if (configuredExe && !isInside(root, configuredExe)) {
|
|
437
|
-
return { removed: false, reason: 'EXTERNAL_BACKEND', freedBytes: 0, paths: [] }
|
|
438
|
-
}
|
|
439
|
-
const usage = this.vectorInstallUsage()
|
|
440
|
-
if (usage.paths.length === 0) {
|
|
441
|
-
return { removed: false, reason: 'NOT_INSTALLED', freedBytes: 0, paths: [] }
|
|
442
|
-
}
|
|
443
|
-
// Stop our child first: on Windows the executable and its DLLs are locked
|
|
444
|
-
// while it runs, so removal would fail with EBUSY/EPERM.
|
|
445
|
-
try { this.embedServer?.stop() } catch { /* best effort */ }
|
|
446
|
-
const removedPaths = []
|
|
447
|
-
for (const p of usage.paths) {
|
|
448
|
-
// Belt and braces: never step outside the install root, and never delete
|
|
449
|
-
// the memory store even if someone pointed storeDir inside it.
|
|
450
|
-
if (!isInside(root, p)) continue
|
|
451
|
-
if (isInside(p, this.cfg.storeDir) || resolve(p) === resolve(this.cfg.storeDir)) continue
|
|
452
|
-
try {
|
|
453
|
-
rmSync(p, { recursive: true, force: true })
|
|
454
|
-
removedPaths.push(p)
|
|
455
|
-
} catch { /* leave it; the panel reports what went */ }
|
|
456
|
-
}
|
|
457
|
-
this.cfg.vector.configured = false
|
|
458
|
-
this.cfg.vector.spawnConfigured = false
|
|
459
|
-
this.cfg.vector.enabled = false
|
|
460
|
-
this.embedServer = null
|
|
461
|
-
return { removed: true, reason: 'REMOVED', freedBytes: usage.bytes, paths: removedPaths }
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
/**
|
|
465
|
-
* Delete every memory record in every scope (the panel's "delete all data"
|
|
466
|
-
* action, used when someone wants the plugin gone without leaving notes
|
|
467
|
-
* behind). Goes through the store so the JSONL + vector sidecars stay
|
|
468
|
-
* consistent instead of ripping files out from under a live store.
|
|
469
|
-
*/
|
|
470
|
-
wipeAllRecords() {
|
|
471
|
-
let removed = 0
|
|
472
|
-
const scopes = []
|
|
473
|
-
for (const scope of this.store.allScopes()) {
|
|
474
|
-
const ids = [...scope.records.keys()]
|
|
475
|
-
if (ids.length === 0) continue
|
|
476
|
-
let local = 0
|
|
477
|
-
for (const id of ids) if (scope.remove(id)) local += 1
|
|
478
|
-
removed += local
|
|
479
|
-
scopes.push(scope.scope)
|
|
480
|
-
}
|
|
481
|
-
try { this.store.flushSync() } catch { /* scheduled flush covers it */ }
|
|
482
|
-
return { removed, scopes }
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
/**
|
|
486
|
-
* Switch the provider over to a backend that is now on disk: fill the spawn
|
|
487
|
-
* config from the installed layout and start it. Idempotent.
|
|
488
|
-
*/
|
|
489
|
-
adoptInstalledBackend() {
|
|
490
|
-
const paths = detectInstalled(this.cfg.vector.installRoot)
|
|
491
|
-
if (!paths) return
|
|
492
|
-
this.cfg.vector.configured = true
|
|
493
|
-
this.cfg.vector.spawnConfigured = true
|
|
494
|
-
this.cfg.vector.enabled = true
|
|
495
|
-
this.cfg.vector.spawn = {
|
|
496
|
-
...this.cfg.vector.spawn,
|
|
497
|
-
enabled: true,
|
|
498
|
-
exe: this.cfg.vector.spawn.exe ?? paths.exe,
|
|
499
|
-
model: this.cfg.vector.spawn.model ?? paths.model,
|
|
500
|
-
cwd: this.cfg.vector.spawn.cwd ?? paths.cwd,
|
|
501
|
-
logFile: this.cfg.vector.spawn.logFile ?? paths.logFile,
|
|
502
|
-
port: this.cfg.vector.spawn.port,
|
|
503
|
-
}
|
|
504
|
-
if (!this.embedServer) {
|
|
505
|
-
this.startEmbedServer()
|
|
506
|
-
return
|
|
507
|
-
}
|
|
508
|
-
void this.ensureEmbedServer(60_000).catch(() => { /* panel shows the outcome */ })
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
/**
|
|
512
|
-
* Hourly expired-record purge + capacity eviction. M4 housekeeping; the
|
|
513
|
-
* timer is unref'd so it never holds the process open, and cleared by
|
|
514
|
-
* flushSync()/dispose.
|
|
515
|
-
*/
|
|
516
|
-
maintenanceTimer = null
|
|
517
|
-
|
|
518
|
-
startMaintenance() {
|
|
519
|
-
this.maintenanceTimer = setInterval(() => {
|
|
520
|
-
this.store.purgeExpired()
|
|
521
|
-
this.store.enforceLimit(this.cfg.maxRecordsPerScope)
|
|
522
|
-
this.store.scheduleFlush(0)
|
|
523
|
-
}, 3_600_000)
|
|
524
|
-
this.maintenanceTimer.unref?.()
|
|
525
|
-
// Pending-embed drain: the store queues ids whose embed failed (backend
|
|
526
|
-
// down at save time, or vectors restored from a jsonl-only backup). Until
|
|
527
|
-
// this timer existed nothing ever drained that queue — a single outage
|
|
528
|
-
// stranded them forever (observable as 待嵌入>0 in the memory panel).
|
|
529
|
-
this.embedRetryTimer = setInterval(() => {
|
|
530
|
-
void this.retryPendingEmbeds().catch(() => {})
|
|
531
|
-
}, 60_000)
|
|
532
|
-
this.embedRetryTimer.unref?.()
|
|
533
|
-
// First drain shortly after boot, when the backend is usually already up.
|
|
534
|
-
const initial = setTimeout(() => {
|
|
535
|
-
this.initialDrainTimer = null
|
|
536
|
-
void this.retryPendingEmbeds().catch(() => {})
|
|
537
|
-
}, 5_000)
|
|
538
|
-
initial.unref?.()
|
|
539
|
-
this.initialDrainTimer = initial
|
|
540
|
-
}
|
|
541
|
-
|
|
542
|
-
embedRetryTimer = null
|
|
543
|
-
initialDrainTimer = null
|
|
544
|
-
/** Owns the llama-server child process when vector.spawn.enabled. */
|
|
545
|
-
embedServer = null
|
|
546
|
-
/** Lazily created by the panel's "download and configure" action. */
|
|
547
|
-
installer = null
|
|
548
|
-
exitHook = null
|
|
549
|
-
|
|
550
|
-
/**
|
|
551
|
-
* Drain the pending-embed queue, probing through the circuit breaker first so
|
|
552
|
-
* a backend that came back online is noticed (backfillEmbeddings itself
|
|
553
|
-
* refuses to touch the network while marked unhealthy, and embed() is the
|
|
554
|
-
* only path that can flip that flag back to healthy).
|
|
555
|
-
*/
|
|
556
|
-
async retryPendingEmbeds() {
|
|
557
|
-
// Vector retrieval opt-in: with no configured backend there is nothing to
|
|
558
|
-
// retry, and probing would only produce failed requests in the log.
|
|
559
|
-
if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return 0
|
|
560
|
-
// Scopes load lazily, so without discovery the queue of an untouched scope
|
|
561
|
-
// (e.g. a project store written by a previous session) is invisible here and
|
|
562
|
-
// would never drain — exactly the 待嵌入>0 the panel showed on 2026-10-10.
|
|
563
|
-
try { this.store.loadDiscovered() } catch { /* discovery is best-effort */ }
|
|
564
|
-
if (this.pendingEmbedCount === 0) return 0
|
|
565
|
-
if (!this.embed.isHealthy) {
|
|
566
|
-
const probe = await this.embed.embed(['ping'], undefined, { timeoutMs: this.cfg.vector.writeTimeoutMs }).catch(() => null)
|
|
567
|
-
if (!probe) return 0
|
|
568
|
-
}
|
|
569
|
-
return this.backfillAll({ throttleMs: 100 })
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
setDefaultScope(scope ) {
|
|
573
|
-
this.defaultScope = scope
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
/**
|
|
577
|
-
* Provide the one-shot LLM caller used by the rerank stage. Passing null
|
|
578
|
-
* (or never calling) keeps retrieval on recall-order only. The cordis
|
|
579
|
-
* wiring adapts `ctx.llm` into this; it is optional by design — the service
|
|
580
|
-
* must never sit PENDING just because no LLM adapter is loaded.
|
|
581
|
-
*/
|
|
582
|
-
setLlmInvoke(invoke ) {
|
|
583
|
-
this.llmInvoke = invoke
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
get activeDefaultScope() {
|
|
587
|
-
return this.defaultScope
|
|
588
|
-
}
|
|
589
|
-
|
|
590
|
-
assertScopeAllowed(scope ) {
|
|
591
|
-
if (scope === 'global' && !this.cfg.globalEnabled) {
|
|
592
|
-
throw memoryError('SCOPE_DISABLED', 'global scope is disabled by configuration')
|
|
593
|
-
}
|
|
594
|
-
if (scope.startsWith('project:') && !this.cfg.projectEnabled) {
|
|
595
|
-
throw memoryError('SCOPE_DISABLED', 'project scope is disabled by configuration')
|
|
596
|
-
}
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
async save(input ) {
|
|
600
|
-
const scope = input.scope ?? this.defaultScope
|
|
601
|
-
this.assertScopeAllowed(scope)
|
|
602
|
-
let content = input.content?.trim()
|
|
603
|
-
if (!content) throw memoryError('CONTENT_EMPTY', 'content must be a non-empty string')
|
|
604
|
-
if (content.length > this.cfg.maxRecordChars) {
|
|
605
|
-
throw memoryError('CONTENT_TOO_LONG', `content exceeds ${this.cfg.maxRecordChars} chars`)
|
|
606
|
-
}
|
|
607
|
-
if (input.supersedes && !this.store.scope(scope).records.get(input.supersedes)) {
|
|
608
|
-
throw memoryError('SUPERSEDES_UNKNOWN', `supersedes target ${input.supersedes} not found in ${scope}`)
|
|
609
|
-
}
|
|
610
|
-
// design §11: keep credentials out of the plaintext store and the embedder
|
|
611
|
-
if (this.cfg.secrets.enabled) {
|
|
612
|
-
const findings = scanSecrets(content)
|
|
613
|
-
if (findings.length > 0) {
|
|
614
|
-
if (this.cfg.secrets.mode === 'refuse') {
|
|
615
|
-
throw memoryError('SECRET_DETECTED', `content matches credential denylist: ${[...new Set(findings.map((f) => f.kind))].join(', ')}`)
|
|
616
|
-
}
|
|
617
|
-
content = redactSecrets(content)
|
|
618
|
-
}
|
|
619
|
-
}
|
|
620
|
-
const { record } = this.store.save(
|
|
621
|
-
{ ...input, scope, content },
|
|
622
|
-
{ importance: input.source?.origin === 'auto' ? 0.4 : 0.8, source: { origin: 'explicit' } },
|
|
623
|
-
)
|
|
624
|
-
void this.embedRecord(record).catch(() => {})
|
|
625
|
-
this.store.scheduleFlush()
|
|
626
|
-
return record
|
|
627
|
-
}
|
|
628
|
-
|
|
629
|
-
async embedRecord(record ) {
|
|
630
|
-
if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return
|
|
631
|
-
const s = this.store.scope(record.scope)
|
|
632
|
-
// save-time budget: one full record, much slower than a search query
|
|
633
|
-
const vec = await this.embed.embed([record.content], undefined, { timeoutMs: this.cfg.vector.writeTimeoutMs }).catch(() => null)
|
|
634
|
-
if (vec?.[0]) s.appendVector(record.id, normalizeInPlace(vec[0]))
|
|
635
|
-
else s.embedPending.add(record.id) // backfill queue; the retry timer drains it
|
|
636
|
-
}
|
|
637
|
-
|
|
638
|
-
/**
|
|
639
|
-
* Retry pending embeddings in batches; returns number embedded.
|
|
640
|
-
* Batches are bounded by count AND total characters, and use the backfill
|
|
641
|
-
* budget: full record contents on CPU take seconds per batch, so the
|
|
642
|
-
* interactive timeout would abort every attempt.
|
|
643
|
-
*/
|
|
644
|
-
async backfillEmbeddings() {
|
|
645
|
-
if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return 0
|
|
646
|
-
if (!this.embed.isHealthy) return 0
|
|
647
|
-
let done = 0
|
|
648
|
-
for (const s of this.store.allScopes()) {
|
|
649
|
-
if (s.embedPending.size === 0) continue
|
|
650
|
-
const texts = []
|
|
651
|
-
const valid = []
|
|
652
|
-
let chars = 0
|
|
653
|
-
for (const id of s.embedPending) {
|
|
654
|
-
if (valid.length >= this.cfg.vector.backfillBatchSize) break
|
|
655
|
-
const t = s.records.get(id)?.content
|
|
656
|
-
if (!t) continue
|
|
657
|
-
if (valid.length > 0 && chars + t.length > this.cfg.vector.backfillMaxChars) break
|
|
658
|
-
texts.push(t)
|
|
659
|
-
valid.push(id)
|
|
660
|
-
chars += t.length
|
|
661
|
-
}
|
|
662
|
-
if (texts.length === 0) continue
|
|
663
|
-
const vecs = await this.embed.embed(texts, undefined, { timeoutMs: this.cfg.vector.backfillTimeoutMs }).catch(() => null)
|
|
664
|
-
if (!vecs) return done
|
|
665
|
-
for (let i = 0; i < valid.length; i++) {
|
|
666
|
-
if (s.records.has(valid[i] ) && vecs[i]) s.appendVector(valid[i] , normalizeInPlace(vecs[i] ))
|
|
667
|
-
s.embedPending.delete(valid[i] )
|
|
668
|
-
done++
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
return done
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
/**
|
|
675
|
-
* First-fill: embed the ENTIRE backlog (unlike backfillEmbeddings' single
|
|
676
|
-
* batch per scope), in throttled batches so a big import does not hammer
|
|
677
|
-
* llama-server or block the caller. Yields to the event loop between
|
|
678
|
-
* batches and stops early on abort/unhealthy embedder.
|
|
679
|
-
*/
|
|
680
|
-
async backfillAll(opts = {}) {
|
|
681
|
-
const throttle = opts.throttleMs ?? 50
|
|
682
|
-
let done = 0
|
|
683
|
-
while (!opts.signal?.aborted) {
|
|
684
|
-
const n = await this.backfillEmbeddings()
|
|
685
|
-
done += n
|
|
686
|
-
if (n === 0) break // per-scope batch was empty → backlog drained
|
|
687
|
-
if (!this.embed.isHealthy) break
|
|
688
|
-
await new Promise((r) => setTimeout(r, throttle))
|
|
689
|
-
}
|
|
690
|
-
return done
|
|
691
|
-
}
|
|
692
|
-
|
|
693
|
-
async search(query ) {
|
|
694
|
-
const topK = query.topK ?? this.cfg.topK
|
|
695
|
-
const recallN = query.recallN ?? this.cfg.recallN
|
|
696
|
-
const now = Date.now()
|
|
697
|
-
const queryTokens = [...new Set(tokenize(query.text))]
|
|
698
|
-
|
|
699
|
-
// candidate scopes: pinned, else project + global
|
|
700
|
-
const stores = []
|
|
701
|
-
if (query.scope) {
|
|
702
|
-
stores.push({ store: this.store.scope(query.scope), isGlobal: query.scope === 'global' })
|
|
703
|
-
} else {
|
|
704
|
-
if (this.cfg.projectEnabled) stores.push({ store: this.store.scope(this.defaultScope), isGlobal: this.defaultScope === 'global' })
|
|
705
|
-
if (this.cfg.globalEnabled) stores.push({ store: this.store.scope('global'), isGlobal: true })
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
// --- channel 1: lexical recall, scored per scope with local IDF ---
|
|
709
|
-
|
|
710
|
-
const candidates = new Map ()
|
|
711
|
-
for (const { store, isGlobal } of stores) {
|
|
712
|
-
const all = [...store.records.values()]
|
|
713
|
-
const tokenSets = all.map((r) => new Set(tokenize(r.content)))
|
|
714
|
-
const stats = buildStats(tokenSets)
|
|
715
|
-
for (let i = 0; i < all.length; i++) {
|
|
716
|
-
const rec = all[i]
|
|
717
|
-
if (rec.expiredAt && rec.expiredAt < now) continue
|
|
718
|
-
const lex = lexicalScore(queryTokens, tokenSets[i] , stats)
|
|
719
|
-
if (lex <= 0) continue
|
|
720
|
-
// Weak-only overlap (no shared word/bigram) must clear the "real hit"
|
|
721
|
-
// floor, not the permissive threshold — kills single-char collisions.
|
|
722
|
-
const floor = isGlobal ? this.cfg.globalThreshold : this.cfg.projectThreshold
|
|
723
|
-
const effective = hasStrongHit(queryTokens, tokenSets[i] ) ? floor : Math.max(floor, REAL_HIT_FLOOR)
|
|
724
|
-
if (lex < effective) continue
|
|
725
|
-
candidates.set(rec.id, { record: rec, recallScore: lex, isGlobal })
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
// --- channel 2: vector recall over embedded records ---
|
|
730
|
-
// Skipped entirely when no backend is configured (lexical recall above
|
|
731
|
-
// already produced the candidates; a request here would just fail).
|
|
732
|
-
const qvec = this.cfg.vector.enabled && this.cfg.vector.configured
|
|
733
|
-
? await this.embed.embed([query.text], query.signal).catch(() => null)
|
|
734
|
-
: null
|
|
735
|
-
if (qvec?.[0]) {
|
|
736
|
-
const vectorScores = new Map ()
|
|
737
|
-
for (const { store } of stores) {
|
|
738
|
-
for (const [id, vec] of store.vectors) {
|
|
739
|
-
if (!store.records.has(id)) continue
|
|
740
|
-
const sim = cosine(qvec[0] , vec)
|
|
741
|
-
if (sim >= this.cfg.vector.minScore) vectorScores.set(id, sim)
|
|
742
|
-
}
|
|
743
|
-
}
|
|
744
|
-
for (const [id, sim] of vectorScores) {
|
|
745
|
-
const existing = candidates.get(id)
|
|
746
|
-
if (existing) {
|
|
747
|
-
existing.recallScore = Math.max(existing.recallScore, sim) // best channel wins
|
|
748
|
-
} else {
|
|
749
|
-
const rec = this.findById(id)
|
|
750
|
-
if (rec && !(rec.expiredAt && rec.expiredAt < now)) {
|
|
751
|
-
if (!(rec.scope === 'global' && sim < this.cfg.globalThreshold)) {
|
|
752
|
-
candidates.set(id, { record: rec, recallScore: sim, isGlobal: rec.scope === 'global' })
|
|
753
|
-
}
|
|
754
|
-
}
|
|
755
|
-
}
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
|
|
759
|
-
// shape with recency/importance, sort, cut
|
|
760
|
-
const supersededIds = new Set ()
|
|
761
|
-
for (const { store } of stores) {
|
|
762
|
-
for (const r of store.records.values()) if (r.supersedes) supersededIds.add(r.supersedes)
|
|
763
|
-
}
|
|
764
|
-
const shape = (c ) =>
|
|
765
|
-
recordScore(c.recallScore, c.record.importance, c.record.accessCount, c.record.lastAccessedAt, {
|
|
766
|
-
now,
|
|
767
|
-
halfLifeDays: this.cfg.halfLifeDays,
|
|
768
|
-
superseded: supersededIds.has(c.record.id),
|
|
769
|
-
})
|
|
770
|
-
|
|
771
|
-
const ranked = [...candidates.values()].sort((a, b) => shape(b) - shape(a)).slice(0, recallN)
|
|
772
|
-
|
|
773
|
-
// --- channel 3: LLM rerank (optional; any skip/failure keeps recall order) ---
|
|
774
|
-
const finalScores = new Map (ranked.map((c) => [c.id, shape(c)]))
|
|
775
|
-
if (this.cfg.llm.enabled && this.llmInvoke && ranked.length >= 2) {
|
|
776
|
-
const asCands = ranked.map((c) => ({ id: c.id, content: c.record.content, recallScore: finalScores.get(c.id) }))
|
|
777
|
-
const cacheKey = RerankCache.key(query.text, ranked.map((c) => c.id), this.cfg.llm.model)
|
|
778
|
-
let outcome = this.rerankCache.get(cacheKey)
|
|
779
|
-
if (!outcome) {
|
|
780
|
-
outcome = await rerank({
|
|
781
|
-
invoke: this.llmInvoke,
|
|
782
|
-
cfg: this.cfg.llm,
|
|
783
|
-
query: query.text,
|
|
784
|
-
candidates: asCands,
|
|
785
|
-
budget: this.rerankBudget,
|
|
786
|
-
signal: query.signal,
|
|
787
|
-
})
|
|
788
|
-
if (outcome) this.rerankCache.set(cacheKey, outcome)
|
|
789
|
-
}
|
|
790
|
-
if (outcome) {
|
|
791
|
-
for (const { id, score } of blendScores(asCands, outcome, this.cfg.llm.weight)) {
|
|
792
|
-
finalScores.set(id, score)
|
|
793
|
-
}
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
|
|
797
|
-
const hits = ranked.map((c) => ({ record: touch(c.record), score: round4(finalScores.get(c.id) ) }))
|
|
798
|
-
|
|
799
|
-
this.store.scheduleFlush()
|
|
800
|
-
return hits.slice(0, topK)
|
|
801
|
-
}
|
|
802
|
-
|
|
803
|
-
findById(id ) {
|
|
804
|
-
for (const s of this.store.allScopes()) {
|
|
805
|
-
const r = s.records.get(id)
|
|
806
|
-
if (r) return r
|
|
807
|
-
}
|
|
808
|
-
return undefined
|
|
809
|
-
}
|
|
810
|
-
|
|
811
|
-
async forget(input ) {
|
|
812
|
-
const ids = []
|
|
813
|
-
if (input.id) {
|
|
814
|
-
for (const s of this.store.allScopes()) {
|
|
815
|
-
if (s.remove(input.id)) ids.push(input.id)
|
|
816
|
-
}
|
|
817
|
-
if (ids.length === 0) throw memoryError('RECORD_NOT_FOUND', `id ${input.id} not found`)
|
|
818
|
-
} else if (input.query) {
|
|
819
|
-
const scope = input.scope ?? this.defaultScope
|
|
820
|
-
ids.push(...this.store.findIdsByQuery(scope, input.query))
|
|
821
|
-
const s = this.store.scope(scope)
|
|
822
|
-
for (const id of ids) s.remove(id)
|
|
823
|
-
if (ids.length === 0) throw memoryError('RECORD_NOT_FOUND', `no record matching query in ${scope}`)
|
|
824
|
-
} else {
|
|
825
|
-
throw memoryError('RECORD_NOT_FOUND', 'forget requires id or query')
|
|
826
|
-
}
|
|
827
|
-
this.store.scheduleFlush()
|
|
828
|
-
return { removed: ids.length, ids }
|
|
829
|
-
}
|
|
830
|
-
|
|
831
|
-
async list(input = {}) {
|
|
832
|
-
const scope = input.scope ?? this.defaultScope
|
|
833
|
-
let records = [...this.store.scope(scope).records.values()]
|
|
834
|
-
if (input.kind) records = records.filter((r) => r.kind === input.kind)
|
|
835
|
-
records.sort((a, b) => b.updatedAt - a.updatedAt)
|
|
836
|
-
return records.slice(0, input.limit ?? 50)
|
|
837
|
-
}
|
|
838
|
-
|
|
839
|
-
/** Force a synchronous flush (dispose/HMR path). */
|
|
840
|
-
flushSync() {
|
|
841
|
-
if (this.maintenanceTimer) {
|
|
842
|
-
clearInterval(this.maintenanceTimer)
|
|
843
|
-
this.maintenanceTimer = null
|
|
844
|
-
}
|
|
845
|
-
if (this.embedRetryTimer) {
|
|
846
|
-
clearInterval(this.embedRetryTimer)
|
|
847
|
-
this.embedRetryTimer = null
|
|
848
|
-
}
|
|
849
|
-
if (this.initialDrainTimer) {
|
|
850
|
-
clearTimeout(this.initialDrainTimer)
|
|
851
|
-
this.initialDrainTimer = null
|
|
852
|
-
}
|
|
853
|
-
if (this.embedServer) this.embedServer.stop()
|
|
854
|
-
if (this.exitHook) {
|
|
855
|
-
process.removeListener('exit', this.exitHook)
|
|
856
|
-
this.exitHook = null
|
|
857
|
-
}
|
|
858
|
-
this.store.purgeExpired()
|
|
859
|
-
this.store.enforceLimit(this.cfg.maxRecordsPerScope)
|
|
860
|
-
this.store.close()
|
|
861
|
-
}
|
|
862
|
-
|
|
863
|
-
get pendingEmbedCount() {
|
|
864
|
-
return this.store.allScopes().reduce((n, s) => n + s.embedPending.size, 0)
|
|
865
|
-
}
|
|
866
|
-
}
|
|
867
|
-
|
|
868
|
-
function touch(record ) {
|
|
869
|
-
record.lastAccessedAt = Date.now()
|
|
870
|
-
record.accessCount += 1
|
|
871
|
-
return record
|
|
872
|
-
}
|
|
873
|
-
|
|
874
|
-
function round4(n ) {
|
|
875
|
-
return Math.round(n * 10_000) / 10_000
|
|
876
|
-
}
|
|
877
|
-
|
|
878
|
-
/** Port of an http(s) endpoint, or null when it cannot be parsed. */
|
|
879
|
-
function portOf(endpoint ) {
|
|
880
|
-
try {
|
|
881
|
-
const port = new URL(endpoint).port
|
|
882
|
-
return port ? Number.parseInt(port, 10) : null
|
|
883
|
-
} catch {
|
|
884
|
-
return null
|
|
885
|
-
}
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
/**
|
|
889
|
-
* Measured score floor of genuine lexical hits; below this only single-char
|
|
890
|
-
* coincidences live. Used to gate weak-only overlaps regardless of config.
|
|
891
|
-
*/
|
|
892
|
-
const REAL_HIT_FLOOR = 0.25
|
|
893
|
-
|
|
894
|
-
function memoryError(code , message ) {
|
|
895
|
-
const err = new Error(`[memory:${code}] ${message}`)
|
|
896
|
-
err.code = code
|
|
897
|
-
return err
|
|
898
|
-
}
|
|
899
|
-
|
|
900
|
-
export function createMemoryService(config = {}) {
|
|
901
|
-
return new LocalMemoryService(config)
|
|
902
|
-
}
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
//# sourceURL=D:\.deepseek\memory-plugin\packages\memory-local\src\index.ts
|
|
1273
|
+
//# sourceURL=source/memory-local/src/index.ts
|