@restregular/dsh-memory 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,909 +1,1273 @@
1
- /**
2
- * dsh-memory-local — local JSONL provider for the DSH memory service.
3
- *
4
- * This entry is intentionally cordis-free so unit tests and standalone tools
5
- * run without the harness. The Cordis wiring (Service subclass, effects,
6
- * plugin apply) lives in ./apply.ts and is only loaded inside DSH.
7
- */
8
-
9
-
10
-
11
-
12
-
13
-
14
-
15
-
16
-
17
- import { MemoryStore, } from './store.js'
18
- import { buildStats, lexicalScore, recordScore, tokenize, cosine, normalizeInPlace, hasStrongHit } from './scoring.js'
19
- import { EmbedClient } from './embed.js'
20
- import { DailyBudget, blendScores, rerank, } from './rerank.js'
21
- import { RerankCache } from './rerank-cache.js'
22
- import { scanSecrets, redactSecrets } from './secrets.js'
23
- import { EmbedServerManager, } from './embed-server.js'
24
- import { VectorInstaller, detectInstalled, installPaths, } from './embed-install.js'
25
- import { existsSync, readdirSync, rmSync, statSync } from 'node:fs'
26
- import { dirname, isAbsolute, join, relative, resolve } from 'node:path'
27
- import { homedir } from 'node:os'
1
+ /**
2
+ * dsh-memory-local — local JSONL provider for the DSH memory service.
3
+ *
4
+ * This entry is intentionally cordis-free so unit tests and standalone tools
5
+ * run without the harness. The Cordis wiring (Service subclass, effects,
6
+ * plugin apply) lives in ./apply.ts and is only loaded inside DSH.
7
+ */
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+ import { MemoryStore, } from './store.js'
18
+ import { buildStats, lexicalScore, recordScore, tokenize, cosine, normalizeInPlace, hasStrongHit } from './scoring.js'
19
+ import { EmbedClient } from './embed.js'
20
+ import { DailyBudget, blendScores, rerank, } from './rerank.js'
21
+ import { RerankCache } from './rerank-cache.js'
22
+ import { scanSecrets, redactSecrets } from './secrets.js'
23
+ import { EmbedServerManager, } from './embed-server.js'
24
+ import { VectorInstaller, detectInstalled, installPaths, } from './embed-install.js'
25
+ import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
26
+ import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'
27
+ import { homedir } from 'node:os'
28
+ import {
29
+ DEFAULT_MODEL_ID,
30
+ EMBEDDING_MODELS,
31
+ findModel,
32
+ findVariant,
33
+
34
+ } from './models.js'
35
+ import { probeHardware, } from './hardware.js'
36
+ import { pickFlavor, recommendModel } from './recommend.js'
37
+
38
+ /** Recursive byte count; 0 for anything unreadable. */
39
+ function pathSize(path ) {
40
+ try {
41
+ const st = statSync(path)
42
+ if (!st.isDirectory()) return st.size
43
+ let total = 0
44
+ for (const entry of readdirSync(path)) total += pathSize(join(path, entry))
45
+ return total
46
+ } catch {
47
+ return 0
48
+ }
49
+ }
50
+
51
+ /** True when `child` is `parent` or lives under it (both resolved). */
52
+ function isInside(parent , child ) {
53
+ const rel = relative(resolve(parent), resolve(child))
54
+ return rel === '' || (!rel.startsWith('..') && !isAbsolute(rel))
55
+ }
56
+
57
+
58
+
59
+
60
+
61
+
62
+
63
+
64
+
65
+
66
+
67
+
68
+
69
+
70
+
71
+
72
+                                 
73
+
74
+
75
+
76
+                              
77
+                  
78
+
79
+
80
+                   
81
+
82
+                       
83
+
84
+
85
+                
86
+
87
+
88
+
89
+
90
+                  
91
+
92
+
93
+                           
94
+
95
+                        
96
+
97
+                        
98
+
99
+  
100
+
101
+
102
+
103
+
104
+
105
+
106
+
107
+
108
+
109
+
110
+
111
+
112
+
113
+
114
+
115
+  
116
+
117
+
118
+
119
+
120
+
121
+  
122
+  
123
+
124
+
125
+
126
+
127
+
128
+
129
+
130
+
131
+  
132
+
133
+
134
+
135
+
136
+
137
+
138
+
139
+
140
+
141
+
142
+
143
+
144
+
145
+
146
+
147
+
148
+
149
+
150
+
151
+
152
+  
153
+
154
+                          
155
+
156
+                     
157
+
158
+
159
+
160
+                
161
+
162
+                           
163
+
164
+             
165
+
166
+                         
167
+
168
+
169
+
170
+
171
+
172
+
173
+
174
+
175
+
176
+
177
+
178
+
179
+
180
+
181
+
182
+
183
+
184
+
185
+
186
+ /** vector 配置节的类型别名,给下面的模型解析helpers 用。 */
187
+
188
+
189
+ /** 面板可以在安装时临时指定的模型选择。 */
190
+
191
+
192
+
193
+
194
+
195
+
196
+
197
+ /**
198
+ * 「面板选过什么」的记忆,落在安装目录里(<installRoot>/selected-model.json)。
199
+ *
200
+ * 为什么需要它:面板没法改写 profile 配置,而"选了模型 → 重启后配置又指回老模型"
201
+ * 会让装好的模型认不出来。这里是**低于配置**的一层记忆:配置里写了 preset/customModel
202
+ * 就听配置的,没写才用这份。
203
+ */
204
+ export const MODEL_STATE_FILE = 'selected-model.json'
205
+
206
+
207
+
208
+
209
+
210
+
211
+
212
+
213
+ function readModelState(root ) {
214
+ try {
215
+ const text = readFileSync(join(root, MODEL_STATE_FILE), 'utf8')
216
+ const parsed = JSON.parse(text)
217
+ return parsed !== null && typeof parsed === 'object' ? parsed : null
218
+ } catch {
219
+ return null
220
+ }
221
+ }
222
+
223
+ function writeModelState(root , state ) {
224
+ try {
225
+ mkdirSync(root, { recursive: true })
226
+ writeFileSync(join(root, MODEL_STATE_FILE), `${JSON.stringify(state, null, 2)}\n`, 'utf8')
227
+ } catch {
228
+ // 记不住不影响本次安装;下次面板仍会显示配置文件里的选择。
229
+ }
230
+ }
231
+
232
+ /** 系统语言(用来判断"中文优先");拿不到就当英文环境。 */
233
+ function safeLocale() {
234
+ try {
235
+ return Intl.DateTimeFormat().resolvedOptions().locale || 'en'
236
+ } catch {
237
+ return 'en'
238
+ }
239
+ }
240
+
241
+ /** 该模型某个档位的体积(不指定档位时用默认档),用于清单展示与磁盘校验。 */
242
+ function variantBytes(model , variantId ) {
243
+ const variant = variantId !== undefined ? model.variants.find((v) => v.id === variantId) : undefined
244
+ return (variant ?? model.variants.find((v) => v.isDefault) ?? model.variants[0] ).bytes
245
+ }
246
+
247
+
248
+
249
+
250
+
251
+
252
+
253
+
254
+
255
+
256
+                            
257
+
258
+
259
+
260
+
261
+
262
+
263
+ /**
264
+ * 把配置里的模型选择解析成安装器要的形态(文件名 / 体积 / 源 / 维度 / pooling / 前缀)。
265
+ *
266
+ * 优先级:customModel(完全自定义) → preset(清单 id) → 用 `model` 字符串去清单里找
267
+ * → 内置默认(bge-m3 Q4_K_M,与老行为逐字一致)。
268
+ * 认不出的 `model` 字符串**不改身份**(请求与边车仍用它),只是安装器退回默认模型 ——
269
+ * 用户指向云端/自建服务时本来就该只写 endpoint + model + dims。
270
+ */
271
+ function resolveModelSelection(v , root ) {
272
+ const state = root !== undefined ? readModelState(root) : null
273
+ const custom = v.customModel ?? state?.customModel
274
+ if (custom !== undefined) {
275
+ let file = custom.file
276
+ if (file === undefined || file === '') {
277
+ try {
278
+ file = basename(new URL(custom.url).pathname) || 'custom-model.gguf'
279
+ } catch {
280
+ file = 'custom-model.gguf'
281
+ }
282
+ }
283
+ return {
284
+ presetId: 'custom',
285
+ variantId: 'custom',
286
+ label: file,
287
+ file,
288
+ ...(custom.bytes !== undefined ? { bytes: custom.bytes } : {}),
289
+ sources: custom.sources ?? [custom.url],
290
+ dims: custom.dims,
291
+ pooling: custom.pooling ?? 'auto',
292
+ modelName: v.model ?? file.replace(/\.gguf$/i, ''),
293
+ ...(v.flavor ?? state?.flavor ? { flavor: v.flavor ?? state?.flavor } : {}),
294
+ }
295
+ }
296
+ // 配置 > 面板的记忆 > 用 model 字符串去清单里找 > 内置默认。
297
+ const presetId = v.preset
298
+ ?? state?.presetId
299
+ ?? (findModel(v.model) !== undefined ? v.model : DEFAULT_MODEL_ID)
300
+ const model = findModel(presetId) ?? findModel(DEFAULT_MODEL_ID)
301
+ const variant = findVariant(model, v.variant ?? state?.variantId)
302
+ return {
303
+ presetId: model.id,
304
+ variantId: variant.id,
305
+ label: `${model.label} · ${variant.label}`,
306
+ file: variant.file,
307
+ bytes: variant.bytes,
308
+ sources: [...variant.sources],
309
+ dims: model.dims,
310
+ pooling: model.pooling,
311
+ modelName: v.model ?? model.id,
312
+ ...(model.queryPrefix !== undefined ? { queryPrefix: model.queryPrefix } : {}),
313
+ ...(model.documentPrefix !== undefined ? { documentPrefix: model.documentPrefix } : {}),
314
+ ...(v.flavor ?? state?.flavor ? { flavor: v.flavor ?? state?.flavor } : {}),
315
+ }
316
+ }
317
+
318
+ /**
319
+ * 接管已装好的后端时,spawn 该用哪个模型文件。
320
+ *
321
+ * 坑:换模型时 `spawn.model` 可能还留着**上一个**模型的路径(显式配置或上次 adopt
322
+ * 写进去的),而 dims/pooling 已经跟着新模型变了 —— 那会启动一个"模型与参数不匹配"
323
+ * 的服务,而且不报错,只是检索质量悄悄变差。所以只有"配置的模型就是当前选中的那个"
324
+ * 才沿用它,否则一律用刚落盘的那份。
325
+ */
326
+ export function pickSpawnModel(currentFile , configuredModel , installedModel ) {
327
+ if (configuredModel !== undefined && basename(configuredModel) === currentFile) return configuredModel
328
+ return installedModel
329
+ }
330
+
331
+ export function resolveConfig(config = {}) {
332
+ const v = config.retrieval?.vector ?? {}
333
+ // Do not touch an embedding backend the user never asked for: pointing at the
334
+ // stock localhost endpoint on a machine that has nothing listening means
335
+ // failed embeds, a tripped breaker and scary log lines for a feature that is
336
+ // optional. Vector retrieval is therefore opt-in by CONFIGURATION — writing
337
+ // `endpoint` (any OpenAI-compatible service), configuring `spawn`, or having
338
+ // the panel's installer put a backend on disk implies intent; an explicit
339
+ // `enabled` always wins.
340
+ const storeDir = config.storeDir ?? '.dsh-memory'
341
+ // Installer target: explicit config → that directory; explicit storeDir →
342
+ // its parent (so a hand-made `<root>/llamacpp` + `<root>/models` layout is
343
+ // recognised as-is); otherwise a stable per-user directory under ~/.dsh.
344
+ const installRoot = v.autoInstall?.dir
345
+ ?? (config.storeDir !== undefined ? dirname(resolve(storeDir)) : join(homedir(), '.dsh', 'memory-vector'))
346
+ // 模型选择:自定义 → 显式 preset → 用 model 字符串去清单里找 → 内置默认。
347
+ // 这一步必须在 detectInstalled 之前:安装判据是「exe + **所选模型的**文件」。
348
+ const selection = resolveModelSelection(v, installRoot)
349
+ const installed = detectInstalled(installRoot, selection.file)
350
+ const endpointConfigured = v.endpoint !== undefined
351
+ // A configured spawn path that no longer exists (plugin uninstalled the
352
+ // backend, or the user moved it) must not count as configured: that would
353
+ // leave the panel showing a managed server that can never start, instead of
354
+ // offering the download button again.
355
+ const spawnExe = v.spawn?.exe
356
+ const spawnExeUsable = spawnExe === undefined || spawnExe === '' || existsSync(spawnExe)
357
+ const spawnConfigured = ((v.spawn?.enabled === true || (spawnExe ?? '') !== '') && spawnExeUsable) || installed !== null
358
+ const configured = endpointConfigured || spawnConfigured
359
+ const vectorEnabled = v.enabled ?? configured
360
+ const flavor = selection.flavor ?? 'cpu'
361
+ return {
362
+ storeDir,
363
+ globalEnabled: config.scopes?.global ?? true,
364
+ projectEnabled: config.scopes?.project ?? true,
365
+ topK: config.retrieval?.topK ?? 5,
366
+ recallN: config.retrieval?.recallN ?? 30,
367
+ globalThreshold: config.retrieval?.globalThreshold ?? 0.12,
368
+ projectThreshold: config.retrieval?.projectThreshold ?? 0.1,
369
+ halfLifeDays: config.retrieval?.recencyHalfLifeDays ?? 30,
370
+ vector: {
371
+ enabled: vectorEnabled,
372
+ configured,
373
+ endpointConfigured,
374
+ spawnConfigured,
375
+ installRoot,
376
+ ...(v.autoInstall?.modelSources ? { installModelSources: v.autoInstall.modelSources } : {}),
377
+ installModel: {
378
+ file: selection.file,
379
+ ...(selection.bytes !== undefined ? { bytes: selection.bytes } : {}),
380
+ ...(selection.sources.length > 0 ? { sources: selection.sources } : {}),
381
+ },
382
+ presetId: selection.presetId,
383
+ variantId: selection.variantId,
384
+ modelLabel: selection.label,
385
+ pooling: v.pooling ?? selection.pooling,
386
+ flavor,
387
+ gpuLayers: v.gpuLayers ?? (flavor === 'cpu' || flavor === 'auto' ? 0 : 99),
388
+ ...(selection.queryPrefix !== undefined ? { queryPrefix: selection.queryPrefix } : {}),
389
+ ...(selection.documentPrefix !== undefined ? { documentPrefix: selection.documentPrefix } : {}),
390
+ endpoint: v.endpoint ?? 'http://127.0.0.1:8078/v1/embeddings',
391
+ model: v.model ?? selection.modelName,
392
+ dims: v.dims ?? selection.dims,
393
+ timeoutMs: v.timeoutMs ?? 300,
394
+ // Full contents are orders of magnitude slower than a short query on CPU
395
+ // (measured 3143ms for three ~600-char records), so write/backfill paths
396
+ // get their own generous budgets instead of the interactive 300ms.
397
+ writeTimeoutMs: v.writeTimeoutMs ?? 10_000,
398
+ backfillTimeoutMs: v.backfillTimeoutMs ?? 30_000,
399
+ backfillMaxChars: v.backfillMaxChars ?? 8_000,
400
+ // BGE CLS baseline noise floor measured ~0.42 on unrelated pairs
401
+ minScore: v.minScore ?? 0.5,
402
+ backfillBatchSize: v.backfillBatchSize ?? 32,
403
+ // Port defaults to the one in the embeddings endpoint so the two can never
404
+ // drift apart; opt-in (enabled:false) keeps dev/test runs from spawning.
405
+ // A backend the panel installed (or a hand-made one at installRoot) is
406
+ // adopted as the spawn target when the user configured no explicit paths.
407
+ spawn: {
408
+ ...(v.spawn ?? {}),
409
+ enabled: v.spawn?.enabled ?? installed !== null,
410
+ ...(installed
411
+ ? {
412
+ exe: v.spawn?.exe ?? installed.exe,
413
+ model: v.spawn?.model ?? installed.model,
414
+ cwd: v.spawn?.cwd ?? installed.cwd,
415
+ logFile: v.spawn?.logFile ?? installed.logFile,
416
+ }
417
+ : {}),
418
+ // 模型自带 pooling 与 GPU 层数:llama-server 拿不到就是错的检索结果,
419
+ // 所以跟着所选模型一起传下去(extraArgs 仍在最后,可覆盖)。
420
+ pooling: v.pooling ?? selection.pooling,
421
+ gpuLayers: v.gpuLayers ?? (flavor === 'cpu' || flavor === 'auto' ? 0 : 99),
422
+ port: v.spawn?.port ?? portOf(v.endpoint ?? 'http://127.0.0.1:8078/v1/embeddings') ?? 8078,
423
+ },
424
+ },
425
+ llm: {
426
+ enabled: config.retrieval?.llm?.enabled ?? false,
427
+ provider: config.retrieval?.llm?.provider ?? '',
428
+ model: config.retrieval?.llm?.model ?? '',
429
+ timeoutMs: config.retrieval?.llm?.timeoutMs ?? 1500,
430
+ dailyBudgetTokens: config.retrieval?.llm?.dailyBudgetTokens ?? 200_000,
431
+ weight: config.retrieval?.llm?.weight ?? 0.6,
432
+ topN: config.retrieval?.llm?.topN ?? 10,
433
+ },
434
+ maxRecordsPerScope: config.limits?.maxRecordsPerScope ?? 2000,
435
+ maxRecordChars: config.limits?.maxRecordChars ?? 4000,
436
+ secrets: {
437
+ enabled: config.secrets?.enabled ?? true,
438
+ mode: config.secrets?.mode ?? 'refuse',
439
+ },
440
+ }
441
+ }
442
+
443
+ export class LocalMemoryService {
444
+ cfg
445
+ store
446
+ embed
447
+ rerankBudget
448
+ rerankCache = new RerankCache()
449
+ /** Host-provided one-shot LLM call; null = rerank disabled (recall order). */
450
+ llmInvoke = null
451
+ /** Where saves land when the caller does not pin a scope. */
452
+ defaultScope = 'global'
453
+
454
+ constructor(config = {}) {
455
+ this.cfg = resolveConfig(config)
456
+ this.rerankBudget = new DailyBudget(this.cfg.llm.dailyBudgetTokens)
457
+ this.store = new MemoryStore(this.cfg.storeDir, { onWarn: (m) => console.warn(`[dsh-memory-local] ${m}`) })
458
+ // Identity BEFORE the eager load (issue #1): ScopeStore skips its sidecar
459
+ // validation entirely while the identity is unset (`if (this.vecIdentity && …)`),
460
+ // so loading first would accept vectors from a previous embedding model
461
+ // whenever the dimensions matched — and the next flush would then rewrite the
462
+ // header as the new model, making the mismatch permanent.
463
+ this.store.setVecIdentity(this.cfg.vector.model, this.cfg.vector.dims)
464
+ this.store.loadAll(this.cfg.projectEnabled && this.cfg.globalEnabled ? [this.defaultScope, 'global'] : [this.defaultScope])
465
+ this.embed = new EmbedClient({
466
+ endpoint: this.cfg.vector.endpoint,
467
+ model: this.cfg.vector.model,
468
+ dims: this.cfg.vector.dims,
469
+ timeoutMs: this.cfg.vector.timeoutMs,
470
+ })
471
+ this.startMaintenance()
472
+ this.startEmbedServer()
473
+ }
474
+
475
+ /** Own the embedding backend's lifetime: spawn on activation, stop on dispose. */
476
+ startEmbedServer() {
477
+ // Nothing configured and retrieval switched off → do not even build the
478
+ // manager, so no probe, no spawn attempt, no misleading panel state.
479
+ if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return
480
+ this.embedServer = new EmbedServerManager(this.cfg.vector.spawn)
481
+ // fire-and-forget: activation must never block on a model load
482
+ void this.ensureEmbedServer(60_000).catch(() => { /* lifecycle never breaks activation */ })
483
+ }
484
+
485
+ /**
486
+ * Start (or adopt) the embedding backend and arm the exit safety net.
487
+ * Shared by the activation path and the panel's manual start button.
488
+ */
489
+ async ensureEmbedServer(waitMs ) {
490
+ const mgr = this.embedServer
491
+ if (!mgr) return 'disabled'
492
+ const state = await mgr.ensureRunning()
493
+ if (state !== 'spawned' && state !== 'adopted-managed') return state
494
+ // hard-exit safety net: cordis disposal covers a clean quit; this covers
495
+ // the host dying without disposing us (the child must not be orphaned)
496
+ if (!this.exitHook) {
497
+ this.exitHook = () => { this.embedServer?.stop() }
498
+ process.once('exit', this.exitHook)
499
+ }
500
+ const ready = await mgr.waitUntilReady(waitMs)
501
+ if (!ready) console.warn('[dsh-memory-local] embed server: 启动超时,检索继续走词法降级')
502
+ return state
503
+ }
504
+
505
+ /** Manual start from the memory panel (same semantics as activation). */
506
+ startEmbedServerManually() {
507
+ return this.ensureEmbedServer(25_000)
508
+ }
509
+
510
+ /**
511
+ * Manual stop from the memory panel. An instance we do not own is left alone
512
+ * (it was started outside the app), so the UI can explain the refusal.
513
+ */
514
+ stopEmbedServerManually() {
515
+ const mgr = this.embedServer
516
+ if (!mgr) return { stopped: false, reason: 'NOT_RUNNING' }
517
+ if (!mgr.isOwned) return { stopped: false, reason: 'EXTERNAL_INSTANCE' }
518
+ return { stopped: mgr.stop(), reason: mgr.isOwned ? 'NOT_RUNNING' : 'STOPPED' }
519
+ }
520
+
521
+ /**
522
+ * Lifecycle snapshot for the memory panel.
523
+ *
524
+ * `mode` is what the panel needs to decide whether a start/stop switch even
525
+ * makes sense:
526
+ * - 'off' nothing configured → no switch, show how to opt in
527
+ * - 'external' we only know an endpoint (cloud or a server the user runs)
528
+ * → no switch, we must not claim ownership of that process
529
+ * - 'managed' spawn is configured → the switch toggles OUR child process
530
+ * `enabled` is the retrieval switch from config; when it is false while a
531
+ * backend IS configured the panel explains the config instead of showing a
532
+ * switch that cannot turn retrieval on.
533
+ */
534
+ async embedServerStatus()
535
+
536
+
537
+
538
+
539
+
540
+                          
541
+
542
+
543
+
544
+                            
545
+
546
+
547
+
548
+
549
+
550
+
551
+
552
+
553
+
554
+
555
+
556
+
557
+
558
+ {
559
+ const mgr = this.embedServer
560
+ const cfg = this.cfg.vector
561
+ const pendingEmbeds = this.pendingEmbedCount
562
+ const mode = !cfg.configured ? 'off' : cfg.spawnConfigured ? 'managed' : 'external'
563
+ const usage = this.vectorInstallUsage()
564
+ const base = {
565
+ enabled: cfg.enabled,
566
+ configured: cfg.configured,
567
+ mode,
568
+ endpoint: cfg.endpoint,
569
+ model: cfg.model,
570
+ modelLabel: cfg.modelLabel,
571
+ presetId: cfg.presetId,
572
+ variantId: cfg.variantId,
573
+ modelFile: cfg.installModel.file,
574
+ dims: cfg.dims,
575
+ pooling: cfg.pooling,
576
+ flavor: cfg.flavor,
577
+ spawnConfigured: cfg.spawnConfigured,
578
+ pendingEmbeds,
579
+ install: this.installSnapshot(),
580
+ maintenance: {
581
+ usageBytes: usage.bytes,
582
+ paths: usage.paths,
583
+ // NOTE: `cfg` is the VECTOR config here, not the whole config — reading
584
+ // cfg.storeDir would silently yield undefined (and JSON drops it).
585
+ storeDir: this.cfg.storeDir,
586
+ records: this.store.allScopes().reduce((n, s) => n + s.records.size, 0),
587
+ },
588
+ }
589
+ if (!mgr) return { ...base, owned: false, pid: null, baseUrl: '', reachable: false }
590
+ const reachable = await mgr.healthCheck()
591
+ return { ...base, ...mgr.info(), reachable, pendingEmbeds }
592
+ }
593
+
594
+ /** Installer state for the panel (idle when nothing was ever requested). */
595
+ installSnapshot() {
596
+ const root = this.cfg.vector.installRoot
597
+ const modelFile = this.cfg.vector.installModel.file
598
+ const installed = detectInstalled(root, modelFile) !== null
599
+ const progress = this.installer?.progress ?? {
600
+ phase: 'idle' ,
601
+ message: '',
602
+ receivedBytes: 0,
603
+ totalBytes: null,
604
+ error: null,
605
+ paths: installed ? installPaths(root, modelFile) : null,
606
+ }
607
+ return { ...progress, installed, root }
608
+ }
609
+
610
+ /**
611
+ * 面板的"模型与硬件"报告:清单 + 当前选择 + 硬件 + 推荐。
612
+ *
613
+ * 硬件探测会 spawn 外部命令(Windows 上还有 PowerShell),所以**只在面板问的时候**跑,
614
+ * 并且结果缓存 10 分钟 —— 绝不在插件激活路径上。
615
+ */
616
+ async modelsReport(force = false)
617
+
618
+
619
+
620
+
621
+
622
+
623
+
624
+
625
+
626
+
627
+ {
628
+ const root = this.cfg.vector.installRoot
629
+ const catalog = EMBEDDING_MODELS.map((model) => ({
630
+ id: model.id,
631
+ label: model.label,
632
+ note: model.note,
633
+ dims: model.dims,
634
+ pooling: model.pooling,
635
+ languages: model.languages,
636
+ contextTokens: model.contextTokens,
637
+ sizeBytes: variantBytes(model, this.cfg.vector.presetId === model.id ? this.cfg.vector.variantId : undefined),
638
+ installed: model.variants.some((variant) => detectInstalled(root, variant.file) !== null),
639
+ // 逐档判断"装没装":bge-m3 装了 Q4_K_M 不等于 Q8_0 也装了。
640
+ // 判据与 detectInstalled 一致 —— 运行时 exe + 该档模型文件都在。
641
+ variants: model.variants.map((variant) => ({
642
+ id: variant.id, label: variant.label, bytes: variant.bytes, file: variant.file,
643
+ installed: detectInstalled(root, variant.file) !== null,
644
+ })),
645
+ }))
646
+ const records = this.store.allScopes().reduce((n, s) => n + s.records.size, 0)
647
+ const locale = safeLocale()
648
+ let hardware = null
649
+ try {
650
+ hardware = await this.hardwareSnapshot(force)
651
+ } catch {
652
+ hardware = null // 探测失败就只显示清单,不给推荐 —— 不猜
653
+ }
654
+ const recommendation = hardware === null
655
+ ? null
656
+ : recommendModel({ hardware, locale, recordCount: records, currentModelId: this.cfg.vector.presetId })
657
+ return {
658
+ catalog,
659
+ current: {
660
+ presetId: this.cfg.vector.presetId,
661
+ variantId: this.cfg.vector.variantId,
662
+ modelLabel: this.cfg.vector.modelLabel,
663
+ file: this.cfg.vector.installModel.file,
664
+ dims: this.cfg.vector.dims,
665
+ pooling: this.cfg.vector.pooling,
666
+ flavor: this.cfg.vector.flavor,
667
+ model: this.cfg.vector.model,
668
+ },
669
+ hardware,
670
+ recommendation,
671
+ locale,
672
+ records,
673
+ }
674
+ }
675
+
676
+ /** 硬件探测结果缓存(默认 10 分钟),'auto' 风味与推荐共用它。 */
677
+ async hardwareSnapshot(force = false) {
678
+ const now = Date.now()
679
+ if (!force && this.hardwareCache !== null && now - this.hardwareAt < 600_000) return this.hardwareCache
680
+ const hw = await probeHardware({ dir: this.cfg.vector.installRoot })
681
+ this.hardwareCache = hw
682
+ this.hardwareAt = now
683
+ return hw
684
+ }
685
+
686
+ /** 把 'auto' 风味解析成具体风味(需要探测;失败则退回 CPU)。 */
687
+ async resolveFlavor() {
688
+ const configured = this.cfg.vector.flavor
689
+ if (configured !== 'auto') return configured
690
+ try {
691
+ return pickFlavor(await this.hardwareSnapshot())
692
+ } catch {
693
+ return 'cpu'
694
+ }
695
+ }
696
+
697
+ /** 面板选了模型/风味之后,把选择写进安装目录的 state 文件并更新运行中的配置。 */
698
+ applyModelSelection(override ) {
699
+ const synthetic = override.custom !== undefined
700
+ ? { customModel: override.custom }
701
+ : { ...(override.preset !== undefined ? { preset: override.preset } : {}), ...(override.variant !== undefined ? { variant: override.variant } : {}) }
702
+ const sel = resolveModelSelection(synthetic)
703
+ const cfg = this.cfg.vector
704
+ cfg.installModel = {
705
+ file: sel.file,
706
+ ...(sel.bytes !== undefined ? { bytes: sel.bytes } : {}),
707
+ ...(sel.sources.length > 0 ? { sources: sel.sources } : {}),
708
+ }
709
+ cfg.presetId = sel.presetId
710
+ cfg.variantId = sel.variantId
711
+ cfg.modelLabel = sel.label
712
+ cfg.pooling = sel.pooling
713
+ cfg.dims = sel.dims
714
+ cfg.model = sel.modelName
715
+ cfg.spawn.pooling = sel.pooling
716
+ if (override.flavor !== undefined) cfg.flavor = override.flavor
717
+ cfg.spawn.gpuLayers = cfg.flavor === 'cpu' || cfg.flavor === 'auto' ? 0 : (cfg.spawn.gpuLayers || 99)
718
+ writeModelState(cfg.installRoot, {
719
+ presetId: sel.presetId === 'custom' ? undefined : sel.presetId,
720
+ variantId: sel.variantId === 'custom' ? undefined : sel.variantId,
721
+ ...(override.custom !== undefined ? { customModel: override.custom } : {}),
722
+ ...(override.flavor !== undefined ? { flavor: override.flavor } : {}),
723
+ })
724
+ return cfg.flavor
725
+ }
726
+
727
+ /**
728
+ * Panel action: download llama.cpp + the GGML model into installRoot, then
729
+ * adopt them (start the managed server). Never runs on its own — a fresh
730
+ * install downloads nothing until the user asks.
731
+ *
732
+ * `override` 是面板的选择(预设/档位/自定义 URL/风味):它会先落进安装目录的
733
+ * state 文件(下次启动仍然生效),再按新选择下载。配置里写了 preset/customModel
734
+ * 的话,配置永远优先 —— state 只是"面板选过什么"的记忆。
735
+ */
736
+ installVectorBackend(override = {}) {
737
+ const flavor = this.applyModelSelection(override)
738
+ const model = this.cfg.vector.installModel
739
+ if (detectInstalled(this.cfg.vector.installRoot, model.file)) {
740
+ this.adoptInstalledBackend()
741
+ return
742
+ }
743
+ // 换模型时把上一个安装器的进度丢掉:它描述的是另一个文件。
744
+ this.installer?.cancel()
745
+ this.installer = new VectorInstaller({
746
+ root: this.cfg.vector.installRoot,
747
+ model: { file: model.file, ...(model.bytes !== undefined ? { bytes: model.bytes } : {}), ...(model.sources !== undefined ? { sources: model.sources } : {}) },
748
+ ...(flavor === 'auto' ? {} : { flavor }),
749
+ ...(this.cfg.vector.installModelSources && model.sources === undefined ? { modelSources: this.cfg.vector.installModelSources } : {}),
750
+ onProgress: (progress) => {
751
+ if (progress.phase === 'done') this.adoptInstalledBackend()
752
+ },
753
+ })
754
+ this.installer.start()
755
+ }
756
+
757
+ /** Panel action: abort an in-flight download. */
758
+ cancelVectorInstall() {
759
+ this.installer?.cancel()
760
+ }
761
+
762
+ /**
763
+ * What the installer put on disk, so the panel can say "释放 xxx MB" before
764
+ * offering to remove it. Counts only our own layout (runtime dir, models dir,
765
+ * log + pid), never the memory store.
766
+ */
767
+ vectorInstallUsage() {
768
+ const paths = installPaths(this.cfg.vector.installRoot, this.cfg.vector.installModel.file)
769
+ const roots = [paths.dir, join(this.cfg.vector.installRoot, 'models')]
770
+ const files = [paths.logFile, `${paths.logFile}.pid`, `${paths.logFile}.err`]
771
+ let bytes = 0
772
+ const seen = []
773
+ for (const p of [...roots, ...files]) {
774
+ if (!existsSync(p)) continue
775
+ seen.push(p)
776
+ bytes += pathSize(p)
777
+ }
778
+ return { bytes, paths: seen }
779
+ }
780
+
781
+ /**
782
+ * Remove the vector backend we installed (runtime + model + logs).
783
+ *
784
+ * Refuses when the configured backend lives outside installRoot: that is a
785
+ * server the user installed and manages, and deleting it would be destroying
786
+ * something we never created. The memory store is never touched — it lives
787
+ * beside the runtime, not inside it.
788
+ */
789
+ uninstallVectorBackend()
790
+
791
+
792
+
793
+
794
+ {
795
+ const root = this.cfg.vector.installRoot
796
+ const configuredExe = this.cfg.vector.spawn.exe
797
+ if (configuredExe && !isInside(root, configuredExe)) {
798
+ return { removed: false, reason: 'EXTERNAL_BACKEND', freedBytes: 0, paths: [] }
799
+ }
800
+ const usage = this.vectorInstallUsage()
801
+ if (usage.paths.length === 0) {
802
+ return { removed: false, reason: 'NOT_INSTALLED', freedBytes: 0, paths: [] }
803
+ }
804
+ // Stop our child first: on Windows the executable and its DLLs are locked
805
+ // while it runs, so removal would fail with EBUSY/EPERM.
806
+ try { this.embedServer?.stop() } catch { /* best effort */ }
807
+ const removedPaths = []
808
+ for (const p of usage.paths) {
809
+ // Belt and braces: never step outside the install root, and never delete
810
+ // the memory store even if someone pointed storeDir inside it.
811
+ if (!isInside(root, p)) continue
812
+ if (isInside(p, this.cfg.storeDir) || resolve(p) === resolve(this.cfg.storeDir)) continue
813
+ try {
814
+ rmSync(p, { recursive: true, force: true })
815
+ removedPaths.push(p)
816
+ } catch { /* leave it; the panel reports what went */ }
817
+ }
818
+ this.cfg.vector.configured = false
819
+ this.cfg.vector.spawnConfigured = false
820
+ this.cfg.vector.enabled = false
821
+ this.embedServer = null
822
+ return { removed: true, reason: 'REMOVED', freedBytes: usage.bytes, paths: removedPaths }
823
+ }
824
+
825
+ /**
826
+ * Delete every memory record in every scope (the panel's "delete all data"
827
+ * action, used when someone wants the plugin gone without leaving notes
828
+ * behind). Goes through the store so the JSONL + vector sidecars stay
829
+ * consistent instead of ripping files out from under a live store.
830
+ */
831
+ wipeAllRecords() {
832
+ let removed = 0
833
+ const scopes = []
834
+ for (const scope of this.store.allScopes()) {
835
+ const ids = [...scope.records.keys()]
836
+ if (ids.length === 0) continue
837
+ let local = 0
838
+ for (const id of ids) if (scope.remove(id)) local += 1
839
+ removed += local
840
+ scopes.push(scope.scope)
841
+ }
842
+ try { this.store.flushSync() } catch { /* scheduled flush covers it */ }
843
+ return { removed, scopes }
844
+ }
845
+
846
+ /**
847
+ * Switch the provider over to a backend that is now on disk: fill the spawn
848
+ * config from the installed layout and start it. Idempotent.
849
+ */
850
+ adoptInstalledBackend() {
851
+ const paths = detectInstalled(this.cfg.vector.installRoot, this.cfg.vector.installModel.file)
852
+ if (!paths) return
853
+ this.cfg.vector.configured = true
854
+ this.cfg.vector.spawnConfigured = true
855
+ this.cfg.vector.enabled = true
856
+ this.cfg.vector.spawn = {
857
+ ...this.cfg.vector.spawn,
858
+ enabled: true,
859
+ exe: this.cfg.vector.spawn.exe ?? paths.exe,
860
+ // 模型路径必须跟着**当前选择的模型**:换了模型就不能继续用上一个的文件(见 pickSpawnModel)。
861
+ model: pickSpawnModel(this.cfg.vector.installModel.file, this.cfg.vector.spawn.model, paths.model),
862
+ cwd: this.cfg.vector.spawn.cwd ?? paths.cwd,
863
+ logFile: this.cfg.vector.spawn.logFile ?? paths.logFile,
864
+ pooling: this.cfg.vector.pooling,
865
+ gpuLayers: this.cfg.vector.flavor === 'cpu' || this.cfg.vector.flavor === 'auto' ? 0 : (this.cfg.vector.spawn.gpuLayers || 99),
866
+ port: this.cfg.vector.spawn.port,
867
+ }
868
+ if (!this.embedServer) {
869
+ this.startEmbedServer()
870
+ return
871
+ }
872
+ void this.ensureEmbedServer(60_000).catch(() => { /* panel shows the outcome */ })
873
+ }
874
+
875
+ /**
876
+ * Hourly expired-record purge + capacity eviction. M4 housekeeping; the
877
+ * timer is unref'd so it never holds the process open, and cleared by
878
+ * flushSync()/dispose.
879
+ */
880
+ maintenanceTimer = null
881
+
882
+ startMaintenance() {
883
+ this.maintenanceTimer = setInterval(() => {
884
+ this.store.purgeExpired()
885
+ this.store.enforceLimit(this.cfg.maxRecordsPerScope)
886
+ this.store.scheduleFlush(0)
887
+ }, 3_600_000)
888
+ this.maintenanceTimer.unref?.()
889
+ // Pending-embed drain: the store queues ids whose embed failed (backend
890
+ // down at save time, or vectors restored from a jsonl-only backup). Until
891
+ // this timer existed nothing ever drained that queue — a single outage
892
+ // stranded them forever (observable as 待嵌入>0 in the memory panel).
893
+ this.embedRetryTimer = setInterval(() => {
894
+ void this.retryPendingEmbeds().catch(() => {})
895
+ }, 60_000)
896
+ this.embedRetryTimer.unref?.()
897
+ // First drain shortly after boot, when the backend is usually already up.
898
+ const initial = setTimeout(() => {
899
+ this.initialDrainTimer = null
900
+ void this.retryPendingEmbeds().catch(() => {})
901
+ }, 5_000)
902
+ initial.unref?.()
903
+ this.initialDrainTimer = initial
904
+ }
905
+
906
+ embedRetryTimer = null
907
+ initialDrainTimer = null
908
+ /** Owns the llama-server child process when vector.spawn.enabled. */
909
+ embedServer = null
910
+ /** Lazily created by the panel's "download and configure" action. */
911
+ installer = null
912
+ /** 硬件探测缓存(面板问了才探测,10 分钟内复用)。 */
913
+ hardwareCache = null
914
+ hardwareAt = 0
915
+ exitHook = null
916
+
917
+ /**
918
+ * Drain the pending-embed queue, probing through the circuit breaker first so
919
+ * a backend that came back online is noticed (backfillEmbeddings itself
920
+ * refuses to touch the network while marked unhealthy, and embed() is the
921
+ * only path that can flip that flag back to healthy).
922
+ */
923
+ async retryPendingEmbeds() {
924
+ // Vector retrieval opt-in: with no configured backend there is nothing to
925
+ // retry, and probing would only produce failed requests in the log.
926
+ if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return 0
927
+ // Scopes load lazily, so without discovery the queue of an untouched scope
928
+ // (e.g. a project store written by a previous session) is invisible here and
929
+ // would never drain — exactly the 待嵌入>0 the panel showed on 2026-10-10.
930
+ try { this.store.loadDiscovered() } catch { /* discovery is best-effort */ }
931
+ if (this.pendingEmbedCount === 0) return 0
932
+ if (!this.embed.isHealthy) {
933
+ const probe = await this.embed.embed(['ping'], undefined, { timeoutMs: this.cfg.vector.writeTimeoutMs }).catch(() => null)
934
+ if (!probe) return 0
935
+ }
936
+ return this.backfillAll({ throttleMs: 100 })
937
+ }
938
+
939
+ setDefaultScope(scope ) {
940
+ this.defaultScope = scope
941
+ }
942
+
943
+ /**
944
+ * Provide the one-shot LLM caller used by the rerank stage. Passing null
945
+ * (or never calling) keeps retrieval on recall-order only. The cordis
946
+ * wiring adapts `ctx.llm` into this; it is optional by design — the service
947
+ * must never sit PENDING just because no LLM adapter is loaded.
948
+ */
949
+ setLlmInvoke(invoke ) {
950
+ this.llmInvoke = invoke
951
+ }
952
+
953
+ get activeDefaultScope() {
954
+ return this.defaultScope
955
+ }
956
+
957
+ assertScopeAllowed(scope ) {
958
+ if (scope === 'global' && !this.cfg.globalEnabled) {
959
+ throw memoryError('SCOPE_DISABLED', 'global scope is disabled by configuration')
960
+ }
961
+ if (scope.startsWith('project:') && !this.cfg.projectEnabled) {
962
+ throw memoryError('SCOPE_DISABLED', 'project scope is disabled by configuration')
963
+ }
964
+ }
965
+
966
+ async save(input ) {
967
+ const scope = input.scope ?? this.defaultScope
968
+ this.assertScopeAllowed(scope)
969
+ let content = input.content?.trim()
970
+ if (!content) throw memoryError('CONTENT_EMPTY', 'content must be a non-empty string')
971
+ if (content.length > this.cfg.maxRecordChars) {
972
+ throw memoryError('CONTENT_TOO_LONG', `content exceeds ${this.cfg.maxRecordChars} chars`)
973
+ }
974
+ if (input.supersedes && !this.store.scope(scope).records.get(input.supersedes)) {
975
+ throw memoryError('SUPERSEDES_UNKNOWN', `supersedes target ${input.supersedes} not found in ${scope}`)
976
+ }
977
+ // design §11: keep credentials out of the plaintext store and the embedder
978
+ if (this.cfg.secrets.enabled) {
979
+ const findings = scanSecrets(content)
980
+ if (findings.length > 0) {
981
+ if (this.cfg.secrets.mode === 'refuse') {
982
+ throw memoryError('SECRET_DETECTED', `content matches credential denylist: ${[...new Set(findings.map((f) => f.kind))].join(', ')}`)
983
+ }
984
+ content = redactSecrets(content)
985
+ }
986
+ }
987
+ const { record } = this.store.save(
988
+ { ...input, scope, content },
989
+ { importance: input.source?.origin === 'auto' ? 0.4 : 0.8, source: { origin: 'explicit' } },
990
+ )
991
+ void this.embedRecord(record).catch(() => {})
992
+ this.store.scheduleFlush()
993
+ return record
994
+ }
995
+
996
+ async embedRecord(record ) {
997
+ if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return
998
+ const s = this.store.scope(record.scope)
999
+ // save-time budget: one full record, much slower than a search query
1000
+ // 记录按"文档"嵌入:有些模型(nomic/e5)要求文档与查询带不同前缀,不带质量会掉。
1001
+ const vec = await this.embed.embed([`${this.cfg.vector.documentPrefix ?? ''}${record.content}`], undefined, { timeoutMs: this.cfg.vector.writeTimeoutMs }).catch(() => null)
1002
+ if (vec?.[0]) s.appendVector(record.id, normalizeInPlace(vec[0]))
1003
+ else s.embedPending.add(record.id) // backfill queue; the retry timer drains it
1004
+ }
1005
+
1006
+ /**
1007
+ * Retry pending embeddings in batches; returns number embedded.
1008
+ * Batches are bounded by count AND total characters, and use the backfill
1009
+ * budget: full record contents on CPU take seconds per batch, so the
1010
+ * interactive timeout would abort every attempt.
1011
+ */
1012
+ async backfillEmbeddings() {
1013
+ if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return 0
1014
+ if (!this.embed.isHealthy) return 0
1015
+ let done = 0
1016
+ for (const s of this.store.allScopes()) {
1017
+ if (s.embedPending.size === 0) continue
1018
+ const texts = []
1019
+ const valid = []
1020
+ let chars = 0
1021
+ for (const id of s.embedPending) {
1022
+ if (valid.length >= this.cfg.vector.backfillBatchSize) break
1023
+ const t = s.records.get(id)?.content
1024
+ if (!t) continue
1025
+ if (valid.length > 0 && chars + t.length > this.cfg.vector.backfillMaxChars) break
1026
+ texts.push(`${this.cfg.vector.documentPrefix ?? ''}${t}`)
1027
+ valid.push(id)
1028
+ chars += t.length
1029
+ }
1030
+ if (texts.length === 0) continue
1031
+ const vecs = await this.embed.embed(texts, undefined, { timeoutMs: this.cfg.vector.backfillTimeoutMs }).catch(() => null)
1032
+ if (!vecs) return done
1033
+ for (let i = 0; i < valid.length; i++) {
1034
+ if (s.records.has(valid[i] ) && vecs[i]) s.appendVector(valid[i] , normalizeInPlace(vecs[i] ))
1035
+ s.embedPending.delete(valid[i] )
1036
+ done++
1037
+ }
1038
+ }
1039
+ return done
1040
+ }
1041
+
1042
+ /**
1043
+ * First-fill: embed the ENTIRE backlog (unlike backfillEmbeddings' single
1044
+ * batch per scope), in throttled batches so a big import does not hammer
1045
+ * llama-server or block the caller. Yields to the event loop between
1046
+ * batches and stops early on abort/unhealthy embedder.
1047
+ */
1048
+ async backfillAll(opts = {}) {
1049
+ const throttle = opts.throttleMs ?? 50
1050
+ let done = 0
1051
+ while (!opts.signal?.aborted) {
1052
+ const n = await this.backfillEmbeddings()
1053
+ done += n
1054
+ if (n === 0) break // per-scope batch was empty → backlog drained
1055
+ if (!this.embed.isHealthy) break
1056
+ await new Promise((r) => setTimeout(r, throttle))
1057
+ }
1058
+ return done
1059
+ }
1060
+
1061
+ async search(query ) {
1062
+ const topK = query.topK ?? this.cfg.topK
1063
+ const recallN = query.recallN ?? this.cfg.recallN
1064
+ const now = Date.now()
1065
+ const queryTokens = [...new Set(tokenize(query.text))]
1066
+
1067
+ // candidate scopes: pinned, else project + global
1068
+ const stores = []
1069
+ if (query.scope) {
1070
+ stores.push({ store: this.store.scope(query.scope), isGlobal: query.scope === 'global' })
1071
+ } else {
1072
+ if (this.cfg.projectEnabled) stores.push({ store: this.store.scope(this.defaultScope), isGlobal: this.defaultScope === 'global' })
1073
+ if (this.cfg.globalEnabled) stores.push({ store: this.store.scope('global'), isGlobal: true })
1074
+ }
1075
+
1076
+ // --- channel 1: lexical recall, scored per scope with local IDF ---
1077
+
1078
+ const candidates = new Map ()
1079
+ for (const { store, isGlobal } of stores) {
1080
+ const all = [...store.records.values()]
1081
+ const tokenSets = all.map((r) => new Set(tokenize(r.content)))
1082
+ const stats = buildStats(tokenSets)
1083
+ for (let i = 0; i < all.length; i++) {
1084
+ const rec = all[i]
1085
+ if (rec.expiredAt && rec.expiredAt < now) continue
1086
+ const lex = lexicalScore(queryTokens, tokenSets[i] , stats)
1087
+ if (lex <= 0) continue
1088
+ // Weak-only overlap (no shared word/bigram) must clear the "real hit"
1089
+ // floor, not the permissive threshold — kills single-char collisions.
1090
+ const floor = isGlobal ? this.cfg.globalThreshold : this.cfg.projectThreshold
1091
+ const effective = hasStrongHit(queryTokens, tokenSets[i] ) ? floor : Math.max(floor, REAL_HIT_FLOOR)
1092
+ if (lex < effective) continue
1093
+ candidates.set(rec.id, { record: rec, recallScore: lex, isGlobal })
1094
+ }
1095
+ }
1096
+
1097
+ // --- channel 2: vector recall over embedded records ---
1098
+ // Skipped entirely when no backend is configured (lexical recall above
1099
+ // already produced the candidates; a request here would just fail).
1100
+ const qvec = this.cfg.vector.enabled && this.cfg.vector.configured
1101
+ ? await this.embed.embed([`${this.cfg.vector.queryPrefix ?? ''}${query.text}`], query.signal).catch(() => null)
1102
+ : null
1103
+ if (qvec?.[0]) {
1104
+ const vectorScores = new Map ()
1105
+ for (const { store } of stores) {
1106
+ for (const [id, vec] of store.vectors) {
1107
+ if (!store.records.has(id)) continue
1108
+ const sim = cosine(qvec[0] , vec)
1109
+ if (sim >= this.cfg.vector.minScore) vectorScores.set(id, sim)
1110
+ }
1111
+ }
1112
+ for (const [id, sim] of vectorScores) {
1113
+ const existing = candidates.get(id)
1114
+ if (existing) {
1115
+ existing.recallScore = Math.max(existing.recallScore, sim) // best channel wins
1116
+ } else {
1117
+ const rec = this.findById(id)
1118
+ if (rec && !(rec.expiredAt && rec.expiredAt < now)) {
1119
+ if (!(rec.scope === 'global' && sim < this.cfg.globalThreshold)) {
1120
+ candidates.set(id, { record: rec, recallScore: sim, isGlobal: rec.scope === 'global' })
1121
+ }
1122
+ }
1123
+ }
1124
+ }
1125
+ }
1126
+
1127
+ // shape with recency/importance, sort, cut
1128
+ const supersededIds = new Set ()
1129
+ for (const { store } of stores) {
1130
+ for (const r of store.records.values()) if (r.supersedes) supersededIds.add(r.supersedes)
1131
+ }
1132
+ const shape = (c ) =>
1133
+ recordScore(c.recallScore, c.record.importance, c.record.accessCount, c.record.lastAccessedAt, {
1134
+ now,
1135
+ halfLifeDays: this.cfg.halfLifeDays,
1136
+ superseded: supersededIds.has(c.record.id),
1137
+ })
1138
+
1139
+ const ranked = [...candidates.values()].sort((a, b) => shape(b) - shape(a)).slice(0, recallN)
1140
+
1141
+ // --- channel 3: LLM rerank (optional; any skip/failure keeps recall order) ---
1142
+ const finalScores = new Map (ranked.map((c) => [c.id, shape(c)]))
1143
+ if (this.cfg.llm.enabled && this.llmInvoke && ranked.length >= 2) {
1144
+ const asCands = ranked.map((c) => ({ id: c.id, content: c.record.content, recallScore: finalScores.get(c.id) }))
1145
+ const cacheKey = RerankCache.key(query.text, ranked.map((c) => c.id), this.cfg.llm.model)
1146
+ let outcome = this.rerankCache.get(cacheKey)
1147
+ if (!outcome) {
1148
+ outcome = await rerank({
1149
+ invoke: this.llmInvoke,
1150
+ cfg: this.cfg.llm,
1151
+ query: query.text,
1152
+ candidates: asCands,
1153
+ budget: this.rerankBudget,
1154
+ signal: query.signal,
1155
+ })
1156
+ if (outcome) this.rerankCache.set(cacheKey, outcome)
1157
+ }
1158
+ if (outcome) {
1159
+ for (const { id, score } of blendScores(asCands, outcome, this.cfg.llm.weight)) {
1160
+ finalScores.set(id, score)
1161
+ }
1162
+ }
1163
+ }
1164
+
1165
+ const hits = ranked.map((c) => ({ record: touch(c.record), score: round4(finalScores.get(c.id) ) }))
1166
+
1167
+ this.store.scheduleFlush()
1168
+ return hits.slice(0, topK)
1169
+ }
1170
+
1171
+ findById(id ) {
1172
+ for (const s of this.store.allScopes()) {
1173
+ const r = s.records.get(id)
1174
+ if (r) return r
1175
+ }
1176
+ return undefined
1177
+ }
1178
+
1179
+ async forget(input ) {
1180
+ const ids = []
1181
+ if (input.id) {
1182
+ for (const s of this.store.allScopes()) {
1183
+ if (s.remove(input.id)) ids.push(input.id)
1184
+ }
1185
+ if (ids.length === 0) throw memoryError('RECORD_NOT_FOUND', `id ${input.id} not found`)
1186
+ } else if (input.query) {
1187
+ const scope = input.scope ?? this.defaultScope
1188
+ ids.push(...this.store.findIdsByQuery(scope, input.query))
1189
+ const s = this.store.scope(scope)
1190
+ for (const id of ids) s.remove(id)
1191
+ if (ids.length === 0) throw memoryError('RECORD_NOT_FOUND', `no record matching query in ${scope}`)
1192
+ } else {
1193
+ throw memoryError('RECORD_NOT_FOUND', 'forget requires id or query')
1194
+ }
1195
+ this.store.scheduleFlush()
1196
+ return { removed: ids.length, ids }
1197
+ }
1198
+
1199
+ async list(input = {}) {
1200
+ const scope = input.scope ?? this.defaultScope
1201
+ let records = [...this.store.scope(scope).records.values()]
1202
+ if (input.kind) records = records.filter((r) => r.kind === input.kind)
1203
+ records.sort((a, b) => b.updatedAt - a.updatedAt)
1204
+ return records.slice(0, input.limit ?? 50)
1205
+ }
1206
+
1207
+ /** Force a synchronous flush (dispose/HMR path). */
1208
+ flushSync() {
1209
+ if (this.maintenanceTimer) {
1210
+ clearInterval(this.maintenanceTimer)
1211
+ this.maintenanceTimer = null
1212
+ }
1213
+ if (this.embedRetryTimer) {
1214
+ clearInterval(this.embedRetryTimer)
1215
+ this.embedRetryTimer = null
1216
+ }
1217
+ if (this.initialDrainTimer) {
1218
+ clearTimeout(this.initialDrainTimer)
1219
+ this.initialDrainTimer = null
1220
+ }
1221
+ if (this.embedServer) this.embedServer.stop()
1222
+ if (this.exitHook) {
1223
+ process.removeListener('exit', this.exitHook)
1224
+ this.exitHook = null
1225
+ }
1226
+ this.store.purgeExpired()
1227
+ this.store.enforceLimit(this.cfg.maxRecordsPerScope)
1228
+ this.store.close()
1229
+ }
1230
+
1231
+ get pendingEmbedCount() {
1232
+ return this.store.allScopes().reduce((n, s) => n + s.embedPending.size, 0)
1233
+ }
1234
+ }
1235
+
1236
+ function touch(record ) {
1237
+ record.lastAccessedAt = Date.now()
1238
+ record.accessCount += 1
1239
+ return record
1240
+ }
1241
+
1242
+ function round4(n ) {
1243
+ return Math.round(n * 10_000) / 10_000
1244
+ }
1245
+
1246
+ /** Port of an http(s) endpoint, or null when it cannot be parsed. */
1247
+ function portOf(endpoint ) {
1248
+ try {
1249
+ const port = new URL(endpoint).port
1250
+ return port ? Number.parseInt(port, 10) : null
1251
+ } catch {
1252
+ return null
1253
+ }
1254
+ }
1255
+
1256
+ /**
1257
+ * Measured score floor of genuine lexical hits; below this only single-char
1258
+ * coincidences live. Used to gate weak-only overlaps regardless of config.
1259
+ */
1260
+ const REAL_HIT_FLOOR = 0.25
1261
+
1262
+ function memoryError(code , message ) {
1263
+ const err = new Error(`[memory:${code}] ${message}`)
1264
+ err.code = code
1265
+ return err
1266
+ }
1267
+
1268
+ export function createMemoryService(config = {}) {
1269
+ return new LocalMemoryService(config)
1270
+ }
28
1271
 
29
- /** Recursive byte count; 0 for anything unreadable. */
30
- function pathSize(path ) {
31
- try {
32
- const st = statSync(path)
33
- if (!st.isDirectory()) return st.size
34
- let total = 0
35
- for (const entry of readdirSync(path)) total += pathSize(join(path, entry))
36
- return total
37
- } catch {
38
- return 0
39
- }
40
- }
41
1272
 
42
- /** True when `child` is `parent` or lives under it (both resolved). */
43
- function isInside(parent , child ) {
44
- const rel = relative(resolve(parent), resolve(child))
45
- return rel === '' || (!rel.startsWith('..') && !isAbsolute(rel))
46
- }
47
-
48
-
49
-
50
-
51
-
52
-
53
-
54
-
55
-
56
-
57
-
58
-
59
-
60
-
61
-
62
-
63
-
64
-
65
-  
66
-
67
-
68
-
69
-
70
-
71
-
72
-
73
-
74
-
75
-
76
-
77
-
78
-
79
-
80
-
81
-  
82
-
83
-
84
-
85
-
86
-
87
-  
88
-  
89
-
90
-
91
-
92
-
93
-
94
-
95
-
96
-
97
-  
98
-
99
-
100
-
101
-
102
-
103
-
104
-
105
-
106
-
107
-
108
-
109
-
110
-
111
-
112
-
113
-
114
-
115
-
116
-
117
-
118
-  
119
-
120
-
121
-
122
-
123
-
124
-
125
-
126
-
127
-
128
-
129
-
130
-
131
-
132
-
133
-
134
-
135
-
136
-
137
- export function resolveConfig(config = {}) {
138
- const v = config.retrieval?.vector ?? {}
139
- // Do not touch an embedding backend the user never asked for: pointing at the
140
- // stock localhost endpoint on a machine that has nothing listening means
141
- // failed embeds, a tripped breaker and scary log lines for a feature that is
142
- // optional. Vector retrieval is therefore opt-in by CONFIGURATION — writing
143
- // `endpoint` (any OpenAI-compatible service), configuring `spawn`, or having
144
- // the panel's installer put a backend on disk implies intent; an explicit
145
- // `enabled` always wins.
146
- const storeDir = config.storeDir ?? '.dsh-memory'
147
- // Installer target: explicit config → that directory; explicit storeDir →
148
- // its parent (so a hand-made `<root>/llamacpp` + `<root>/models` layout is
149
- // recognised as-is); otherwise a stable per-user directory under ~/.dsh.
150
- const installRoot = v.autoInstall?.dir
151
- ?? (config.storeDir !== undefined ? dirname(resolve(storeDir)) : join(homedir(), '.dsh', 'memory-vector'))
152
- const installed = detectInstalled(installRoot)
153
- const endpointConfigured = v.endpoint !== undefined
154
- // A configured spawn path that no longer exists (plugin uninstalled the
155
- // backend, or the user moved it) must not count as configured: that would
156
- // leave the panel showing a managed server that can never start, instead of
157
- // offering the download button again.
158
- const spawnExe = v.spawn?.exe
159
- const spawnExeUsable = spawnExe === undefined || spawnExe === '' || existsSync(spawnExe)
160
- const spawnConfigured = ((v.spawn?.enabled === true || (spawnExe ?? '') !== '') && spawnExeUsable) || installed !== null
161
- const configured = endpointConfigured || spawnConfigured
162
- const vectorEnabled = v.enabled ?? configured
163
- return {
164
- storeDir,
165
- globalEnabled: config.scopes?.global ?? true,
166
- projectEnabled: config.scopes?.project ?? true,
167
- topK: config.retrieval?.topK ?? 5,
168
- recallN: config.retrieval?.recallN ?? 30,
169
- globalThreshold: config.retrieval?.globalThreshold ?? 0.12,
170
- projectThreshold: config.retrieval?.projectThreshold ?? 0.1,
171
- halfLifeDays: config.retrieval?.recencyHalfLifeDays ?? 30,
172
- vector: {
173
- enabled: vectorEnabled,
174
- configured,
175
- endpointConfigured,
176
- spawnConfigured,
177
- installRoot,
178
- ...(v.autoInstall?.modelSources ? { installModelSources: v.autoInstall.modelSources } : {}),
179
- endpoint: v.endpoint ?? 'http://127.0.0.1:8078/v1/embeddings',
180
- model: v.model ?? 'bge-m3',
181
- dims: v.dims ?? 1024,
182
- timeoutMs: v.timeoutMs ?? 300,
183
- // Full contents are orders of magnitude slower than a short query on CPU
184
- // (measured 3143ms for three ~600-char records), so write/backfill paths
185
- // get their own generous budgets instead of the interactive 300ms.
186
- writeTimeoutMs: v.writeTimeoutMs ?? 10_000,
187
- backfillTimeoutMs: v.backfillTimeoutMs ?? 30_000,
188
- backfillMaxChars: v.backfillMaxChars ?? 8_000,
189
- // BGE CLS baseline noise floor measured ~0.42 on unrelated pairs
190
- minScore: v.minScore ?? 0.5,
191
- backfillBatchSize: v.backfillBatchSize ?? 32,
192
- // Port defaults to the one in the embeddings endpoint so the two can never
193
- // drift apart; opt-in (enabled:false) keeps dev/test runs from spawning.
194
- // A backend the panel installed (or a hand-made one at installRoot) is
195
- // adopted as the spawn target when the user configured no explicit paths.
196
- spawn: {
197
- ...(v.spawn ?? {}),
198
- enabled: v.spawn?.enabled ?? installed !== null,
199
- ...(installed
200
- ? {
201
- exe: v.spawn?.exe ?? installed.exe,
202
- model: v.spawn?.model ?? installed.model,
203
- cwd: v.spawn?.cwd ?? installed.cwd,
204
- logFile: v.spawn?.logFile ?? installed.logFile,
205
- }
206
- : {}),
207
- port: v.spawn?.port ?? portOf(v.endpoint ?? 'http://127.0.0.1:8078/v1/embeddings') ?? 8078,
208
- },
209
- },
210
- llm: {
211
- enabled: config.retrieval?.llm?.enabled ?? false,
212
- provider: config.retrieval?.llm?.provider ?? '',
213
- model: config.retrieval?.llm?.model ?? '',
214
- timeoutMs: config.retrieval?.llm?.timeoutMs ?? 1500,
215
- dailyBudgetTokens: config.retrieval?.llm?.dailyBudgetTokens ?? 200_000,
216
- weight: config.retrieval?.llm?.weight ?? 0.6,
217
- topN: config.retrieval?.llm?.topN ?? 10,
218
- },
219
- maxRecordsPerScope: config.limits?.maxRecordsPerScope ?? 2000,
220
- maxRecordChars: config.limits?.maxRecordChars ?? 4000,
221
- secrets: {
222
- enabled: config.secrets?.enabled ?? true,
223
- mode: config.secrets?.mode ?? 'refuse',
224
- },
225
- }
226
- }
227
-
228
- export class LocalMemoryService {
229
- cfg
230
- store
231
- embed
232
- rerankBudget
233
- rerankCache = new RerankCache()
234
- /** Host-provided one-shot LLM call; null = rerank disabled (recall order). */
235
- llmInvoke = null
236
- /** Where saves land when the caller does not pin a scope. */
237
- defaultScope = 'global'
238
-
239
- constructor(config = {}) {
240
- this.cfg = resolveConfig(config)
241
- this.rerankBudget = new DailyBudget(this.cfg.llm.dailyBudgetTokens)
242
- this.store = new MemoryStore(this.cfg.storeDir, { onWarn: (m) => console.warn(`[dsh-memory-local] ${m}`) })
243
- // Identity BEFORE the eager load (issue #1): ScopeStore skips its sidecar
244
- // validation entirely while the identity is unset (`if (this.vecIdentity && …)`),
245
- // so loading first would accept vectors from a previous embedding model
246
- // whenever the dimensions matched — and the next flush would then rewrite the
247
- // header as the new model, making the mismatch permanent.
248
- this.store.setVecIdentity(this.cfg.vector.model, this.cfg.vector.dims)
249
- this.store.loadAll(this.cfg.projectEnabled && this.cfg.globalEnabled ? [this.defaultScope, 'global'] : [this.defaultScope])
250
- this.embed = new EmbedClient({
251
- endpoint: this.cfg.vector.endpoint,
252
- model: this.cfg.vector.model,
253
- dims: this.cfg.vector.dims,
254
- timeoutMs: this.cfg.vector.timeoutMs,
255
- })
256
- this.startMaintenance()
257
- this.startEmbedServer()
258
- }
259
-
260
- /** Own the embedding backend's lifetime: spawn on activation, stop on dispose. */
261
- startEmbedServer() {
262
- // Nothing configured and retrieval switched off → do not even build the
263
- // manager, so no probe, no spawn attempt, no misleading panel state.
264
- if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return
265
- this.embedServer = new EmbedServerManager(this.cfg.vector.spawn)
266
- // fire-and-forget: activation must never block on a model load
267
- void this.ensureEmbedServer(60_000).catch(() => { /* lifecycle never breaks activation */ })
268
- }
269
-
270
- /**
271
- * Start (or adopt) the embedding backend and arm the exit safety net.
272
- * Shared by the activation path and the panel's manual start button.
273
- */
274
- async ensureEmbedServer(waitMs ) {
275
- const mgr = this.embedServer
276
- if (!mgr) return 'disabled'
277
- const state = await mgr.ensureRunning()
278
- if (state !== 'spawned' && state !== 'adopted-managed') return state
279
- // hard-exit safety net: cordis disposal covers a clean quit; this covers
280
- // the host dying without disposing us (the child must not be orphaned)
281
- if (!this.exitHook) {
282
- this.exitHook = () => { this.embedServer?.stop() }
283
- process.once('exit', this.exitHook)
284
- }
285
- const ready = await mgr.waitUntilReady(waitMs)
286
- if (!ready) console.warn('[dsh-memory-local] embed server: 启动超时,检索继续走词法降级')
287
- return state
288
- }
289
-
290
- /** Manual start from the memory panel (same semantics as activation). */
291
- startEmbedServerManually() {
292
- return this.ensureEmbedServer(25_000)
293
- }
294
-
295
- /**
296
- * Manual stop from the memory panel. An instance we do not own is left alone
297
- * (it was started outside the app), so the UI can explain the refusal.
298
- */
299
- stopEmbedServerManually() {
300
- const mgr = this.embedServer
301
- if (!mgr) return { stopped: false, reason: 'NOT_RUNNING' }
302
- if (!mgr.isOwned) return { stopped: false, reason: 'EXTERNAL_INSTANCE' }
303
- return { stopped: mgr.stop(), reason: mgr.isOwned ? 'NOT_RUNNING' : 'STOPPED' }
304
- }
305
-
306
- /**
307
- * Lifecycle snapshot for the memory panel.
308
- *
309
- * `mode` is what the panel needs to decide whether a start/stop switch even
310
- * makes sense:
311
- * - 'off' nothing configured → no switch, show how to opt in
312
- * - 'external' we only know an endpoint (cloud or a server the user runs)
313
- * → no switch, we must not claim ownership of that process
314
- * - 'managed' spawn is configured → the switch toggles OUR child process
315
- * `enabled` is the retrieval switch from config; when it is false while a
316
- * backend IS configured the panel explains the config instead of showing a
317
- * switch that cannot turn retrieval on.
318
- */
319
- async embedServerStatus()
320
-
321
-
322
-
323
-
324
-
325
-
326
-
327
-
328
-
329
-
330
-
331
-
332
-
333
-
334
- {
335
- const mgr = this.embedServer
336
- const cfg = this.cfg.vector
337
- const pendingEmbeds = this.pendingEmbedCount
338
- const mode = !cfg.configured ? 'off' : cfg.spawnConfigured ? 'managed' : 'external'
339
- const usage = this.vectorInstallUsage()
340
- const base = {
341
- enabled: cfg.enabled,
342
- configured: cfg.configured,
343
- mode,
344
- endpoint: cfg.endpoint,
345
- model: cfg.model,
346
- spawnConfigured: cfg.spawnConfigured,
347
- pendingEmbeds,
348
- install: this.installSnapshot(),
349
- maintenance: {
350
- usageBytes: usage.bytes,
351
- paths: usage.paths,
352
- // NOTE: `cfg` is the VECTOR config here, not the whole config — reading
353
- // cfg.storeDir would silently yield undefined (and JSON drops it).
354
- storeDir: this.cfg.storeDir,
355
- records: this.store.allScopes().reduce((n, s) => n + s.records.size, 0),
356
- },
357
- }
358
- if (!mgr) return { ...base, owned: false, pid: null, baseUrl: '', reachable: false }
359
- const reachable = await mgr.healthCheck()
360
- return { ...base, ...mgr.info(), reachable, pendingEmbeds }
361
- }
362
-
363
- /** Installer state for the panel (idle when nothing was ever requested). */
364
- installSnapshot() {
365
- const root = this.cfg.vector.installRoot
366
- const installed = detectInstalled(root) !== null
367
- const progress = this.installer?.progress ?? {
368
- phase: 'idle' ,
369
- message: '',
370
- receivedBytes: 0,
371
- totalBytes: null,
372
- error: null,
373
- paths: installed ? installPaths(root) : null,
374
- }
375
- return { ...progress, installed, root }
376
- }
377
-
378
- /**
379
- * Panel action: download llama.cpp + the GGML model into installRoot, then
380
- * adopt them (start the managed server). Never runs on its own — a fresh
381
- * install downloads nothing until the user asks.
382
- */
383
- installVectorBackend() {
384
- if (detectInstalled(this.cfg.vector.installRoot)) {
385
- this.adoptInstalledBackend()
386
- return
387
- }
388
- if (!this.installer) {
389
- this.installer = new VectorInstaller({
390
- root: this.cfg.vector.installRoot,
391
- ...(this.cfg.vector.installModelSources ? { modelSources: this.cfg.vector.installModelSources } : {}),
392
- onProgress: (progress) => {
393
- if (progress.phase === 'done') this.adoptInstalledBackend()
394
- },
395
- })
396
- }
397
- this.installer.start()
398
- }
399
-
400
- /** Panel action: abort an in-flight download. */
401
- cancelVectorInstall() {
402
- this.installer?.cancel()
403
- }
404
-
405
- /**
406
- * What the installer put on disk, so the panel can say "释放 xxx MB" before
407
- * offering to remove it. Counts only our own layout (runtime dir, models dir,
408
- * log + pid), never the memory store.
409
- */
410
- vectorInstallUsage() {
411
- const paths = installPaths(this.cfg.vector.installRoot)
412
- const roots = [paths.dir, join(this.cfg.vector.installRoot, 'models')]
413
- const files = [paths.logFile, `${paths.logFile}.pid`, `${paths.logFile}.err`]
414
- let bytes = 0
415
- const seen = []
416
- for (const p of [...roots, ...files]) {
417
- if (!existsSync(p)) continue
418
- seen.push(p)
419
- bytes += pathSize(p)
420
- }
421
- return { bytes, paths: seen }
422
- }
423
-
424
- /**
425
- * Remove the vector backend we installed (runtime + model + logs).
426
- *
427
- * Refuses when the configured backend lives outside installRoot: that is a
428
- * server the user installed and manages, and deleting it would be destroying
429
- * something we never created. The memory store is never touched — it lives
430
- * beside the runtime, not inside it.
431
- */
432
- uninstallVectorBackend()
433
-
434
-
435
-
436
-
437
- {
438
- const root = this.cfg.vector.installRoot
439
- const configuredExe = this.cfg.vector.spawn.exe
440
- if (configuredExe && !isInside(root, configuredExe)) {
441
- return { removed: false, reason: 'EXTERNAL_BACKEND', freedBytes: 0, paths: [] }
442
- }
443
- const usage = this.vectorInstallUsage()
444
- if (usage.paths.length === 0) {
445
- return { removed: false, reason: 'NOT_INSTALLED', freedBytes: 0, paths: [] }
446
- }
447
- // Stop our child first: on Windows the executable and its DLLs are locked
448
- // while it runs, so removal would fail with EBUSY/EPERM.
449
- try { this.embedServer?.stop() } catch { /* best effort */ }
450
- const removedPaths = []
451
- for (const p of usage.paths) {
452
- // Belt and braces: never step outside the install root, and never delete
453
- // the memory store even if someone pointed storeDir inside it.
454
- if (!isInside(root, p)) continue
455
- if (isInside(p, this.cfg.storeDir) || resolve(p) === resolve(this.cfg.storeDir)) continue
456
- try {
457
- rmSync(p, { recursive: true, force: true })
458
- removedPaths.push(p)
459
- } catch { /* leave it; the panel reports what went */ }
460
- }
461
- this.cfg.vector.configured = false
462
- this.cfg.vector.spawnConfigured = false
463
- this.cfg.vector.enabled = false
464
- this.embedServer = null
465
- return { removed: true, reason: 'REMOVED', freedBytes: usage.bytes, paths: removedPaths }
466
- }
467
-
468
- /**
469
- * Delete every memory record in every scope (the panel's "delete all data"
470
- * action, used when someone wants the plugin gone without leaving notes
471
- * behind). Goes through the store so the JSONL + vector sidecars stay
472
- * consistent instead of ripping files out from under a live store.
473
- */
474
- wipeAllRecords() {
475
- let removed = 0
476
- const scopes = []
477
- for (const scope of this.store.allScopes()) {
478
- const ids = [...scope.records.keys()]
479
- if (ids.length === 0) continue
480
- let local = 0
481
- for (const id of ids) if (scope.remove(id)) local += 1
482
- removed += local
483
- scopes.push(scope.scope)
484
- }
485
- try { this.store.flushSync() } catch { /* scheduled flush covers it */ }
486
- return { removed, scopes }
487
- }
488
-
489
- /**
490
- * Switch the provider over to a backend that is now on disk: fill the spawn
491
- * config from the installed layout and start it. Idempotent.
492
- */
493
- adoptInstalledBackend() {
494
- const paths = detectInstalled(this.cfg.vector.installRoot)
495
- if (!paths) return
496
- this.cfg.vector.configured = true
497
- this.cfg.vector.spawnConfigured = true
498
- this.cfg.vector.enabled = true
499
- this.cfg.vector.spawn = {
500
- ...this.cfg.vector.spawn,
501
- enabled: true,
502
- exe: this.cfg.vector.spawn.exe ?? paths.exe,
503
- model: this.cfg.vector.spawn.model ?? paths.model,
504
- cwd: this.cfg.vector.spawn.cwd ?? paths.cwd,
505
- logFile: this.cfg.vector.spawn.logFile ?? paths.logFile,
506
- port: this.cfg.vector.spawn.port,
507
- }
508
- if (!this.embedServer) {
509
- this.startEmbedServer()
510
- return
511
- }
512
- void this.ensureEmbedServer(60_000).catch(() => { /* panel shows the outcome */ })
513
- }
514
-
515
- /**
516
- * Hourly expired-record purge + capacity eviction. M4 housekeeping; the
517
- * timer is unref'd so it never holds the process open, and cleared by
518
- * flushSync()/dispose.
519
- */
520
- maintenanceTimer = null
521
-
522
- startMaintenance() {
523
- this.maintenanceTimer = setInterval(() => {
524
- this.store.purgeExpired()
525
- this.store.enforceLimit(this.cfg.maxRecordsPerScope)
526
- this.store.scheduleFlush(0)
527
- }, 3_600_000)
528
- this.maintenanceTimer.unref?.()
529
- // Pending-embed drain: the store queues ids whose embed failed (backend
530
- // down at save time, or vectors restored from a jsonl-only backup). Until
531
- // this timer existed nothing ever drained that queue — a single outage
532
- // stranded them forever (observable as 待嵌入>0 in the memory panel).
533
- this.embedRetryTimer = setInterval(() => {
534
- void this.retryPendingEmbeds().catch(() => {})
535
- }, 60_000)
536
- this.embedRetryTimer.unref?.()
537
- // First drain shortly after boot, when the backend is usually already up.
538
- const initial = setTimeout(() => {
539
- this.initialDrainTimer = null
540
- void this.retryPendingEmbeds().catch(() => {})
541
- }, 5_000)
542
- initial.unref?.()
543
- this.initialDrainTimer = initial
544
- }
545
-
546
- embedRetryTimer = null
547
- initialDrainTimer = null
548
- /** Owns the llama-server child process when vector.spawn.enabled. */
549
- embedServer = null
550
- /** Lazily created by the panel's "download and configure" action. */
551
- installer = null
552
- exitHook = null
553
-
554
- /**
555
- * Drain the pending-embed queue, probing through the circuit breaker first so
556
- * a backend that came back online is noticed (backfillEmbeddings itself
557
- * refuses to touch the network while marked unhealthy, and embed() is the
558
- * only path that can flip that flag back to healthy).
559
- */
560
- async retryPendingEmbeds() {
561
- // Vector retrieval opt-in: with no configured backend there is nothing to
562
- // retry, and probing would only produce failed requests in the log.
563
- if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return 0
564
- // Scopes load lazily, so without discovery the queue of an untouched scope
565
- // (e.g. a project store written by a previous session) is invisible here and
566
- // would never drain — exactly the 待嵌入>0 the panel showed on 2026-10-10.
567
- try { this.store.loadDiscovered() } catch { /* discovery is best-effort */ }
568
- if (this.pendingEmbedCount === 0) return 0
569
- if (!this.embed.isHealthy) {
570
- const probe = await this.embed.embed(['ping'], undefined, { timeoutMs: this.cfg.vector.writeTimeoutMs }).catch(() => null)
571
- if (!probe) return 0
572
- }
573
- return this.backfillAll({ throttleMs: 100 })
574
- }
575
-
576
- setDefaultScope(scope ) {
577
- this.defaultScope = scope
578
- }
579
-
580
- /**
581
- * Provide the one-shot LLM caller used by the rerank stage. Passing null
582
- * (or never calling) keeps retrieval on recall-order only. The cordis
583
- * wiring adapts `ctx.llm` into this; it is optional by design — the service
584
- * must never sit PENDING just because no LLM adapter is loaded.
585
- */
586
- setLlmInvoke(invoke ) {
587
- this.llmInvoke = invoke
588
- }
589
-
590
- get activeDefaultScope() {
591
- return this.defaultScope
592
- }
593
-
594
- assertScopeAllowed(scope ) {
595
- if (scope === 'global' && !this.cfg.globalEnabled) {
596
- throw memoryError('SCOPE_DISABLED', 'global scope is disabled by configuration')
597
- }
598
- if (scope.startsWith('project:') && !this.cfg.projectEnabled) {
599
- throw memoryError('SCOPE_DISABLED', 'project scope is disabled by configuration')
600
- }
601
- }
602
-
603
- async save(input ) {
604
- const scope = input.scope ?? this.defaultScope
605
- this.assertScopeAllowed(scope)
606
- let content = input.content?.trim()
607
- if (!content) throw memoryError('CONTENT_EMPTY', 'content must be a non-empty string')
608
- if (content.length > this.cfg.maxRecordChars) {
609
- throw memoryError('CONTENT_TOO_LONG', `content exceeds ${this.cfg.maxRecordChars} chars`)
610
- }
611
- if (input.supersedes && !this.store.scope(scope).records.get(input.supersedes)) {
612
- throw memoryError('SUPERSEDES_UNKNOWN', `supersedes target ${input.supersedes} not found in ${scope}`)
613
- }
614
- // design §11: keep credentials out of the plaintext store and the embedder
615
- if (this.cfg.secrets.enabled) {
616
- const findings = scanSecrets(content)
617
- if (findings.length > 0) {
618
- if (this.cfg.secrets.mode === 'refuse') {
619
- throw memoryError('SECRET_DETECTED', `content matches credential denylist: ${[...new Set(findings.map((f) => f.kind))].join(', ')}`)
620
- }
621
- content = redactSecrets(content)
622
- }
623
- }
624
- const { record } = this.store.save(
625
- { ...input, scope, content },
626
- { importance: input.source?.origin === 'auto' ? 0.4 : 0.8, source: { origin: 'explicit' } },
627
- )
628
- void this.embedRecord(record).catch(() => {})
629
- this.store.scheduleFlush()
630
- return record
631
- }
632
-
633
- async embedRecord(record ) {
634
- if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return
635
- const s = this.store.scope(record.scope)
636
- // save-time budget: one full record, much slower than a search query
637
- const vec = await this.embed.embed([record.content], undefined, { timeoutMs: this.cfg.vector.writeTimeoutMs }).catch(() => null)
638
- if (vec?.[0]) s.appendVector(record.id, normalizeInPlace(vec[0]))
639
- else s.embedPending.add(record.id) // backfill queue; the retry timer drains it
640
- }
641
-
642
- /**
643
- * Retry pending embeddings in batches; returns number embedded.
644
- * Batches are bounded by count AND total characters, and use the backfill
645
- * budget: full record contents on CPU take seconds per batch, so the
646
- * interactive timeout would abort every attempt.
647
- */
648
- async backfillEmbeddings() {
649
- if (!this.cfg.vector.enabled || !this.cfg.vector.configured) return 0
650
- if (!this.embed.isHealthy) return 0
651
- let done = 0
652
- for (const s of this.store.allScopes()) {
653
- if (s.embedPending.size === 0) continue
654
- const texts = []
655
- const valid = []
656
- let chars = 0
657
- for (const id of s.embedPending) {
658
- if (valid.length >= this.cfg.vector.backfillBatchSize) break
659
- const t = s.records.get(id)?.content
660
- if (!t) continue
661
- if (valid.length > 0 && chars + t.length > this.cfg.vector.backfillMaxChars) break
662
- texts.push(t)
663
- valid.push(id)
664
- chars += t.length
665
- }
666
- if (texts.length === 0) continue
667
- const vecs = await this.embed.embed(texts, undefined, { timeoutMs: this.cfg.vector.backfillTimeoutMs }).catch(() => null)
668
- if (!vecs) return done
669
- for (let i = 0; i < valid.length; i++) {
670
- if (s.records.has(valid[i] ) && vecs[i]) s.appendVector(valid[i] , normalizeInPlace(vecs[i] ))
671
- s.embedPending.delete(valid[i] )
672
- done++
673
- }
674
- }
675
- return done
676
- }
677
-
678
- /**
679
- * First-fill: embed the ENTIRE backlog (unlike backfillEmbeddings' single
680
- * batch per scope), in throttled batches so a big import does not hammer
681
- * llama-server or block the caller. Yields to the event loop between
682
- * batches and stops early on abort/unhealthy embedder.
683
- */
684
- async backfillAll(opts = {}) {
685
- const throttle = opts.throttleMs ?? 50
686
- let done = 0
687
- while (!opts.signal?.aborted) {
688
- const n = await this.backfillEmbeddings()
689
- done += n
690
- if (n === 0) break // per-scope batch was empty → backlog drained
691
- if (!this.embed.isHealthy) break
692
- await new Promise((r) => setTimeout(r, throttle))
693
- }
694
- return done
695
- }
696
-
697
- async search(query ) {
698
- const topK = query.topK ?? this.cfg.topK
699
- const recallN = query.recallN ?? this.cfg.recallN
700
- const now = Date.now()
701
- const queryTokens = [...new Set(tokenize(query.text))]
702
-
703
- // candidate scopes: pinned, else project + global
704
- const stores = []
705
- if (query.scope) {
706
- stores.push({ store: this.store.scope(query.scope), isGlobal: query.scope === 'global' })
707
- } else {
708
- if (this.cfg.projectEnabled) stores.push({ store: this.store.scope(this.defaultScope), isGlobal: this.defaultScope === 'global' })
709
- if (this.cfg.globalEnabled) stores.push({ store: this.store.scope('global'), isGlobal: true })
710
- }
711
-
712
- // --- channel 1: lexical recall, scored per scope with local IDF ---
713
-
714
- const candidates = new Map ()
715
- for (const { store, isGlobal } of stores) {
716
- const all = [...store.records.values()]
717
- const tokenSets = all.map((r) => new Set(tokenize(r.content)))
718
- const stats = buildStats(tokenSets)
719
- for (let i = 0; i < all.length; i++) {
720
- const rec = all[i]
721
- if (rec.expiredAt && rec.expiredAt < now) continue
722
- const lex = lexicalScore(queryTokens, tokenSets[i] , stats)
723
- if (lex <= 0) continue
724
- // Weak-only overlap (no shared word/bigram) must clear the "real hit"
725
- // floor, not the permissive threshold — kills single-char collisions.
726
- const floor = isGlobal ? this.cfg.globalThreshold : this.cfg.projectThreshold
727
- const effective = hasStrongHit(queryTokens, tokenSets[i] ) ? floor : Math.max(floor, REAL_HIT_FLOOR)
728
- if (lex < effective) continue
729
- candidates.set(rec.id, { record: rec, recallScore: lex, isGlobal })
730
- }
731
- }
732
-
733
- // --- channel 2: vector recall over embedded records ---
734
- // Skipped entirely when no backend is configured (lexical recall above
735
- // already produced the candidates; a request here would just fail).
736
- const qvec = this.cfg.vector.enabled && this.cfg.vector.configured
737
- ? await this.embed.embed([query.text], query.signal).catch(() => null)
738
- : null
739
- if (qvec?.[0]) {
740
- const vectorScores = new Map ()
741
- for (const { store } of stores) {
742
- for (const [id, vec] of store.vectors) {
743
- if (!store.records.has(id)) continue
744
- const sim = cosine(qvec[0] , vec)
745
- if (sim >= this.cfg.vector.minScore) vectorScores.set(id, sim)
746
- }
747
- }
748
- for (const [id, sim] of vectorScores) {
749
- const existing = candidates.get(id)
750
- if (existing) {
751
- existing.recallScore = Math.max(existing.recallScore, sim) // best channel wins
752
- } else {
753
- const rec = this.findById(id)
754
- if (rec && !(rec.expiredAt && rec.expiredAt < now)) {
755
- if (!(rec.scope === 'global' && sim < this.cfg.globalThreshold)) {
756
- candidates.set(id, { record: rec, recallScore: sim, isGlobal: rec.scope === 'global' })
757
- }
758
- }
759
- }
760
- }
761
- }
762
-
763
- // shape with recency/importance, sort, cut
764
- const supersededIds = new Set ()
765
- for (const { store } of stores) {
766
- for (const r of store.records.values()) if (r.supersedes) supersededIds.add(r.supersedes)
767
- }
768
- const shape = (c ) =>
769
- recordScore(c.recallScore, c.record.importance, c.record.accessCount, c.record.lastAccessedAt, {
770
- now,
771
- halfLifeDays: this.cfg.halfLifeDays,
772
- superseded: supersededIds.has(c.record.id),
773
- })
774
-
775
- const ranked = [...candidates.values()].sort((a, b) => shape(b) - shape(a)).slice(0, recallN)
776
-
777
- // --- channel 3: LLM rerank (optional; any skip/failure keeps recall order) ---
778
- const finalScores = new Map (ranked.map((c) => [c.id, shape(c)]))
779
- if (this.cfg.llm.enabled && this.llmInvoke && ranked.length >= 2) {
780
- const asCands = ranked.map((c) => ({ id: c.id, content: c.record.content, recallScore: finalScores.get(c.id) }))
781
- const cacheKey = RerankCache.key(query.text, ranked.map((c) => c.id), this.cfg.llm.model)
782
- let outcome = this.rerankCache.get(cacheKey)
783
- if (!outcome) {
784
- outcome = await rerank({
785
- invoke: this.llmInvoke,
786
- cfg: this.cfg.llm,
787
- query: query.text,
788
- candidates: asCands,
789
- budget: this.rerankBudget,
790
- signal: query.signal,
791
- })
792
- if (outcome) this.rerankCache.set(cacheKey, outcome)
793
- }
794
- if (outcome) {
795
- for (const { id, score } of blendScores(asCands, outcome, this.cfg.llm.weight)) {
796
- finalScores.set(id, score)
797
- }
798
- }
799
- }
800
-
801
- const hits = ranked.map((c) => ({ record: touch(c.record), score: round4(finalScores.get(c.id) ) }))
802
-
803
- this.store.scheduleFlush()
804
- return hits.slice(0, topK)
805
- }
806
-
807
- findById(id ) {
808
- for (const s of this.store.allScopes()) {
809
- const r = s.records.get(id)
810
- if (r) return r
811
- }
812
- return undefined
813
- }
814
-
815
- async forget(input ) {
816
- const ids = []
817
- if (input.id) {
818
- for (const s of this.store.allScopes()) {
819
- if (s.remove(input.id)) ids.push(input.id)
820
- }
821
- if (ids.length === 0) throw memoryError('RECORD_NOT_FOUND', `id ${input.id} not found`)
822
- } else if (input.query) {
823
- const scope = input.scope ?? this.defaultScope
824
- ids.push(...this.store.findIdsByQuery(scope, input.query))
825
- const s = this.store.scope(scope)
826
- for (const id of ids) s.remove(id)
827
- if (ids.length === 0) throw memoryError('RECORD_NOT_FOUND', `no record matching query in ${scope}`)
828
- } else {
829
- throw memoryError('RECORD_NOT_FOUND', 'forget requires id or query')
830
- }
831
- this.store.scheduleFlush()
832
- return { removed: ids.length, ids }
833
- }
834
-
835
- async list(input = {}) {
836
- const scope = input.scope ?? this.defaultScope
837
- let records = [...this.store.scope(scope).records.values()]
838
- if (input.kind) records = records.filter((r) => r.kind === input.kind)
839
- records.sort((a, b) => b.updatedAt - a.updatedAt)
840
- return records.slice(0, input.limit ?? 50)
841
- }
842
-
843
- /** Force a synchronous flush (dispose/HMR path). */
844
- flushSync() {
845
- if (this.maintenanceTimer) {
846
- clearInterval(this.maintenanceTimer)
847
- this.maintenanceTimer = null
848
- }
849
- if (this.embedRetryTimer) {
850
- clearInterval(this.embedRetryTimer)
851
- this.embedRetryTimer = null
852
- }
853
- if (this.initialDrainTimer) {
854
- clearTimeout(this.initialDrainTimer)
855
- this.initialDrainTimer = null
856
- }
857
- if (this.embedServer) this.embedServer.stop()
858
- if (this.exitHook) {
859
- process.removeListener('exit', this.exitHook)
860
- this.exitHook = null
861
- }
862
- this.store.purgeExpired()
863
- this.store.enforceLimit(this.cfg.maxRecordsPerScope)
864
- this.store.close()
865
- }
866
-
867
- get pendingEmbedCount() {
868
- return this.store.allScopes().reduce((n, s) => n + s.embedPending.size, 0)
869
- }
870
- }
871
-
872
- function touch(record ) {
873
- record.lastAccessedAt = Date.now()
874
- record.accessCount += 1
875
- return record
876
- }
877
-
878
- function round4(n ) {
879
- return Math.round(n * 10_000) / 10_000
880
- }
881
-
882
- /** Port of an http(s) endpoint, or null when it cannot be parsed. */
883
- function portOf(endpoint ) {
884
- try {
885
- const port = new URL(endpoint).port
886
- return port ? Number.parseInt(port, 10) : null
887
- } catch {
888
- return null
889
- }
890
- }
891
-
892
- /**
893
- * Measured score floor of genuine lexical hits; below this only single-char
894
- * coincidences live. Used to gate weak-only overlaps regardless of config.
895
- */
896
- const REAL_HIT_FLOOR = 0.25
897
-
898
- function memoryError(code , message ) {
899
- const err = new Error(`[memory:${code}] ${message}`)
900
- err.code = code
901
- return err
902
- }
903
-
904
- export function createMemoryService(config = {}) {
905
- return new LocalMemoryService(config)
906
- }
907
-
908
-
909
- //# sourceURL=D:\.deepseek\memory-plugin\packages\memory-local\src\index.ts
1273
+ //# sourceURL=source/memory-local/src/index.ts