@restregular/dsh-memory 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +69 -2
- package/cordis.patch.yml +10 -0
- package/lib/inject/apply.js +82 -0
- package/lib/inject/inject.js +127 -0
- package/lib/provider/apply.js +108 -0
- package/lib/provider/embed-server.js +296 -0
- package/lib/provider/embed.js +105 -0
- package/lib/provider/index.js +612 -0
- package/lib/provider/rerank-cache.js +64 -0
- package/lib/provider/rerank.js +169 -0
- package/lib/provider/scoring.js +145 -0
- package/lib/provider/secrets.js +57 -0
- package/lib/provider/store.js +485 -0
- package/lib/tools/apply.js +106 -0
- package/lib/tools/index.js +224 -0
- package/package.json +49 -4
|
@@ -0,0 +1,612 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-memory-local — local JSONL provider for the DSH memory service.
|
|
3
|
+
*
|
|
4
|
+
* This entry is intentionally cordis-free so unit tests and standalone tools
|
|
5
|
+
* run without the harness. The Cordis wiring (Service subclass, effects,
|
|
6
|
+
* plugin apply) lives in ./apply.ts and is only loaded inside DSH.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
import { MemoryStore, } from './store.js'
|
|
18
|
+
import { buildStats, lexicalScore, recordScore, tokenize, cosine, normalizeInPlace, hasStrongHit } from './scoring.js'
|
|
19
|
+
import { EmbedClient } from './embed.js'
|
|
20
|
+
import { DailyBudget, blendScores, rerank, } from './rerank.js'
|
|
21
|
+
import { RerankCache } from './rerank-cache.js'
|
|
22
|
+
import { scanSecrets, redactSecrets } from './secrets.js'
|
|
23
|
+
import { EmbedServerManager, } from './embed-server.js'
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
export function resolveConfig(config = {}) {
|
|
91
|
+
const v = config.retrieval?.vector ?? {}
|
|
92
|
+
return {
|
|
93
|
+
storeDir: config.storeDir ?? '.dsh-memory',
|
|
94
|
+
globalEnabled: config.scopes?.global ?? true,
|
|
95
|
+
projectEnabled: config.scopes?.project ?? true,
|
|
96
|
+
topK: config.retrieval?.topK ?? 5,
|
|
97
|
+
recallN: config.retrieval?.recallN ?? 30,
|
|
98
|
+
globalThreshold: config.retrieval?.globalThreshold ?? 0.12,
|
|
99
|
+
projectThreshold: config.retrieval?.projectThreshold ?? 0.1,
|
|
100
|
+
halfLifeDays: config.retrieval?.recencyHalfLifeDays ?? 30,
|
|
101
|
+
vector: {
|
|
102
|
+
enabled: v.enabled ?? true,
|
|
103
|
+
endpoint: v.endpoint ?? 'http://127.0.0.1:8078/v1/embeddings',
|
|
104
|
+
model: v.model ?? 'bge-m3',
|
|
105
|
+
dims: v.dims ?? 1024,
|
|
106
|
+
timeoutMs: v.timeoutMs ?? 300,
|
|
107
|
+
// Full contents are orders of magnitude slower than a short query on CPU
|
|
108
|
+
// (measured 3143ms for three ~600-char records), so write/backfill paths
|
|
109
|
+
// get their own generous budgets instead of the interactive 300ms.
|
|
110
|
+
writeTimeoutMs: v.writeTimeoutMs ?? 10_000,
|
|
111
|
+
backfillTimeoutMs: v.backfillTimeoutMs ?? 30_000,
|
|
112
|
+
backfillMaxChars: v.backfillMaxChars ?? 8_000,
|
|
113
|
+
// BGE CLS baseline noise floor measured ~0.42 on unrelated pairs
|
|
114
|
+
minScore: v.minScore ?? 0.5,
|
|
115
|
+
backfillBatchSize: v.backfillBatchSize ?? 32,
|
|
116
|
+
// Port defaults to the one in the embeddings endpoint so the two can never
|
|
117
|
+
// drift apart; opt-in (enabled:false) keeps dev/test runs from spawning.
|
|
118
|
+
spawn: {
|
|
119
|
+
...(v.spawn ?? {}),
|
|
120
|
+
enabled: v.spawn?.enabled ?? false,
|
|
121
|
+
port: v.spawn?.port ?? portOf(v.endpoint ?? 'http://127.0.0.1:8078/v1/embeddings') ?? 8078,
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
llm: {
|
|
125
|
+
enabled: config.retrieval?.llm?.enabled ?? false,
|
|
126
|
+
provider: config.retrieval?.llm?.provider ?? '',
|
|
127
|
+
model: config.retrieval?.llm?.model ?? '',
|
|
128
|
+
timeoutMs: config.retrieval?.llm?.timeoutMs ?? 1500,
|
|
129
|
+
dailyBudgetTokens: config.retrieval?.llm?.dailyBudgetTokens ?? 200_000,
|
|
130
|
+
weight: config.retrieval?.llm?.weight ?? 0.6,
|
|
131
|
+
topN: config.retrieval?.llm?.topN ?? 10,
|
|
132
|
+
},
|
|
133
|
+
maxRecordsPerScope: config.limits?.maxRecordsPerScope ?? 2000,
|
|
134
|
+
maxRecordChars: config.limits?.maxRecordChars ?? 4000,
|
|
135
|
+
secrets: {
|
|
136
|
+
enabled: config.secrets?.enabled ?? true,
|
|
137
|
+
mode: config.secrets?.mode ?? 'refuse',
|
|
138
|
+
},
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export class LocalMemoryService {
|
|
143
|
+
cfg
|
|
144
|
+
store
|
|
145
|
+
embed
|
|
146
|
+
rerankBudget
|
|
147
|
+
rerankCache = new RerankCache()
|
|
148
|
+
/** Host-provided one-shot LLM call; null = rerank disabled (recall order). */
|
|
149
|
+
llmInvoke = null
|
|
150
|
+
/** Where saves land when the caller does not pin a scope. */
|
|
151
|
+
defaultScope = 'global'
|
|
152
|
+
|
|
153
|
+
constructor(config = {}) {
|
|
154
|
+
this.cfg = resolveConfig(config)
|
|
155
|
+
this.rerankBudget = new DailyBudget(this.cfg.llm.dailyBudgetTokens)
|
|
156
|
+
this.store = new MemoryStore(this.cfg.storeDir, { onWarn: (m) => console.warn(`[dsh-memory-local] ${m}`) })
|
|
157
|
+
this.store.loadAll(this.cfg.projectEnabled && this.cfg.globalEnabled ? [this.defaultScope, 'global'] : [this.defaultScope])
|
|
158
|
+
this.embed = new EmbedClient({
|
|
159
|
+
endpoint: this.cfg.vector.endpoint,
|
|
160
|
+
model: this.cfg.vector.model,
|
|
161
|
+
dims: this.cfg.vector.dims,
|
|
162
|
+
timeoutMs: this.cfg.vector.timeoutMs,
|
|
163
|
+
})
|
|
164
|
+
// sidecar validation: model/dims change discards old vectors on load
|
|
165
|
+
this.store.setVecIdentity(this.cfg.vector.model, this.cfg.vector.dims)
|
|
166
|
+
this.startMaintenance()
|
|
167
|
+
this.startEmbedServer()
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Own the embedding backend's lifetime: spawn on activation, stop on dispose. */
|
|
171
|
+
startEmbedServer() {
|
|
172
|
+
this.embedServer = new EmbedServerManager(this.cfg.vector.spawn)
|
|
173
|
+
// fire-and-forget: activation must never block on a model load
|
|
174
|
+
void this.ensureEmbedServer(60_000).catch(() => { /* lifecycle never breaks activation */ })
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Start (or adopt) the embedding backend and arm the exit safety net.
|
|
179
|
+
* Shared by the activation path and the panel's manual start button.
|
|
180
|
+
*/
|
|
181
|
+
async ensureEmbedServer(waitMs ) {
|
|
182
|
+
const mgr = this.embedServer
|
|
183
|
+
if (!mgr) return 'disabled'
|
|
184
|
+
const state = await mgr.ensureRunning()
|
|
185
|
+
if (state !== 'spawned' && state !== 'adopted-managed') return state
|
|
186
|
+
// hard-exit safety net: cordis disposal covers a clean quit; this covers
|
|
187
|
+
// the host dying without disposing us (the child must not be orphaned)
|
|
188
|
+
if (!this.exitHook) {
|
|
189
|
+
this.exitHook = () => { this.embedServer?.stop() }
|
|
190
|
+
process.once('exit', this.exitHook)
|
|
191
|
+
}
|
|
192
|
+
const ready = await mgr.waitUntilReady(waitMs)
|
|
193
|
+
if (!ready) console.warn('[dsh-memory-local] embed server: 启动超时,检索继续走词法降级')
|
|
194
|
+
return state
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Manual start from the memory panel (same semantics as activation). */
|
|
198
|
+
startEmbedServerManually() {
|
|
199
|
+
return this.ensureEmbedServer(25_000)
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Manual stop from the memory panel. An instance we do not own is left alone
|
|
204
|
+
* (it was started outside the app), so the UI can explain the refusal.
|
|
205
|
+
*/
|
|
206
|
+
stopEmbedServerManually() {
|
|
207
|
+
const mgr = this.embedServer
|
|
208
|
+
if (!mgr) return { stopped: false, reason: 'NOT_RUNNING' }
|
|
209
|
+
if (!mgr.isOwned) return { stopped: false, reason: 'EXTERNAL_INSTANCE' }
|
|
210
|
+
return { stopped: mgr.stop(), reason: mgr.isOwned ? 'NOT_RUNNING' : 'STOPPED' }
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Lifecycle snapshot for the memory panel. */
|
|
214
|
+
async embedServerStatus()
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
{
|
|
222
|
+
const mgr = this.embedServer
|
|
223
|
+
const pendingEmbeds = this.pendingEmbedCount
|
|
224
|
+
if (!mgr) return { enabled: false, owned: false, pid: null, baseUrl: '', reachable: false, pendingEmbeds }
|
|
225
|
+
const reachable = await mgr.healthCheck()
|
|
226
|
+
return { ...mgr.info(), reachable, pendingEmbeds }
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Hourly expired-record purge + capacity eviction. M4 housekeeping; the
|
|
231
|
+
* timer is unref'd so it never holds the process open, and cleared by
|
|
232
|
+
* flushSync()/dispose.
|
|
233
|
+
*/
|
|
234
|
+
maintenanceTimer = null
|
|
235
|
+
|
|
236
|
+
startMaintenance() {
|
|
237
|
+
this.maintenanceTimer = setInterval(() => {
|
|
238
|
+
this.store.purgeExpired()
|
|
239
|
+
this.store.enforceLimit(this.cfg.maxRecordsPerScope)
|
|
240
|
+
this.store.scheduleFlush(0)
|
|
241
|
+
}, 3_600_000)
|
|
242
|
+
this.maintenanceTimer.unref?.()
|
|
243
|
+
// Pending-embed drain: the store queues ids whose embed failed (backend
|
|
244
|
+
// down at save time, or vectors restored from a jsonl-only backup). Until
|
|
245
|
+
// this timer existed nothing ever drained that queue — a single outage
|
|
246
|
+
// stranded them forever (observable as 待嵌入>0 in the memory panel).
|
|
247
|
+
this.embedRetryTimer = setInterval(() => {
|
|
248
|
+
void this.retryPendingEmbeds().catch(() => {})
|
|
249
|
+
}, 60_000)
|
|
250
|
+
this.embedRetryTimer.unref?.()
|
|
251
|
+
// First drain shortly after boot, when the backend is usually already up.
|
|
252
|
+
const initial = setTimeout(() => {
|
|
253
|
+
this.initialDrainTimer = null
|
|
254
|
+
void this.retryPendingEmbeds().catch(() => {})
|
|
255
|
+
}, 5_000)
|
|
256
|
+
initial.unref?.()
|
|
257
|
+
this.initialDrainTimer = initial
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
embedRetryTimer = null
|
|
261
|
+
initialDrainTimer = null
|
|
262
|
+
/** Owns the llama-server child process when vector.spawn.enabled. */
|
|
263
|
+
embedServer = null
|
|
264
|
+
exitHook = null
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Drain the pending-embed queue, probing through the circuit breaker first so
|
|
268
|
+
* a backend that came back online is noticed (backfillEmbeddings itself
|
|
269
|
+
* refuses to touch the network while marked unhealthy, and embed() is the
|
|
270
|
+
* only path that can flip that flag back to healthy).
|
|
271
|
+
*/
|
|
272
|
+
async retryPendingEmbeds() {
|
|
273
|
+
// Scopes load lazily, so without discovery the queue of an untouched scope
|
|
274
|
+
// (e.g. a project store written by a previous session) is invisible here and
|
|
275
|
+
// would never drain — exactly the 待嵌入>0 the panel showed on 2026-10-10.
|
|
276
|
+
try { this.store.loadDiscovered() } catch { /* discovery is best-effort */ }
|
|
277
|
+
if (this.pendingEmbedCount === 0) return 0
|
|
278
|
+
if (!this.embed.isHealthy) {
|
|
279
|
+
const probe = await this.embed.embed(['ping'], undefined, { timeoutMs: this.cfg.vector.writeTimeoutMs }).catch(() => null)
|
|
280
|
+
if (!probe) return 0
|
|
281
|
+
}
|
|
282
|
+
return this.backfillAll({ throttleMs: 100 })
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
setDefaultScope(scope ) {
|
|
286
|
+
this.defaultScope = scope
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Provide the one-shot LLM caller used by the rerank stage. Passing null
|
|
291
|
+
* (or never calling) keeps retrieval on recall-order only. The cordis
|
|
292
|
+
* wiring adapts `ctx.llm` into this; it is optional by design — the service
|
|
293
|
+
* must never sit PENDING just because no LLM adapter is loaded.
|
|
294
|
+
*/
|
|
295
|
+
setLlmInvoke(invoke ) {
|
|
296
|
+
this.llmInvoke = invoke
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
get activeDefaultScope() {
|
|
300
|
+
return this.defaultScope
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
assertScopeAllowed(scope ) {
|
|
304
|
+
if (scope === 'global' && !this.cfg.globalEnabled) {
|
|
305
|
+
throw memoryError('SCOPE_DISABLED', 'global scope is disabled by configuration')
|
|
306
|
+
}
|
|
307
|
+
if (scope.startsWith('project:') && !this.cfg.projectEnabled) {
|
|
308
|
+
throw memoryError('SCOPE_DISABLED', 'project scope is disabled by configuration')
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
async save(input ) {
|
|
313
|
+
const scope = input.scope ?? this.defaultScope
|
|
314
|
+
this.assertScopeAllowed(scope)
|
|
315
|
+
let content = input.content?.trim()
|
|
316
|
+
if (!content) throw memoryError('CONTENT_EMPTY', 'content must be a non-empty string')
|
|
317
|
+
if (content.length > this.cfg.maxRecordChars) {
|
|
318
|
+
throw memoryError('CONTENT_TOO_LONG', `content exceeds ${this.cfg.maxRecordChars} chars`)
|
|
319
|
+
}
|
|
320
|
+
if (input.supersedes && !this.store.scope(scope).records.get(input.supersedes)) {
|
|
321
|
+
throw memoryError('SUPERSEDES_UNKNOWN', `supersedes target ${input.supersedes} not found in ${scope}`)
|
|
322
|
+
}
|
|
323
|
+
// design §11: keep credentials out of the plaintext store and the embedder
|
|
324
|
+
if (this.cfg.secrets.enabled) {
|
|
325
|
+
const findings = scanSecrets(content)
|
|
326
|
+
if (findings.length > 0) {
|
|
327
|
+
if (this.cfg.secrets.mode === 'refuse') {
|
|
328
|
+
throw memoryError('SECRET_DETECTED', `content matches credential denylist: ${[...new Set(findings.map((f) => f.kind))].join(', ')}`)
|
|
329
|
+
}
|
|
330
|
+
content = redactSecrets(content)
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
const { record } = this.store.save(
|
|
334
|
+
{ ...input, scope, content },
|
|
335
|
+
{ importance: input.source?.origin === 'auto' ? 0.4 : 0.8, source: { origin: 'explicit' } },
|
|
336
|
+
)
|
|
337
|
+
void this.embedRecord(record).catch(() => {})
|
|
338
|
+
this.store.scheduleFlush()
|
|
339
|
+
return record
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
async embedRecord(record ) {
|
|
343
|
+
const s = this.store.scope(record.scope)
|
|
344
|
+
// save-time budget: one full record, much slower than a search query
|
|
345
|
+
const vec = await this.embed.embed([record.content], undefined, { timeoutMs: this.cfg.vector.writeTimeoutMs }).catch(() => null)
|
|
346
|
+
if (vec?.[0]) s.appendVector(record.id, normalizeInPlace(vec[0]))
|
|
347
|
+
else s.embedPending.add(record.id) // backfill queue; the retry timer drains it
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Retry pending embeddings in batches; returns number embedded.
|
|
352
|
+
* Batches are bounded by count AND total characters, and use the backfill
|
|
353
|
+
* budget: full record contents on CPU take seconds per batch, so the
|
|
354
|
+
* interactive timeout would abort every attempt.
|
|
355
|
+
*/
|
|
356
|
+
async backfillEmbeddings() {
|
|
357
|
+
if (!this.embed.isHealthy) return 0
|
|
358
|
+
let done = 0
|
|
359
|
+
for (const s of this.store.allScopes()) {
|
|
360
|
+
if (s.embedPending.size === 0) continue
|
|
361
|
+
const texts = []
|
|
362
|
+
const valid = []
|
|
363
|
+
let chars = 0
|
|
364
|
+
for (const id of s.embedPending) {
|
|
365
|
+
if (valid.length >= this.cfg.vector.backfillBatchSize) break
|
|
366
|
+
const t = s.records.get(id)?.content
|
|
367
|
+
if (!t) continue
|
|
368
|
+
if (valid.length > 0 && chars + t.length > this.cfg.vector.backfillMaxChars) break
|
|
369
|
+
texts.push(t)
|
|
370
|
+
valid.push(id)
|
|
371
|
+
chars += t.length
|
|
372
|
+
}
|
|
373
|
+
if (texts.length === 0) continue
|
|
374
|
+
const vecs = await this.embed.embed(texts, undefined, { timeoutMs: this.cfg.vector.backfillTimeoutMs }).catch(() => null)
|
|
375
|
+
if (!vecs) return done
|
|
376
|
+
for (let i = 0; i < valid.length; i++) {
|
|
377
|
+
if (s.records.has(valid[i] ) && vecs[i]) s.appendVector(valid[i] , normalizeInPlace(vecs[i] ))
|
|
378
|
+
s.embedPending.delete(valid[i] )
|
|
379
|
+
done++
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
return done
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* First-fill: embed the ENTIRE backlog (unlike backfillEmbeddings' single
|
|
387
|
+
* batch per scope), in throttled batches so a big import does not hammer
|
|
388
|
+
* llama-server or block the caller. Yields to the event loop between
|
|
389
|
+
* batches and stops early on abort/unhealthy embedder.
|
|
390
|
+
*/
|
|
391
|
+
async backfillAll(opts = {}) {
|
|
392
|
+
const throttle = opts.throttleMs ?? 50
|
|
393
|
+
let done = 0
|
|
394
|
+
while (!opts.signal?.aborted) {
|
|
395
|
+
const n = await this.backfillEmbeddings()
|
|
396
|
+
done += n
|
|
397
|
+
if (n === 0) break // per-scope batch was empty → backlog drained
|
|
398
|
+
if (!this.embed.isHealthy) break
|
|
399
|
+
await new Promise((r) => setTimeout(r, throttle))
|
|
400
|
+
}
|
|
401
|
+
return done
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
async search(query ) {
|
|
405
|
+
const topK = query.topK ?? this.cfg.topK
|
|
406
|
+
const recallN = query.recallN ?? this.cfg.recallN
|
|
407
|
+
const now = Date.now()
|
|
408
|
+
const queryTokens = [...new Set(tokenize(query.text))]
|
|
409
|
+
|
|
410
|
+
// candidate scopes: pinned, else project + global
|
|
411
|
+
const stores = []
|
|
412
|
+
if (query.scope) {
|
|
413
|
+
stores.push({ store: this.store.scope(query.scope), isGlobal: query.scope === 'global' })
|
|
414
|
+
} else {
|
|
415
|
+
if (this.cfg.projectEnabled) stores.push({ store: this.store.scope(this.defaultScope), isGlobal: this.defaultScope === 'global' })
|
|
416
|
+
if (this.cfg.globalEnabled) stores.push({ store: this.store.scope('global'), isGlobal: true })
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// --- channel 1: lexical recall, scored per scope with local IDF ---
|
|
420
|
+
|
|
421
|
+
const candidates = new Map ()
|
|
422
|
+
for (const { store, isGlobal } of stores) {
|
|
423
|
+
const all = [...store.records.values()]
|
|
424
|
+
const tokenSets = all.map((r) => new Set(tokenize(r.content)))
|
|
425
|
+
const stats = buildStats(tokenSets)
|
|
426
|
+
for (let i = 0; i < all.length; i++) {
|
|
427
|
+
const rec = all[i]
|
|
428
|
+
if (rec.expiredAt && rec.expiredAt < now) continue
|
|
429
|
+
const lex = lexicalScore(queryTokens, tokenSets[i] , stats)
|
|
430
|
+
if (lex <= 0) continue
|
|
431
|
+
// Weak-only overlap (no shared word/bigram) must clear the "real hit"
|
|
432
|
+
// floor, not the permissive threshold — kills single-char collisions.
|
|
433
|
+
const floor = isGlobal ? this.cfg.globalThreshold : this.cfg.projectThreshold
|
|
434
|
+
const effective = hasStrongHit(queryTokens, tokenSets[i] ) ? floor : Math.max(floor, REAL_HIT_FLOOR)
|
|
435
|
+
if (lex < effective) continue
|
|
436
|
+
candidates.set(rec.id, { record: rec, recallScore: lex, isGlobal })
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// --- channel 2: vector recall over embedded records ---
|
|
441
|
+
const qvec = await this.embed.embed([query.text], query.signal).catch(() => null)
|
|
442
|
+
if (qvec?.[0]) {
|
|
443
|
+
const vectorScores = new Map ()
|
|
444
|
+
for (const { store } of stores) {
|
|
445
|
+
for (const [id, vec] of store.vectors) {
|
|
446
|
+
if (!store.records.has(id)) continue
|
|
447
|
+
const sim = cosine(qvec[0] , vec)
|
|
448
|
+
if (sim >= this.cfg.vector.minScore) vectorScores.set(id, sim)
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
for (const [id, sim] of vectorScores) {
|
|
452
|
+
const existing = candidates.get(id)
|
|
453
|
+
if (existing) {
|
|
454
|
+
existing.recallScore = Math.max(existing.recallScore, sim) // best channel wins
|
|
455
|
+
} else {
|
|
456
|
+
const rec = this.findById(id)
|
|
457
|
+
if (rec && !(rec.expiredAt && rec.expiredAt < now)) {
|
|
458
|
+
if (!(rec.scope === 'global' && sim < this.cfg.globalThreshold)) {
|
|
459
|
+
candidates.set(id, { record: rec, recallScore: sim, isGlobal: rec.scope === 'global' })
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
// shape with recency/importance, sort, cut
|
|
467
|
+
const supersededIds = new Set ()
|
|
468
|
+
for (const { store } of stores) {
|
|
469
|
+
for (const r of store.records.values()) if (r.supersedes) supersededIds.add(r.supersedes)
|
|
470
|
+
}
|
|
471
|
+
const shape = (c ) =>
|
|
472
|
+
recordScore(c.recallScore, c.record.importance, c.record.accessCount, c.record.lastAccessedAt, {
|
|
473
|
+
now,
|
|
474
|
+
halfLifeDays: this.cfg.halfLifeDays,
|
|
475
|
+
superseded: supersededIds.has(c.record.id),
|
|
476
|
+
})
|
|
477
|
+
|
|
478
|
+
const ranked = [...candidates.values()].sort((a, b) => shape(b) - shape(a)).slice(0, recallN)
|
|
479
|
+
|
|
480
|
+
// --- channel 3: LLM rerank (optional; any skip/failure keeps recall order) ---
|
|
481
|
+
const finalScores = new Map (ranked.map((c) => [c.id, shape(c)]))
|
|
482
|
+
if (this.cfg.llm.enabled && this.llmInvoke && ranked.length >= 2) {
|
|
483
|
+
const asCands = ranked.map((c) => ({ id: c.id, content: c.record.content, recallScore: finalScores.get(c.id) }))
|
|
484
|
+
const cacheKey = RerankCache.key(query.text, ranked.map((c) => c.id), this.cfg.llm.model)
|
|
485
|
+
let outcome = this.rerankCache.get(cacheKey)
|
|
486
|
+
if (!outcome) {
|
|
487
|
+
outcome = await rerank({
|
|
488
|
+
invoke: this.llmInvoke,
|
|
489
|
+
cfg: this.cfg.llm,
|
|
490
|
+
query: query.text,
|
|
491
|
+
candidates: asCands,
|
|
492
|
+
budget: this.rerankBudget,
|
|
493
|
+
signal: query.signal,
|
|
494
|
+
})
|
|
495
|
+
if (outcome) this.rerankCache.set(cacheKey, outcome)
|
|
496
|
+
}
|
|
497
|
+
if (outcome) {
|
|
498
|
+
for (const { id, score } of blendScores(asCands, outcome, this.cfg.llm.weight)) {
|
|
499
|
+
finalScores.set(id, score)
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
const hits = ranked.map((c) => ({ record: touch(c.record), score: round4(finalScores.get(c.id) ) }))
|
|
505
|
+
|
|
506
|
+
this.store.scheduleFlush()
|
|
507
|
+
return hits.slice(0, topK)
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
findById(id ) {
|
|
511
|
+
for (const s of this.store.allScopes()) {
|
|
512
|
+
const r = s.records.get(id)
|
|
513
|
+
if (r) return r
|
|
514
|
+
}
|
|
515
|
+
return undefined
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
async forget(input ) {
|
|
519
|
+
const ids = []
|
|
520
|
+
if (input.id) {
|
|
521
|
+
for (const s of this.store.allScopes()) {
|
|
522
|
+
if (s.remove(input.id)) ids.push(input.id)
|
|
523
|
+
}
|
|
524
|
+
if (ids.length === 0) throw memoryError('RECORD_NOT_FOUND', `id ${input.id} not found`)
|
|
525
|
+
} else if (input.query) {
|
|
526
|
+
const scope = input.scope ?? this.defaultScope
|
|
527
|
+
ids.push(...this.store.findIdsByQuery(scope, input.query))
|
|
528
|
+
const s = this.store.scope(scope)
|
|
529
|
+
for (const id of ids) s.remove(id)
|
|
530
|
+
if (ids.length === 0) throw memoryError('RECORD_NOT_FOUND', `no record matching query in ${scope}`)
|
|
531
|
+
} else {
|
|
532
|
+
throw memoryError('RECORD_NOT_FOUND', 'forget requires id or query')
|
|
533
|
+
}
|
|
534
|
+
this.store.scheduleFlush()
|
|
535
|
+
return { removed: ids.length, ids }
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
async list(input = {}) {
|
|
539
|
+
const scope = input.scope ?? this.defaultScope
|
|
540
|
+
let records = [...this.store.scope(scope).records.values()]
|
|
541
|
+
if (input.kind) records = records.filter((r) => r.kind === input.kind)
|
|
542
|
+
records.sort((a, b) => b.updatedAt - a.updatedAt)
|
|
543
|
+
return records.slice(0, input.limit ?? 50)
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
/** Force a synchronous flush (dispose/HMR path). */
|
|
547
|
+
flushSync() {
|
|
548
|
+
if (this.maintenanceTimer) {
|
|
549
|
+
clearInterval(this.maintenanceTimer)
|
|
550
|
+
this.maintenanceTimer = null
|
|
551
|
+
}
|
|
552
|
+
if (this.embedRetryTimer) {
|
|
553
|
+
clearInterval(this.embedRetryTimer)
|
|
554
|
+
this.embedRetryTimer = null
|
|
555
|
+
}
|
|
556
|
+
if (this.initialDrainTimer) {
|
|
557
|
+
clearTimeout(this.initialDrainTimer)
|
|
558
|
+
this.initialDrainTimer = null
|
|
559
|
+
}
|
|
560
|
+
if (this.embedServer) this.embedServer.stop()
|
|
561
|
+
if (this.exitHook) {
|
|
562
|
+
process.removeListener('exit', this.exitHook)
|
|
563
|
+
this.exitHook = null
|
|
564
|
+
}
|
|
565
|
+
this.store.purgeExpired()
|
|
566
|
+
this.store.enforceLimit(this.cfg.maxRecordsPerScope)
|
|
567
|
+
this.store.close()
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
get pendingEmbedCount() {
|
|
571
|
+
return this.store.allScopes().reduce((n, s) => n + s.embedPending.size, 0)
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
function touch(record ) {
|
|
576
|
+
record.lastAccessedAt = Date.now()
|
|
577
|
+
record.accessCount += 1
|
|
578
|
+
return record
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
function round4(n ) {
|
|
582
|
+
return Math.round(n * 10_000) / 10_000
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/** Port of an http(s) endpoint, or null when it cannot be parsed. */
|
|
586
|
+
function portOf(endpoint ) {
|
|
587
|
+
try {
|
|
588
|
+
const port = new URL(endpoint).port
|
|
589
|
+
return port ? Number.parseInt(port, 10) : null
|
|
590
|
+
} catch {
|
|
591
|
+
return null
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/**
|
|
596
|
+
* Measured score floor of genuine lexical hits; below this only single-char
|
|
597
|
+
* coincidences live. Used to gate weak-only overlaps regardless of config.
|
|
598
|
+
*/
|
|
599
|
+
const REAL_HIT_FLOOR = 0.25
|
|
600
|
+
|
|
601
|
+
function memoryError(code , message ) {
|
|
602
|
+
const err = new Error(`[memory:${code}] ${message}`)
|
|
603
|
+
err.code = code
|
|
604
|
+
return err
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
export function createMemoryService(config = {}) {
|
|
608
|
+
return new LocalMemoryService(config)
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
//# sourceURL=D:\.deepseek\memory-plugin\packages\memory-local\src\index.ts
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rerank cache (design §12): identical (query, candidate set, judge model)
|
|
3
|
+
* repeats — common during iterative agent loops — reuse the previous LLM
|
|
4
|
+
* verdict instead of spending budget again. Entries are invalidated when the
|
|
5
|
+
* candidate set membership changes (id set hash), capped, and TTL-expires.
|
|
6
|
+
*/
|
|
7
|
+
import { createHash } from 'node:crypto'
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
export class RerankCache {
|
|
16
|
+
entries = new Map ()
|
|
17
|
+
maxEntries
|
|
18
|
+
ttlMs
|
|
19
|
+
|
|
20
|
+
constructor(maxEntries = 256, ttlMs = 30 * 60_000) {
|
|
21
|
+
this.maxEntries = maxEntries
|
|
22
|
+
this.ttlMs = ttlMs
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
static key(query , candidateIds , judgeModel ) {
|
|
26
|
+
return createHash('sha256')
|
|
27
|
+
.update(query)
|
|
28
|
+
.update('\u0000')
|
|
29
|
+
.update([...candidateIds].sort().join(','))
|
|
30
|
+
.update('\u0000')
|
|
31
|
+
.update(judgeModel)
|
|
32
|
+
.digest('hex')
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
get(key , now = Date.now()) {
|
|
36
|
+
const e = this.entries.get(key)
|
|
37
|
+
if (!e) return null
|
|
38
|
+
if (now - e.at > this.ttlMs) {
|
|
39
|
+
this.entries.delete(key)
|
|
40
|
+
return null
|
|
41
|
+
}
|
|
42
|
+
return e.outcome
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
set(key , outcome , now = Date.now()) {
|
|
46
|
+
// simple FIFO cap; insertion order in Map is stable
|
|
47
|
+
if (this.entries.size >= this.maxEntries) {
|
|
48
|
+
const oldest = this.entries.keys().next().value
|
|
49
|
+
if (oldest !== undefined) this.entries.delete(oldest)
|
|
50
|
+
}
|
|
51
|
+
this.entries.set(key, { outcome, at: now })
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
get size() {
|
|
55
|
+
return this.entries.size
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
clear() {
|
|
59
|
+
this.entries.clear()
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
//# sourceURL=D:\.deepseek\memory-plugin\packages\memory-local\src\rerank-cache.ts
|