@restregular/dsh-memory 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,485 +1,491 @@
1
- /**
2
- * JSONL-backed store. One file per scope, append-on-write semantics with
3
- * atomic compaction (tmp + rename). Vectors live in memory for M1; the .vec
4
- * sidecar format arrives in M1.5.
5
- */
6
- import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from 'node:fs'
7
- import { createHash, randomUUID } from 'node:crypto'
8
- import { Buffer } from 'node:buffer'
9
- import { dirname, join } from 'node:path'
10
-
11
-
12
-
13
-
14
-
15
-
16
- export function scopeFileName(scope ) { if (scope === 'global') return 'global.jsonl'
17
- const hash = createHash('sha1').update(scope).digest('hex')
18
- return join('projects', `${hash}.jsonl`)
19
- }
20
-
21
- /** Derive the scope for records loaded from a file path. */
22
- export function scopeFromFile(file , knownScopes ) {
23
- if (file.endsWith('global.jsonl')) return 'global'
24
- const base = file.split(/[\\/]/).pop() ?? ''
25
- for (const [scope, name] of knownScopes) if (name === base) return scope
26
- return null
27
- }
28
-
29
- /**
30
- * Read the first well-formed record's scope from a scope file. Project files are
31
- * named by hash, so this is the only way back to the scope key without the
32
- * provider having created that scope in this process already.
33
- */
34
- export function firstScopeInFile(file ) {
35
- let text
36
- try {
37
- text = readFileSync(file, 'utf8')
38
- } catch {
39
- return null
40
- }
41
- for (const line of text.split('\n')) {
42
- const trimmed = line.trim()
43
- if (!trimmed) continue
44
- try {
45
- const rec = JSON.parse(trimmed)
46
- if (typeof rec.scope === 'string' && rec.scope) return rec.scope
47
- } catch {
48
- // skip malformed line, keep looking
49
- }
50
- }
51
- return null
52
- }
53
-
54
- export function normalizeContent(content ) {
55
- return content.replace(/\s+/g, ' ').trim().toLowerCase()
56
- }
57
-
58
- export function contentHash(scope , content ) {
59
- return createHash('sha1').update(`${scope}::${normalizeContent(content)}`).digest('hex')
60
- }
61
-
62
- export function newId() {
63
- return randomUUID()
64
- }
65
-
66
- export class ScopeStore {
67
- scope
68
- records = new Map ()
69
- vectors = new Map ()
70
- /** ids saved while the embed backend was down; retried opportunistically. */
71
- embedPending = new Set ()
72
- dirty = false
73
- /** true once vectors were touched since the last .vec flush. */
74
- vectorsDirty = false
75
- vecFile
76
- events
77
- /** Embedding identity for sidecar validation; set via setVecIdentity(). */
78
- vecIdentity = ''
79
-
80
- constructor(scope , root , events = {}) {
81
- this.scope = scope
82
- this.vecFile = join(root, scopeFileName(scope).replace(/\.jsonl$/, '.vec'))
83
- this.events = events
84
- }
85
-
86
- /**
87
- * Declare which model+dims produced the in-memory vectors. When it differs
88
- * from the sidecar header on load, the sidecar is discarded and every record
89
- * re-enters the embed queue — old vectors for a new model are wrong, not
90
- * merely stale.
91
- */
92
- setVecIdentity(model , dims ) {
93
- this.vecIdentity = `${model}\t${dims}`
94
- }
95
-
96
- get size() {
97
- return this.records.size
98
- }
99
-
100
- insert(record ) {
101
- this.records.set(record.id, record)
102
- this.dirty = true
103
- }
104
-
105
- remove(id ) {
106
- const had = this.records.delete(id)
107
- if (had) {
108
- this.vectors.delete(id)
109
- this.embedPending.delete(id)
110
- this.dirty = true
111
- this.vectorsDirty = true
112
- }
113
- return had
114
- }
115
-
116
- setVector(id , vec ) {
117
- this.vectors.set(id, vec)
118
- this.vectorsDirty = true
119
- }
120
-
121
- /**
122
- * Append a record to the sidecar: `<id> \t <base64 dim vec>` per line.
123
- * Vectors are written immediately on embed (single-line append) and fully
124
- * rewritten on delete/compact — embeddings are expensive to recompute.
125
- * A freshly created file starts with the `#model\tdims` identity header.
126
- */
127
- appendVector(id , vec ) {
128
- this.setVector(id, vec)
129
- try {
130
- mkdirSync(dirname(this.vecFile), { recursive: true })
131
- if (this.vecIdentity && this.readHeader() !== this.vecIdentity) {
132
- // missing, legacy, or wrong-model file: rewrite fresh from memory
133
- this.writeSidecar()
134
- this.vectorsDirty = false
135
- return
136
- }
137
- appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
138
- this.vectorsDirty = false
139
- } catch (err) {
140
- this.vectorsDirty = true // full rewrite on next flushVectors
141
- this.events.onWarn?.(`vec: append failed: ${(err ).message}`)
142
- }
143
- }
144
-
145
- /** First line of the sidecar if it is a header; '' when absent/headerless. */
146
- readHeader() {
147
- if (!existsSync(this.vecFile)) return ''
148
- try {
149
- const first = readFileSync(this.vecFile, 'utf8').split('\n', 1)[0] ?? ''
150
- return first.startsWith('#') ? first.slice(1) : ''
151
- } catch {
152
- return ''
153
- }
154
- }
155
-
156
- /** Atomic full rewrite: identity header + every in-memory vector. */
157
- writeSidecar() {
158
- mkdirSync(dirname(this.vecFile), { recursive: true })
159
- const tmp = `${this.vecFile}.tmp`
160
- const lines = this.vecIdentity ? [`#${this.vecIdentity}`] : []
161
- for (const [id, v] of this.vectors) lines.push(`${id}\t${encodeVec(v)}`)
162
- writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
163
- renameSync(tmp, this.vecFile)
164
- }
165
-
166
- /** Rewrite the whole sidecar (after deletes / compaction / append failures). */
167
- flushVectors() {
168
- if (!this.vectorsDirty) return
169
- try {
170
- this.writeSidecar()
171
- this.vectorsDirty = false
172
- } catch (err) {
173
- this.events.onWarn?.(`vec: flush failed: ${(err ).message}`)
174
- }
175
- }
176
-
177
- /**
178
- * Load vectors from the sidecar. The first line must be a `#model\tdims`
179
- * header when an identity is configured — mismatch or a legacy headerless
180
- * file discards the whole sidecar (re-embed). Unknown ids are skipped.
181
- */
182
- loadVectors() {
183
- if (!existsSync(this.vecFile)) return
184
- try {
185
- const text = readFileSync(this.vecFile, 'utf8')
186
- let sawHeader = false
187
- let sawData = false
188
- const discard = (why ) => {
189
- this.events.onWarn?.(`vec: ${why}; discarding sidecar for re-embed`)
190
- this.vectors.clear()
191
- this.vectorsDirty = true
192
- }
193
- for (const line of text.split('\n')) {
194
- const trimmed = line.trim()
195
- if (!trimmed) continue
196
- if (trimmed.startsWith('#')) {
197
- if (this.vecIdentity && trimmed.slice(1) !== this.vecIdentity) {
198
- discard(`identity mismatch (${trimmed.slice(1)} != ${this.vecIdentity})`)
199
- return
200
- }
201
- sawHeader = true
202
- continue
203
- }
204
- if (!sawData) {
205
- sawData = true
206
- if (this.vecIdentity && !sawHeader) {
207
- discard('legacy sidecar without identity header')
208
- return
209
- }
210
- }
211
- const tab = trimmed.indexOf('\t')
212
- if (tab <= 0) continue
213
- const id = trimmed.slice(0, tab)
214
- if (!this.records.has(id)) continue // stale vector for a deleted record
215
- const vec = decodeVec(trimmed.slice(tab + 1))
216
- if (vec) this.vectors.set(id, vec)
217
- }
218
- } catch (err) {
219
- this.events.onWarn?.(`vec: load failed: ${(err ).message}`)
220
- }
221
- }
222
- }
223
-
224
- // --- base64 vector codec (4 bytes per float, bit-exact) ---
225
-
226
- function encodeVec(vec ) {
227
- return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength).toString('base64')
228
- }
229
-
230
- function decodeVec(text ) {
231
- try {
232
- const buf = Buffer.from(text, 'base64')
233
- if (buf.length === 0 || buf.length % 4 !== 0) return null
234
- return new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4)
235
- } catch {
236
- return null
237
- }
238
- }
239
-
240
-
241
-
242
-
243
-
244
- export class MemoryStore {
245
- root
246
- scopes = new Map ()
247
- /** scopes whose jsonl has been loaded from disk (lazy, once per scope). */
248
- loaded = new Set ()
249
- /** scope -> its jsonl filename, so reload can map files back to scopes. */
250
- fileNames = new Map ()
251
- flushTimer = null
252
- events
253
- /** `model\tdims` handed to every ScopeStore for sidecar validation. */
254
- vecIdentity = ''
255
-
256
- constructor(root , events = {}) {
257
- this.root = root
258
- this.events = events
259
- mkdirSync(join(root, 'projects'), { recursive: true })
260
- }
261
-
262
- /** Declare the embedding identity; applies to existing and future scopes. */
263
- setVecIdentity(model , dims ) {
264
- this.vecIdentity = `${model}\t${dims}`
265
- for (const s of this.scopes.values()) s.setVecIdentity(model, dims)
266
- }
267
-
268
- /** Preload specific scopes eagerly (e.g. defaults); others load on first access. */
269
- loadAll(wanted ) {
270
- for (const scope of wanted) this.scope(scope)
271
- }
272
-
273
- loadScope(scope ) {
274
- const file = this.fileFor(scope)
275
- if (!existsSync(file)) return
276
- let text
277
- try {
278
- text = readFileSync(file, 'utf8')
279
- } catch (err) {
280
- this.events.onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
281
- return
282
- }
283
- const store = this.scope(scope)
284
- let lineNo = 0
285
- for (const line of text.split('\n')) {
286
- lineNo++
287
- const trimmed = line.trim()
288
- if (!trimmed) continue
289
- try {
290
- const rec = JSON.parse(trimmed)
291
- if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
292
- throw new Error('bad shape')
293
- }
294
- if (rec.scope !== scope) continue // file/scope mismatch: ignore, will be rewritten on flush
295
- store.records.set(rec.id, rec)
296
- } catch {
297
- this.events.onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
298
- }
299
- }
300
- // sidecar vectors second; anything still un-embedded joins the backfill queue
301
- store.loadVectors()
302
- for (const id of store.records.keys()) {
303
- if (!store.vectors.has(id)) store.embedPending.add(id)
304
- }
305
- store.dirty = false
306
- }
307
-
308
- scope(scope ) {
309
- let s = this.scopes.get(scope)
310
- if (!s) {
311
- s = new ScopeStore(scope, this.root, this.events)
312
- if (this.vecIdentity) s.setVecIdentity(...(this.vecIdentity.split('\t') ))
313
- this.scopes.set(scope, s)
314
- this.fileNames.set(scope, scopeFileName(scope))
315
- }
316
- if (!this.loaded.has(scope)) {
317
- this.loaded.add(scope)
318
- this.loadScope(scope)
319
- }
320
- return s
321
- }
322
-
323
- allScopes() {
324
- return [...this.scopes.values()]
325
- }
326
-
327
- /**
328
- * Eagerly load every scope that has data on disk and return the scope keys.
329
- *
330
- * Scopes are otherwise created lazily, so a long-lived process only knows the
331
- * scopes it has already touched — a management UI (the memory panel) would
332
- * silently miss project memories written in earlier sessions. Project files
333
- * are named by scope hash, so the scope key is recovered from the first
334
- * record inside each file.
335
- */
336
- loadDiscovered() {
337
- const discovered = []
338
- if (existsSync(join(this.root, 'global.jsonl'))) discovered.push('global')
339
- const projDir = join(this.root, 'projects')
340
- if (existsSync(projDir)) {
341
- for (const file of readdirSync(projDir)) {
342
- if (!file.endsWith('.jsonl')) continue
343
- const scope = firstScopeInFile(join(projDir, file))
344
- if (scope) discovered.push(scope)
345
- }
346
- }
347
- for (const scope of discovered) this.scope(scope) // triggers lazy disk load
348
- return discovered
349
- }
350
-
351
- fileFor(scope ) {
352
- return join(this.root, this.fileNames.get(scope) ?? scopeFileName(scope))
353
- }
354
-
355
- anyDirty() {
356
- for (const s of this.scopes.values()) if (s.dirty) return true
357
- return false
358
- }
359
-
360
- scheduleFlush(delayMs = 500) {
361
- if (this.flushTimer) return
362
- this.flushTimer = setTimeout(() => {
363
- this.flushTimer = null
364
- this.flush()
365
- }, delayMs)
366
- // a debounced flush must not hold the event loop open
367
- this.flushTimer.unref?.()
368
- }
369
-
370
- /** Write dirty scopes atomically (tmp + rename). */
371
- flush() {
372
- for (const s of this.scopes.values()) {
373
- if (!s.dirty) continue
374
- const file = this.fileFor(s.scope)
375
- mkdirSync(dirname(file), { recursive: true })
376
- const tmp = `${file}.tmp`
377
- const lines = [...s.records.values()].map((r) => JSON.stringify(r))
378
- writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
379
- renameSync(tmp, file)
380
- s.dirty = false
381
- }
382
- // vectors flush too: they are the expensive-to-recompute half
383
- for (const s of this.scopes.values()) s.flushVectors()
384
- }
385
-
386
- /** Synchronous final flush — call on dispose. */
387
- close() {
388
- if (this.flushTimer) {
389
- clearTimeout(this.flushTimer)
390
- this.flushTimer = null
391
- }
392
- this.flush()
393
- }
394
-
395
- /**
396
- * Dedupe-aware save: identical normalized content in the scope refreshes the
397
- * existing record instead of duplicating it.
398
- */
399
- save(
400
- input ,
401
- defaults ,
402
- ) {
403
- const s = this.scope(input.scope)
404
- const hash = contentHash(input.scope, input.content)
405
- for (const rec of s.records.values()) {
406
- if (contentHash(rec.scope, rec.content) === hash) {
407
- rec.updatedAt = Date.now()
408
- rec.accessCount += 1
409
- rec.importance = Math.max(rec.importance, input.importance ?? defaults.importance)
410
- if (input.supersedes) rec.supersedes = input.supersedes
411
- s.dirty = true
412
- return { record: rec, deduped: true }
413
- }
414
- }
415
- const now = Date.now()
416
- const record = {
417
- id: newId(),
418
- scope: input.scope,
419
- kind: input.kind,
420
- content: input.content,
421
- tags: input.tags ?? [],
422
- source: { ...defaults.source, ...input.source },
423
- createdAt: now,
424
- updatedAt: now,
425
- lastAccessedAt: now,
426
- accessCount: 0,
427
- importance: input.importance ?? defaults.importance,
428
- ...(input.supersedes ? { supersedes: input.supersedes } : {}),
429
- ...(input.expiredAt ? { expiredAt: input.expiredAt } : {}),
430
- }
431
- s.insert(record)
432
- return { record, deduped: false }
433
- }
434
-
435
- /** Substring match over normalized content, used by forget-by-query. */
436
- findIdsByQuery(scope , query ) {
437
- const s = this.scope(scope)
438
- const needle = normalizeContent(query)
439
- if (!needle) return []
440
- const ids = []
441
- for (const rec of s.records.values()) {
442
- if (normalizeContent(rec.content).includes(needle)) ids.push(rec.id)
443
- }
444
- return ids
445
- }
446
-
447
- /** Remove every expired record across scopes; returns removed ids. */
448
- purgeExpired(now = Date.now()) {
449
- const removed = []
450
- for (const s of this.scopes.values()) {
451
- for (const rec of [...s.records.values()]) {
452
- if (rec.expiredAt !== undefined && rec.expiredAt < now) {
453
- s.remove(rec.id)
454
- removed.push(rec.id)
455
- }
456
- }
457
- }
458
- return removed
459
- }
460
-
461
- /**
462
- * Evict lowest-ranked records until each scope is within its cap. Ranking:
463
- * recency (30d half-life) × importance — keep what retrieval would favor.
464
- */
465
- enforceLimit(maxPerScope , now = Date.now()) {
466
- const removed = []
467
- for (const s of this.scopes.values()) {
468
- if (s.records.size <= maxPerScope) continue
469
- const ranked = [...s.records.values()]
470
- .map((r) => ({
471
- id: r.id,
472
- rank: Math.pow(0.5, Math.max(0, now - r.updatedAt) / 86_400_000 / 30) * (0.1 + r.importance),
473
- }))
474
- .sort((a, b) => b.rank - a.rank)
475
- for (const r of ranked.slice(maxPerScope)) {
476
- s.remove(r.id)
477
- removed.push(r.id)
478
- }
479
- }
480
- return removed
481
- }
482
- }
483
-
484
-
485
- //# sourceURL=D:\.deepseek\memory-plugin\packages\memory-local\src\store.ts
1
+ /**
2
+ * JSONL-backed store. One file per scope, append-on-write semantics with
3
+ * atomic compaction (tmp + rename). Vectors live in memory for M1; the .vec
4
+ * sidecar format arrives in M1.5.
5
+ */
6
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from 'node:fs'
7
+ import { createHash, randomUUID } from 'node:crypto'
8
+ import { Buffer } from 'node:buffer'
9
+ import { dirname, join } from 'node:path'
10
+
11
+
12
+
13
+
14
+
15
+
16
+ export function scopeFileName(scope ) { if (scope === 'global') return 'global.jsonl'
17
+ const hash = createHash('sha1').update(scope).digest('hex')
18
+ return join('projects', `${hash}.jsonl`)
19
+ }
20
+
21
+ /** Derive the scope for records loaded from a file path. */
22
+ export function scopeFromFile(file , knownScopes ) {
23
+ if (file.endsWith('global.jsonl')) return 'global'
24
+ const base = file.split(/[\\/]/).pop() ?? ''
25
+ for (const [scope, name] of knownScopes) if (name === base) return scope
26
+ return null
27
+ }
28
+
29
+ /**
30
+ * Read the first well-formed record's scope from a scope file. Project files are
31
+ * named by hash, so this is the only way back to the scope key without the
32
+ * provider having created that scope in this process already.
33
+ */
34
+ export function firstScopeInFile(file ) {
35
+ let text
36
+ try {
37
+ text = readFileSync(file, 'utf8')
38
+ } catch {
39
+ return null
40
+ }
41
+ for (const line of text.split('\n')) {
42
+ const trimmed = line.trim()
43
+ if (!trimmed) continue
44
+ try {
45
+ const rec = JSON.parse(trimmed)
46
+ if (typeof rec.scope === 'string' && rec.scope) return rec.scope
47
+ } catch {
48
+ // skip malformed line, keep looking
49
+ }
50
+ }
51
+ return null
52
+ }
53
+
54
+ export function normalizeContent(content ) {
55
+ return content.replace(/\s+/g, ' ').trim().toLowerCase()
56
+ }
57
+
58
+ export function contentHash(scope , content ) {
59
+ return createHash('sha1').update(`${scope}::${normalizeContent(content)}`).digest('hex')
60
+ }
61
+
62
+ export function newId() {
63
+ return randomUUID()
64
+ }
65
+
66
+ export class ScopeStore {
67
+ scope
68
+ records = new Map ()
69
+ vectors = new Map ()
70
+ /** ids saved while the embed backend was down; retried opportunistically. */
71
+ embedPending = new Set ()
72
+ dirty = false
73
+ /** true once vectors were touched since the last .vec flush. */
74
+ vectorsDirty = false
75
+ vecFile
76
+ events
77
+ /** Embedding identity for sidecar validation; set via setVecIdentity(). */
78
+ vecIdentity = ''
79
+
80
+ constructor(scope , root , events = {}) {
81
+ this.scope = scope
82
+ this.vecFile = join(root, scopeFileName(scope).replace(/\.jsonl$/, '.vec'))
83
+ this.events = events
84
+ }
85
+
86
+ /**
87
+ * Declare which model+dims produced the in-memory vectors. When it differs
88
+ * from the sidecar header on load, the sidecar is discarded and every record
89
+ * re-enters the embed queue — old vectors for a new model are wrong, not
90
+ * merely stale.
91
+ */
92
+ setVecIdentity(model , dims ) {
93
+ this.vecIdentity = `${model}\t${dims}`
94
+ }
95
+
96
+ get size() {
97
+ return this.records.size
98
+ }
99
+
100
+ insert(record ) {
101
+ this.records.set(record.id, record)
102
+ this.dirty = true
103
+ }
104
+
105
+ remove(id ) {
106
+ const had = this.records.delete(id)
107
+ if (had) {
108
+ this.vectors.delete(id)
109
+ this.embedPending.delete(id)
110
+ this.dirty = true
111
+ this.vectorsDirty = true
112
+ }
113
+ return had
114
+ }
115
+
116
+ setVector(id , vec ) {
117
+ this.vectors.set(id, vec)
118
+ this.vectorsDirty = true
119
+ }
120
+
121
+ /**
122
+ * Append a record to the sidecar: `<id> \t <base64 dim vec>` per line.
123
+ * Vectors are written immediately on embed (single-line append) and fully
124
+ * rewritten on delete/compact — embeddings are expensive to recompute.
125
+ * A freshly created file starts with the `#model\tdims` identity header.
126
+ */
127
+ appendVector(id , vec ) {
128
+ this.setVector(id, vec)
129
+ try {
130
+ mkdirSync(dirname(this.vecFile), { recursive: true })
131
+ if (this.vecIdentity && this.readHeader() !== this.vecIdentity) {
132
+ // missing, legacy, or wrong-model file: rewrite fresh from memory
133
+ this.writeSidecar()
134
+ this.vectorsDirty = false
135
+ return
136
+ }
137
+ appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
138
+ this.vectorsDirty = false
139
+ } catch (err) {
140
+ this.vectorsDirty = true // full rewrite on next flushVectors
141
+ this.events.onWarn?.(`vec: append failed: ${(err ).message}`)
142
+ }
143
+ }
144
+
145
+ /** First line of the sidecar if it is a header; '' when absent/headerless. */
146
+ readHeader() {
147
+ if (!existsSync(this.vecFile)) return ''
148
+ try {
149
+ const first = readFileSync(this.vecFile, 'utf8').split('\n', 1)[0] ?? ''
150
+ return first.startsWith('#') ? first.slice(1) : ''
151
+ } catch {
152
+ return ''
153
+ }
154
+ }
155
+
156
+ /** Atomic full rewrite: identity header + every in-memory vector. */
157
+ writeSidecar() {
158
+ mkdirSync(dirname(this.vecFile), { recursive: true })
159
+ const tmp = `${this.vecFile}.tmp`
160
+ const lines = this.vecIdentity ? [`#${this.vecIdentity}`] : []
161
+ for (const [id, v] of this.vectors) lines.push(`${id}\t${encodeVec(v)}`)
162
+ writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
163
+ renameSync(tmp, this.vecFile)
164
+ }
165
+
166
+ /** Rewrite the whole sidecar (after deletes / compaction / append failures). */
167
+ flushVectors() {
168
+ if (!this.vectorsDirty) return
169
+ try {
170
+ this.writeSidecar()
171
+ this.vectorsDirty = false
172
+ } catch (err) {
173
+ this.events.onWarn?.(`vec: flush failed: ${(err ).message}`)
174
+ }
175
+ }
176
+
177
+ /**
178
+ * Load vectors from the sidecar. The first line must be a `#model\tdims`
179
+ * header when an identity is configured — mismatch or a legacy headerless
180
+ * file discards the whole sidecar (re-embed). Unknown ids are skipped.
181
+ */
182
+ loadVectors() {
183
+ if (!existsSync(this.vecFile)) return
184
+ try {
185
+ const text = readFileSync(this.vecFile, 'utf8')
186
+ let sawHeader = false
187
+ let sawData = false
188
+ const discard = (why ) => {
189
+ this.events.onWarn?.(`vec: ${why}; discarding sidecar for re-embed`)
190
+ this.vectors.clear()
191
+ this.vectorsDirty = true
192
+ }
193
+ for (const line of text.split('\n')) {
194
+ const trimmed = line.trim()
195
+ if (!trimmed) continue
196
+ if (trimmed.startsWith('#')) {
197
+ if (this.vecIdentity && trimmed.slice(1) !== this.vecIdentity) {
198
+ discard(`identity mismatch (${trimmed.slice(1)} != ${this.vecIdentity})`)
199
+ return
200
+ }
201
+ // Loading before the identity is declared means NO validation happens
202
+ // (the guards below are all identity-gated). That ordering bug shipped
203
+ // once — see RestRegular/dsh-memory#1 — so make any recurrence visible.
204
+ if (!this.vecIdentity) {
205
+ this.events.onWarn?.(`${this.scope}: loading a vector sidecar before setVecIdentity(); its ${trimmed.slice(1)} vectors are accepted unvalidated`)
206
+ }
207
+ sawHeader = true
208
+ continue
209
+ }
210
+ if (!sawData) {
211
+ sawData = true
212
+ if (this.vecIdentity && !sawHeader) {
213
+ discard('legacy sidecar without identity header')
214
+ return
215
+ }
216
+ }
217
+ const tab = trimmed.indexOf('\t')
218
+ if (tab <= 0) continue
219
+ const id = trimmed.slice(0, tab)
220
+ if (!this.records.has(id)) continue // stale vector for a deleted record
221
+ const vec = decodeVec(trimmed.slice(tab + 1))
222
+ if (vec) this.vectors.set(id, vec)
223
+ }
224
+ } catch (err) {
225
+ this.events.onWarn?.(`vec: load failed: ${(err ).message}`)
226
+ }
227
+ }
228
+ }
229
+
230
+ // --- base64 vector codec (4 bytes per float, bit-exact) ---
231
+
232
+ function encodeVec(vec ) {
233
+ return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength).toString('base64')
234
+ }
235
+
236
+ function decodeVec(text ) {
237
+ try {
238
+ const buf = Buffer.from(text, 'base64')
239
+ if (buf.length === 0 || buf.length % 4 !== 0) return null
240
+ return new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4)
241
+ } catch {
242
+ return null
243
+ }
244
+ }
245
+
246
+
247
+
248
+
249
+
250
+ export class MemoryStore {
251
+ root
252
+ scopes = new Map ()
253
+ /** scopes whose jsonl has been loaded from disk (lazy, once per scope). */
254
+ loaded = new Set ()
255
+ /** scope -> its jsonl filename, so reload can map files back to scopes. */
256
+ fileNames = new Map ()
257
+ flushTimer = null
258
+ events
259
+ /** `model\tdims` handed to every ScopeStore for sidecar validation. */
260
+ vecIdentity = ''
261
+
262
+ constructor(root , events = {}) {
263
+ this.root = root
264
+ this.events = events
265
+ mkdirSync(join(root, 'projects'), { recursive: true })
266
+ }
267
+
268
+ /** Declare the embedding identity; applies to existing and future scopes. */
269
+ setVecIdentity(model , dims ) {
270
+ this.vecIdentity = `${model}\t${dims}`
271
+ for (const s of this.scopes.values()) s.setVecIdentity(model, dims)
272
+ }
273
+
274
+ /** Preload specific scopes eagerly (e.g. defaults); others load on first access. */
275
+ loadAll(wanted ) {
276
+ for (const scope of wanted) this.scope(scope)
277
+ }
278
+
279
+ loadScope(scope ) {
280
+ const file = this.fileFor(scope)
281
+ if (!existsSync(file)) return
282
+ let text
283
+ try {
284
+ text = readFileSync(file, 'utf8')
285
+ } catch (err) {
286
+ this.events.onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
287
+ return
288
+ }
289
+ const store = this.scope(scope)
290
+ let lineNo = 0
291
+ for (const line of text.split('\n')) {
292
+ lineNo++
293
+ const trimmed = line.trim()
294
+ if (!trimmed) continue
295
+ try {
296
+ const rec = JSON.parse(trimmed)
297
+ if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
298
+ throw new Error('bad shape')
299
+ }
300
+ if (rec.scope !== scope) continue // file/scope mismatch: ignore, will be rewritten on flush
301
+ store.records.set(rec.id, rec)
302
+ } catch {
303
+ this.events.onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
304
+ }
305
+ }
306
+ // sidecar vectors second; anything still un-embedded joins the backfill queue
307
+ store.loadVectors()
308
+ for (const id of store.records.keys()) {
309
+ if (!store.vectors.has(id)) store.embedPending.add(id)
310
+ }
311
+ store.dirty = false
312
+ }
313
+
314
+ scope(scope ) {
315
+ let s = this.scopes.get(scope)
316
+ if (!s) {
317
+ s = new ScopeStore(scope, this.root, this.events)
318
+ if (this.vecIdentity) s.setVecIdentity(...(this.vecIdentity.split('\t') ))
319
+ this.scopes.set(scope, s)
320
+ this.fileNames.set(scope, scopeFileName(scope))
321
+ }
322
+ if (!this.loaded.has(scope)) {
323
+ this.loaded.add(scope)
324
+ this.loadScope(scope)
325
+ }
326
+ return s
327
+ }
328
+
329
+ allScopes() {
330
+ return [...this.scopes.values()]
331
+ }
332
+
333
+ /**
334
+ * Eagerly load every scope that has data on disk and return the scope keys.
335
+ *
336
+ * Scopes are otherwise created lazily, so a long-lived process only knows the
337
+ * scopes it has already touched — a management UI (the memory panel) would
338
+ * silently miss project memories written in earlier sessions. Project files
339
+ * are named by scope hash, so the scope key is recovered from the first
340
+ * record inside each file.
341
+ */
342
+ loadDiscovered() {
343
+ const discovered = []
344
+ if (existsSync(join(this.root, 'global.jsonl'))) discovered.push('global')
345
+ const projDir = join(this.root, 'projects')
346
+ if (existsSync(projDir)) {
347
+ for (const file of readdirSync(projDir)) {
348
+ if (!file.endsWith('.jsonl')) continue
349
+ const scope = firstScopeInFile(join(projDir, file))
350
+ if (scope) discovered.push(scope)
351
+ }
352
+ }
353
+ for (const scope of discovered) this.scope(scope) // triggers lazy disk load
354
+ return discovered
355
+ }
356
+
357
+ fileFor(scope ) {
358
+ return join(this.root, this.fileNames.get(scope) ?? scopeFileName(scope))
359
+ }
360
+
361
+ anyDirty() {
362
+ for (const s of this.scopes.values()) if (s.dirty) return true
363
+ return false
364
+ }
365
+
366
+ scheduleFlush(delayMs = 500) {
367
+ if (this.flushTimer) return
368
+ this.flushTimer = setTimeout(() => {
369
+ this.flushTimer = null
370
+ this.flush()
371
+ }, delayMs)
372
+ // a debounced flush must not hold the event loop open
373
+ this.flushTimer.unref?.()
374
+ }
375
+
376
+ /** Write dirty scopes atomically (tmp + rename). */
377
+ flush() {
378
+ for (const s of this.scopes.values()) {
379
+ if (!s.dirty) continue
380
+ const file = this.fileFor(s.scope)
381
+ mkdirSync(dirname(file), { recursive: true })
382
+ const tmp = `${file}.tmp`
383
+ const lines = [...s.records.values()].map((r) => JSON.stringify(r))
384
+ writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
385
+ renameSync(tmp, file)
386
+ s.dirty = false
387
+ }
388
+ // vectors flush too: they are the expensive-to-recompute half
389
+ for (const s of this.scopes.values()) s.flushVectors()
390
+ }
391
+
392
+ /** Synchronous final flush — call on dispose. */
393
+ close() {
394
+ if (this.flushTimer) {
395
+ clearTimeout(this.flushTimer)
396
+ this.flushTimer = null
397
+ }
398
+ this.flush()
399
+ }
400
+
401
+ /**
402
+ * Dedupe-aware save: identical normalized content in the scope refreshes the
403
+ * existing record instead of duplicating it.
404
+ */
405
+ save(
406
+ input ,
407
+ defaults ,
408
+ ) {
409
+ const s = this.scope(input.scope)
410
+ const hash = contentHash(input.scope, input.content)
411
+ for (const rec of s.records.values()) {
412
+ if (contentHash(rec.scope, rec.content) === hash) {
413
+ rec.updatedAt = Date.now()
414
+ rec.accessCount += 1
415
+ rec.importance = Math.max(rec.importance, input.importance ?? defaults.importance)
416
+ if (input.supersedes) rec.supersedes = input.supersedes
417
+ s.dirty = true
418
+ return { record: rec, deduped: true }
419
+ }
420
+ }
421
+ const now = Date.now()
422
+ const record = {
423
+ id: newId(),
424
+ scope: input.scope,
425
+ kind: input.kind,
426
+ content: input.content,
427
+ tags: input.tags ?? [],
428
+ source: { ...defaults.source, ...input.source },
429
+ createdAt: now,
430
+ updatedAt: now,
431
+ lastAccessedAt: now,
432
+ accessCount: 0,
433
+ importance: input.importance ?? defaults.importance,
434
+ ...(input.supersedes ? { supersedes: input.supersedes } : {}),
435
+ ...(input.expiredAt ? { expiredAt: input.expiredAt } : {}),
436
+ }
437
+ s.insert(record)
438
+ return { record, deduped: false }
439
+ }
440
+
441
+ /** Substring match over normalized content, used by forget-by-query. */
442
+ findIdsByQuery(scope , query ) {
443
+ const s = this.scope(scope)
444
+ const needle = normalizeContent(query)
445
+ if (!needle) return []
446
+ const ids = []
447
+ for (const rec of s.records.values()) {
448
+ if (normalizeContent(rec.content).includes(needle)) ids.push(rec.id)
449
+ }
450
+ return ids
451
+ }
452
+
453
+ /** Remove every expired record across scopes; returns removed ids. */
454
+ purgeExpired(now = Date.now()) {
455
+ const removed = []
456
+ for (const s of this.scopes.values()) {
457
+ for (const rec of [...s.records.values()]) {
458
+ if (rec.expiredAt !== undefined && rec.expiredAt < now) {
459
+ s.remove(rec.id)
460
+ removed.push(rec.id)
461
+ }
462
+ }
463
+ }
464
+ return removed
465
+ }
466
+
467
+ /**
468
+ * Evict lowest-ranked records until each scope is within its cap. Ranking:
469
+ * recency (30d half-life) × importance — keep what retrieval would favor.
470
+ */
471
+ enforceLimit(maxPerScope , now = Date.now()) {
472
+ const removed = []
473
+ for (const s of this.scopes.values()) {
474
+ if (s.records.size <= maxPerScope) continue
475
+ const ranked = [...s.records.values()]
476
+ .map((r) => ({
477
+ id: r.id,
478
+ rank: Math.pow(0.5, Math.max(0, now - r.updatedAt) / 86_400_000 / 30) * (0.1 + r.importance),
479
+ }))
480
+ .sort((a, b) => b.rank - a.rank)
481
+ for (const r of ranked.slice(maxPerScope)) {
482
+ s.remove(r.id)
483
+ removed.push(r.id)
484
+ }
485
+ }
486
+ return removed
487
+ }
488
+ }
489
+
490
+
491
+ //# sourceURL=source/memory-local/src/store.ts