@restregular/dsh-memory 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,491 +1,491 @@
1
- /**
2
- * JSONL-backed store. One file per scope, append-on-write semantics with
3
- * atomic compaction (tmp + rename). Vectors live in memory for M1; the .vec
4
- * sidecar format arrives in M1.5.
5
- */
6
- import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from 'node:fs'
7
- import { createHash, randomUUID } from 'node:crypto'
8
- import { Buffer } from 'node:buffer'
9
- import { dirname, join } from 'node:path'
10
-
11
-
12
-
13
-
14
-
15
-
16
- export function scopeFileName(scope ) { if (scope === 'global') return 'global.jsonl'
17
- const hash = createHash('sha1').update(scope).digest('hex')
18
- return join('projects', `${hash}.jsonl`)
19
- }
20
-
21
- /** Derive the scope for records loaded from a file path. */
22
- export function scopeFromFile(file , knownScopes ) {
23
- if (file.endsWith('global.jsonl')) return 'global'
24
- const base = file.split(/[\\/]/).pop() ?? ''
25
- for (const [scope, name] of knownScopes) if (name === base) return scope
26
- return null
27
- }
28
-
29
- /**
30
- * Read the first well-formed record's scope from a scope file. Project files are
31
- * named by hash, so this is the only way back to the scope key without the
32
- * provider having created that scope in this process already.
33
- */
34
- export function firstScopeInFile(file ) {
35
- let text
36
- try {
37
- text = readFileSync(file, 'utf8')
38
- } catch {
39
- return null
40
- }
41
- for (const line of text.split('\n')) {
42
- const trimmed = line.trim()
43
- if (!trimmed) continue
44
- try {
45
- const rec = JSON.parse(trimmed)
46
- if (typeof rec.scope === 'string' && rec.scope) return rec.scope
47
- } catch {
48
- // skip malformed line, keep looking
49
- }
50
- }
51
- return null
52
- }
53
-
54
- export function normalizeContent(content ) {
55
- return content.replace(/\s+/g, ' ').trim().toLowerCase()
56
- }
57
-
58
- export function contentHash(scope , content ) {
59
- return createHash('sha1').update(`${scope}::${normalizeContent(content)}`).digest('hex')
60
- }
61
-
62
- export function newId() {
63
- return randomUUID()
64
- }
65
-
66
- export class ScopeStore {
67
- scope
68
- records = new Map ()
69
- vectors = new Map ()
70
- /** ids saved while the embed backend was down; retried opportunistically. */
71
- embedPending = new Set ()
72
- dirty = false
73
- /** true once vectors were touched since the last .vec flush. */
74
- vectorsDirty = false
75
- vecFile
76
- events
77
- /** Embedding identity for sidecar validation; set via setVecIdentity(). */
78
- vecIdentity = ''
79
-
80
- constructor(scope , root , events = {}) {
81
- this.scope = scope
82
- this.vecFile = join(root, scopeFileName(scope).replace(/\.jsonl$/, '.vec'))
83
- this.events = events
84
- }
85
-
86
- /**
87
- * Declare which model+dims produced the in-memory vectors. When it differs
88
- * from the sidecar header on load, the sidecar is discarded and every record
89
- * re-enters the embed queue — old vectors for a new model are wrong, not
90
- * merely stale.
91
- */
92
- setVecIdentity(model , dims ) {
93
- this.vecIdentity = `${model}\t${dims}`
94
- }
95
-
96
- get size() {
97
- return this.records.size
98
- }
99
-
100
- insert(record ) {
101
- this.records.set(record.id, record)
102
- this.dirty = true
103
- }
104
-
105
- remove(id ) {
106
- const had = this.records.delete(id)
107
- if (had) {
108
- this.vectors.delete(id)
109
- this.embedPending.delete(id)
110
- this.dirty = true
111
- this.vectorsDirty = true
112
- }
113
- return had
114
- }
115
-
116
- setVector(id , vec ) {
117
- this.vectors.set(id, vec)
118
- this.vectorsDirty = true
119
- }
120
-
121
- /**
122
- * Append a record to the sidecar: `<id> \t <base64 dim vec>` per line.
123
- * Vectors are written immediately on embed (single-line append) and fully
124
- * rewritten on delete/compact — embeddings are expensive to recompute.
125
- * A freshly created file starts with the `#model\tdims` identity header.
126
- */
127
- appendVector(id , vec ) {
128
- this.setVector(id, vec)
129
- try {
130
- mkdirSync(dirname(this.vecFile), { recursive: true })
131
- if (this.vecIdentity && this.readHeader() !== this.vecIdentity) {
132
- // missing, legacy, or wrong-model file: rewrite fresh from memory
133
- this.writeSidecar()
134
- this.vectorsDirty = false
135
- return
136
- }
137
- appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
138
- this.vectorsDirty = false
139
- } catch (err) {
140
- this.vectorsDirty = true // full rewrite on next flushVectors
141
- this.events.onWarn?.(`vec: append failed: ${(err ).message}`)
142
- }
143
- }
144
-
145
- /** First line of the sidecar if it is a header; '' when absent/headerless. */
146
- readHeader() {
147
- if (!existsSync(this.vecFile)) return ''
148
- try {
149
- const first = readFileSync(this.vecFile, 'utf8').split('\n', 1)[0] ?? ''
150
- return first.startsWith('#') ? first.slice(1) : ''
151
- } catch {
152
- return ''
153
- }
154
- }
155
-
156
- /** Atomic full rewrite: identity header + every in-memory vector. */
157
- writeSidecar() {
158
- mkdirSync(dirname(this.vecFile), { recursive: true })
159
- const tmp = `${this.vecFile}.tmp`
160
- const lines = this.vecIdentity ? [`#${this.vecIdentity}`] : []
161
- for (const [id, v] of this.vectors) lines.push(`${id}\t${encodeVec(v)}`)
162
- writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
163
- renameSync(tmp, this.vecFile)
164
- }
165
-
166
- /** Rewrite the whole sidecar (after deletes / compaction / append failures). */
167
- flushVectors() {
168
- if (!this.vectorsDirty) return
169
- try {
170
- this.writeSidecar()
171
- this.vectorsDirty = false
172
- } catch (err) {
173
- this.events.onWarn?.(`vec: flush failed: ${(err ).message}`)
174
- }
175
- }
176
-
177
- /**
178
- * Load vectors from the sidecar. The first line must be a `#model\tdims`
179
- * header when an identity is configured — mismatch or a legacy headerless
180
- * file discards the whole sidecar (re-embed). Unknown ids are skipped.
181
- */
182
- loadVectors() {
183
- if (!existsSync(this.vecFile)) return
184
- try {
185
- const text = readFileSync(this.vecFile, 'utf8')
186
- let sawHeader = false
187
- let sawData = false
188
- const discard = (why ) => {
189
- this.events.onWarn?.(`vec: ${why}; discarding sidecar for re-embed`)
190
- this.vectors.clear()
191
- this.vectorsDirty = true
192
- }
193
- for (const line of text.split('\n')) {
194
- const trimmed = line.trim()
195
- if (!trimmed) continue
196
- if (trimmed.startsWith('#')) {
197
- if (this.vecIdentity && trimmed.slice(1) !== this.vecIdentity) {
198
- discard(`identity mismatch (${trimmed.slice(1)} != ${this.vecIdentity})`)
199
- return
200
- }
201
- // Loading before the identity is declared means NO validation happens
202
- // (the guards below are all identity-gated). That ordering bug shipped
203
- // once — see RestRegular/dsh-memory#1 — so make any recurrence visible.
204
- if (!this.vecIdentity) {
205
- this.events.onWarn?.(`${this.scope}: loading a vector sidecar before setVecIdentity(); its ${trimmed.slice(1)} vectors are accepted unvalidated`)
206
- }
207
- sawHeader = true
208
- continue
209
- }
210
- if (!sawData) {
211
- sawData = true
212
- if (this.vecIdentity && !sawHeader) {
213
- discard('legacy sidecar without identity header')
214
- return
215
- }
216
- }
217
- const tab = trimmed.indexOf('\t')
218
- if (tab <= 0) continue
219
- const id = trimmed.slice(0, tab)
220
- if (!this.records.has(id)) continue // stale vector for a deleted record
221
- const vec = decodeVec(trimmed.slice(tab + 1))
222
- if (vec) this.vectors.set(id, vec)
223
- }
224
- } catch (err) {
225
- this.events.onWarn?.(`vec: load failed: ${(err ).message}`)
226
- }
227
- }
228
- }
229
-
230
- // --- base64 vector codec (4 bytes per float, bit-exact) ---
231
-
232
- function encodeVec(vec ) {
233
- return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength).toString('base64')
234
- }
235
-
236
- function decodeVec(text ) {
237
- try {
238
- const buf = Buffer.from(text, 'base64')
239
- if (buf.length === 0 || buf.length % 4 !== 0) return null
240
- return new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4)
241
- } catch {
242
- return null
243
- }
244
- }
245
-
246
-
247
-
248
-
249
-
250
- export class MemoryStore {
251
- root
252
- scopes = new Map ()
253
- /** scopes whose jsonl has been loaded from disk (lazy, once per scope). */
254
- loaded = new Set ()
255
- /** scope -> its jsonl filename, so reload can map files back to scopes. */
256
- fileNames = new Map ()
257
- flushTimer = null
258
- events
259
- /** `model\tdims` handed to every ScopeStore for sidecar validation. */
260
- vecIdentity = ''
261
-
262
- constructor(root , events = {}) {
263
- this.root = root
264
- this.events = events
265
- mkdirSync(join(root, 'projects'), { recursive: true })
266
- }
267
-
268
- /** Declare the embedding identity; applies to existing and future scopes. */
269
- setVecIdentity(model , dims ) {
270
- this.vecIdentity = `${model}\t${dims}`
271
- for (const s of this.scopes.values()) s.setVecIdentity(model, dims)
272
- }
273
-
274
- /** Preload specific scopes eagerly (e.g. defaults); others load on first access. */
275
- loadAll(wanted ) {
276
- for (const scope of wanted) this.scope(scope)
277
- }
278
-
279
- loadScope(scope ) {
280
- const file = this.fileFor(scope)
281
- if (!existsSync(file)) return
282
- let text
283
- try {
284
- text = readFileSync(file, 'utf8')
285
- } catch (err) {
286
- this.events.onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
287
- return
288
- }
289
- const store = this.scope(scope)
290
- let lineNo = 0
291
- for (const line of text.split('\n')) {
292
- lineNo++
293
- const trimmed = line.trim()
294
- if (!trimmed) continue
295
- try {
296
- const rec = JSON.parse(trimmed)
297
- if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
298
- throw new Error('bad shape')
299
- }
300
- if (rec.scope !== scope) continue // file/scope mismatch: ignore, will be rewritten on flush
301
- store.records.set(rec.id, rec)
302
- } catch {
303
- this.events.onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
304
- }
305
- }
306
- // sidecar vectors second; anything still un-embedded joins the backfill queue
307
- store.loadVectors()
308
- for (const id of store.records.keys()) {
309
- if (!store.vectors.has(id)) store.embedPending.add(id)
310
- }
311
- store.dirty = false
312
- }
313
-
314
- scope(scope ) {
315
- let s = this.scopes.get(scope)
316
- if (!s) {
317
- s = new ScopeStore(scope, this.root, this.events)
318
- if (this.vecIdentity) s.setVecIdentity(...(this.vecIdentity.split('\t') ))
319
- this.scopes.set(scope, s)
320
- this.fileNames.set(scope, scopeFileName(scope))
321
- }
322
- if (!this.loaded.has(scope)) {
323
- this.loaded.add(scope)
324
- this.loadScope(scope)
325
- }
326
- return s
327
- }
328
-
329
- allScopes() {
330
- return [...this.scopes.values()]
331
- }
332
-
333
- /**
334
- * Eagerly load every scope that has data on disk and return the scope keys.
335
- *
336
- * Scopes are otherwise created lazily, so a long-lived process only knows the
337
- * scopes it has already touched — a management UI (the memory panel) would
338
- * silently miss project memories written in earlier sessions. Project files
339
- * are named by scope hash, so the scope key is recovered from the first
340
- * record inside each file.
341
- */
342
- loadDiscovered() {
343
- const discovered = []
344
- if (existsSync(join(this.root, 'global.jsonl'))) discovered.push('global')
345
- const projDir = join(this.root, 'projects')
346
- if (existsSync(projDir)) {
347
- for (const file of readdirSync(projDir)) {
348
- if (!file.endsWith('.jsonl')) continue
349
- const scope = firstScopeInFile(join(projDir, file))
350
- if (scope) discovered.push(scope)
351
- }
352
- }
353
- for (const scope of discovered) this.scope(scope) // triggers lazy disk load
354
- return discovered
355
- }
356
-
357
- fileFor(scope ) {
358
- return join(this.root, this.fileNames.get(scope) ?? scopeFileName(scope))
359
- }
360
-
361
- anyDirty() {
362
- for (const s of this.scopes.values()) if (s.dirty) return true
363
- return false
364
- }
365
-
366
- scheduleFlush(delayMs = 500) {
367
- if (this.flushTimer) return
368
- this.flushTimer = setTimeout(() => {
369
- this.flushTimer = null
370
- this.flush()
371
- }, delayMs)
372
- // a debounced flush must not hold the event loop open
373
- this.flushTimer.unref?.()
374
- }
375
-
376
- /** Write dirty scopes atomically (tmp + rename). */
377
- flush() {
378
- for (const s of this.scopes.values()) {
379
- if (!s.dirty) continue
380
- const file = this.fileFor(s.scope)
381
- mkdirSync(dirname(file), { recursive: true })
382
- const tmp = `${file}.tmp`
383
- const lines = [...s.records.values()].map((r) => JSON.stringify(r))
384
- writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
385
- renameSync(tmp, file)
386
- s.dirty = false
387
- }
388
- // vectors flush too: they are the expensive-to-recompute half
389
- for (const s of this.scopes.values()) s.flushVectors()
390
- }
391
-
392
- /** Synchronous final flush — call on dispose. */
393
- close() {
394
- if (this.flushTimer) {
395
- clearTimeout(this.flushTimer)
396
- this.flushTimer = null
397
- }
398
- this.flush()
399
- }
400
-
401
- /**
402
- * Dedupe-aware save: identical normalized content in the scope refreshes the
403
- * existing record instead of duplicating it.
404
- */
405
- save(
406
- input ,
407
- defaults ,
408
- ) {
409
- const s = this.scope(input.scope)
410
- const hash = contentHash(input.scope, input.content)
411
- for (const rec of s.records.values()) {
412
- if (contentHash(rec.scope, rec.content) === hash) {
413
- rec.updatedAt = Date.now()
414
- rec.accessCount += 1
415
- rec.importance = Math.max(rec.importance, input.importance ?? defaults.importance)
416
- if (input.supersedes) rec.supersedes = input.supersedes
417
- s.dirty = true
418
- return { record: rec, deduped: true }
419
- }
420
- }
421
- const now = Date.now()
422
- const record = {
423
- id: newId(),
424
- scope: input.scope,
425
- kind: input.kind,
426
- content: input.content,
427
- tags: input.tags ?? [],
428
- source: { ...defaults.source, ...input.source },
429
- createdAt: now,
430
- updatedAt: now,
431
- lastAccessedAt: now,
432
- accessCount: 0,
433
- importance: input.importance ?? defaults.importance,
434
- ...(input.supersedes ? { supersedes: input.supersedes } : {}),
435
- ...(input.expiredAt ? { expiredAt: input.expiredAt } : {}),
436
- }
437
- s.insert(record)
438
- return { record, deduped: false }
439
- }
440
-
441
- /** Substring match over normalized content, used by forget-by-query. */
442
- findIdsByQuery(scope , query ) {
443
- const s = this.scope(scope)
444
- const needle = normalizeContent(query)
445
- if (!needle) return []
446
- const ids = []
447
- for (const rec of s.records.values()) {
448
- if (normalizeContent(rec.content).includes(needle)) ids.push(rec.id)
449
- }
450
- return ids
451
- }
452
-
453
- /** Remove every expired record across scopes; returns removed ids. */
454
- purgeExpired(now = Date.now()) {
455
- const removed = []
456
- for (const s of this.scopes.values()) {
457
- for (const rec of [...s.records.values()]) {
458
- if (rec.expiredAt !== undefined && rec.expiredAt < now) {
459
- s.remove(rec.id)
460
- removed.push(rec.id)
461
- }
462
- }
463
- }
464
- return removed
465
- }
466
-
467
- /**
468
- * Evict lowest-ranked records until each scope is within its cap. Ranking:
469
- * recency (30d half-life) × importance — keep what retrieval would favor.
470
- */
471
- enforceLimit(maxPerScope , now = Date.now()) {
472
- const removed = []
473
- for (const s of this.scopes.values()) {
474
- if (s.records.size <= maxPerScope) continue
475
- const ranked = [...s.records.values()]
476
- .map((r) => ({
477
- id: r.id,
478
- rank: Math.pow(0.5, Math.max(0, now - r.updatedAt) / 86_400_000 / 30) * (0.1 + r.importance),
479
- }))
480
- .sort((a, b) => b.rank - a.rank)
481
- for (const r of ranked.slice(maxPerScope)) {
482
- s.remove(r.id)
483
- removed.push(r.id)
484
- }
485
- }
486
- return removed
487
- }
488
- }
489
-
490
-
491
- //# sourceURL=D:\.deepseek\memory-plugin\packages\memory-local\src\store.ts
1
+ /**
2
+ * JSONL-backed store. One file per scope, append-on-write semantics with
3
+ * atomic compaction (tmp + rename). Vectors live in memory for M1; the .vec
4
+ * sidecar format arrives in M1.5.
5
+ */
6
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from 'node:fs'
7
+ import { createHash, randomUUID } from 'node:crypto'
8
+ import { Buffer } from 'node:buffer'
9
+ import { dirname, join } from 'node:path'
10
+
11
+
12
+
13
+
14
+
15
+
16
+ export function scopeFileName(scope ) { if (scope === 'global') return 'global.jsonl'
17
+ const hash = createHash('sha1').update(scope).digest('hex')
18
+ return join('projects', `${hash}.jsonl`)
19
+ }
20
+
21
+ /** Derive the scope for records loaded from a file path. */
22
+ export function scopeFromFile(file , knownScopes ) {
23
+ if (file.endsWith('global.jsonl')) return 'global'
24
+ const base = file.split(/[\\/]/).pop() ?? ''
25
+ for (const [scope, name] of knownScopes) if (name === base) return scope
26
+ return null
27
+ }
28
+
29
+ /**
30
+ * Read the first well-formed record's scope from a scope file. Project files are
31
+ * named by hash, so this is the only way back to the scope key without the
32
+ * provider having created that scope in this process already.
33
+ */
34
+ export function firstScopeInFile(file ) {
35
+ let text
36
+ try {
37
+ text = readFileSync(file, 'utf8')
38
+ } catch {
39
+ return null
40
+ }
41
+ for (const line of text.split('\n')) {
42
+ const trimmed = line.trim()
43
+ if (!trimmed) continue
44
+ try {
45
+ const rec = JSON.parse(trimmed)
46
+ if (typeof rec.scope === 'string' && rec.scope) return rec.scope
47
+ } catch {
48
+ // skip malformed line, keep looking
49
+ }
50
+ }
51
+ return null
52
+ }
53
+
54
+ export function normalizeContent(content ) {
55
+ return content.replace(/\s+/g, ' ').trim().toLowerCase()
56
+ }
57
+
58
+ export function contentHash(scope , content ) {
59
+ return createHash('sha1').update(`${scope}::${normalizeContent(content)}`).digest('hex')
60
+ }
61
+
62
+ export function newId() {
63
+ return randomUUID()
64
+ }
65
+
66
+ export class ScopeStore {
67
+ scope
68
+ records = new Map ()
69
+ vectors = new Map ()
70
+ /** ids saved while the embed backend was down; retried opportunistically. */
71
+ embedPending = new Set ()
72
+ dirty = false
73
+ /** true once vectors were touched since the last .vec flush. */
74
+ vectorsDirty = false
75
+ vecFile
76
+ events
77
+ /** Embedding identity for sidecar validation; set via setVecIdentity(). */
78
+ vecIdentity = ''
79
+
80
+ constructor(scope , root , events = {}) {
81
+ this.scope = scope
82
+ this.vecFile = join(root, scopeFileName(scope).replace(/\.jsonl$/, '.vec'))
83
+ this.events = events
84
+ }
85
+
86
+ /**
87
+ * Declare which model+dims produced the in-memory vectors. When it differs
88
+ * from the sidecar header on load, the sidecar is discarded and every record
89
+ * re-enters the embed queue — old vectors for a new model are wrong, not
90
+ * merely stale.
91
+ */
92
+ setVecIdentity(model , dims ) {
93
+ this.vecIdentity = `${model}\t${dims}`
94
+ }
95
+
96
+ get size() {
97
+ return this.records.size
98
+ }
99
+
100
+ insert(record ) {
101
+ this.records.set(record.id, record)
102
+ this.dirty = true
103
+ }
104
+
105
+ remove(id ) {
106
+ const had = this.records.delete(id)
107
+ if (had) {
108
+ this.vectors.delete(id)
109
+ this.embedPending.delete(id)
110
+ this.dirty = true
111
+ this.vectorsDirty = true
112
+ }
113
+ return had
114
+ }
115
+
116
+ setVector(id , vec ) {
117
+ this.vectors.set(id, vec)
118
+ this.vectorsDirty = true
119
+ }
120
+
121
+ /**
122
+ * Append a record to the sidecar: `<id> \t <base64 dim vec>` per line.
123
+ * Vectors are written immediately on embed (single-line append) and fully
124
+ * rewritten on delete/compact — embeddings are expensive to recompute.
125
+ * A freshly created file starts with the `#model\tdims` identity header.
126
+ */
127
+ appendVector(id , vec ) {
128
+ this.setVector(id, vec)
129
+ try {
130
+ mkdirSync(dirname(this.vecFile), { recursive: true })
131
+ if (this.vecIdentity && this.readHeader() !== this.vecIdentity) {
132
+ // missing, legacy, or wrong-model file: rewrite fresh from memory
133
+ this.writeSidecar()
134
+ this.vectorsDirty = false
135
+ return
136
+ }
137
+ appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
138
+ this.vectorsDirty = false
139
+ } catch (err) {
140
+ this.vectorsDirty = true // full rewrite on next flushVectors
141
+ this.events.onWarn?.(`vec: append failed: ${(err ).message}`)
142
+ }
143
+ }
144
+
145
+ /** First line of the sidecar if it is a header; '' when absent/headerless. */
146
+ readHeader() {
147
+ if (!existsSync(this.vecFile)) return ''
148
+ try {
149
+ const first = readFileSync(this.vecFile, 'utf8').split('\n', 1)[0] ?? ''
150
+ return first.startsWith('#') ? first.slice(1) : ''
151
+ } catch {
152
+ return ''
153
+ }
154
+ }
155
+
156
+ /** Atomic full rewrite: identity header + every in-memory vector. */
157
+ writeSidecar() {
158
+ mkdirSync(dirname(this.vecFile), { recursive: true })
159
+ const tmp = `${this.vecFile}.tmp`
160
+ const lines = this.vecIdentity ? [`#${this.vecIdentity}`] : []
161
+ for (const [id, v] of this.vectors) lines.push(`${id}\t${encodeVec(v)}`)
162
+ writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
163
+ renameSync(tmp, this.vecFile)
164
+ }
165
+
166
+ /** Rewrite the whole sidecar (after deletes / compaction / append failures). */
167
+ flushVectors() {
168
+ if (!this.vectorsDirty) return
169
+ try {
170
+ this.writeSidecar()
171
+ this.vectorsDirty = false
172
+ } catch (err) {
173
+ this.events.onWarn?.(`vec: flush failed: ${(err ).message}`)
174
+ }
175
+ }
176
+
177
+ /**
178
+ * Load vectors from the sidecar. The first line must be a `#model\tdims`
179
+ * header when an identity is configured — mismatch or a legacy headerless
180
+ * file discards the whole sidecar (re-embed). Unknown ids are skipped.
181
+ */
182
+ loadVectors() {
183
+ if (!existsSync(this.vecFile)) return
184
+ try {
185
+ const text = readFileSync(this.vecFile, 'utf8')
186
+ let sawHeader = false
187
+ let sawData = false
188
+ const discard = (why ) => {
189
+ this.events.onWarn?.(`vec: ${why}; discarding sidecar for re-embed`)
190
+ this.vectors.clear()
191
+ this.vectorsDirty = true
192
+ }
193
+ for (const line of text.split('\n')) {
194
+ const trimmed = line.trim()
195
+ if (!trimmed) continue
196
+ if (trimmed.startsWith('#')) {
197
+ if (this.vecIdentity && trimmed.slice(1) !== this.vecIdentity) {
198
+ discard(`identity mismatch (${trimmed.slice(1)} != ${this.vecIdentity})`)
199
+ return
200
+ }
201
+ // Loading before the identity is declared means NO validation happens
202
+ // (the guards below are all identity-gated). That ordering bug shipped
203
+ // once — see RestRegular/dsh-memory#1 — so make any recurrence visible.
204
+ if (!this.vecIdentity) {
205
+ this.events.onWarn?.(`${this.scope}: loading a vector sidecar before setVecIdentity(); its ${trimmed.slice(1)} vectors are accepted unvalidated`)
206
+ }
207
+ sawHeader = true
208
+ continue
209
+ }
210
+ if (!sawData) {
211
+ sawData = true
212
+ if (this.vecIdentity && !sawHeader) {
213
+ discard('legacy sidecar without identity header')
214
+ return
215
+ }
216
+ }
217
+ const tab = trimmed.indexOf('\t')
218
+ if (tab <= 0) continue
219
+ const id = trimmed.slice(0, tab)
220
+ if (!this.records.has(id)) continue // stale vector for a deleted record
221
+ const vec = decodeVec(trimmed.slice(tab + 1))
222
+ if (vec) this.vectors.set(id, vec)
223
+ }
224
+ } catch (err) {
225
+ this.events.onWarn?.(`vec: load failed: ${(err ).message}`)
226
+ }
227
+ }
228
+ }
229
+
230
+ // --- base64 vector codec (4 bytes per float, bit-exact) ---
231
+
232
+ function encodeVec(vec ) {
233
+ return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength).toString('base64')
234
+ }
235
+
236
+ function decodeVec(text ) {
237
+ try {
238
+ const buf = Buffer.from(text, 'base64')
239
+ if (buf.length === 0 || buf.length % 4 !== 0) return null
240
+ return new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4)
241
+ } catch {
242
+ return null
243
+ }
244
+ }
245
+
246
+
247
+
248
+
249
+
250
+ export class MemoryStore {
251
+ root
252
+ scopes = new Map ()
253
+ /** scopes whose jsonl has been loaded from disk (lazy, once per scope). */
254
+ loaded = new Set ()
255
+ /** scope -> its jsonl filename, so reload can map files back to scopes. */
256
+ fileNames = new Map ()
257
+ flushTimer = null
258
+ events
259
+ /** `model\tdims` handed to every ScopeStore for sidecar validation. */
260
+ vecIdentity = ''
261
+
262
+ constructor(root , events = {}) {
263
+ this.root = root
264
+ this.events = events
265
+ mkdirSync(join(root, 'projects'), { recursive: true })
266
+ }
267
+
268
+ /** Declare the embedding identity; applies to existing and future scopes. */
269
+ setVecIdentity(model , dims ) {
270
+ this.vecIdentity = `${model}\t${dims}`
271
+ for (const s of this.scopes.values()) s.setVecIdentity(model, dims)
272
+ }
273
+
274
+ /** Preload specific scopes eagerly (e.g. defaults); others load on first access. */
275
+ loadAll(wanted ) {
276
+ for (const scope of wanted) this.scope(scope)
277
+ }
278
+
279
+ loadScope(scope ) {
280
+ const file = this.fileFor(scope)
281
+ if (!existsSync(file)) return
282
+ let text
283
+ try {
284
+ text = readFileSync(file, 'utf8')
285
+ } catch (err) {
286
+ this.events.onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
287
+ return
288
+ }
289
+ const store = this.scope(scope)
290
+ let lineNo = 0
291
+ for (const line of text.split('\n')) {
292
+ lineNo++
293
+ const trimmed = line.trim()
294
+ if (!trimmed) continue
295
+ try {
296
+ const rec = JSON.parse(trimmed)
297
+ if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
298
+ throw new Error('bad shape')
299
+ }
300
+ if (rec.scope !== scope) continue // file/scope mismatch: ignore, will be rewritten on flush
301
+ store.records.set(rec.id, rec)
302
+ } catch {
303
+ this.events.onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
304
+ }
305
+ }
306
+ // sidecar vectors second; anything still un-embedded joins the backfill queue
307
+ store.loadVectors()
308
+ for (const id of store.records.keys()) {
309
+ if (!store.vectors.has(id)) store.embedPending.add(id)
310
+ }
311
+ store.dirty = false
312
+ }
313
+
314
+ scope(scope ) {
315
+ let s = this.scopes.get(scope)
316
+ if (!s) {
317
+ s = new ScopeStore(scope, this.root, this.events)
318
+ if (this.vecIdentity) s.setVecIdentity(...(this.vecIdentity.split('\t') ))
319
+ this.scopes.set(scope, s)
320
+ this.fileNames.set(scope, scopeFileName(scope))
321
+ }
322
+ if (!this.loaded.has(scope)) {
323
+ this.loaded.add(scope)
324
+ this.loadScope(scope)
325
+ }
326
+ return s
327
+ }
328
+
329
+ allScopes() {
330
+ return [...this.scopes.values()]
331
+ }
332
+
333
+ /**
334
+ * Eagerly load every scope that has data on disk and return the scope keys.
335
+ *
336
+ * Scopes are otherwise created lazily, so a long-lived process only knows the
337
+ * scopes it has already touched — a management UI (the memory panel) would
338
+ * silently miss project memories written in earlier sessions. Project files
339
+ * are named by scope hash, so the scope key is recovered from the first
340
+ * record inside each file.
341
+ */
342
+ loadDiscovered() {
343
+ const discovered = []
344
+ if (existsSync(join(this.root, 'global.jsonl'))) discovered.push('global')
345
+ const projDir = join(this.root, 'projects')
346
+ if (existsSync(projDir)) {
347
+ for (const file of readdirSync(projDir)) {
348
+ if (!file.endsWith('.jsonl')) continue
349
+ const scope = firstScopeInFile(join(projDir, file))
350
+ if (scope) discovered.push(scope)
351
+ }
352
+ }
353
+ for (const scope of discovered) this.scope(scope) // triggers lazy disk load
354
+ return discovered
355
+ }
356
+
357
+ fileFor(scope ) {
358
+ return join(this.root, this.fileNames.get(scope) ?? scopeFileName(scope))
359
+ }
360
+
361
+ anyDirty() {
362
+ for (const s of this.scopes.values()) if (s.dirty) return true
363
+ return false
364
+ }
365
+
366
+ scheduleFlush(delayMs = 500) {
367
+ if (this.flushTimer) return
368
+ this.flushTimer = setTimeout(() => {
369
+ this.flushTimer = null
370
+ this.flush()
371
+ }, delayMs)
372
+ // a debounced flush must not hold the event loop open
373
+ this.flushTimer.unref?.()
374
+ }
375
+
376
+ /** Write dirty scopes atomically (tmp + rename). */
377
+ flush() {
378
+ for (const s of this.scopes.values()) {
379
+ if (!s.dirty) continue
380
+ const file = this.fileFor(s.scope)
381
+ mkdirSync(dirname(file), { recursive: true })
382
+ const tmp = `${file}.tmp`
383
+ const lines = [...s.records.values()].map((r) => JSON.stringify(r))
384
+ writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
385
+ renameSync(tmp, file)
386
+ s.dirty = false
387
+ }
388
+ // vectors flush too: they are the expensive-to-recompute half
389
+ for (const s of this.scopes.values()) s.flushVectors()
390
+ }
391
+
392
+ /** Synchronous final flush — call on dispose. */
393
+ close() {
394
+ if (this.flushTimer) {
395
+ clearTimeout(this.flushTimer)
396
+ this.flushTimer = null
397
+ }
398
+ this.flush()
399
+ }
400
+
401
+ /**
402
+ * Dedupe-aware save: identical normalized content in the scope refreshes the
403
+ * existing record instead of duplicating it.
404
+ */
405
+ save(
406
+ input ,
407
+ defaults ,
408
+ ) {
409
+ const s = this.scope(input.scope)
410
+ const hash = contentHash(input.scope, input.content)
411
+ for (const rec of s.records.values()) {
412
+ if (contentHash(rec.scope, rec.content) === hash) {
413
+ rec.updatedAt = Date.now()
414
+ rec.accessCount += 1
415
+ rec.importance = Math.max(rec.importance, input.importance ?? defaults.importance)
416
+ if (input.supersedes) rec.supersedes = input.supersedes
417
+ s.dirty = true
418
+ return { record: rec, deduped: true }
419
+ }
420
+ }
421
+ const now = Date.now()
422
+ const record = {
423
+ id: newId(),
424
+ scope: input.scope,
425
+ kind: input.kind,
426
+ content: input.content,
427
+ tags: input.tags ?? [],
428
+ source: { ...defaults.source, ...input.source },
429
+ createdAt: now,
430
+ updatedAt: now,
431
+ lastAccessedAt: now,
432
+ accessCount: 0,
433
+ importance: input.importance ?? defaults.importance,
434
+ ...(input.supersedes ? { supersedes: input.supersedes } : {}),
435
+ ...(input.expiredAt ? { expiredAt: input.expiredAt } : {}),
436
+ }
437
+ s.insert(record)
438
+ return { record, deduped: false }
439
+ }
440
+
441
+ /** Substring match over normalized content, used by forget-by-query. */
442
+ findIdsByQuery(scope , query ) {
443
+ const s = this.scope(scope)
444
+ const needle = normalizeContent(query)
445
+ if (!needle) return []
446
+ const ids = []
447
+ for (const rec of s.records.values()) {
448
+ if (normalizeContent(rec.content).includes(needle)) ids.push(rec.id)
449
+ }
450
+ return ids
451
+ }
452
+
453
+ /** Remove every expired record across scopes; returns removed ids. */
454
+ purgeExpired(now = Date.now()) {
455
+ const removed = []
456
+ for (const s of this.scopes.values()) {
457
+ for (const rec of [...s.records.values()]) {
458
+ if (rec.expiredAt !== undefined && rec.expiredAt < now) {
459
+ s.remove(rec.id)
460
+ removed.push(rec.id)
461
+ }
462
+ }
463
+ }
464
+ return removed
465
+ }
466
+
467
+ /**
468
+ * Evict lowest-ranked records until each scope is within its cap. Ranking:
469
+ * recency (30d half-life) × importance — keep what retrieval would favor.
470
+ */
471
+ enforceLimit(maxPerScope , now = Date.now()) {
472
+ const removed = []
473
+ for (const s of this.scopes.values()) {
474
+ if (s.records.size <= maxPerScope) continue
475
+ const ranked = [...s.records.values()]
476
+ .map((r) => ({
477
+ id: r.id,
478
+ rank: Math.pow(0.5, Math.max(0, now - r.updatedAt) / 86_400_000 / 30) * (0.1 + r.importance),
479
+ }))
480
+ .sort((a, b) => b.rank - a.rank)
481
+ for (const r of ranked.slice(maxPerScope)) {
482
+ s.remove(r.id)
483
+ removed.push(r.id)
484
+ }
485
+ }
486
+ return removed
487
+ }
488
+ }
489
+
490
+
491
+ //# sourceURL=source/memory-local/src/store.ts