@restregular/dsh-memory 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,485 @@
1
+ /**
2
+ * JSONL-backed store. One file per scope, append-on-write semantics with
3
+ * atomic compaction (tmp + rename). Vectors live in memory for M1; the .vec
4
+ * sidecar format arrives in M1.5.
5
+ */
6
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from 'node:fs'
7
+ import { createHash, randomUUID } from 'node:crypto'
8
+ import { Buffer } from 'node:buffer'
9
+ import { dirname, join } from 'node:path'
10
+
11
+
12
+
13
+
14
+
15
+
16
+ export function scopeFileName(scope ) { if (scope === 'global') return 'global.jsonl'
17
+ const hash = createHash('sha1').update(scope).digest('hex')
18
+ return join('projects', `${hash}.jsonl`)
19
+ }
20
+
21
+ /** Derive the scope for records loaded from a file path. */
22
+ export function scopeFromFile(file , knownScopes ) {
23
+ if (file.endsWith('global.jsonl')) return 'global'
24
+ const base = file.split(/[\\/]/).pop() ?? ''
25
+ for (const [scope, name] of knownScopes) if (name === base) return scope
26
+ return null
27
+ }
28
+
29
+ /**
30
+ * Read the first well-formed record's scope from a scope file. Project files are
31
+ * named by hash, so this is the only way back to the scope key without the
32
+ * provider having created that scope in this process already.
33
+ */
34
+ export function firstScopeInFile(file ) {
35
+ let text
36
+ try {
37
+ text = readFileSync(file, 'utf8')
38
+ } catch {
39
+ return null
40
+ }
41
+ for (const line of text.split('\n')) {
42
+ const trimmed = line.trim()
43
+ if (!trimmed) continue
44
+ try {
45
+ const rec = JSON.parse(trimmed)
46
+ if (typeof rec.scope === 'string' && rec.scope) return rec.scope
47
+ } catch {
48
+ // skip malformed line, keep looking
49
+ }
50
+ }
51
+ return null
52
+ }
53
+
54
+ export function normalizeContent(content ) {
55
+ return content.replace(/\s+/g, ' ').trim().toLowerCase()
56
+ }
57
+
58
+ export function contentHash(scope , content ) {
59
+ return createHash('sha1').update(`${scope}::${normalizeContent(content)}`).digest('hex')
60
+ }
61
+
62
+ export function newId() {
63
+ return randomUUID()
64
+ }
65
+
66
+ export class ScopeStore {
67
+ scope
68
+ records = new Map ()
69
+ vectors = new Map ()
70
+ /** ids saved while the embed backend was down; retried opportunistically. */
71
+ embedPending = new Set ()
72
+ dirty = false
73
+ /** true once vectors were touched since the last .vec flush. */
74
+ vectorsDirty = false
75
+ vecFile
76
+ events
77
+ /** Embedding identity for sidecar validation; set via setVecIdentity(). */
78
+ vecIdentity = ''
79
+
80
+ constructor(scope , root , events = {}) {
81
+ this.scope = scope
82
+ this.vecFile = join(root, scopeFileName(scope).replace(/\.jsonl$/, '.vec'))
83
+ this.events = events
84
+ }
85
+
86
+ /**
87
+ * Declare which model+dims produced the in-memory vectors. When it differs
88
+ * from the sidecar header on load, the sidecar is discarded and every record
89
+ * re-enters the embed queue — old vectors for a new model are wrong, not
90
+ * merely stale.
91
+ */
92
+ setVecIdentity(model , dims ) {
93
+ this.vecIdentity = `${model}\t${dims}`
94
+ }
95
+
96
+ get size() {
97
+ return this.records.size
98
+ }
99
+
100
+ insert(record ) {
101
+ this.records.set(record.id, record)
102
+ this.dirty = true
103
+ }
104
+
105
+ remove(id ) {
106
+ const had = this.records.delete(id)
107
+ if (had) {
108
+ this.vectors.delete(id)
109
+ this.embedPending.delete(id)
110
+ this.dirty = true
111
+ this.vectorsDirty = true
112
+ }
113
+ return had
114
+ }
115
+
116
+ setVector(id , vec ) {
117
+ this.vectors.set(id, vec)
118
+ this.vectorsDirty = true
119
+ }
120
+
121
+ /**
122
+ * Append a record to the sidecar: `<id> \t <base64 dim vec>` per line.
123
+ * Vectors are written immediately on embed (single-line append) and fully
124
+ * rewritten on delete/compact — embeddings are expensive to recompute.
125
+ * A freshly created file starts with the `#model\tdims` identity header.
126
+ */
127
+ appendVector(id , vec ) {
128
+ this.setVector(id, vec)
129
+ try {
130
+ mkdirSync(dirname(this.vecFile), { recursive: true })
131
+ if (this.vecIdentity && this.readHeader() !== this.vecIdentity) {
132
+ // missing, legacy, or wrong-model file: rewrite fresh from memory
133
+ this.writeSidecar()
134
+ this.vectorsDirty = false
135
+ return
136
+ }
137
+ appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
138
+ this.vectorsDirty = false
139
+ } catch (err) {
140
+ this.vectorsDirty = true // full rewrite on next flushVectors
141
+ this.events.onWarn?.(`vec: append failed: ${(err ).message}`)
142
+ }
143
+ }
144
+
145
+ /** First line of the sidecar if it is a header; '' when absent/headerless. */
146
+ readHeader() {
147
+ if (!existsSync(this.vecFile)) return ''
148
+ try {
149
+ const first = readFileSync(this.vecFile, 'utf8').split('\n', 1)[0] ?? ''
150
+ return first.startsWith('#') ? first.slice(1) : ''
151
+ } catch {
152
+ return ''
153
+ }
154
+ }
155
+
156
+ /** Atomic full rewrite: identity header + every in-memory vector. */
157
+ writeSidecar() {
158
+ mkdirSync(dirname(this.vecFile), { recursive: true })
159
+ const tmp = `${this.vecFile}.tmp`
160
+ const lines = this.vecIdentity ? [`#${this.vecIdentity}`] : []
161
+ for (const [id, v] of this.vectors) lines.push(`${id}\t${encodeVec(v)}`)
162
+ writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
163
+ renameSync(tmp, this.vecFile)
164
+ }
165
+
166
+ /** Rewrite the whole sidecar (after deletes / compaction / append failures). */
167
+ flushVectors() {
168
+ if (!this.vectorsDirty) return
169
+ try {
170
+ this.writeSidecar()
171
+ this.vectorsDirty = false
172
+ } catch (err) {
173
+ this.events.onWarn?.(`vec: flush failed: ${(err ).message}`)
174
+ }
175
+ }
176
+
177
+ /**
178
+ * Load vectors from the sidecar. The first line must be a `#model\tdims`
179
+ * header when an identity is configured — mismatch or a legacy headerless
180
+ * file discards the whole sidecar (re-embed). Unknown ids are skipped.
181
+ */
182
+ loadVectors() {
183
+ if (!existsSync(this.vecFile)) return
184
+ try {
185
+ const text = readFileSync(this.vecFile, 'utf8')
186
+ let sawHeader = false
187
+ let sawData = false
188
+ const discard = (why ) => {
189
+ this.events.onWarn?.(`vec: ${why}; discarding sidecar for re-embed`)
190
+ this.vectors.clear()
191
+ this.vectorsDirty = true
192
+ }
193
+ for (const line of text.split('\n')) {
194
+ const trimmed = line.trim()
195
+ if (!trimmed) continue
196
+ if (trimmed.startsWith('#')) {
197
+ if (this.vecIdentity && trimmed.slice(1) !== this.vecIdentity) {
198
+ discard(`identity mismatch (${trimmed.slice(1)} != ${this.vecIdentity})`)
199
+ return
200
+ }
201
+ sawHeader = true
202
+ continue
203
+ }
204
+ if (!sawData) {
205
+ sawData = true
206
+ if (this.vecIdentity && !sawHeader) {
207
+ discard('legacy sidecar without identity header')
208
+ return
209
+ }
210
+ }
211
+ const tab = trimmed.indexOf('\t')
212
+ if (tab <= 0) continue
213
+ const id = trimmed.slice(0, tab)
214
+ if (!this.records.has(id)) continue // stale vector for a deleted record
215
+ const vec = decodeVec(trimmed.slice(tab + 1))
216
+ if (vec) this.vectors.set(id, vec)
217
+ }
218
+ } catch (err) {
219
+ this.events.onWarn?.(`vec: load failed: ${(err ).message}`)
220
+ }
221
+ }
222
+ }
223
+
224
+ // --- base64 vector codec (4 bytes per float, bit-exact) ---
225
+
226
+ function encodeVec(vec ) {
227
+ return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength).toString('base64')
228
+ }
229
+
230
+ function decodeVec(text ) {
231
+ try {
232
+ const buf = Buffer.from(text, 'base64')
233
+ if (buf.length === 0 || buf.length % 4 !== 0) return null
234
+ return new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4)
235
+ } catch {
236
+ return null
237
+ }
238
+ }
239
+
240
+
241
+
242
+
243
+
244
+ export class MemoryStore {
245
+ root
246
+ scopes = new Map ()
247
+ /** scopes whose jsonl has been loaded from disk (lazy, once per scope). */
248
+ loaded = new Set ()
249
+ /** scope -> its jsonl filename, so reload can map files back to scopes. */
250
+ fileNames = new Map ()
251
+ flushTimer = null
252
+ events
253
+ /** `model\tdims` handed to every ScopeStore for sidecar validation. */
254
+ vecIdentity = ''
255
+
256
+ constructor(root , events = {}) {
257
+ this.root = root
258
+ this.events = events
259
+ mkdirSync(join(root, 'projects'), { recursive: true })
260
+ }
261
+
262
+ /** Declare the embedding identity; applies to existing and future scopes. */
263
+ setVecIdentity(model , dims ) {
264
+ this.vecIdentity = `${model}\t${dims}`
265
+ for (const s of this.scopes.values()) s.setVecIdentity(model, dims)
266
+ }
267
+
268
+ /** Preload specific scopes eagerly (e.g. defaults); others load on first access. */
269
+ loadAll(wanted ) {
270
+ for (const scope of wanted) this.scope(scope)
271
+ }
272
+
273
+ loadScope(scope ) {
274
+ const file = this.fileFor(scope)
275
+ if (!existsSync(file)) return
276
+ let text
277
+ try {
278
+ text = readFileSync(file, 'utf8')
279
+ } catch (err) {
280
+ this.events.onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
281
+ return
282
+ }
283
+ const store = this.scope(scope)
284
+ let lineNo = 0
285
+ for (const line of text.split('\n')) {
286
+ lineNo++
287
+ const trimmed = line.trim()
288
+ if (!trimmed) continue
289
+ try {
290
+ const rec = JSON.parse(trimmed)
291
+ if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
292
+ throw new Error('bad shape')
293
+ }
294
+ if (rec.scope !== scope) continue // file/scope mismatch: ignore, will be rewritten on flush
295
+ store.records.set(rec.id, rec)
296
+ } catch {
297
+ this.events.onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
298
+ }
299
+ }
300
+ // sidecar vectors second; anything still un-embedded joins the backfill queue
301
+ store.loadVectors()
302
+ for (const id of store.records.keys()) {
303
+ if (!store.vectors.has(id)) store.embedPending.add(id)
304
+ }
305
+ store.dirty = false
306
+ }
307
+
308
+ scope(scope ) {
309
+ let s = this.scopes.get(scope)
310
+ if (!s) {
311
+ s = new ScopeStore(scope, this.root, this.events)
312
+ if (this.vecIdentity) s.setVecIdentity(...(this.vecIdentity.split('\t') ))
313
+ this.scopes.set(scope, s)
314
+ this.fileNames.set(scope, scopeFileName(scope))
315
+ }
316
+ if (!this.loaded.has(scope)) {
317
+ this.loaded.add(scope)
318
+ this.loadScope(scope)
319
+ }
320
+ return s
321
+ }
322
+
323
+ allScopes() {
324
+ return [...this.scopes.values()]
325
+ }
326
+
327
+ /**
328
+ * Eagerly load every scope that has data on disk and return the scope keys.
329
+ *
330
+ * Scopes are otherwise created lazily, so a long-lived process only knows the
331
+ * scopes it has already touched — a management UI (the memory panel) would
332
+ * silently miss project memories written in earlier sessions. Project files
333
+ * are named by scope hash, so the scope key is recovered from the first
334
+ * record inside each file.
335
+ */
336
+ loadDiscovered() {
337
+ const discovered = []
338
+ if (existsSync(join(this.root, 'global.jsonl'))) discovered.push('global')
339
+ const projDir = join(this.root, 'projects')
340
+ if (existsSync(projDir)) {
341
+ for (const file of readdirSync(projDir)) {
342
+ if (!file.endsWith('.jsonl')) continue
343
+ const scope = firstScopeInFile(join(projDir, file))
344
+ if (scope) discovered.push(scope)
345
+ }
346
+ }
347
+ for (const scope of discovered) this.scope(scope) // triggers lazy disk load
348
+ return discovered
349
+ }
350
+
351
+ fileFor(scope ) {
352
+ return join(this.root, this.fileNames.get(scope) ?? scopeFileName(scope))
353
+ }
354
+
355
+ anyDirty() {
356
+ for (const s of this.scopes.values()) if (s.dirty) return true
357
+ return false
358
+ }
359
+
360
+ scheduleFlush(delayMs = 500) {
361
+ if (this.flushTimer) return
362
+ this.flushTimer = setTimeout(() => {
363
+ this.flushTimer = null
364
+ this.flush()
365
+ }, delayMs)
366
+ // a debounced flush must not hold the event loop open
367
+ this.flushTimer.unref?.()
368
+ }
369
+
370
+ /** Write dirty scopes atomically (tmp + rename). */
371
+ flush() {
372
+ for (const s of this.scopes.values()) {
373
+ if (!s.dirty) continue
374
+ const file = this.fileFor(s.scope)
375
+ mkdirSync(dirname(file), { recursive: true })
376
+ const tmp = `${file}.tmp`
377
+ const lines = [...s.records.values()].map((r) => JSON.stringify(r))
378
+ writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
379
+ renameSync(tmp, file)
380
+ s.dirty = false
381
+ }
382
+ // vectors flush too: they are the expensive-to-recompute half
383
+ for (const s of this.scopes.values()) s.flushVectors()
384
+ }
385
+
386
+ /** Synchronous final flush — call on dispose. */
387
+ close() {
388
+ if (this.flushTimer) {
389
+ clearTimeout(this.flushTimer)
390
+ this.flushTimer = null
391
+ }
392
+ this.flush()
393
+ }
394
+
395
+ /**
396
+ * Dedupe-aware save: identical normalized content in the scope refreshes the
397
+ * existing record instead of duplicating it.
398
+ */
399
+ save(
400
+ input ,
401
+ defaults ,
402
+ ) {
403
+ const s = this.scope(input.scope)
404
+ const hash = contentHash(input.scope, input.content)
405
+ for (const rec of s.records.values()) {
406
+ if (contentHash(rec.scope, rec.content) === hash) {
407
+ rec.updatedAt = Date.now()
408
+ rec.accessCount += 1
409
+ rec.importance = Math.max(rec.importance, input.importance ?? defaults.importance)
410
+ if (input.supersedes) rec.supersedes = input.supersedes
411
+ s.dirty = true
412
+ return { record: rec, deduped: true }
413
+ }
414
+ }
415
+ const now = Date.now()
416
+ const record = {
417
+ id: newId(),
418
+ scope: input.scope,
419
+ kind: input.kind,
420
+ content: input.content,
421
+ tags: input.tags ?? [],
422
+ source: { ...defaults.source, ...input.source },
423
+ createdAt: now,
424
+ updatedAt: now,
425
+ lastAccessedAt: now,
426
+ accessCount: 0,
427
+ importance: input.importance ?? defaults.importance,
428
+ ...(input.supersedes ? { supersedes: input.supersedes } : {}),
429
+ ...(input.expiredAt ? { expiredAt: input.expiredAt } : {}),
430
+ }
431
+ s.insert(record)
432
+ return { record, deduped: false }
433
+ }
434
+
435
+ /** Substring match over normalized content, used by forget-by-query. */
436
+ findIdsByQuery(scope , query ) {
437
+ const s = this.scope(scope)
438
+ const needle = normalizeContent(query)
439
+ if (!needle) return []
440
+ const ids = []
441
+ for (const rec of s.records.values()) {
442
+ if (normalizeContent(rec.content).includes(needle)) ids.push(rec.id)
443
+ }
444
+ return ids
445
+ }
446
+
447
+ /** Remove every expired record across scopes; returns removed ids. */
448
+ purgeExpired(now = Date.now()) {
449
+ const removed = []
450
+ for (const s of this.scopes.values()) {
451
+ for (const rec of [...s.records.values()]) {
452
+ if (rec.expiredAt !== undefined && rec.expiredAt < now) {
453
+ s.remove(rec.id)
454
+ removed.push(rec.id)
455
+ }
456
+ }
457
+ }
458
+ return removed
459
+ }
460
+
461
+ /**
462
+ * Evict lowest-ranked records until each scope is within its cap. Ranking:
463
+ * recency (30d half-life) × importance — keep what retrieval would favor.
464
+ */
465
+ enforceLimit(maxPerScope , now = Date.now()) {
466
+ const removed = []
467
+ for (const s of this.scopes.values()) {
468
+ if (s.records.size <= maxPerScope) continue
469
+ const ranked = [...s.records.values()]
470
+ .map((r) => ({
471
+ id: r.id,
472
+ rank: Math.pow(0.5, Math.max(0, now - r.updatedAt) / 86_400_000 / 30) * (0.1 + r.importance),
473
+ }))
474
+ .sort((a, b) => b.rank - a.rank)
475
+ for (const r of ranked.slice(maxPerScope)) {
476
+ s.remove(r.id)
477
+ removed.push(r.id)
478
+ }
479
+ }
480
+ return removed
481
+ }
482
+ }
483
+
484
+
485
+ //# sourceURL=D:\.deepseek\memory-plugin\packages\memory-local\src\store.ts
@@ -0,0 +1,106 @@
1
+ /**
2
+ * Cordis wiring for dsh-tool-memory. Loaded only inside DSH: wraps the pure
3
+ * tool specs with the official defineTool DSL and registers them.
4
+ *
5
+ * inject: ['tools', 'memory'] is REQUIRED:
6
+ * - 'tools' → the tool registry must be ready
7
+ * - 'memory' → this fiber stays PENDING until the provider provides the
8
+ * service, so `ctx.memory` is guaranteed active when apply runs (accessing
9
+ * it without declaring made apply throw and the entry silently die)
10
+ */
11
+ import { defineTool } from '@deepseek-ai/dsh-tools'
12
+ import {
13
+ memoryTools,
14
+
15
+
16
+
17
+ } from './index.js'
18
+
19
+ export const name = 'dsh-tool-memory'
20
+ export const inject = ['tools', 'memory']
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
29
+ export function apply(ctx , config = {}) {
30
+ const memory = ctx.memory
31
+ const projectScope = (config.projectScope ?? `project:${process.cwd()}`)
32
+ const opts = {
33
+ getService: () => memory,
34
+ defaultScope: () => (config.defaultScope === 'global' ? 'global' : projectScope),
35
+ }
36
+ const tools = memoryTools(opts)
37
+
38
+ ctx.tools.register(defineTool({
39
+ name: tools.memory_save.name,
40
+ description: tools.memory_save.description,
41
+ parameters: tools.memory_save.parameters ,
42
+ output: {
43
+ schema: { type: 'json' },
44
+ render: (_args , value ) => [{
45
+ type: 'text',
46
+ text: value.deduped
47
+ ? `已更新既有记忆(内容相同,去重):${value.id} @ ${value.scope}`
48
+ : `已保存记忆 ${value.id} @ ${value.scope}`,
49
+ }],
50
+ },
51
+ execute: (args ) => tools.memory_save.execute(args),
52
+ }))
53
+
54
+ ctx.tools.register(defineTool({
55
+ name: tools.memory_search.name,
56
+ description: tools.memory_search.description,
57
+ parameters: tools.memory_search.parameters ,
58
+ output: {
59
+ schema: { type: 'json' },
60
+ render: (_args , value ) => [{
61
+ type: 'text',
62
+ text: value.records.length === 0
63
+ ? '没有找到相关记忆。'
64
+ : value.records
65
+ .map((h) => `[${h.score.toFixed(3)}] (${h.scope}/${h.kind}) ${h.content}`)
66
+ .join('\n'),
67
+ }],
68
+ },
69
+ execute: (args ) => tools.memory_search.execute(args),
70
+ }))
71
+
72
+ ctx.tools.register(defineTool({
73
+ name: tools.memory_forget.name,
74
+ description: tools.memory_forget.description,
75
+ parameters: tools.memory_forget.parameters ,
76
+ output: {
77
+ schema: { type: 'json' },
78
+ render: (_args , value ) => [{
79
+ type: 'text',
80
+ text: value.removed === 0 ? '没有匹配的记忆。' : `已删除 ${value.removed} 条记忆。`,
81
+ }],
82
+ },
83
+ execute: (args ) => tools.memory_forget.execute(args),
84
+ }))
85
+
86
+ ctx.tools.register(defineTool({
87
+ name: tools.memory_list.name,
88
+ description: tools.memory_list.description,
89
+ parameters: tools.memory_list.parameters ,
90
+ output: {
91
+ schema: { type: 'json' },
92
+ render: (_args , value ) => [{
93
+ type: 'text',
94
+ text: value.records.length === 0
95
+ ? '该作用域还没有记忆。'
96
+ : value.records
97
+ .map((r) => `(${r.scope}/${r.kind}) ${r.content}`)
98
+ .join('\n'),
99
+ }],
100
+ },
101
+ execute: (args ) => tools.memory_list.execute(args),
102
+ }))
103
+ }
104
+
105
+
106
+ //# sourceURL=D:\.deepseek\memory-plugin\packages\tool-memory\src\apply.ts