@restregular/dsh-memory 0.9.9 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -2
- package/lib/panel.js +2 -1
- package/lib/provider/index.js +4 -3
- package/lib/provider/store.js +343 -66
- package/lib/tools/apply.js +11 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -141,9 +141,10 @@ memory_save({
|
|
|
141
141
|
| --- | --- |
|
|
142
142
|
| `<storeDir>/global.jsonl` | global 作用域的记忆,一行一条 |
|
|
143
143
|
| `<storeDir>/projects/<sha1(scope)>.jsonl` | 其它作用域(会话 / 项目 / 自定义)各一个文件;文件名是 scope 键的 SHA-1,所以名字里带路径分隔符也不可能穿越 |
|
|
144
|
-
| `<storeDir>/projects/<sha1(scope)>.vec` | 向量边车。首行是身份头 `#<model>\t<dims
|
|
144
|
+
| `<storeDir>/projects/<sha1(scope)>.vec` | 向量边车。首行是身份头 `#<model>\t<dims>\t<pooling>`(0.10.0 起;老文件只有前两段,仍兼容),后面每行 `id\t<base64 向量>` |
|
|
145
145
|
| `<storeDir>/settings.json` | 面板「设置」里改过的注入开关(配置里的值是默认值,这里覆盖它) |
|
|
146
146
|
| `<storeDir>/scopes.json` | 项目作用域的别名(显示名 → `project:<路径>`),同一路径同名幂等,不同路径同名拒绝 |
|
|
147
|
+
| `<storeDir>/*.lock` | 写入期间的临时锁文件(同一 scope 的跨进程写锁),写完立即删除 |
|
|
147
148
|
| `<installRoot>/llamacpp/` | llama-server 运行时(exe + 依赖 dll + `llama-server.log`) |
|
|
148
149
|
| `<installRoot>/models/` | 下载的 GGUF 模型 |
|
|
149
150
|
| `<installRoot>/selected-model.json` | 面板里选过哪一档(比配置优先级低) |
|
|
@@ -154,7 +155,9 @@ scope 键的三种形态:`global`、`project:<cwd>`、`session:<会话 id>`,其
|
|
|
154
155
|
|
|
155
156
|
`installRoot` 默认 `~/.dsh/memory-vector`;显式配了 `storeDir` 时取**它的父目录**(所以手写的 `<root>/llamacpp` + `<root>/models` 布局会被直接认出来)。
|
|
156
157
|
|
|
157
|
-
> **换模型 = 向量作废重建**。`.vec` 的身份头记录 `model` 与 `dims
|
|
158
|
+
> **换模型 = 向量作废重建**。`.vec` 的身份头记录 `model`、`dims` 与 `pooling`(0.10.0 起),任一对不上就整份丢弃并重新嵌入;重建期间检索暂时只用词法。面板会在换之前告诉你涉及多少条记忆。老边车只有 `model` + `dims` 两段,按兼容接受,下一次写向量时自动升级成三段。
|
|
159
|
+
|
|
160
|
+
**多个进程共用一个 `storeDir`**(0.10.0 起):比如插件 + 一个 MCP server,或者同时开两个 DSH 实例。写入按 scope 走锁文件、每次都"锁内重新读盘再合并本次增量",不再用内存快照整份覆盖磁盘;读路径每秒 stat 一次,别的进程刚写的记录不用重启就能看到。实测 4 个进程 × 20 条并发写同一个 scope 零丢失。代价是写路径多了一次读盘 + 一次 stat,量级远小于一次嵌入。已知取舍:同内容去重(`save` 的 dedupe)仍是进程内判断,两个进程同时写**完全相同**的内容理论上会落两条。
|
|
158
161
|
|
|
159
162
|
## 配置 / Configuration
|
|
160
163
|
|
|
@@ -349,6 +352,18 @@ tools/build-single-package.mjs 构建(产物契约检查 + 可选加载验证)
|
|
|
349
352
|
|
|
350
353
|
## 版本
|
|
351
354
|
|
|
355
|
+
- **0.10.0** —— 存储层可多进程共用 + 向量身份带上 pooling(issue #3):
|
|
356
|
+
- **跨进程安全**:不再"每个 scope 只读盘一次 + 内存快照整份重写"。每个 scope 一把锁文件(`openSync(…, 'wx')`,Windows 上 `EPERM`/`EACCES` 也按"已存在"处理),`flush()` 在锁内**重新读盘**、只把本次的增量(新增/更新/删除)合并进去;tmp 文件名带 pid + 随机后缀,两个进程并发写不会撞名破坏原子性。陈旧锁(持有者已死或 10s 未动)会被接管。
|
|
357
|
+
- **读路径能看见别的进程的写入**:每 1s 限流 stat 一次,`mtime:size` 变了就重新读盘,不用重启。
|
|
358
|
+
- **向量边车同样先读后写**:整份重写前先合并磁盘上同身份的条目,不覆盖别人刚追加的向量。
|
|
359
|
+
- **`pooling` 进身份头**:`.vec` 首行从 `#model\tdims` 变成 `#model\tdims\tpooling`。同一个模型换 `--pooling` 就是一个新的向量空间(实测 qwen3-embedding 上 `last` 与 `cls` 的余弦只有 0.35),以前校验会静默放行。**老边车(两段)按兼容接受**,不做全量重嵌;下一次写向量时自动升级成三段。
|
|
360
|
+
- **删空 scope 的判定改成"合并之后"**:别的进程写进去的记录不会被误判成"删空"而连文件一起删掉。
|
|
361
|
+
- 顺手修掉 `wipeAllRecords()` 里的 `store.flushSync()`(MemoryStore 没有这个方法,异常被 catch 吞掉,实际没有同步落盘)。
|
|
362
|
+
- 新增测试:`multiprocess.test.ts`(两实例交错写、删除不复活、跨实例可见性、**4 个真实子进程并发写零丢失**、不留 `.lock`/`.tmp`)与 `pooling-identity.test.ts`(三段身份、旧头兼容与升级、换 pooling 作废、无头边车丢弃)。
|
|
363
|
+
- **0.9.10** —— 修复 `memory_get` 没有注册:
|
|
364
|
+
- `memoryTools()` 一直有 `memory_get` 的实现,但工具注册层漏了 `register(tools.memory_get, …)`,所以模型侧只能看到 save/search/forget/list 四个工具。
|
|
365
|
+
- 现在注册完整五个工具;`memory_get` 输出完整 id、作用域、类型、标签和全文,找不到的 id 单独列出。
|
|
366
|
+
- 新增静态守卫测试:`memoryTools` 里的每个工具都必须在 `apply.ts` 注册,防止再次“实现了但没注册”。
|
|
352
367
|
- **0.9.9** —— 收敛自动召回作用域与相关性:
|
|
353
368
|
- 自动召回只检索 **global + 当前会话 + 当前工作区** 三个标准作用域。
|
|
354
369
|
- 不再把 `inject.extraScopes` 的自定义作用域或 fork 父会话带进自动召回;普通每轮召回仍按原配置。
|
package/lib/panel.js
CHANGED
|
@@ -456,7 +456,8 @@ async function handle(ctx, routePath, req, res) {
|
|
|
456
456
|
if (body.tags !== undefined) rec.tags = Array.isArray(body.tags) ? body.tags.map(String) : []
|
|
457
457
|
if (body.supersededBy !== undefined) rec.supersedes = body.supersededBy === null ? undefined : String(body.supersededBy)
|
|
458
458
|
rec.updatedAt = Date.now()
|
|
459
|
-
|
|
459
|
+
// 改动本身要进增量(flush 只落增量,不再整份快照覆盖);向量作废靠 vectorsDirty。
|
|
460
|
+
s.insert(rec)
|
|
460
461
|
store.scheduleFlush()
|
|
461
462
|
// 立即重建向量(不等 60s 的定时回填);失败就留在 pending 里由重试兜底。
|
|
462
463
|
if (reindex && typeof memory.reindexRecord === 'function') {
|
package/lib/provider/index.js
CHANGED
|
@@ -508,7 +508,9 @@ export class LocalMemoryService {
|
|
|
508
508
|
// so loading first would accept vectors from a previous embedding model
|
|
509
509
|
// whenever the dimensions matched — and the next flush would then rewrite the
|
|
510
510
|
// header as the new model, making the mismatch permanent.
|
|
511
|
-
|
|
511
|
+
// pooling 也进身份(issue #3):同一个模型换 --pooling 等于换了一个向量空间
|
|
512
|
+
// (last 与 cls 的余弦只有 0.35),以前它不在 identity 里,校验会静默放行。
|
|
513
|
+
this.store.setVecIdentity(this.cfg.vector.model, this.cfg.vector.dims, this.cfg.vector.pooling)
|
|
512
514
|
// 会话 scope 是按会话动态生成的,**不能**在这里 eager 加载(加载谁?);只预载
|
|
513
515
|
// project 与 global 这两个稳定作用域。其余按需 peek/loadDiscovered。
|
|
514
516
|
// eager 只加载"已存在"的 scope(peek),不会为不存在的 scope 建空文件。
|
|
@@ -993,7 +995,7 @@ export class LocalMemoryService {
|
|
|
993
995
|
removed += local
|
|
994
996
|
scopes.push(scope.scope)
|
|
995
997
|
}
|
|
996
|
-
try { this.store.
|
|
998
|
+
try { this.store.flush() } catch { /* scheduled flush covers it */ }
|
|
997
999
|
return { removed, scopes }
|
|
998
1000
|
}
|
|
999
1001
|
|
|
@@ -1357,7 +1359,6 @@ export class LocalMemoryService {
|
|
|
1357
1359
|
total += 1
|
|
1358
1360
|
}
|
|
1359
1361
|
s.vectorsDirty = true
|
|
1360
|
-
s.dirty = true
|
|
1361
1362
|
}
|
|
1362
1363
|
this.store.scheduleFlush()
|
|
1363
1364
|
void this.backfillAll({ throttleMs: 50 }).catch(() => { /* 失败留给 60s 重试 */ })
|
package/lib/provider/store.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* atomic compaction (tmp + rename). Vectors live in memory for M1; the .vec
|
|
4
4
|
* sidecar format arrives in M1.5.
|
|
5
5
|
*/
|
|
6
|
-
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
6
|
+
import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
7
7
|
import { createHash, randomUUID } from 'node:crypto'
|
|
8
8
|
import { Buffer } from 'node:buffer'
|
|
9
9
|
import { dirname, join } from 'node:path'
|
|
@@ -76,22 +76,195 @@ export function newId() {
|
|
|
76
76
|
return randomUUID()
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
+
// ---------------------------------------------------------------- 跨进程写锁
|
|
80
|
+
|
|
81
|
+
/** 等锁的上限:超过就告警并"不带锁照写"(宁可少一次合并,也不要丢掉这次写入)。 */
|
|
82
|
+
const LOCK_TIMEOUT_MS = 5_000
|
|
83
|
+
/** 超过这个时长没被碰过的锁视为陈旧(进程被 kill -9 时不会自己清理)。 */
|
|
84
|
+
const LOCK_STALE_MS = 10_000
|
|
85
|
+
|
|
86
|
+
/** 同步睡眠,不占 CPU:锁等待用(必须同步 —— flush/close 的调用方都是同步路径)。 */
|
|
87
|
+
function sleepSync(ms ) {
|
|
88
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function processAlive(pid ) {
|
|
92
|
+
try {
|
|
93
|
+
process.kill(pid, 0)
|
|
94
|
+
return true
|
|
95
|
+
} catch (err) {
|
|
96
|
+
// EPERM = 进程存在但没有权限发信号
|
|
97
|
+
return (err ).code === 'EPERM'
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** 锁是不是陈旧的(拥有者已死 / 太久没动)?是则删掉并返回 true。 */
|
|
102
|
+
function breakStaleLock(lockPath ) {
|
|
103
|
+
let age
|
|
104
|
+
try {
|
|
105
|
+
age = Date.now() - statSync(lockPath).mtimeMs
|
|
106
|
+
} catch {
|
|
107
|
+
return true // 已经不在了,直接重试
|
|
108
|
+
}
|
|
109
|
+
if (age > LOCK_STALE_MS) {
|
|
110
|
+
try { rmSync(lockPath, { force: true }) } catch { /* 下一轮再试 */ }
|
|
111
|
+
return true
|
|
112
|
+
}
|
|
113
|
+
let pid = 0
|
|
114
|
+
try { pid = Number.parseInt(readFileSync(lockPath, 'utf8').trim(), 10) } catch { pid = 0 }
|
|
115
|
+
if (pid > 0 && !processAlive(pid)) {
|
|
116
|
+
try { rmSync(lockPath, { force: true }) } catch { /* 下一轮再试 */ }
|
|
117
|
+
return true
|
|
118
|
+
}
|
|
119
|
+
return false
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** 本进程已持有的锁(可重入:flush 持锁时会再调 writeSidecar,不能自己把自己锁死)。 */
|
|
123
|
+
const heldLocks = new Map ()
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* 取某个 scope 的写锁(锁文件 = `${数据文件}.lock`,`openSync(wx)` 的 O_EXCL 语义)。
|
|
127
|
+
*
|
|
128
|
+
* 为什么需要它:`loadScope()` 只读盘一次、`flush()` 又全量重写 —— 两个进程共用一个
|
|
129
|
+
* storeDir 时,后 flush 的那个会用自己那份旧快照把对方的记录整条抹掉(见 issue #3)。
|
|
130
|
+
* 现在所有写都是"锁内重新读盘 → 合并增量 → 原子写",锁只是把这两步串起来。
|
|
131
|
+
*
|
|
132
|
+
* Windows:`openSync(wx)` 在"文件已存在、但正处于删除过程中"时报 EPERM/EACCES 而不是
|
|
133
|
+
* EEXIST,所以这两个码也按"已存在"处理,否则并发下会直接崩。
|
|
134
|
+
*/
|
|
135
|
+
function withScopeLock (lockPath , fn , onWarn ) {
|
|
136
|
+
const depth = heldLocks.get(lockPath)
|
|
137
|
+
if (depth !== undefined) {
|
|
138
|
+
heldLocks.set(lockPath, depth + 1)
|
|
139
|
+
try {
|
|
140
|
+
return fn()
|
|
141
|
+
} finally {
|
|
142
|
+
const left = (heldLocks.get(lockPath) ?? 1) - 1
|
|
143
|
+
if (left <= 0) heldLocks.delete(lockPath)
|
|
144
|
+
else heldLocks.set(lockPath, left)
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
mkdirSync(dirname(lockPath), { recursive: true })
|
|
148
|
+
let fd = null
|
|
149
|
+
let delay = 2
|
|
150
|
+
const deadline = Date.now() + LOCK_TIMEOUT_MS
|
|
151
|
+
while (fd === null) {
|
|
152
|
+
try {
|
|
153
|
+
fd = openSync(lockPath, 'wx')
|
|
154
|
+
} catch (err) {
|
|
155
|
+
const code = (err ).code
|
|
156
|
+
if (code !== 'EEXIST' && code !== 'EPERM' && code !== 'EACCES') throw err
|
|
157
|
+
const broke = breakStaleLock(lockPath)
|
|
158
|
+
if (!broke && Date.now() >= deadline) {
|
|
159
|
+
onWarn?.(`store: lock ${lockPath} timed out after ${LOCK_TIMEOUT_MS}ms; writing without it`)
|
|
160
|
+
return fn()
|
|
161
|
+
}
|
|
162
|
+
if (!broke) {
|
|
163
|
+
sleepSync(delay)
|
|
164
|
+
delay = Math.min(delay * 2, 50)
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
try { writeFileSync(fd, String(process.pid), 'utf8') } catch { /* pid 只是给接管用的提示 */ }
|
|
169
|
+
heldLocks.set(lockPath, 1)
|
|
170
|
+
try {
|
|
171
|
+
return fn()
|
|
172
|
+
} finally {
|
|
173
|
+
heldLocks.delete(lockPath)
|
|
174
|
+
try { closeSync(fd) } catch { /* 已经关了 */ }
|
|
175
|
+
try { rmSync(lockPath, { force: true }) } catch { /* 下一次 breakStale 会兜底 */ }
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** tmp 名带 pid + 随机后缀:两个进程并发写同一个 scope 时不会撞名破坏原子性。 */
|
|
180
|
+
function tmpPathFor(file ) {
|
|
181
|
+
return `${file}.${process.pid}.${randomUUID().slice(0, 8)}.tmp`
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** 原子写整个 JSONL(tmp + rename)。调用方必须持有该 scope 的锁。 */
|
|
185
|
+
function writeJsonlAtomic(file , records ) {
|
|
186
|
+
mkdirSync(dirname(file), { recursive: true })
|
|
187
|
+
const tmp = tmpPathFor(file)
|
|
188
|
+
const lines = [...records.values()].map((r) => JSON.stringify(r))
|
|
189
|
+
writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
|
|
190
|
+
renameSync(tmp, file)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** 文件的时间戳快照,用来判断"别的进程有没有改过它"。 */
|
|
194
|
+
function stampOf(file ) {
|
|
195
|
+
try {
|
|
196
|
+
const st = statSync(file)
|
|
197
|
+
return `${st.mtimeMs}:${st.size}`
|
|
198
|
+
} catch {
|
|
199
|
+
return 'missing'
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** 读一个 scope 文件成 Map(坏行跳过;scope 不符的行忽略,flush 时会清掉)。 */
|
|
204
|
+
function readRecords(file , scope , onWarn ) {
|
|
205
|
+
const out = new Map ()
|
|
206
|
+
let text
|
|
207
|
+
try {
|
|
208
|
+
text = readFileSync(file, 'utf8')
|
|
209
|
+
} catch (err) {
|
|
210
|
+
if ((err ).code !== 'ENOENT') onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
|
|
211
|
+
return out
|
|
212
|
+
}
|
|
213
|
+
let lineNo = 0
|
|
214
|
+
for (const line of text.split('\n')) {
|
|
215
|
+
lineNo++
|
|
216
|
+
const trimmed = line.trim()
|
|
217
|
+
if (!trimmed) continue
|
|
218
|
+
try {
|
|
219
|
+
const rec = JSON.parse(trimmed)
|
|
220
|
+
if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
|
|
221
|
+
throw new Error('bad shape')
|
|
222
|
+
}
|
|
223
|
+
if (rec.scope !== scope) continue
|
|
224
|
+
out.set(rec.id, rec)
|
|
225
|
+
} catch {
|
|
226
|
+
onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return out
|
|
230
|
+
}
|
|
231
|
+
|
|
79
232
|
export class ScopeStore {
|
|
80
233
|
scope
|
|
81
234
|
records = new Map ()
|
|
82
235
|
vectors = new Map ()
|
|
83
236
|
/** ids saved while the embed backend was down; retried opportunistically. */
|
|
84
237
|
embedPending = new Set ()
|
|
85
|
-
|
|
238
|
+
/**
|
|
239
|
+
* 待落盘的**增量**(不是整份快照)。flush 时把它合并进"锁内重新读到的磁盘内容",
|
|
240
|
+
* 而不是拿内存覆盖磁盘 —— 这是多个进程共用同一个 storeDir 时互不抹数据的前提。
|
|
241
|
+
*/
|
|
242
|
+
upserts = new Map ()
|
|
243
|
+
/** 本进程删掉的 id;flush 时从磁盘内容里移除。 */
|
|
244
|
+
deletes = new Set ()
|
|
86
245
|
/** true once vectors were touched since the last .vec flush. */
|
|
87
246
|
vectorsDirty = false
|
|
88
247
|
/** 数据文件路径(写、以及删空后移除都走它)。 */
|
|
89
248
|
file
|
|
249
|
+
/** 最近一次与磁盘同步后的 `mtime:size`;用于发现别的进程改过这个 scope。 */
|
|
250
|
+
stamp = ''
|
|
251
|
+
/** 上次检查外部改动的时间(读路径限流用)。 */
|
|
252
|
+
checkedAt = 0
|
|
90
253
|
vecFile
|
|
91
254
|
events
|
|
92
255
|
/** Embedding identity for sidecar validation; set via setVecIdentity(). */
|
|
93
256
|
vecIdentity = ''
|
|
94
257
|
|
|
258
|
+
/** 有增量待写才算脏(只有向量变动时不脏,走 vectorsDirty)。 */
|
|
259
|
+
get dirty() {
|
|
260
|
+
return this.upserts.size > 0 || this.deletes.size > 0
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** 这个 scope 的跨进程写锁(JSONL 与 .vec 共用一把,避免两边交叉覆盖)。 */
|
|
264
|
+
get lockPath() {
|
|
265
|
+
return `${this.file}.lock`
|
|
266
|
+
}
|
|
267
|
+
|
|
95
268
|
constructor(scope , root , events = {}) {
|
|
96
269
|
this.scope = scope
|
|
97
270
|
this.file = join(root, scopeFileName(scope))
|
|
@@ -105,8 +278,26 @@ export class ScopeStore {
|
|
|
105
278
|
* re-enters the embed queue — old vectors for a new model are wrong, not
|
|
106
279
|
* merely stale.
|
|
107
280
|
*/
|
|
108
|
-
setVecIdentity(model , dims ) {
|
|
109
|
-
this.vecIdentity =
|
|
281
|
+
setVecIdentity(model , dims , pooling ) {
|
|
282
|
+
this.vecIdentity = pooling !== undefined && pooling !== '' && pooling !== 'auto'
|
|
283
|
+
? `${model}\t${dims}\t${pooling}`
|
|
284
|
+
: `${model}\t${dims}`
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* 边车身份对不对。老边车只有 `model\tdims` 两段(0.10.0 之前不记 pooling),
|
|
289
|
+
* 缺 pooling 时按兼容接受 —— 不能因为多记了一个字段就让所有人的向量全量作废。
|
|
290
|
+
* 两边都有 pooling 时才严格比:这正是 issue #3 里那条「用错 --pooling 会静默
|
|
291
|
+
* 把整份边车作废、而且 identity 校验会放行」的修法。
|
|
292
|
+
*/
|
|
293
|
+
identityMatches(header ) {
|
|
294
|
+
if (!this.vecIdentity) return true
|
|
295
|
+
if (!header) return false
|
|
296
|
+
const mine = this.vecIdentity.split('\t')
|
|
297
|
+
const theirs = header.split('\t')
|
|
298
|
+
if (mine[0] !== theirs[0] || mine[1] !== theirs[1]) return false
|
|
299
|
+
if (mine[2] !== undefined && theirs[2] !== undefined) return mine[2] === theirs[2]
|
|
300
|
+
return true
|
|
110
301
|
}
|
|
111
302
|
|
|
112
303
|
get size() {
|
|
@@ -115,7 +306,8 @@ export class ScopeStore {
|
|
|
115
306
|
|
|
116
307
|
insert(record ) {
|
|
117
308
|
this.records.set(record.id, record)
|
|
118
|
-
this.
|
|
309
|
+
this.upserts.set(record.id, record)
|
|
310
|
+
this.deletes.delete(record.id)
|
|
119
311
|
}
|
|
120
312
|
|
|
121
313
|
remove(id ) {
|
|
@@ -123,12 +315,19 @@ export class ScopeStore {
|
|
|
123
315
|
if (had) {
|
|
124
316
|
this.vectors.delete(id)
|
|
125
317
|
this.embedPending.delete(id)
|
|
126
|
-
this.
|
|
318
|
+
this.upserts.delete(id)
|
|
319
|
+
this.deletes.add(id)
|
|
127
320
|
this.vectorsDirty = true
|
|
128
321
|
}
|
|
129
322
|
return had
|
|
130
323
|
}
|
|
131
324
|
|
|
325
|
+
/** 合并后把缓存换成磁盘上的内容(增量已并入),清掉已落盘的那部分增量。 */
|
|
326
|
+
replaceRecords(next ) {
|
|
327
|
+
this.records.clear()
|
|
328
|
+
for (const [id, rec] of next) this.records.set(id, rec)
|
|
329
|
+
}
|
|
330
|
+
|
|
132
331
|
/**
|
|
133
332
|
* Delete this scope's data files(.jsonl 与 .vec 边车)。
|
|
134
333
|
*
|
|
@@ -165,14 +364,19 @@ export class ScopeStore {
|
|
|
165
364
|
this.setVector(id, vec)
|
|
166
365
|
try {
|
|
167
366
|
mkdirSync(dirname(this.vecFile), { recursive: true })
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
367
|
+
withScopeLock(this.lockPath, () => {
|
|
368
|
+
const header = this.readHeader()
|
|
369
|
+
// 缺文件 / 别人的模型 / 旧的两段头(得把 pooling 记进去)→ 整份重写
|
|
370
|
+
const staleHeader = this.vecIdentity !== '' &&
|
|
371
|
+
(!this.identityMatches(header) || header.split('\t').length < this.vecIdentity.split('\t').length)
|
|
372
|
+
if (staleHeader) {
|
|
373
|
+
this.writeSidecarLocked()
|
|
374
|
+
this.vectorsDirty = false
|
|
375
|
+
return
|
|
376
|
+
}
|
|
377
|
+
appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
|
|
171
378
|
this.vectorsDirty = false
|
|
172
|
-
|
|
173
|
-
}
|
|
174
|
-
appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
|
|
175
|
-
this.vectorsDirty = false
|
|
379
|
+
}, this.events.onWarn)
|
|
176
380
|
} catch (err) {
|
|
177
381
|
this.vectorsDirty = true // full rewrite on next flushVectors
|
|
178
382
|
this.events.onWarn?.(`vec: append failed: ${(err ).message}`)
|
|
@@ -190,16 +394,53 @@ export class ScopeStore {
|
|
|
190
394
|
}
|
|
191
395
|
}
|
|
192
396
|
|
|
193
|
-
/** Atomic full rewrite: identity header + every in-memory vector. */
|
|
397
|
+
/** Atomic full rewrite: identity header + every in-memory vector (merged with disk). */
|
|
194
398
|
writeSidecar() {
|
|
399
|
+
withScopeLock(this.lockPath, () => this.writeSidecarLocked(), this.events.onWarn)
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** 持锁版本:先读盘上同身份的条目再合并,别的进程刚追加的向量不会被覆盖掉。 */
|
|
403
|
+
writeSidecarLocked() {
|
|
195
404
|
mkdirSync(dirname(this.vecFile), { recursive: true })
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
405
|
+
const lines = []
|
|
406
|
+
if (this.vecIdentity) lines.push(`#${this.vecIdentity}`)
|
|
407
|
+
const merged = this.readSidecarEntries()
|
|
408
|
+
for (const [id, v] of this.vectors) merged.set(id, `${id}\t${encodeVec(v)}`)
|
|
409
|
+
for (const line of merged.values()) lines.push(line)
|
|
410
|
+
const tmp = tmpPathFor(this.vecFile)
|
|
199
411
|
writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
|
|
200
412
|
renameSync(tmp, this.vecFile)
|
|
201
413
|
}
|
|
202
414
|
|
|
415
|
+
/**
|
|
416
|
+
* 磁盘边车里"身份一致"的条目(id → 原始行)。身份不同就返回空(整份重写),
|
|
417
|
+
* 没头的旧文件也不合并 —— 那些向量不属于当前模型。
|
|
418
|
+
*/
|
|
419
|
+
readSidecarEntries() {
|
|
420
|
+
const out = new Map ()
|
|
421
|
+
if (!existsSync(this.vecFile)) return out
|
|
422
|
+
try {
|
|
423
|
+
const text = readFileSync(this.vecFile, 'utf8')
|
|
424
|
+
let sawHeader = false
|
|
425
|
+
for (const line of text.split('\n')) {
|
|
426
|
+
const trimmed = line.trim()
|
|
427
|
+
if (!trimmed) continue
|
|
428
|
+
if (trimmed.startsWith('#')) {
|
|
429
|
+
sawHeader = true
|
|
430
|
+
if (!this.identityMatches(trimmed.slice(1))) return new Map()
|
|
431
|
+
continue
|
|
432
|
+
}
|
|
433
|
+
if (!sawHeader && this.vecIdentity) return new Map()
|
|
434
|
+
const tab = trimmed.indexOf('\t')
|
|
435
|
+
if (tab <= 0) continue
|
|
436
|
+
out.set(trimmed.slice(0, tab), trimmed)
|
|
437
|
+
}
|
|
438
|
+
} catch {
|
|
439
|
+
return new Map()
|
|
440
|
+
}
|
|
441
|
+
return out
|
|
442
|
+
}
|
|
443
|
+
|
|
203
444
|
/** Rewrite the whole sidecar (after deletes / compaction / append failures). */
|
|
204
445
|
flushVectors() {
|
|
205
446
|
if (!this.vectorsDirty) return
|
|
@@ -231,7 +472,7 @@ export class ScopeStore {
|
|
|
231
472
|
const trimmed = line.trim()
|
|
232
473
|
if (!trimmed) continue
|
|
233
474
|
if (trimmed.startsWith('#')) {
|
|
234
|
-
if (this.
|
|
475
|
+
if (!this.identityMatches(trimmed.slice(1))) {
|
|
235
476
|
discard(`identity mismatch (${trimmed.slice(1)} != ${this.vecIdentity})`)
|
|
236
477
|
return
|
|
237
478
|
}
|
|
@@ -282,6 +523,11 @@ function decodeVec(text ) {
|
|
|
282
523
|
|
|
283
524
|
|
|
284
525
|
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
|
|
285
531
|
|
|
286
532
|
|
|
287
533
|
export class MemoryStore {
|
|
@@ -293,19 +539,23 @@ export class MemoryStore {
|
|
|
293
539
|
fileNames = new Map ()
|
|
294
540
|
flushTimer = null
|
|
295
541
|
events
|
|
296
|
-
|
|
542
|
+
refreshTtlMs
|
|
543
|
+
/** `model\tdims[\tpooling]` handed to every ScopeStore for sidecar validation. */
|
|
297
544
|
vecIdentity = ''
|
|
298
545
|
|
|
299
546
|
constructor(root , events = {}) {
|
|
300
547
|
this.root = root
|
|
301
548
|
this.events = events
|
|
549
|
+
this.refreshTtlMs = events.refreshTtlMs ?? 1_000
|
|
302
550
|
mkdirSync(join(root, 'projects'), { recursive: true })
|
|
303
551
|
}
|
|
304
552
|
|
|
305
553
|
/** Declare the embedding identity; applies to existing and future scopes. */
|
|
306
|
-
setVecIdentity(model , dims ) {
|
|
307
|
-
this.vecIdentity =
|
|
308
|
-
|
|
554
|
+
setVecIdentity(model , dims , pooling ) {
|
|
555
|
+
this.vecIdentity = pooling !== undefined && pooling !== '' && pooling !== 'auto'
|
|
556
|
+
? `${model}\t${dims}\t${pooling}`
|
|
557
|
+
: `${model}\t${dims}`
|
|
558
|
+
for (const s of this.scopes.values()) s.setVecIdentity(model, dims, pooling)
|
|
309
559
|
}
|
|
310
560
|
|
|
311
561
|
/** Preload specific scopes eagerly (e.g. defaults); others load on first access. */
|
|
@@ -315,50 +565,55 @@ export class MemoryStore {
|
|
|
315
565
|
|
|
316
566
|
loadScope(scope ) {
|
|
317
567
|
const file = this.fileFor(scope)
|
|
318
|
-
if (!fileExists(file)) return
|
|
319
|
-
let text
|
|
320
|
-
try {
|
|
321
|
-
text = readFileSync(file, 'utf8')
|
|
322
|
-
} catch (err) {
|
|
323
|
-
this.events.onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
|
|
324
|
-
return
|
|
325
|
-
}
|
|
326
568
|
const store = this.scope(scope)
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
if (!trimmed) continue
|
|
332
|
-
try {
|
|
333
|
-
const rec = JSON.parse(trimmed)
|
|
334
|
-
if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
|
|
335
|
-
throw new Error('bad shape')
|
|
336
|
-
}
|
|
337
|
-
if (rec.scope !== scope) continue // file/scope mismatch: ignore, will be rewritten on flush
|
|
338
|
-
store.records.set(rec.id, rec)
|
|
339
|
-
} catch {
|
|
340
|
-
this.events.onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
|
|
341
|
-
}
|
|
569
|
+
if (!fileExists(file)) {
|
|
570
|
+
store.stamp = 'missing'
|
|
571
|
+
store.checkedAt = Date.now()
|
|
572
|
+
return
|
|
342
573
|
}
|
|
574
|
+
store.replaceRecords(readRecords(file, scope, (m) => this.events.onWarn?.(m)))
|
|
343
575
|
// sidecar vectors second; anything still un-embedded joins the backfill queue
|
|
344
576
|
store.loadVectors()
|
|
345
577
|
for (const id of store.records.keys()) {
|
|
346
578
|
if (!store.vectors.has(id)) store.embedPending.add(id)
|
|
347
579
|
}
|
|
348
|
-
store.
|
|
580
|
+
store.stamp = stampOf(file)
|
|
581
|
+
store.checkedAt = Date.now()
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
/**
|
|
585
|
+
* 别的进程可能改了同一个 scope:读路径按 refreshTtlMs 限流地 stat 一次,
|
|
586
|
+
* mtime/size 变了就重新读盘(还没落盘的增量交给 flush 在锁内合并)。
|
|
587
|
+
*/
|
|
588
|
+
refreshFromDisk(s ) {
|
|
589
|
+
if (this.refreshTtlMs <= 0) return
|
|
590
|
+
const now = Date.now()
|
|
591
|
+
if (now - s.checkedAt < this.refreshTtlMs) return
|
|
592
|
+
s.checkedAt = now
|
|
593
|
+
if (s.dirty) return
|
|
594
|
+
const file = this.fileFor(s.scope)
|
|
595
|
+
const stamp = stampOf(file)
|
|
596
|
+
if (stamp === s.stamp) return
|
|
597
|
+
s.replaceRecords(stamp === 'missing' ? new Map() : readRecords(file, s.scope, (m) => this.events.onWarn?.(m)))
|
|
598
|
+
s.stamp = stamp
|
|
349
599
|
}
|
|
350
600
|
|
|
351
601
|
scope(scope ) {
|
|
352
602
|
let s = this.scopes.get(scope)
|
|
353
603
|
if (!s) {
|
|
354
604
|
s = new ScopeStore(scope, this.root, this.events)
|
|
355
|
-
if (this.vecIdentity)
|
|
605
|
+
if (this.vecIdentity) {
|
|
606
|
+
const [model, dims, pooling] = this.vecIdentity.split('\t')
|
|
607
|
+
s.setVecIdentity(model , Number(dims), pooling)
|
|
608
|
+
}
|
|
356
609
|
this.scopes.set(scope, s)
|
|
357
610
|
this.fileNames.set(scope, scopeFileName(scope))
|
|
358
611
|
}
|
|
359
612
|
if (!this.loaded.has(scope)) {
|
|
360
613
|
this.loaded.add(scope)
|
|
361
614
|
this.loadScope(scope)
|
|
615
|
+
} else {
|
|
616
|
+
this.refreshFromDisk(s)
|
|
362
617
|
}
|
|
363
618
|
return s
|
|
364
619
|
}
|
|
@@ -372,7 +627,10 @@ export class MemoryStore {
|
|
|
372
627
|
*/
|
|
373
628
|
peek(scope ) {
|
|
374
629
|
const existing = this.scopes.get(scope)
|
|
375
|
-
if (existing)
|
|
630
|
+
if (existing) {
|
|
631
|
+
this.refreshFromDisk(existing)
|
|
632
|
+
return existing
|
|
633
|
+
}
|
|
376
634
|
const file = join(this.root, scopeFileName(scope))
|
|
377
635
|
if (!fileExists(file)) return undefined
|
|
378
636
|
return this.scope(scope)
|
|
@@ -452,33 +710,52 @@ export class MemoryStore {
|
|
|
452
710
|
}
|
|
453
711
|
|
|
454
712
|
/**
|
|
455
|
-
* Write
|
|
713
|
+
* Write the pending deltas of every dirty scope.
|
|
456
714
|
*
|
|
457
|
-
*
|
|
458
|
-
*
|
|
459
|
-
*
|
|
715
|
+
* 每个 scope 走一次"锁内重新读盘 → 合并增量 → 原子写":不再用内存快照整份覆盖,
|
|
716
|
+
* 所以另一个进程(比如把同一个 storeDir 挂在 MCP server 上)刚写的记录不会消失。
|
|
717
|
+
* 一个被删空的 scope(合并后 0 条记录)**不再落 0 字节文件**,而是把 .jsonl 与 .vec
|
|
718
|
+
* 一起删掉 —— 但前提是磁盘上确实也空了(别人写的记录不算"删空")。
|
|
719
|
+
* `global` 永不自动删除(见 isRemovable)。
|
|
460
720
|
*/
|
|
461
721
|
flush() {
|
|
462
722
|
for (const s of this.scopes.values()) {
|
|
463
723
|
if (!s.dirty) continue
|
|
464
|
-
|
|
465
|
-
s.removeFiles()
|
|
466
|
-
s.dirty = false
|
|
467
|
-
s.vectorsDirty = false
|
|
468
|
-
continue
|
|
469
|
-
}
|
|
470
|
-
const file = this.fileFor(s.scope)
|
|
471
|
-
mkdirSync(dirname(file), { recursive: true })
|
|
472
|
-
const tmp = `${file}.tmp`
|
|
473
|
-
const lines = [...s.records.values()].map((r) => JSON.stringify(r))
|
|
474
|
-
writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
|
|
475
|
-
renameSync(tmp, file)
|
|
476
|
-
s.dirty = false
|
|
724
|
+
this.flushScope(s)
|
|
477
725
|
}
|
|
478
726
|
// vectors flush too: they are the expensive-to-recompute half
|
|
479
727
|
for (const s of this.scopes.values()) s.flushVectors()
|
|
480
728
|
}
|
|
481
729
|
|
|
730
|
+
/** 锁内合并一个 scope 的增量(同步;调用方不持有这把锁)。 */
|
|
731
|
+
flushScope(s ) {
|
|
732
|
+
const file = this.fileFor(s.scope)
|
|
733
|
+
// 增量在等锁期间不会变(同步执行),先拷一份,锁里只做合并与写
|
|
734
|
+
const upserts = new Map(s.upserts)
|
|
735
|
+
const deletes = new Set(s.deletes)
|
|
736
|
+
withScopeLock(s.lockPath, () => {
|
|
737
|
+
const merged = fileExists(file)
|
|
738
|
+
? readRecords(file, s.scope, (m) => this.events.onWarn?.(m))
|
|
739
|
+
: new Map ()
|
|
740
|
+
for (const [id, rec] of upserts) merged.set(id, rec)
|
|
741
|
+
for (const id of deletes) merged.delete(id)
|
|
742
|
+
if (merged.size === 0 && this.isRemovable(s.scope)) {
|
|
743
|
+
s.removeFiles()
|
|
744
|
+
// 边车已经被一起删掉了;别让随后的 flushVectors 又把它写回来
|
|
745
|
+
s.vectors.clear()
|
|
746
|
+
s.embedPending.clear()
|
|
747
|
+
s.vectorsDirty = false
|
|
748
|
+
} else {
|
|
749
|
+
writeJsonlAtomic(file, merged)
|
|
750
|
+
}
|
|
751
|
+
s.replaceRecords(merged)
|
|
752
|
+
for (const id of upserts.keys()) s.upserts.delete(id)
|
|
753
|
+
for (const id of deletes) s.deletes.delete(id)
|
|
754
|
+
s.stamp = stampOf(file)
|
|
755
|
+
s.checkedAt = Date.now()
|
|
756
|
+
}, this.events.onWarn)
|
|
757
|
+
}
|
|
758
|
+
|
|
482
759
|
/** Synchronous final flush — call on dispose. */
|
|
483
760
|
close() {
|
|
484
761
|
if (this.flushTimer) {
|
|
@@ -504,7 +781,7 @@ export class MemoryStore {
|
|
|
504
781
|
rec.accessCount += 1
|
|
505
782
|
rec.importance = Math.max(rec.importance, input.importance ?? defaults.importance)
|
|
506
783
|
if (input.supersedes) rec.supersedes = input.supersedes
|
|
507
|
-
s.
|
|
784
|
+
s.insert(rec) // 去重命中的也是一次写:进增量,flush 时落盘
|
|
508
785
|
return { record: rec, deduped: true }
|
|
509
786
|
}
|
|
510
787
|
}
|
package/lib/tools/apply.js
CHANGED
|
@@ -120,6 +120,17 @@ export function apply(ctx , config = {}) {
|
|
|
120
120
|
const summary = scopeSummary(value, agent)
|
|
121
121
|
return summary ? `${body}\n${summary}` : body
|
|
122
122
|
})
|
|
123
|
+
|
|
124
|
+
register(tools.memory_get, (value, _args, exec) => {
|
|
125
|
+
const agent = (exec )?.agent
|
|
126
|
+
const body = value.records.map((r) => {
|
|
127
|
+
const tags = Array.isArray(r.tags) && r.tags.length > 0 ? ` · ${r.tags.join(', ')}` : ''
|
|
128
|
+
return `[${r.id}] (${renderScope(r.scope, agent)}/${r.kind}${tags})\n${r.content}`
|
|
129
|
+
}).join('\n\n')
|
|
130
|
+
const missing = value.missing.length > 0 ? `未找到:${value.missing.join(', ')}` : ''
|
|
131
|
+
if (body === '') return missing || '没有找到记忆。'
|
|
132
|
+
return missing === '' ? body : `${body}\n\n${missing}`
|
|
133
|
+
})
|
|
123
134
|
}
|
|
124
135
|
|
|
125
136
|
|