@restregular/dsh-memory 0.9.10 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +483 -468
- package/lib/client.js +13 -8
- package/lib/panel.js +2 -1
- package/lib/provider/index.js +4 -3
- package/lib/provider/store.js +343 -66
- package/package.json +1 -1
package/lib/client.js
CHANGED
|
@@ -63,12 +63,15 @@ window.__ModuleLoader__.load({
|
|
|
63
63
|
'.memp button:focus{outline:none}',
|
|
64
64
|
'.memp button:focus-visible{outline:2px solid var(--dsw-alias-brand-primary);outline-offset:1px}',
|
|
65
65
|
'.memp button:disabled{opacity:.5;cursor:default;border-color:var(--dsw-alias-border-l1);color:var(--dsw-alias-label-primary)}',
|
|
66
|
-
// 服务器开关(switch 造型)
|
|
66
|
+
// 服务器开关(switch 造型)。配色照抄宿主 [role=switch]:
|
|
67
|
+
// - 关闭态轨道 border-l3(border-l1 只有 ~6% 透明度,深浅色下都几乎看不见);
|
|
68
|
+
// - 圆点关闭态 switch-thumb、开启态 label-primary-foreground。
|
|
69
|
+
// 深色主题的 brand-primary 是近白色,圆点写死 #fff 会直接融进轨道里。
|
|
67
70
|
'.memp .toggle{display:inline-flex;align-items:center;gap:8px;padding:4px 12px 4px 5px}',
|
|
68
|
-
'.memp .toggle-track{width:34px;height:18px;border-radius:999px;background:var(--dsw-alias-border-
|
|
71
|
+
'.memp .toggle-track{width:34px;height:18px;border-radius:999px;background:var(--dsw-alias-border-l3);position:relative;transition:background .15s ease;flex:none;display:inline-block}',
|
|
69
72
|
'.memp .toggle.on .toggle-track{background:var(--dsw-alias-brand-primary)}',
|
|
70
|
-
'.memp .toggle-knob{position:absolute;top:2px;left:2px;width:14px;height:14px;border-radius:50%;background
|
|
71
|
-
'.memp .toggle.on .toggle-knob{transform:translateX(16px)}',
|
|
73
|
+
'.memp .toggle-knob{position:absolute;top:2px;left:2px;width:14px;height:14px;border-radius:50%;background:var(--dsw-alias-switch-thumb);box-shadow:0 1px 3px rgba(0,0,0,.25);transition:transform .15s ease,background .15s ease}',
|
|
74
|
+
'.memp .toggle.on .toggle-knob{transform:translateX(16px);background:var(--dsw-alias-label-primary-foreground)}',
|
|
72
75
|
// 记忆列表:一条一行时信息密度优先
|
|
73
76
|
'.memp .list{display:flex;flex-direction:column;gap:8px}',
|
|
74
77
|
'.memp .rec{border:1px solid var(--dsw-alias-border-l1);border-radius:10px;padding:9px 12px;background:var(--dsw-alias-bg-layer-1)}',
|
|
@@ -158,11 +161,13 @@ window.__ModuleLoader__.load({
|
|
|
158
161
|
'.memp .setrow .set-desc{flex:1 1 100%;color:var(--dsw-alias-label-secondary);font-size:12px;line-height:1.5;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}',
|
|
159
162
|
'.memp .setrow .set-field{display:inline-flex;align-items:center;gap:5px;font-size:12px;color:var(--dsw-alias-label-secondary)}',
|
|
160
163
|
'.memp .setrow input[type=number]{width:78px}',
|
|
161
|
-
// 开关:与向量服务的 toggle 同一套视觉(轨道 + 圆点)
|
|
162
|
-
|
|
164
|
+
// 开关:与向量服务的 toggle 同一套视觉(轨道 + 圆点),配色同宿主 switch。
|
|
165
|
+
// 深色主题下开启态轨道是近白色,圆点必须用 label-primary-foreground 才看得见;
|
|
166
|
+
// 关闭态轨道用 border-l3、圆点用 switch-thumb,避免"只剩一个白点"。
|
|
167
|
+
'.memp .switch{position:relative;width:34px;height:20px;padding:0;border:0;border-radius:999px;background:var(--dsw-alias-border-l3);cursor:pointer;transition:background .12s ease}',
|
|
163
168
|
'.memp .switch.on{background:var(--dsw-alias-brand-primary)}',
|
|
164
|
-
'.memp .switch .knob{position:absolute;top:2px;left:2px;width:16px;height:16px;border-radius:50%;background
|
|
165
|
-
'.memp .switch.on .knob{left:16px}',
|
|
169
|
+
'.memp .switch .knob{position:absolute;top:2px;left:2px;width:16px;height:16px;border-radius:50%;background:var(--dsw-alias-switch-thumb);transition:left .12s ease,background .12s ease;box-shadow:0 1px 2px rgba(0,0,0,.25)}',
|
|
170
|
+
'.memp .switch.on .knob{left:16px;background:var(--dsw-alias-label-primary-foreground)}',
|
|
166
171
|
'.memp .setfoot{display:flex;align-items:center;gap:10px;padding-top:12px}',
|
|
167
172
|
// 新版设置页:卡片 + 可见 label + 状态文案 + 字段说明
|
|
168
173
|
'.memp .setp{display:flex;flex-direction:column;gap:10px}',
|
package/lib/panel.js
CHANGED
|
@@ -456,7 +456,8 @@ async function handle(ctx, routePath, req, res) {
|
|
|
456
456
|
if (body.tags !== undefined) rec.tags = Array.isArray(body.tags) ? body.tags.map(String) : []
|
|
457
457
|
if (body.supersededBy !== undefined) rec.supersedes = body.supersededBy === null ? undefined : String(body.supersededBy)
|
|
458
458
|
rec.updatedAt = Date.now()
|
|
459
|
-
|
|
459
|
+
// 改动本身要进增量(flush 只落增量,不再整份快照覆盖);向量作废靠 vectorsDirty。
|
|
460
|
+
s.insert(rec)
|
|
460
461
|
store.scheduleFlush()
|
|
461
462
|
// 立即重建向量(不等 60s 的定时回填);失败就留在 pending 里由重试兜底。
|
|
462
463
|
if (reindex && typeof memory.reindexRecord === 'function') {
|
package/lib/provider/index.js
CHANGED
|
@@ -508,7 +508,9 @@ export class LocalMemoryService {
|
|
|
508
508
|
// so loading first would accept vectors from a previous embedding model
|
|
509
509
|
// whenever the dimensions matched — and the next flush would then rewrite the
|
|
510
510
|
// header as the new model, making the mismatch permanent.
|
|
511
|
-
|
|
511
|
+
// pooling 也进身份(issue #3):同一个模型换 --pooling 等于换了一个向量空间
|
|
512
|
+
// (last 与 cls 的余弦只有 0.35),以前它不在 identity 里,校验会静默放行。
|
|
513
|
+
this.store.setVecIdentity(this.cfg.vector.model, this.cfg.vector.dims, this.cfg.vector.pooling)
|
|
512
514
|
// 会话 scope 是按会话动态生成的,**不能**在这里 eager 加载(加载谁?);只预载
|
|
513
515
|
// project 与 global 这两个稳定作用域。其余按需 peek/loadDiscovered。
|
|
514
516
|
// eager 只加载"已存在"的 scope(peek),不会为不存在的 scope 建空文件。
|
|
@@ -993,7 +995,7 @@ export class LocalMemoryService {
|
|
|
993
995
|
removed += local
|
|
994
996
|
scopes.push(scope.scope)
|
|
995
997
|
}
|
|
996
|
-
try { this.store.
|
|
998
|
+
try { this.store.flush() } catch { /* scheduled flush covers it */ }
|
|
997
999
|
return { removed, scopes }
|
|
998
1000
|
}
|
|
999
1001
|
|
|
@@ -1357,7 +1359,6 @@ export class LocalMemoryService {
|
|
|
1357
1359
|
total += 1
|
|
1358
1360
|
}
|
|
1359
1361
|
s.vectorsDirty = true
|
|
1360
|
-
s.dirty = true
|
|
1361
1362
|
}
|
|
1362
1363
|
this.store.scheduleFlush()
|
|
1363
1364
|
void this.backfillAll({ throttleMs: 50 }).catch(() => { /* 失败留给 60s 重试 */ })
|
package/lib/provider/store.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* atomic compaction (tmp + rename). Vectors live in memory for M1; the .vec
|
|
4
4
|
* sidecar format arrives in M1.5.
|
|
5
5
|
*/
|
|
6
|
-
import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
6
|
+
import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
7
7
|
import { createHash, randomUUID } from 'node:crypto'
|
|
8
8
|
import { Buffer } from 'node:buffer'
|
|
9
9
|
import { dirname, join } from 'node:path'
|
|
@@ -76,22 +76,195 @@ export function newId() {
|
|
|
76
76
|
return randomUUID()
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
+
// ---------------------------------------------------------------- 跨进程写锁
|
|
80
|
+
|
|
81
|
+
/** 等锁的上限:超过就告警并"不带锁照写"(宁可少一次合并,也不要丢掉这次写入)。 */
|
|
82
|
+
const LOCK_TIMEOUT_MS = 5_000
|
|
83
|
+
/** 超过这个时长没被碰过的锁视为陈旧(进程被 kill -9 时不会自己清理)。 */
|
|
84
|
+
const LOCK_STALE_MS = 10_000
|
|
85
|
+
|
|
86
|
+
/** 同步睡眠,不占 CPU:锁等待用(必须同步 —— flush/close 的调用方都是同步路径)。 */
|
|
87
|
+
function sleepSync(ms ) {
|
|
88
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function processAlive(pid ) {
|
|
92
|
+
try {
|
|
93
|
+
process.kill(pid, 0)
|
|
94
|
+
return true
|
|
95
|
+
} catch (err) {
|
|
96
|
+
// EPERM = 进程存在但没有权限发信号
|
|
97
|
+
return (err ).code === 'EPERM'
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** 锁是不是陈旧的(拥有者已死 / 太久没动)?是则删掉并返回 true。 */
|
|
102
|
+
function breakStaleLock(lockPath ) {
|
|
103
|
+
let age
|
|
104
|
+
try {
|
|
105
|
+
age = Date.now() - statSync(lockPath).mtimeMs
|
|
106
|
+
} catch {
|
|
107
|
+
return true // 已经不在了,直接重试
|
|
108
|
+
}
|
|
109
|
+
if (age > LOCK_STALE_MS) {
|
|
110
|
+
try { rmSync(lockPath, { force: true }) } catch { /* 下一轮再试 */ }
|
|
111
|
+
return true
|
|
112
|
+
}
|
|
113
|
+
let pid = 0
|
|
114
|
+
try { pid = Number.parseInt(readFileSync(lockPath, 'utf8').trim(), 10) } catch { pid = 0 }
|
|
115
|
+
if (pid > 0 && !processAlive(pid)) {
|
|
116
|
+
try { rmSync(lockPath, { force: true }) } catch { /* 下一轮再试 */ }
|
|
117
|
+
return true
|
|
118
|
+
}
|
|
119
|
+
return false
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** 本进程已持有的锁(可重入:flush 持锁时会再调 writeSidecar,不能自己把自己锁死)。 */
|
|
123
|
+
const heldLocks = new Map ()
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* 取某个 scope 的写锁(锁文件 = `${数据文件}.lock`,`openSync(wx)` 的 O_EXCL 语义)。
|
|
127
|
+
*
|
|
128
|
+
* 为什么需要它:`loadScope()` 只读盘一次、`flush()` 又全量重写 —— 两个进程共用一个
|
|
129
|
+
* storeDir 时,后 flush 的那个会用自己那份旧快照把对方的记录整条抹掉(见 issue #3)。
|
|
130
|
+
* 现在所有写都是"锁内重新读盘 → 合并增量 → 原子写",锁只是把这两步串起来。
|
|
131
|
+
*
|
|
132
|
+
* Windows:`openSync(wx)` 在"文件已存在、但正处于删除过程中"时报 EPERM/EACCES 而不是
|
|
133
|
+
* EEXIST,所以这两个码也按"已存在"处理,否则并发下会直接崩。
|
|
134
|
+
*/
|
|
135
|
+
function withScopeLock (lockPath , fn , onWarn ) {
|
|
136
|
+
const depth = heldLocks.get(lockPath)
|
|
137
|
+
if (depth !== undefined) {
|
|
138
|
+
heldLocks.set(lockPath, depth + 1)
|
|
139
|
+
try {
|
|
140
|
+
return fn()
|
|
141
|
+
} finally {
|
|
142
|
+
const left = (heldLocks.get(lockPath) ?? 1) - 1
|
|
143
|
+
if (left <= 0) heldLocks.delete(lockPath)
|
|
144
|
+
else heldLocks.set(lockPath, left)
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
mkdirSync(dirname(lockPath), { recursive: true })
|
|
148
|
+
let fd = null
|
|
149
|
+
let delay = 2
|
|
150
|
+
const deadline = Date.now() + LOCK_TIMEOUT_MS
|
|
151
|
+
while (fd === null) {
|
|
152
|
+
try {
|
|
153
|
+
fd = openSync(lockPath, 'wx')
|
|
154
|
+
} catch (err) {
|
|
155
|
+
const code = (err ).code
|
|
156
|
+
if (code !== 'EEXIST' && code !== 'EPERM' && code !== 'EACCES') throw err
|
|
157
|
+
const broke = breakStaleLock(lockPath)
|
|
158
|
+
if (!broke && Date.now() >= deadline) {
|
|
159
|
+
onWarn?.(`store: lock ${lockPath} timed out after ${LOCK_TIMEOUT_MS}ms; writing without it`)
|
|
160
|
+
return fn()
|
|
161
|
+
}
|
|
162
|
+
if (!broke) {
|
|
163
|
+
sleepSync(delay)
|
|
164
|
+
delay = Math.min(delay * 2, 50)
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
try { writeFileSync(fd, String(process.pid), 'utf8') } catch { /* pid 只是给接管用的提示 */ }
|
|
169
|
+
heldLocks.set(lockPath, 1)
|
|
170
|
+
try {
|
|
171
|
+
return fn()
|
|
172
|
+
} finally {
|
|
173
|
+
heldLocks.delete(lockPath)
|
|
174
|
+
try { closeSync(fd) } catch { /* 已经关了 */ }
|
|
175
|
+
try { rmSync(lockPath, { force: true }) } catch { /* 下一次 breakStale 会兜底 */ }
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** tmp 名带 pid + 随机后缀:两个进程并发写同一个 scope 时不会撞名破坏原子性。 */
|
|
180
|
+
function tmpPathFor(file ) {
|
|
181
|
+
return `${file}.${process.pid}.${randomUUID().slice(0, 8)}.tmp`
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** 原子写整个 JSONL(tmp + rename)。调用方必须持有该 scope 的锁。 */
|
|
185
|
+
function writeJsonlAtomic(file , records ) {
|
|
186
|
+
mkdirSync(dirname(file), { recursive: true })
|
|
187
|
+
const tmp = tmpPathFor(file)
|
|
188
|
+
const lines = [...records.values()].map((r) => JSON.stringify(r))
|
|
189
|
+
writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
|
|
190
|
+
renameSync(tmp, file)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** 文件的时间戳快照,用来判断"别的进程有没有改过它"。 */
|
|
194
|
+
function stampOf(file ) {
|
|
195
|
+
try {
|
|
196
|
+
const st = statSync(file)
|
|
197
|
+
return `${st.mtimeMs}:${st.size}`
|
|
198
|
+
} catch {
|
|
199
|
+
return 'missing'
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** 读一个 scope 文件成 Map(坏行跳过;scope 不符的行忽略,flush 时会清掉)。 */
|
|
204
|
+
function readRecords(file , scope , onWarn ) {
|
|
205
|
+
const out = new Map ()
|
|
206
|
+
let text
|
|
207
|
+
try {
|
|
208
|
+
text = readFileSync(file, 'utf8')
|
|
209
|
+
} catch (err) {
|
|
210
|
+
if ((err ).code !== 'ENOENT') onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
|
|
211
|
+
return out
|
|
212
|
+
}
|
|
213
|
+
let lineNo = 0
|
|
214
|
+
for (const line of text.split('\n')) {
|
|
215
|
+
lineNo++
|
|
216
|
+
const trimmed = line.trim()
|
|
217
|
+
if (!trimmed) continue
|
|
218
|
+
try {
|
|
219
|
+
const rec = JSON.parse(trimmed)
|
|
220
|
+
if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
|
|
221
|
+
throw new Error('bad shape')
|
|
222
|
+
}
|
|
223
|
+
if (rec.scope !== scope) continue
|
|
224
|
+
out.set(rec.id, rec)
|
|
225
|
+
} catch {
|
|
226
|
+
onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return out
|
|
230
|
+
}
|
|
231
|
+
|
|
79
232
|
export class ScopeStore {
|
|
80
233
|
scope
|
|
81
234
|
records = new Map ()
|
|
82
235
|
vectors = new Map ()
|
|
83
236
|
/** ids saved while the embed backend was down; retried opportunistically. */
|
|
84
237
|
embedPending = new Set ()
|
|
85
|
-
|
|
238
|
+
/**
|
|
239
|
+
* 待落盘的**增量**(不是整份快照)。flush 时把它合并进"锁内重新读到的磁盘内容",
|
|
240
|
+
* 而不是拿内存覆盖磁盘 —— 这是多个进程共用同一个 storeDir 时互不抹数据的前提。
|
|
241
|
+
*/
|
|
242
|
+
upserts = new Map ()
|
|
243
|
+
/** 本进程删掉的 id;flush 时从磁盘内容里移除。 */
|
|
244
|
+
deletes = new Set ()
|
|
86
245
|
/** true once vectors were touched since the last .vec flush. */
|
|
87
246
|
vectorsDirty = false
|
|
88
247
|
/** 数据文件路径(写、以及删空后移除都走它)。 */
|
|
89
248
|
file
|
|
249
|
+
/** 最近一次与磁盘同步后的 `mtime:size`;用于发现别的进程改过这个 scope。 */
|
|
250
|
+
stamp = ''
|
|
251
|
+
/** 上次检查外部改动的时间(读路径限流用)。 */
|
|
252
|
+
checkedAt = 0
|
|
90
253
|
vecFile
|
|
91
254
|
events
|
|
92
255
|
/** Embedding identity for sidecar validation; set via setVecIdentity(). */
|
|
93
256
|
vecIdentity = ''
|
|
94
257
|
|
|
258
|
+
/** 有增量待写才算脏(只有向量变动时不脏,走 vectorsDirty)。 */
|
|
259
|
+
get dirty() {
|
|
260
|
+
return this.upserts.size > 0 || this.deletes.size > 0
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** 这个 scope 的跨进程写锁(JSONL 与 .vec 共用一把,避免两边交叉覆盖)。 */
|
|
264
|
+
get lockPath() {
|
|
265
|
+
return `${this.file}.lock`
|
|
266
|
+
}
|
|
267
|
+
|
|
95
268
|
constructor(scope , root , events = {}) {
|
|
96
269
|
this.scope = scope
|
|
97
270
|
this.file = join(root, scopeFileName(scope))
|
|
@@ -105,8 +278,26 @@ export class ScopeStore {
|
|
|
105
278
|
* re-enters the embed queue — old vectors for a new model are wrong, not
|
|
106
279
|
* merely stale.
|
|
107
280
|
*/
|
|
108
|
-
setVecIdentity(model , dims ) {
|
|
109
|
-
this.vecIdentity =
|
|
281
|
+
setVecIdentity(model , dims , pooling ) {
|
|
282
|
+
this.vecIdentity = pooling !== undefined && pooling !== '' && pooling !== 'auto'
|
|
283
|
+
? `${model}\t${dims}\t${pooling}`
|
|
284
|
+
: `${model}\t${dims}`
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* 边车身份对不对。老边车只有 `model\tdims` 两段(0.10.0 之前不记 pooling),
|
|
289
|
+
* 缺 pooling 时按兼容接受 —— 不能因为多记了一个字段就让所有人的向量全量作废。
|
|
290
|
+
* 两边都有 pooling 时才严格比:这正是 issue #3 里那条「用错 --pooling 会静默
|
|
291
|
+
* 把整份边车作废、而且 identity 校验会放行」的修法。
|
|
292
|
+
*/
|
|
293
|
+
identityMatches(header ) {
|
|
294
|
+
if (!this.vecIdentity) return true
|
|
295
|
+
if (!header) return false
|
|
296
|
+
const mine = this.vecIdentity.split('\t')
|
|
297
|
+
const theirs = header.split('\t')
|
|
298
|
+
if (mine[0] !== theirs[0] || mine[1] !== theirs[1]) return false
|
|
299
|
+
if (mine[2] !== undefined && theirs[2] !== undefined) return mine[2] === theirs[2]
|
|
300
|
+
return true
|
|
110
301
|
}
|
|
111
302
|
|
|
112
303
|
get size() {
|
|
@@ -115,7 +306,8 @@ export class ScopeStore {
|
|
|
115
306
|
|
|
116
307
|
insert(record ) {
|
|
117
308
|
this.records.set(record.id, record)
|
|
118
|
-
this.
|
|
309
|
+
this.upserts.set(record.id, record)
|
|
310
|
+
this.deletes.delete(record.id)
|
|
119
311
|
}
|
|
120
312
|
|
|
121
313
|
remove(id ) {
|
|
@@ -123,12 +315,19 @@ export class ScopeStore {
|
|
|
123
315
|
if (had) {
|
|
124
316
|
this.vectors.delete(id)
|
|
125
317
|
this.embedPending.delete(id)
|
|
126
|
-
this.
|
|
318
|
+
this.upserts.delete(id)
|
|
319
|
+
this.deletes.add(id)
|
|
127
320
|
this.vectorsDirty = true
|
|
128
321
|
}
|
|
129
322
|
return had
|
|
130
323
|
}
|
|
131
324
|
|
|
325
|
+
/** 合并后把缓存换成磁盘上的内容(增量已并入),清掉已落盘的那部分增量。 */
|
|
326
|
+
replaceRecords(next ) {
|
|
327
|
+
this.records.clear()
|
|
328
|
+
for (const [id, rec] of next) this.records.set(id, rec)
|
|
329
|
+
}
|
|
330
|
+
|
|
132
331
|
/**
|
|
133
332
|
* Delete this scope's data files(.jsonl 与 .vec 边车)。
|
|
134
333
|
*
|
|
@@ -165,14 +364,19 @@ export class ScopeStore {
|
|
|
165
364
|
this.setVector(id, vec)
|
|
166
365
|
try {
|
|
167
366
|
mkdirSync(dirname(this.vecFile), { recursive: true })
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
367
|
+
withScopeLock(this.lockPath, () => {
|
|
368
|
+
const header = this.readHeader()
|
|
369
|
+
// 缺文件 / 别人的模型 / 旧的两段头(得把 pooling 记进去)→ 整份重写
|
|
370
|
+
const staleHeader = this.vecIdentity !== '' &&
|
|
371
|
+
(!this.identityMatches(header) || header.split('\t').length < this.vecIdentity.split('\t').length)
|
|
372
|
+
if (staleHeader) {
|
|
373
|
+
this.writeSidecarLocked()
|
|
374
|
+
this.vectorsDirty = false
|
|
375
|
+
return
|
|
376
|
+
}
|
|
377
|
+
appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
|
|
171
378
|
this.vectorsDirty = false
|
|
172
|
-
|
|
173
|
-
}
|
|
174
|
-
appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
|
|
175
|
-
this.vectorsDirty = false
|
|
379
|
+
}, this.events.onWarn)
|
|
176
380
|
} catch (err) {
|
|
177
381
|
this.vectorsDirty = true // full rewrite on next flushVectors
|
|
178
382
|
this.events.onWarn?.(`vec: append failed: ${(err ).message}`)
|
|
@@ -190,16 +394,53 @@ export class ScopeStore {
|
|
|
190
394
|
}
|
|
191
395
|
}
|
|
192
396
|
|
|
193
|
-
/** Atomic full rewrite: identity header + every in-memory vector. */
|
|
397
|
+
/** Atomic full rewrite: identity header + every in-memory vector (merged with disk). */
|
|
194
398
|
writeSidecar() {
|
|
399
|
+
withScopeLock(this.lockPath, () => this.writeSidecarLocked(), this.events.onWarn)
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** 持锁版本:先读盘上同身份的条目再合并,别的进程刚追加的向量不会被覆盖掉。 */
|
|
403
|
+
writeSidecarLocked() {
|
|
195
404
|
mkdirSync(dirname(this.vecFile), { recursive: true })
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
405
|
+
const lines = []
|
|
406
|
+
if (this.vecIdentity) lines.push(`#${this.vecIdentity}`)
|
|
407
|
+
const merged = this.readSidecarEntries()
|
|
408
|
+
for (const [id, v] of this.vectors) merged.set(id, `${id}\t${encodeVec(v)}`)
|
|
409
|
+
for (const line of merged.values()) lines.push(line)
|
|
410
|
+
const tmp = tmpPathFor(this.vecFile)
|
|
199
411
|
writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
|
|
200
412
|
renameSync(tmp, this.vecFile)
|
|
201
413
|
}
|
|
202
414
|
|
|
415
|
+
/**
|
|
416
|
+
* 磁盘边车里"身份一致"的条目(id → 原始行)。身份不同就返回空(整份重写),
|
|
417
|
+
* 没头的旧文件也不合并 —— 那些向量不属于当前模型。
|
|
418
|
+
*/
|
|
419
|
+
readSidecarEntries() {
|
|
420
|
+
const out = new Map ()
|
|
421
|
+
if (!existsSync(this.vecFile)) return out
|
|
422
|
+
try {
|
|
423
|
+
const text = readFileSync(this.vecFile, 'utf8')
|
|
424
|
+
let sawHeader = false
|
|
425
|
+
for (const line of text.split('\n')) {
|
|
426
|
+
const trimmed = line.trim()
|
|
427
|
+
if (!trimmed) continue
|
|
428
|
+
if (trimmed.startsWith('#')) {
|
|
429
|
+
sawHeader = true
|
|
430
|
+
if (!this.identityMatches(trimmed.slice(1))) return new Map()
|
|
431
|
+
continue
|
|
432
|
+
}
|
|
433
|
+
if (!sawHeader && this.vecIdentity) return new Map()
|
|
434
|
+
const tab = trimmed.indexOf('\t')
|
|
435
|
+
if (tab <= 0) continue
|
|
436
|
+
out.set(trimmed.slice(0, tab), trimmed)
|
|
437
|
+
}
|
|
438
|
+
} catch {
|
|
439
|
+
return new Map()
|
|
440
|
+
}
|
|
441
|
+
return out
|
|
442
|
+
}
|
|
443
|
+
|
|
203
444
|
/** Rewrite the whole sidecar (after deletes / compaction / append failures). */
|
|
204
445
|
flushVectors() {
|
|
205
446
|
if (!this.vectorsDirty) return
|
|
@@ -231,7 +472,7 @@ export class ScopeStore {
|
|
|
231
472
|
const trimmed = line.trim()
|
|
232
473
|
if (!trimmed) continue
|
|
233
474
|
if (trimmed.startsWith('#')) {
|
|
234
|
-
if (this.
|
|
475
|
+
if (!this.identityMatches(trimmed.slice(1))) {
|
|
235
476
|
discard(`identity mismatch (${trimmed.slice(1)} != ${this.vecIdentity})`)
|
|
236
477
|
return
|
|
237
478
|
}
|
|
@@ -282,6 +523,11 @@ function decodeVec(text ) {
|
|
|
282
523
|
|
|
283
524
|
|
|
284
525
|
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
|
|
285
531
|
|
|
286
532
|
|
|
287
533
|
export class MemoryStore {
|
|
@@ -293,19 +539,23 @@ export class MemoryStore {
|
|
|
293
539
|
fileNames = new Map ()
|
|
294
540
|
flushTimer = null
|
|
295
541
|
events
|
|
296
|
-
|
|
542
|
+
refreshTtlMs
|
|
543
|
+
/** `model\tdims[\tpooling]` handed to every ScopeStore for sidecar validation. */
|
|
297
544
|
vecIdentity = ''
|
|
298
545
|
|
|
299
546
|
constructor(root , events = {}) {
|
|
300
547
|
this.root = root
|
|
301
548
|
this.events = events
|
|
549
|
+
this.refreshTtlMs = events.refreshTtlMs ?? 1_000
|
|
302
550
|
mkdirSync(join(root, 'projects'), { recursive: true })
|
|
303
551
|
}
|
|
304
552
|
|
|
305
553
|
/** Declare the embedding identity; applies to existing and future scopes. */
|
|
306
|
-
setVecIdentity(model , dims ) {
|
|
307
|
-
this.vecIdentity =
|
|
308
|
-
|
|
554
|
+
setVecIdentity(model , dims , pooling ) {
|
|
555
|
+
this.vecIdentity = pooling !== undefined && pooling !== '' && pooling !== 'auto'
|
|
556
|
+
? `${model}\t${dims}\t${pooling}`
|
|
557
|
+
: `${model}\t${dims}`
|
|
558
|
+
for (const s of this.scopes.values()) s.setVecIdentity(model, dims, pooling)
|
|
309
559
|
}
|
|
310
560
|
|
|
311
561
|
/** Preload specific scopes eagerly (e.g. defaults); others load on first access. */
|
|
@@ -315,50 +565,55 @@ export class MemoryStore {
|
|
|
315
565
|
|
|
316
566
|
loadScope(scope ) {
|
|
317
567
|
const file = this.fileFor(scope)
|
|
318
|
-
if (!fileExists(file)) return
|
|
319
|
-
let text
|
|
320
|
-
try {
|
|
321
|
-
text = readFileSync(file, 'utf8')
|
|
322
|
-
} catch (err) {
|
|
323
|
-
this.events.onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
|
|
324
|
-
return
|
|
325
|
-
}
|
|
326
568
|
const store = this.scope(scope)
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
if (!trimmed) continue
|
|
332
|
-
try {
|
|
333
|
-
const rec = JSON.parse(trimmed)
|
|
334
|
-
if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
|
|
335
|
-
throw new Error('bad shape')
|
|
336
|
-
}
|
|
337
|
-
if (rec.scope !== scope) continue // file/scope mismatch: ignore, will be rewritten on flush
|
|
338
|
-
store.records.set(rec.id, rec)
|
|
339
|
-
} catch {
|
|
340
|
-
this.events.onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
|
|
341
|
-
}
|
|
569
|
+
if (!fileExists(file)) {
|
|
570
|
+
store.stamp = 'missing'
|
|
571
|
+
store.checkedAt = Date.now()
|
|
572
|
+
return
|
|
342
573
|
}
|
|
574
|
+
store.replaceRecords(readRecords(file, scope, (m) => this.events.onWarn?.(m)))
|
|
343
575
|
// sidecar vectors second; anything still un-embedded joins the backfill queue
|
|
344
576
|
store.loadVectors()
|
|
345
577
|
for (const id of store.records.keys()) {
|
|
346
578
|
if (!store.vectors.has(id)) store.embedPending.add(id)
|
|
347
579
|
}
|
|
348
|
-
store.
|
|
580
|
+
store.stamp = stampOf(file)
|
|
581
|
+
store.checkedAt = Date.now()
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
/**
|
|
585
|
+
* 别的进程可能改了同一个 scope:读路径按 refreshTtlMs 限流地 stat 一次,
|
|
586
|
+
* mtime/size 变了就重新读盘(还没落盘的增量交给 flush 在锁内合并)。
|
|
587
|
+
*/
|
|
588
|
+
refreshFromDisk(s ) {
|
|
589
|
+
if (this.refreshTtlMs <= 0) return
|
|
590
|
+
const now = Date.now()
|
|
591
|
+
if (now - s.checkedAt < this.refreshTtlMs) return
|
|
592
|
+
s.checkedAt = now
|
|
593
|
+
if (s.dirty) return
|
|
594
|
+
const file = this.fileFor(s.scope)
|
|
595
|
+
const stamp = stampOf(file)
|
|
596
|
+
if (stamp === s.stamp) return
|
|
597
|
+
s.replaceRecords(stamp === 'missing' ? new Map() : readRecords(file, s.scope, (m) => this.events.onWarn?.(m)))
|
|
598
|
+
s.stamp = stamp
|
|
349
599
|
}
|
|
350
600
|
|
|
351
601
|
scope(scope ) {
|
|
352
602
|
let s = this.scopes.get(scope)
|
|
353
603
|
if (!s) {
|
|
354
604
|
s = new ScopeStore(scope, this.root, this.events)
|
|
355
|
-
if (this.vecIdentity)
|
|
605
|
+
if (this.vecIdentity) {
|
|
606
|
+
const [model, dims, pooling] = this.vecIdentity.split('\t')
|
|
607
|
+
s.setVecIdentity(model , Number(dims), pooling)
|
|
608
|
+
}
|
|
356
609
|
this.scopes.set(scope, s)
|
|
357
610
|
this.fileNames.set(scope, scopeFileName(scope))
|
|
358
611
|
}
|
|
359
612
|
if (!this.loaded.has(scope)) {
|
|
360
613
|
this.loaded.add(scope)
|
|
361
614
|
this.loadScope(scope)
|
|
615
|
+
} else {
|
|
616
|
+
this.refreshFromDisk(s)
|
|
362
617
|
}
|
|
363
618
|
return s
|
|
364
619
|
}
|
|
@@ -372,7 +627,10 @@ export class MemoryStore {
|
|
|
372
627
|
*/
|
|
373
628
|
peek(scope ) {
|
|
374
629
|
const existing = this.scopes.get(scope)
|
|
375
|
-
if (existing)
|
|
630
|
+
if (existing) {
|
|
631
|
+
this.refreshFromDisk(existing)
|
|
632
|
+
return existing
|
|
633
|
+
}
|
|
376
634
|
const file = join(this.root, scopeFileName(scope))
|
|
377
635
|
if (!fileExists(file)) return undefined
|
|
378
636
|
return this.scope(scope)
|
|
@@ -452,33 +710,52 @@ export class MemoryStore {
|
|
|
452
710
|
}
|
|
453
711
|
|
|
454
712
|
/**
|
|
455
|
-
* Write
|
|
713
|
+
* Write the pending deltas of every dirty scope.
|
|
456
714
|
*
|
|
457
|
-
*
|
|
458
|
-
*
|
|
459
|
-
*
|
|
715
|
+
* 每个 scope 走一次"锁内重新读盘 → 合并增量 → 原子写":不再用内存快照整份覆盖,
|
|
716
|
+
* 所以另一个进程(比如把同一个 storeDir 挂在 MCP server 上)刚写的记录不会消失。
|
|
717
|
+
* 一个被删空的 scope(合并后 0 条记录)**不再落 0 字节文件**,而是把 .jsonl 与 .vec
|
|
718
|
+
* 一起删掉 —— 但前提是磁盘上确实也空了(别人写的记录不算"删空")。
|
|
719
|
+
* `global` 永不自动删除(见 isRemovable)。
|
|
460
720
|
*/
|
|
461
721
|
flush() {
|
|
462
722
|
for (const s of this.scopes.values()) {
|
|
463
723
|
if (!s.dirty) continue
|
|
464
|
-
|
|
465
|
-
s.removeFiles()
|
|
466
|
-
s.dirty = false
|
|
467
|
-
s.vectorsDirty = false
|
|
468
|
-
continue
|
|
469
|
-
}
|
|
470
|
-
const file = this.fileFor(s.scope)
|
|
471
|
-
mkdirSync(dirname(file), { recursive: true })
|
|
472
|
-
const tmp = `${file}.tmp`
|
|
473
|
-
const lines = [...s.records.values()].map((r) => JSON.stringify(r))
|
|
474
|
-
writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
|
|
475
|
-
renameSync(tmp, file)
|
|
476
|
-
s.dirty = false
|
|
724
|
+
this.flushScope(s)
|
|
477
725
|
}
|
|
478
726
|
// vectors flush too: they are the expensive-to-recompute half
|
|
479
727
|
for (const s of this.scopes.values()) s.flushVectors()
|
|
480
728
|
}
|
|
481
729
|
|
|
730
|
+
/** 锁内合并一个 scope 的增量(同步;调用方不持有这把锁)。 */
|
|
731
|
+
flushScope(s ) {
|
|
732
|
+
const file = this.fileFor(s.scope)
|
|
733
|
+
// 增量在等锁期间不会变(同步执行),先拷一份,锁里只做合并与写
|
|
734
|
+
const upserts = new Map(s.upserts)
|
|
735
|
+
const deletes = new Set(s.deletes)
|
|
736
|
+
withScopeLock(s.lockPath, () => {
|
|
737
|
+
const merged = fileExists(file)
|
|
738
|
+
? readRecords(file, s.scope, (m) => this.events.onWarn?.(m))
|
|
739
|
+
: new Map ()
|
|
740
|
+
for (const [id, rec] of upserts) merged.set(id, rec)
|
|
741
|
+
for (const id of deletes) merged.delete(id)
|
|
742
|
+
if (merged.size === 0 && this.isRemovable(s.scope)) {
|
|
743
|
+
s.removeFiles()
|
|
744
|
+
// 边车已经被一起删掉了;别让随后的 flushVectors 又把它写回来
|
|
745
|
+
s.vectors.clear()
|
|
746
|
+
s.embedPending.clear()
|
|
747
|
+
s.vectorsDirty = false
|
|
748
|
+
} else {
|
|
749
|
+
writeJsonlAtomic(file, merged)
|
|
750
|
+
}
|
|
751
|
+
s.replaceRecords(merged)
|
|
752
|
+
for (const id of upserts.keys()) s.upserts.delete(id)
|
|
753
|
+
for (const id of deletes) s.deletes.delete(id)
|
|
754
|
+
s.stamp = stampOf(file)
|
|
755
|
+
s.checkedAt = Date.now()
|
|
756
|
+
}, this.events.onWarn)
|
|
757
|
+
}
|
|
758
|
+
|
|
482
759
|
/** Synchronous final flush — call on dispose. */
|
|
483
760
|
close() {
|
|
484
761
|
if (this.flushTimer) {
|
|
@@ -504,7 +781,7 @@ export class MemoryStore {
|
|
|
504
781
|
rec.accessCount += 1
|
|
505
782
|
rec.importance = Math.max(rec.importance, input.importance ?? defaults.importance)
|
|
506
783
|
if (input.supersedes) rec.supersedes = input.supersedes
|
|
507
|
-
s.
|
|
784
|
+
s.insert(rec) // 去重命中的也是一次写:进增量,flush 时落盘
|
|
508
785
|
return { record: rec, deduped: true }
|
|
509
786
|
}
|
|
510
787
|
}
|