@restregular/dsh-memory 0.9.10 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/client.js CHANGED
@@ -63,12 +63,15 @@ window.__ModuleLoader__.load({
63
63
  '.memp button:focus{outline:none}',
64
64
  '.memp button:focus-visible{outline:2px solid var(--dsw-alias-brand-primary);outline-offset:1px}',
65
65
  '.memp button:disabled{opacity:.5;cursor:default;border-color:var(--dsw-alias-border-l1);color:var(--dsw-alias-label-primary)}',
66
- // 服务器开关(switch 造型)
66
+ // 服务器开关(switch 造型)。配色照抄宿主 [role=switch]:
67
+ // - 关闭态轨道 border-l3(border-l1 只有 ~6% 透明度,深浅色下都几乎看不见);
68
+ // - 圆点关闭态 switch-thumb、开启态 label-primary-foreground。
69
+ // 深色主题的 brand-primary 是近白色,圆点写死 #fff 会直接融进轨道里。
67
70
  '.memp .toggle{display:inline-flex;align-items:center;gap:8px;padding:4px 12px 4px 5px}',
68
- '.memp .toggle-track{width:34px;height:18px;border-radius:999px;background:var(--dsw-alias-border-l1);position:relative;transition:background .15s ease;flex:none;display:inline-block}',
71
+ '.memp .toggle-track{width:34px;height:18px;border-radius:999px;background:var(--dsw-alias-border-l3);position:relative;transition:background .15s ease;flex:none;display:inline-block}',
69
72
  '.memp .toggle.on .toggle-track{background:var(--dsw-alias-brand-primary)}',
70
- '.memp .toggle-knob{position:absolute;top:2px;left:2px;width:14px;height:14px;border-radius:50%;background:#fff;box-shadow:0 1px 3px rgba(0,0,0,.25);transition:transform .15s ease}',
71
- '.memp .toggle.on .toggle-knob{transform:translateX(16px)}',
73
+ '.memp .toggle-knob{position:absolute;top:2px;left:2px;width:14px;height:14px;border-radius:50%;background:var(--dsw-alias-switch-thumb);box-shadow:0 1px 3px rgba(0,0,0,.25);transition:transform .15s ease,background .15s ease}',
74
+ '.memp .toggle.on .toggle-knob{transform:translateX(16px);background:var(--dsw-alias-label-primary-foreground)}',
72
75
  // 记忆列表:一条一行时信息密度优先
73
76
  '.memp .list{display:flex;flex-direction:column;gap:8px}',
74
77
  '.memp .rec{border:1px solid var(--dsw-alias-border-l1);border-radius:10px;padding:9px 12px;background:var(--dsw-alias-bg-layer-1)}',
@@ -158,11 +161,13 @@ window.__ModuleLoader__.load({
158
161
  '.memp .setrow .set-desc{flex:1 1 100%;color:var(--dsw-alias-label-secondary);font-size:12px;line-height:1.5;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}',
159
162
  '.memp .setrow .set-field{display:inline-flex;align-items:center;gap:5px;font-size:12px;color:var(--dsw-alias-label-secondary)}',
160
163
  '.memp .setrow input[type=number]{width:78px}',
161
- // 开关:与向量服务的 toggle 同一套视觉(轨道 + 圆点)
162
- '.memp .switch{position:relative;width:34px;height:20px;padding:0;border:0;border-radius:999px;background:var(--dsw-alias-border-l1);cursor:pointer;transition:background .12s ease}',
164
+ // 开关:与向量服务的 toggle 同一套视觉(轨道 + 圆点),配色同宿主 switch。
165
+ // 深色主题下开启态轨道是近白色,圆点必须用 label-primary-foreground 才看得见;
166
+ // 关闭态轨道用 border-l3、圆点用 switch-thumb,避免"只剩一个白点"。
167
+ '.memp .switch{position:relative;width:34px;height:20px;padding:0;border:0;border-radius:999px;background:var(--dsw-alias-border-l3);cursor:pointer;transition:background .12s ease}',
163
168
  '.memp .switch.on{background:var(--dsw-alias-brand-primary)}',
164
- '.memp .switch .knob{position:absolute;top:2px;left:2px;width:16px;height:16px;border-radius:50%;background:#fff;transition:left .12s ease;box-shadow:0 1px 2px rgba(0,0,0,.25)}',
165
- '.memp .switch.on .knob{left:16px}',
169
+ '.memp .switch .knob{position:absolute;top:2px;left:2px;width:16px;height:16px;border-radius:50%;background:var(--dsw-alias-switch-thumb);transition:left .12s ease,background .12s ease;box-shadow:0 1px 2px rgba(0,0,0,.25)}',
170
+ '.memp .switch.on .knob{left:16px;background:var(--dsw-alias-label-primary-foreground)}',
166
171
  '.memp .setfoot{display:flex;align-items:center;gap:10px;padding-top:12px}',
167
172
  // 新版设置页:卡片 + 可见 label + 状态文案 + 字段说明
168
173
  '.memp .setp{display:flex;flex-direction:column;gap:10px}',
package/lib/panel.js CHANGED
@@ -456,7 +456,8 @@ async function handle(ctx, routePath, req, res) {
456
456
  if (body.tags !== undefined) rec.tags = Array.isArray(body.tags) ? body.tags.map(String) : []
457
457
  if (body.supersededBy !== undefined) rec.supersedes = body.supersededBy === null ? undefined : String(body.supersededBy)
458
458
  rec.updatedAt = Date.now()
459
- s.dirty = true
459
+ // 改动本身要进增量(flush 只落增量,不再整份快照覆盖);向量作废靠 vectorsDirty。
460
+ s.insert(rec)
460
461
  store.scheduleFlush()
461
462
  // 立即重建向量(不等 60s 的定时回填);失败就留在 pending 里由重试兜底。
462
463
  if (reindex && typeof memory.reindexRecord === 'function') {
@@ -508,7 +508,9 @@ export class LocalMemoryService {
508
508
  // so loading first would accept vectors from a previous embedding model
509
509
  // whenever the dimensions matched — and the next flush would then rewrite the
510
510
  // header as the new model, making the mismatch permanent.
511
- this.store.setVecIdentity(this.cfg.vector.model, this.cfg.vector.dims)
511
+ // pooling 也进身份(issue #3):同一个模型换 --pooling 等于换了一个向量空间
512
+ // (last 与 cls 的余弦只有 0.35),以前它不在 identity 里,校验会静默放行。
513
+ this.store.setVecIdentity(this.cfg.vector.model, this.cfg.vector.dims, this.cfg.vector.pooling)
512
514
  // 会话 scope 是按会话动态生成的,**不能**在这里 eager 加载(加载谁?);只预载
513
515
  // project 与 global 这两个稳定作用域。其余按需 peek/loadDiscovered。
514
516
  // eager 只加载"已存在"的 scope(peek),不会为不存在的 scope 建空文件。
@@ -993,7 +995,7 @@ export class LocalMemoryService {
993
995
  removed += local
994
996
  scopes.push(scope.scope)
995
997
  }
996
- try { this.store.flushSync() } catch { /* scheduled flush covers it */ }
998
+ try { this.store.flush() } catch { /* scheduled flush covers it */ }
997
999
  return { removed, scopes }
998
1000
  }
999
1001
 
@@ -1357,7 +1359,6 @@ export class LocalMemoryService {
1357
1359
  total += 1
1358
1360
  }
1359
1361
  s.vectorsDirty = true
1360
- s.dirty = true
1361
1362
  }
1362
1363
  this.store.scheduleFlush()
1363
1364
  void this.backfillAll({ throttleMs: 50 }).catch(() => { /* 失败留给 60s 重试 */ })
@@ -3,7 +3,7 @@
3
3
  * atomic compaction (tmp + rename). Vectors live in memory for M1; the .vec
4
4
  * sidecar format arrives in M1.5.
5
5
  */
6
- import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
6
+ import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
7
7
  import { createHash, randomUUID } from 'node:crypto'
8
8
  import { Buffer } from 'node:buffer'
9
9
  import { dirname, join } from 'node:path'
@@ -76,22 +76,195 @@ export function newId() {
76
76
  return randomUUID()
77
77
  }
78
78
 
79
+ // ---------------------------------------------------------------- 跨进程写锁
80
+
81
+ /** 等锁的上限:超过就告警并"不带锁照写"(宁可少一次合并,也不要丢掉这次写入)。 */
82
+ const LOCK_TIMEOUT_MS = 5_000
83
+ /** 超过这个时长没被碰过的锁视为陈旧(进程被 kill -9 时不会自己清理)。 */
84
+ const LOCK_STALE_MS = 10_000
85
+
86
+ /** 同步睡眠,不占 CPU:锁等待用(必须同步 —— flush/close 的调用方都是同步路径)。 */
87
+ function sleepSync(ms ) {
88
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
89
+ }
90
+
91
+ function processAlive(pid ) {
92
+ try {
93
+ process.kill(pid, 0)
94
+ return true
95
+ } catch (err) {
96
+ // EPERM = 进程存在但没有权限发信号
97
+ return (err ).code === 'EPERM'
98
+ }
99
+ }
100
+
101
+ /** 锁是不是陈旧的(拥有者已死 / 太久没动)?是则删掉并返回 true。 */
102
+ function breakStaleLock(lockPath ) {
103
+ let age
104
+ try {
105
+ age = Date.now() - statSync(lockPath).mtimeMs
106
+ } catch {
107
+ return true // 已经不在了,直接重试
108
+ }
109
+ if (age > LOCK_STALE_MS) {
110
+ try { rmSync(lockPath, { force: true }) } catch { /* 下一轮再试 */ }
111
+ return true
112
+ }
113
+ let pid = 0
114
+ try { pid = Number.parseInt(readFileSync(lockPath, 'utf8').trim(), 10) } catch { pid = 0 }
115
+ if (pid > 0 && !processAlive(pid)) {
116
+ try { rmSync(lockPath, { force: true }) } catch { /* 下一轮再试 */ }
117
+ return true
118
+ }
119
+ return false
120
+ }
121
+
122
+ /** 本进程已持有的锁(可重入:flush 持锁时会再调 writeSidecar,不能自己把自己锁死)。 */
123
+ const heldLocks = new Map ()
124
+
125
+ /**
126
+ * 取某个 scope 的写锁(锁文件 = `${数据文件}.lock`,`openSync(wx)` 的 O_EXCL 语义)。
127
+ *
128
+ * 为什么需要它:`loadScope()` 只读盘一次、`flush()` 又全量重写 —— 两个进程共用一个
129
+ * storeDir 时,后 flush 的那个会用自己那份旧快照把对方的记录整条抹掉(见 issue #3)。
130
+ * 现在所有写都是"锁内重新读盘 → 合并增量 → 原子写",锁只是把这两步串起来。
131
+ *
132
+ * Windows:`openSync(wx)` 在"文件已存在、但正处于删除过程中"时报 EPERM/EACCES 而不是
133
+ * EEXIST,所以这两个码也按"已存在"处理,否则并发下会直接崩。
134
+ */
135
+ function withScopeLock (lockPath , fn , onWarn ) {
136
+ const depth = heldLocks.get(lockPath)
137
+ if (depth !== undefined) {
138
+ heldLocks.set(lockPath, depth + 1)
139
+ try {
140
+ return fn()
141
+ } finally {
142
+ const left = (heldLocks.get(lockPath) ?? 1) - 1
143
+ if (left <= 0) heldLocks.delete(lockPath)
144
+ else heldLocks.set(lockPath, left)
145
+ }
146
+ }
147
+ mkdirSync(dirname(lockPath), { recursive: true })
148
+ let fd = null
149
+ let delay = 2
150
+ const deadline = Date.now() + LOCK_TIMEOUT_MS
151
+ while (fd === null) {
152
+ try {
153
+ fd = openSync(lockPath, 'wx')
154
+ } catch (err) {
155
+ const code = (err ).code
156
+ if (code !== 'EEXIST' && code !== 'EPERM' && code !== 'EACCES') throw err
157
+ const broke = breakStaleLock(lockPath)
158
+ if (!broke && Date.now() >= deadline) {
159
+ onWarn?.(`store: lock ${lockPath} timed out after ${LOCK_TIMEOUT_MS}ms; writing without it`)
160
+ return fn()
161
+ }
162
+ if (!broke) {
163
+ sleepSync(delay)
164
+ delay = Math.min(delay * 2, 50)
165
+ }
166
+ }
167
+ }
168
+ try { writeFileSync(fd, String(process.pid), 'utf8') } catch { /* pid 只是给接管用的提示 */ }
169
+ heldLocks.set(lockPath, 1)
170
+ try {
171
+ return fn()
172
+ } finally {
173
+ heldLocks.delete(lockPath)
174
+ try { closeSync(fd) } catch { /* 已经关了 */ }
175
+ try { rmSync(lockPath, { force: true }) } catch { /* 下一次 breakStale 会兜底 */ }
176
+ }
177
+ }
178
+
179
+ /** tmp 名带 pid + 随机后缀:两个进程并发写同一个 scope 时不会撞名破坏原子性。 */
180
+ function tmpPathFor(file ) {
181
+ return `${file}.${process.pid}.${randomUUID().slice(0, 8)}.tmp`
182
+ }
183
+
184
+ /** 原子写整个 JSONL(tmp + rename)。调用方必须持有该 scope 的锁。 */
185
+ function writeJsonlAtomic(file , records ) {
186
+ mkdirSync(dirname(file), { recursive: true })
187
+ const tmp = tmpPathFor(file)
188
+ const lines = [...records.values()].map((r) => JSON.stringify(r))
189
+ writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
190
+ renameSync(tmp, file)
191
+ }
192
+
193
+ /** 文件的时间戳快照,用来判断"别的进程有没有改过它"。 */
194
+ function stampOf(file ) {
195
+ try {
196
+ const st = statSync(file)
197
+ return `${st.mtimeMs}:${st.size}`
198
+ } catch {
199
+ return 'missing'
200
+ }
201
+ }
202
+
203
+ /** 读一个 scope 文件成 Map(坏行跳过;scope 不符的行忽略,flush 时会清掉)。 */
204
+ function readRecords(file , scope , onWarn ) {
205
+ const out = new Map ()
206
+ let text
207
+ try {
208
+ text = readFileSync(file, 'utf8')
209
+ } catch (err) {
210
+ if ((err ).code !== 'ENOENT') onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
211
+ return out
212
+ }
213
+ let lineNo = 0
214
+ for (const line of text.split('\n')) {
215
+ lineNo++
216
+ const trimmed = line.trim()
217
+ if (!trimmed) continue
218
+ try {
219
+ const rec = JSON.parse(trimmed)
220
+ if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
221
+ throw new Error('bad shape')
222
+ }
223
+ if (rec.scope !== scope) continue
224
+ out.set(rec.id, rec)
225
+ } catch {
226
+ onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
227
+ }
228
+ }
229
+ return out
230
+ }
231
+
79
232
  export class ScopeStore {
80
233
  scope
81
234
  records = new Map ()
82
235
  vectors = new Map ()
83
236
  /** ids saved while the embed backend was down; retried opportunistically. */
84
237
  embedPending = new Set ()
85
- dirty = false
238
+ /**
239
+ * 待落盘的**增量**(不是整份快照)。flush 时把它合并进"锁内重新读到的磁盘内容",
240
+ * 而不是拿内存覆盖磁盘 —— 这是多个进程共用同一个 storeDir 时互不抹数据的前提。
241
+ */
242
+ upserts = new Map ()
243
+ /** 本进程删掉的 id;flush 时从磁盘内容里移除。 */
244
+ deletes = new Set ()
86
245
  /** true once vectors were touched since the last .vec flush. */
87
246
  vectorsDirty = false
88
247
  /** 数据文件路径(写、以及删空后移除都走它)。 */
89
248
  file
249
+ /** 最近一次与磁盘同步后的 `mtime:size`;用于发现别的进程改过这个 scope。 */
250
+ stamp = ''
251
+ /** 上次检查外部改动的时间(读路径限流用)。 */
252
+ checkedAt = 0
90
253
  vecFile
91
254
  events
92
255
  /** Embedding identity for sidecar validation; set via setVecIdentity(). */
93
256
  vecIdentity = ''
94
257
 
258
+ /** 有增量待写才算脏(只有向量变动时不脏,走 vectorsDirty)。 */
259
+ get dirty() {
260
+ return this.upserts.size > 0 || this.deletes.size > 0
261
+ }
262
+
263
+ /** 这个 scope 的跨进程写锁(JSONL 与 .vec 共用一把,避免两边交叉覆盖)。 */
264
+ get lockPath() {
265
+ return `${this.file}.lock`
266
+ }
267
+
95
268
  constructor(scope , root , events = {}) {
96
269
  this.scope = scope
97
270
  this.file = join(root, scopeFileName(scope))
@@ -105,8 +278,26 @@ export class ScopeStore {
105
278
  * re-enters the embed queue — old vectors for a new model are wrong, not
106
279
  * merely stale.
107
280
  */
108
- setVecIdentity(model , dims ) {
109
- this.vecIdentity = `${model}\t${dims}`
281
+ setVecIdentity(model , dims , pooling ) {
282
+ this.vecIdentity = pooling !== undefined && pooling !== '' && pooling !== 'auto'
283
+ ? `${model}\t${dims}\t${pooling}`
284
+ : `${model}\t${dims}`
285
+ }
286
+
287
+ /**
288
+ * 边车身份对不对。老边车只有 `model\tdims` 两段(0.10.0 之前不记 pooling),
289
+ * 缺 pooling 时按兼容接受 —— 不能因为多记了一个字段就让所有人的向量全量作废。
290
+ * 两边都有 pooling 时才严格比:这正是 issue #3 里那条「用错 --pooling 会静默
291
+ * 把整份边车作废、而且 identity 校验会放行」的修法。
292
+ */
293
+ identityMatches(header ) {
294
+ if (!this.vecIdentity) return true
295
+ if (!header) return false
296
+ const mine = this.vecIdentity.split('\t')
297
+ const theirs = header.split('\t')
298
+ if (mine[0] !== theirs[0] || mine[1] !== theirs[1]) return false
299
+ if (mine[2] !== undefined && theirs[2] !== undefined) return mine[2] === theirs[2]
300
+ return true
110
301
  }
111
302
 
112
303
  get size() {
@@ -115,7 +306,8 @@ export class ScopeStore {
115
306
 
116
307
  insert(record ) {
117
308
  this.records.set(record.id, record)
118
- this.dirty = true
309
+ this.upserts.set(record.id, record)
310
+ this.deletes.delete(record.id)
119
311
  }
120
312
 
121
313
  remove(id ) {
@@ -123,12 +315,19 @@ export class ScopeStore {
123
315
  if (had) {
124
316
  this.vectors.delete(id)
125
317
  this.embedPending.delete(id)
126
- this.dirty = true
318
+ this.upserts.delete(id)
319
+ this.deletes.add(id)
127
320
  this.vectorsDirty = true
128
321
  }
129
322
  return had
130
323
  }
131
324
 
325
+ /** 合并后把缓存换成磁盘上的内容(增量已并入),清掉已落盘的那部分增量。 */
326
+ replaceRecords(next ) {
327
+ this.records.clear()
328
+ for (const [id, rec] of next) this.records.set(id, rec)
329
+ }
330
+
132
331
  /**
133
332
  * Delete this scope's data files(.jsonl 与 .vec 边车)。
134
333
  *
@@ -165,14 +364,19 @@ export class ScopeStore {
165
364
  this.setVector(id, vec)
166
365
  try {
167
366
  mkdirSync(dirname(this.vecFile), { recursive: true })
168
- if (this.vecIdentity && this.readHeader() !== this.vecIdentity) {
169
- // missing, legacy, or wrong-model file: rewrite fresh from memory
170
- this.writeSidecar()
367
+ withScopeLock(this.lockPath, () => {
368
+ const header = this.readHeader()
369
+ // 缺文件 / 别人的模型 / 旧的两段头(得把 pooling 记进去)→ 整份重写
370
+ const staleHeader = this.vecIdentity !== '' &&
371
+ (!this.identityMatches(header) || header.split('\t').length < this.vecIdentity.split('\t').length)
372
+ if (staleHeader) {
373
+ this.writeSidecarLocked()
374
+ this.vectorsDirty = false
375
+ return
376
+ }
377
+ appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
171
378
  this.vectorsDirty = false
172
- return
173
- }
174
- appendFileSync(this.vecFile, `${id}\t${encodeVec(vec)}\n`, 'utf8')
175
- this.vectorsDirty = false
379
+ }, this.events.onWarn)
176
380
  } catch (err) {
177
381
  this.vectorsDirty = true // full rewrite on next flushVectors
178
382
  this.events.onWarn?.(`vec: append failed: ${(err ).message}`)
@@ -190,16 +394,53 @@ export class ScopeStore {
190
394
  }
191
395
  }
192
396
 
193
- /** Atomic full rewrite: identity header + every in-memory vector. */
397
+ /** Atomic full rewrite: identity header + every in-memory vector (merged with disk). */
194
398
  writeSidecar() {
399
+ withScopeLock(this.lockPath, () => this.writeSidecarLocked(), this.events.onWarn)
400
+ }
401
+
402
+ /** 持锁版本:先读盘上同身份的条目再合并,别的进程刚追加的向量不会被覆盖掉。 */
403
+ writeSidecarLocked() {
195
404
  mkdirSync(dirname(this.vecFile), { recursive: true })
196
- const tmp = `${this.vecFile}.tmp`
197
- const lines = this.vecIdentity ? [`#${this.vecIdentity}`] : []
198
- for (const [id, v] of this.vectors) lines.push(`${id}\t${encodeVec(v)}`)
405
+ const lines = []
406
+ if (this.vecIdentity) lines.push(`#${this.vecIdentity}`)
407
+ const merged = this.readSidecarEntries()
408
+ for (const [id, v] of this.vectors) merged.set(id, `${id}\t${encodeVec(v)}`)
409
+ for (const line of merged.values()) lines.push(line)
410
+ const tmp = tmpPathFor(this.vecFile)
199
411
  writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
200
412
  renameSync(tmp, this.vecFile)
201
413
  }
202
414
 
415
+ /**
416
+ * 磁盘边车里"身份一致"的条目(id → 原始行)。身份不同就返回空(整份重写),
417
+ * 没头的旧文件也不合并 —— 那些向量不属于当前模型。
418
+ */
419
+ readSidecarEntries() {
420
+ const out = new Map ()
421
+ if (!existsSync(this.vecFile)) return out
422
+ try {
423
+ const text = readFileSync(this.vecFile, 'utf8')
424
+ let sawHeader = false
425
+ for (const line of text.split('\n')) {
426
+ const trimmed = line.trim()
427
+ if (!trimmed) continue
428
+ if (trimmed.startsWith('#')) {
429
+ sawHeader = true
430
+ if (!this.identityMatches(trimmed.slice(1))) return new Map()
431
+ continue
432
+ }
433
+ if (!sawHeader && this.vecIdentity) return new Map()
434
+ const tab = trimmed.indexOf('\t')
435
+ if (tab <= 0) continue
436
+ out.set(trimmed.slice(0, tab), trimmed)
437
+ }
438
+ } catch {
439
+ return new Map()
440
+ }
441
+ return out
442
+ }
443
+
203
444
  /** Rewrite the whole sidecar (after deletes / compaction / append failures). */
204
445
  flushVectors() {
205
446
  if (!this.vectorsDirty) return
@@ -231,7 +472,7 @@ export class ScopeStore {
231
472
  const trimmed = line.trim()
232
473
  if (!trimmed) continue
233
474
  if (trimmed.startsWith('#')) {
234
- if (this.vecIdentity && trimmed.slice(1) !== this.vecIdentity) {
475
+ if (!this.identityMatches(trimmed.slice(1))) {
235
476
  discard(`identity mismatch (${trimmed.slice(1)} != ${this.vecIdentity})`)
236
477
  return
237
478
  }
@@ -282,6 +523,11 @@ function decodeVec(text ) {
282
523
 
283
524
 
284
525
 
526
+
527
+                            
528
+                              
529
+
530
+
285
531
 
286
532
 
287
533
  export class MemoryStore {
@@ -293,19 +539,23 @@ export class MemoryStore {
293
539
  fileNames = new Map ()
294
540
  flushTimer = null
295
541
  events
296
- /** `model\tdims` handed to every ScopeStore for sidecar validation. */
542
+ refreshTtlMs
543
+ /** `model\tdims[\tpooling]` handed to every ScopeStore for sidecar validation. */
297
544
  vecIdentity = ''
298
545
 
299
546
  constructor(root , events = {}) {
300
547
  this.root = root
301
548
  this.events = events
549
+ this.refreshTtlMs = events.refreshTtlMs ?? 1_000
302
550
  mkdirSync(join(root, 'projects'), { recursive: true })
303
551
  }
304
552
 
305
553
  /** Declare the embedding identity; applies to existing and future scopes. */
306
- setVecIdentity(model , dims ) {
307
- this.vecIdentity = `${model}\t${dims}`
308
- for (const s of this.scopes.values()) s.setVecIdentity(model, dims)
554
+ setVecIdentity(model , dims , pooling ) {
555
+ this.vecIdentity = pooling !== undefined && pooling !== '' && pooling !== 'auto'
556
+ ? `${model}\t${dims}\t${pooling}`
557
+ : `${model}\t${dims}`
558
+ for (const s of this.scopes.values()) s.setVecIdentity(model, dims, pooling)
309
559
  }
310
560
 
311
561
  /** Preload specific scopes eagerly (e.g. defaults); others load on first access. */
@@ -315,50 +565,55 @@ export class MemoryStore {
315
565
 
316
566
  loadScope(scope ) {
317
567
  const file = this.fileFor(scope)
318
- if (!fileExists(file)) return
319
- let text
320
- try {
321
- text = readFileSync(file, 'utf8')
322
- } catch (err) {
323
- this.events.onWarn?.(`store: cannot read ${file}: ${(err ).message}`)
324
- return
325
- }
326
568
  const store = this.scope(scope)
327
- let lineNo = 0
328
- for (const line of text.split('\n')) {
329
- lineNo++
330
- const trimmed = line.trim()
331
- if (!trimmed) continue
332
- try {
333
- const rec = JSON.parse(trimmed)
334
- if (!rec || typeof rec.id !== 'string' || typeof rec.content !== 'string' || typeof rec.scope !== 'string') {
335
- throw new Error('bad shape')
336
- }
337
- if (rec.scope !== scope) continue // file/scope mismatch: ignore, will be rewritten on flush
338
- store.records.set(rec.id, rec)
339
- } catch {
340
- this.events.onWarn?.(`store: skipping malformed ${file}:${lineNo}`)
341
- }
569
+ if (!fileExists(file)) {
570
+ store.stamp = 'missing'
571
+ store.checkedAt = Date.now()
572
+ return
342
573
  }
574
+ store.replaceRecords(readRecords(file, scope, (m) => this.events.onWarn?.(m)))
343
575
  // sidecar vectors second; anything still un-embedded joins the backfill queue
344
576
  store.loadVectors()
345
577
  for (const id of store.records.keys()) {
346
578
  if (!store.vectors.has(id)) store.embedPending.add(id)
347
579
  }
348
- store.dirty = false
580
+ store.stamp = stampOf(file)
581
+ store.checkedAt = Date.now()
582
+ }
583
+
584
+ /**
585
+ * 别的进程可能改了同一个 scope:读路径按 refreshTtlMs 限流地 stat 一次,
586
+ * mtime/size 变了就重新读盘(还没落盘的增量交给 flush 在锁内合并)。
587
+ */
588
+ refreshFromDisk(s ) {
589
+ if (this.refreshTtlMs <= 0) return
590
+ const now = Date.now()
591
+ if (now - s.checkedAt < this.refreshTtlMs) return
592
+ s.checkedAt = now
593
+ if (s.dirty) return
594
+ const file = this.fileFor(s.scope)
595
+ const stamp = stampOf(file)
596
+ if (stamp === s.stamp) return
597
+ s.replaceRecords(stamp === 'missing' ? new Map() : readRecords(file, s.scope, (m) => this.events.onWarn?.(m)))
598
+ s.stamp = stamp
349
599
  }
350
600
 
351
601
  scope(scope ) {
352
602
  let s = this.scopes.get(scope)
353
603
  if (!s) {
354
604
  s = new ScopeStore(scope, this.root, this.events)
355
- if (this.vecIdentity) s.setVecIdentity(...(this.vecIdentity.split('\t') ))
605
+ if (this.vecIdentity) {
606
+ const [model, dims, pooling] = this.vecIdentity.split('\t')
607
+ s.setVecIdentity(model , Number(dims), pooling)
608
+ }
356
609
  this.scopes.set(scope, s)
357
610
  this.fileNames.set(scope, scopeFileName(scope))
358
611
  }
359
612
  if (!this.loaded.has(scope)) {
360
613
  this.loaded.add(scope)
361
614
  this.loadScope(scope)
615
+ } else {
616
+ this.refreshFromDisk(s)
362
617
  }
363
618
  return s
364
619
  }
@@ -372,7 +627,10 @@ export class MemoryStore {
372
627
  */
373
628
  peek(scope ) {
374
629
  const existing = this.scopes.get(scope)
375
- if (existing) return existing
630
+ if (existing) {
631
+ this.refreshFromDisk(existing)
632
+ return existing
633
+ }
376
634
  const file = join(this.root, scopeFileName(scope))
377
635
  if (!fileExists(file)) return undefined
378
636
  return this.scope(scope)
@@ -452,33 +710,52 @@ export class MemoryStore {
452
710
  }
453
711
 
454
712
  /**
455
- * Write dirty scopes atomically (tmp + rename).
713
+ * Write the pending deltas of every dirty scope.
456
714
  *
457
- * 一个被删空的 scope(内存里 0 条记录)**不再落一个 0 字节文件**,而是把它的 .jsonl 与
458
- * .vec 一起删掉:空文件没有任何信息,反而在面板里留下一条 `(0)` 的作用域、在磁盘上长期堆积。
459
- * 作用域是懒加载的,下次往同一个名字写入时会重新创建。`global` 永不自动删除(见 isRemovable)。
715
+ * 每个 scope 走一次"锁内重新读盘 → 合并增量 → 原子写":不再用内存快照整份覆盖,
716
+ * 所以另一个进程(比如把同一个 storeDir 挂在 MCP server 上)刚写的记录不会消失。
717
+ * 一个被删空的 scope(合并后 0 条记录)**不再落 0 字节文件**,而是把 .jsonl 与 .vec
718
+ * 一起删掉 —— 但前提是磁盘上确实也空了(别人写的记录不算"删空")。
719
+ * `global` 永不自动删除(见 isRemovable)。
460
720
  */
461
721
  flush() {
462
722
  for (const s of this.scopes.values()) {
463
723
  if (!s.dirty) continue
464
- if (s.records.size === 0 && this.isRemovable(s.scope)) {
465
- s.removeFiles()
466
- s.dirty = false
467
- s.vectorsDirty = false
468
- continue
469
- }
470
- const file = this.fileFor(s.scope)
471
- mkdirSync(dirname(file), { recursive: true })
472
- const tmp = `${file}.tmp`
473
- const lines = [...s.records.values()].map((r) => JSON.stringify(r))
474
- writeFileSync(tmp, lines.length ? `${lines.join('\n')}\n` : '', 'utf8')
475
- renameSync(tmp, file)
476
- s.dirty = false
724
+ this.flushScope(s)
477
725
  }
478
726
  // vectors flush too: they are the expensive-to-recompute half
479
727
  for (const s of this.scopes.values()) s.flushVectors()
480
728
  }
481
729
 
730
+ /** 锁内合并一个 scope 的增量(同步;调用方不持有这把锁)。 */
731
+ flushScope(s ) {
732
+ const file = this.fileFor(s.scope)
733
+ // 增量在等锁期间不会变(同步执行),先拷一份,锁里只做合并与写
734
+ const upserts = new Map(s.upserts)
735
+ const deletes = new Set(s.deletes)
736
+ withScopeLock(s.lockPath, () => {
737
+ const merged = fileExists(file)
738
+ ? readRecords(file, s.scope, (m) => this.events.onWarn?.(m))
739
+ : new Map ()
740
+ for (const [id, rec] of upserts) merged.set(id, rec)
741
+ for (const id of deletes) merged.delete(id)
742
+ if (merged.size === 0 && this.isRemovable(s.scope)) {
743
+ s.removeFiles()
744
+ // 边车已经被一起删掉了;别让随后的 flushVectors 又把它写回来
745
+ s.vectors.clear()
746
+ s.embedPending.clear()
747
+ s.vectorsDirty = false
748
+ } else {
749
+ writeJsonlAtomic(file, merged)
750
+ }
751
+ s.replaceRecords(merged)
752
+ for (const id of upserts.keys()) s.upserts.delete(id)
753
+ for (const id of deletes) s.deletes.delete(id)
754
+ s.stamp = stampOf(file)
755
+ s.checkedAt = Date.now()
756
+ }, this.events.onWarn)
757
+ }
758
+
482
759
  /** Synchronous final flush — call on dispose. */
483
760
  close() {
484
761
  if (this.flushTimer) {
@@ -504,7 +781,7 @@ export class MemoryStore {
504
781
  rec.accessCount += 1
505
782
  rec.importance = Math.max(rec.importance, input.importance ?? defaults.importance)
506
783
  if (input.supersedes) rec.supersedes = input.supersedes
507
- s.dirty = true
784
+ s.insert(rec) // 去重命中的也是一次写:进增量,flush 时落盘
508
785
  return { record: rec, deduped: true }
509
786
  }
510
787
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@restregular/dsh-memory",
3
- "version": "0.9.10",
3
+ "version": "0.10.1",
4
4
  "description": "DeepSeek Harness 本地长期记忆(单包单入口):JSONL 存储 + 本地向量检索(BGE-M3)+ 五个模型工具 + 自动召回注入 + 管理面板。",
5
5
  "icon": "./icon.svg",
6
6
  "type": "module",