dsh-memory-tiers 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +420 -2460
- package/README.md +140 -268
- package/client.mjs +701 -608
- package/index.mjs +939 -826
- package/{memory5.mjs → memory.mjs} +5280 -3403
- package/package.json +61 -63
- package/settings.mjs +335 -327
- package/silicon.mjs +24 -22
- package/memory6.mjs +0 -1553
- package/reindex.mjs +0 -162
package/memory6.mjs
DELETED
|
@@ -1,1553 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* memory6.mjs —— 六层 · 六个文件(split 布局)
|
|
3
|
-
*
|
|
4
|
-
* 和 memory5 的区别(设计见 ARCHITECTURE.md 第 26 节):
|
|
5
|
-
*
|
|
6
|
-
* 1. 六个文件六层:L1.json / L2.json / L3.json / DL1.json / DL2.json / DL3.json
|
|
7
|
-
* L1.tmp/ 是 L1 的暂存区(每端一个 append-only 文件),不算第七层。
|
|
8
|
-
* 2. 端只写 L1.tmp/<自己>.jsonl(追加、不调模型);整理者拿钥匙后才动其余文件。
|
|
9
|
-
* 3. 谁先结束谁拿钥匙:open('wx') 抢锁 → rename 认领两边的 temp → 并进 L1 →
|
|
10
|
-
* 按整轮溢出(L1 旧半;L2 旧半 + DL1 全部)→ L3 快照 → 放钥匙。
|
|
11
|
-
* 4. attend / pin / forget 走 signals/<writer>.jsonl,整理者统一消费。
|
|
12
|
-
* 5. 检索按层读文件;detail 顺着摘要的 sources 去 DL2 取原文。
|
|
13
|
-
*
|
|
14
|
-
* 硬约束(破一条就会丢记忆):
|
|
15
|
-
* - L1.tmp 只追加、追完就关(不长期持有句柄,否则 rename 会失败)。
|
|
16
|
-
* - 先写目标层(L2/DL1/DL2/DL3),最后写 L1;崩了重跑靠 id 去重。
|
|
17
|
-
* - 不直接删别人的 temp:rename 认领成功才算自己的,失败就跳过。
|
|
18
|
-
* - 同一 id 只允许出现在一层(读侧也按 id 去重)。
|
|
19
|
-
*/
|
|
20
|
-
|
|
21
|
-
import {
|
|
22
|
-
readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync,
|
|
23
|
-
renameSync, unlinkSync, appendFileSync, statSync, utimesSync,
|
|
24
|
-
openSync, closeSync,
|
|
25
|
-
} from 'node:fs'
|
|
26
|
-
import { join } from 'node:path'
|
|
27
|
-
import {
|
|
28
|
-
LAYER, DEFAULTS, makeItem, heatOf, touchHeat, halfLifeOf,
|
|
29
|
-
cosine, dayKey, DAY, isDeep,
|
|
30
|
-
} from './memory5.mjs'
|
|
31
|
-
|
|
32
|
-
// ============================================================
|
|
33
|
-
// 配置
|
|
34
|
-
// ============================================================
|
|
35
|
-
|
|
36
|
-
export const DEFAULTS6 = {
|
|
37
|
-
...DEFAULTS,
|
|
38
|
-
|
|
39
|
-
// L1 容量:达到就触发整理(不是注入配额)。用户 2026-11-01 定。
|
|
40
|
-
hotMaxChars: 3000,
|
|
41
|
-
hotMaxItems: 200,
|
|
42
|
-
// L1 注入按端配额:自己最多占多少,剩下的给另一端(另一端为空时自己也只到这条线)。
|
|
43
|
-
hotQuotaOwn: 0.7,
|
|
44
|
-
|
|
45
|
-
// 只整理"安静了这么久"的轮次,避免聊到一半被腰斩。
|
|
46
|
-
quietMs: 10 * 60 * 1000,
|
|
47
|
-
// 整理触发:一端结束 + 这个定时器补漏。
|
|
48
|
-
organizeEveryMs: 5 * 60 * 1000,
|
|
49
|
-
// 钥匙多久没心跳算过期(进程崩了也不至于把记忆冻结)。
|
|
50
|
-
staleLockMs: 10 * 60 * 1000,
|
|
51
|
-
// 单轮整理硬超时(模型卡住时别一直占着钥匙)。
|
|
52
|
-
maxOrganizeMs: 5 * 60 * 1000,
|
|
53
|
-
|
|
54
|
-
// 会话正文截断(按整轮拼起来再截,不是每条消息截)。
|
|
55
|
-
botCaptureChars: 400,
|
|
56
|
-
toolCaptureChars: 500,
|
|
57
|
-
|
|
58
|
-
// 摘要批次按 (origin, scene, channel) 分组,不跨场景。
|
|
59
|
-
fusionPrefilter: 0.82,
|
|
60
|
-
forgetThreshold: 0.25,
|
|
61
|
-
longMaxChars: 4000,
|
|
62
|
-
consolidateMaxChars: 2500,
|
|
63
|
-
consolidateMaxItems: 120,
|
|
64
|
-
// 第二次整理(L2 溢出)时,`+` 最多留这么多条在 L2;超出的进 DL1。
|
|
65
|
-
// DL2 只放"被摘要引用的原文",`+` 永远不进 DL2。
|
|
66
|
-
l2KeepMax: 10,
|
|
67
|
-
|
|
68
|
-
// —— 只读层缓存(search / buildBlock / stats)——
|
|
69
|
-
// 深层有十万条时,每次检索都要 JSON.parse 几十 MB,实测 0.4~1.7 秒。
|
|
70
|
-
// 缓存解析结果后是毫秒级。三条上限都是为了"宁可慢,不可崩":
|
|
71
|
-
cacheMaxFileBytes: 64 * 1024 * 1024, // 单文件超过 64MB 不缓存(约 20 万条)
|
|
72
|
-
cacheMaxTotalBytes: 384 * 1024 * 1024, // 缓存总量上限(超出按 LRU 淘汰)
|
|
73
|
-
cacheHeapLimitBytes: 1.5 * 1024 * 1024 * 1024, // 堆已用超过 1.5GB 就放弃缓存
|
|
74
|
-
// 每层的预算占比:按"谁最常被读"分,而不是先到先得。
|
|
75
|
-
// 不分预算的话,一个超大的 DL1 会把 DL2/DL3 全挤出去,
|
|
76
|
-
// 每次检索都在"重读 → 淘汰 → 再重读"之间颠簸(实测比不缓存还慢)。
|
|
77
|
-
// 注意这是**上限**:某层超预算就不缓存它,不会去挤别的层。
|
|
78
|
-
cacheLayerShare: { DL3: 0.4, DL2: 0.3, DL1: 0.3 },
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
const DATA_LAYERS = ['L1', 'L2', 'DL1', 'DL2', 'DL3']
|
|
82
|
-
const ALL_LAYERS = ['L1', 'L2', 'L3', 'DL1', 'DL2', 'DL3']
|
|
83
|
-
|
|
84
|
-
const sanitizeWriter = (w) =>
|
|
85
|
-
String(w || '').trim().replace(/[^A-Za-z0-9_-]/g, '').slice(0, 32) || 'default'
|
|
86
|
-
|
|
87
|
-
const cmpId = (a, b) => (a < b ? -1 : a > b ? 1 : 0)
|
|
88
|
-
const chars = (list) => list.reduce((s, it) => s + String(it?.text || '').length, 0)
|
|
89
|
-
const uniq = (a) => [...new Set(a.filter(Boolean))]
|
|
90
|
-
|
|
91
|
-
function srcKey(items) {
|
|
92
|
-
const s = items.map((it) => it.id).sort().join(',')
|
|
93
|
-
let h = 5381
|
|
94
|
-
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) | 0
|
|
95
|
-
return `k${(h >>> 0).toString(36)}`
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
/**
|
|
99
|
-
* 按轮次取旧的一半:从旧到新整轮拿,直到剩下的字数 ≤ cap/2。
|
|
100
|
-
* 一轮特别大时会把整层取空,这是允许的(用户 2026-11-01 确认)。
|
|
101
|
-
*
|
|
102
|
-
* 导出是为了单测能直接验"不切断问答"这件事。
|
|
103
|
-
*/
|
|
104
|
-
export function pickTurnBatch(items, cap, { orderKey = (it) => it.createdAt, quietBefore = 0, maxItems = Infinity } = {}) {
|
|
105
|
-
if (!items.length) return []
|
|
106
|
-
const total = chars(items)
|
|
107
|
-
if (total <= cap && items.length <= maxItems) return []
|
|
108
|
-
const groups = new Map()
|
|
109
|
-
for (const it of items) {
|
|
110
|
-
const k = it.turnId || it.id
|
|
111
|
-
if (!groups.has(k)) groups.set(k, [])
|
|
112
|
-
groups.get(k).push(it)
|
|
113
|
-
}
|
|
114
|
-
const turns = [...groups.values()]
|
|
115
|
-
.map((g) => ({ items: g, t: Math.min(...g.map(orderKey)) }))
|
|
116
|
-
.sort((a, b) => a.t - b.t)
|
|
117
|
-
// 钉住的轮次永远取不走,不能算进"剩余"里——否则它们占着额度,
|
|
118
|
-
// 其余轮次会被全部取空(不丢数据,但会把 L1 白搬一遍)。
|
|
119
|
-
const pinnedTurns = turns.filter((t) => t.items.some((it) => it.pinned))
|
|
120
|
-
const pinnedChars = pinnedTurns.reduce((s, t) => s + chars(t.items), 0)
|
|
121
|
-
const pinnedItems = pinnedTurns.reduce((s, t) => s + t.items.length, 0)
|
|
122
|
-
let remaining = total - pinnedChars
|
|
123
|
-
let remainingItems = items.length - pinnedItems
|
|
124
|
-
const charTarget = cap / 2
|
|
125
|
-
const itemTarget = maxItems === Infinity ? Infinity : Math.ceil(maxItems / 2)
|
|
126
|
-
const out = []
|
|
127
|
-
for (const turn of turns) {
|
|
128
|
-
if (remaining <= charTarget && remainingItems <= itemTarget) break
|
|
129
|
-
if (turn.items.some((it) => it.pinned)) continue // 钉住的轮次不参与溢出
|
|
130
|
-
if (quietBefore && Math.max(...turn.items.map((it) => it.createdAt)) > quietBefore) break
|
|
131
|
-
out.push(...turn.items)
|
|
132
|
-
remaining -= chars(turn.items)
|
|
133
|
-
remainingItems -= turn.items.length
|
|
134
|
-
}
|
|
135
|
-
return out
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
// ============================================================
|
|
139
|
-
// 小工具
|
|
140
|
-
// ============================================================
|
|
141
|
-
|
|
142
|
-
function atomicWrite(p, body) {
|
|
143
|
-
const tmp = `${p}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 8)}`
|
|
144
|
-
let lastErr = null
|
|
145
|
-
for (let i = 0; i < 6; i++) {
|
|
146
|
-
try {
|
|
147
|
-
writeFileSync(tmp, body, 'utf8')
|
|
148
|
-
renameSync(tmp, p)
|
|
149
|
-
return true
|
|
150
|
-
} catch (e) {
|
|
151
|
-
lastErr = e
|
|
152
|
-
if (!['EBUSY', 'EPERM', 'EACCES', 'ENOENT'].includes(e.code)) break
|
|
153
|
-
// Windows:目标正被读,等一下再试。
|
|
154
|
-
// ★ 不用 while 空转:Node 是单线程,空转 30ms 会把整个进程占满(所有请求都卡住)。
|
|
155
|
-
// Atomics.wait 是同步睡眠,让出 CPU;环境不支持时退回限次空转。
|
|
156
|
-
try {
|
|
157
|
-
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 30)
|
|
158
|
-
} catch {
|
|
159
|
-
const until = Date.now() + 10
|
|
160
|
-
while (Date.now() < until) { /* 退化:最多 10ms */ }
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
try { unlinkSync(tmp) } catch { /* 清不掉也无所谓 */ }
|
|
165
|
-
throw new Error(`写文件失败 ${p}: ${lastErr?.message || lastErr}`)
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
function readJsonl(p) {
|
|
169
|
-
let raw
|
|
170
|
-
try { raw = readFileSync(p, 'utf8') } catch { return { items: [], bad: 0 } }
|
|
171
|
-
const items = []
|
|
172
|
-
let bad = 0
|
|
173
|
-
for (const line of raw.split(/\r?\n/)) {
|
|
174
|
-
if (!line.trim()) continue
|
|
175
|
-
try { items.push(JSON.parse(line)) } catch { bad++ }
|
|
176
|
-
}
|
|
177
|
-
return { items, bad }
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
function appendJsonl(p, items) {
|
|
181
|
-
if (!items.length) return
|
|
182
|
-
mkdirSync(join(p, '..'), { recursive: true })
|
|
183
|
-
appendFileSync(p, items.map((it) => JSON.stringify(it)).join('\n') + '\n', 'utf8')
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
// ============================================================
|
|
187
|
-
// 主体
|
|
188
|
-
// ============================================================
|
|
189
|
-
|
|
190
|
-
export function createMemory6(o = {}) {
|
|
191
|
-
const cfg = { ...DEFAULTS6, ...(o.cfg || {}) }
|
|
192
|
-
const dir = o.dir || '.memory6'
|
|
193
|
-
const writer = sanitizeWriter(o.writer)
|
|
194
|
-
const tick = o.now || (() => Date.now())
|
|
195
|
-
const embed = typeof o.embed === 'function' ? o.embed : null
|
|
196
|
-
const vectorsDisabled = !embed
|
|
197
|
-
|
|
198
|
-
const paths = {
|
|
199
|
-
L1: join(dir, 'L1.json'),
|
|
200
|
-
L2: join(dir, 'L2.json'),
|
|
201
|
-
L3: join(dir, 'L3.json'),
|
|
202
|
-
DL1: join(dir, 'DL1.json'),
|
|
203
|
-
DL2: join(dir, 'DL2.json'),
|
|
204
|
-
DL3: join(dir, 'DL3.json'),
|
|
205
|
-
}
|
|
206
|
-
const tmpDir = join(dir, 'L1.tmp')
|
|
207
|
-
const sigDir = join(dir, 'signals')
|
|
208
|
-
const vecDir = join(dir, 'vectors')
|
|
209
|
-
const vecPath = join(vecDir, 'deep-part-0000.json')
|
|
210
|
-
const vecMetaPath = join(vecDir, 'deep-meta.json')
|
|
211
|
-
const lockPath = join(dir, 'organize.lock')
|
|
212
|
-
const statePath = join(dir, 'organize.state.json')
|
|
213
|
-
const appliedPath = join(sigDir, 'applied.json')
|
|
214
|
-
const auditPath = join(dir, 'audit.log')
|
|
215
|
-
const gravePath = join(dir, 'graveyard', 'forgotten.jsonl')
|
|
216
|
-
|
|
217
|
-
for (const p of [dir, tmpDir, sigDir, vecDir, join(dir, 'graveyard')]) {
|
|
218
|
-
try { mkdirSync(p, { recursive: true }) } catch { /* 已存在 */ }
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
// —— 模型调用计数(三个数分开记,和 memory5 同口径) ——
|
|
222
|
-
const calls = {
|
|
223
|
-
embed: { rounds: 0, items: 0 },
|
|
224
|
-
summarize: { rounds: 0, items: 0 },
|
|
225
|
-
relation: { rounds: 0, items: 0 },
|
|
226
|
-
rerank: { rounds: 0, items: 0 },
|
|
227
|
-
}
|
|
228
|
-
const bump = (k, n = 1) => { calls[k].rounds++; calls[k].items += n }
|
|
229
|
-
|
|
230
|
-
function audit(line) {
|
|
231
|
-
const safe = String(line).replace(/[\r\n]+/g, ' ')
|
|
232
|
-
try { appendFileSync(auditPath, `${new Date().toISOString()} ${safe}\n`, 'utf8') } catch { /* 审计失败不挡主流程 */ }
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
// ============================================================
|
|
236
|
-
// 文件读写
|
|
237
|
-
// ============================================================
|
|
238
|
-
|
|
239
|
-
function readLayer(name, { strict = false } = {}) {
|
|
240
|
-
const p = paths[name]
|
|
241
|
-
let raw
|
|
242
|
-
try { raw = readFileSync(p, 'utf8') } catch (e) {
|
|
243
|
-
if (e.code === 'ENOENT') return []
|
|
244
|
-
if (strict) throw e
|
|
245
|
-
audit(`READ_FAIL ${name} ${e.message}`)
|
|
246
|
-
return []
|
|
247
|
-
}
|
|
248
|
-
try {
|
|
249
|
-
const j = JSON.parse(raw)
|
|
250
|
-
return Array.isArray(j?.items) ? j.items : []
|
|
251
|
-
} catch (e) {
|
|
252
|
-
// 读侧绝不把损坏的层当空:抛出去,调用方决定降级
|
|
253
|
-
if (strict) throw new Error(`层文件损坏 ${name}: ${e.message}`)
|
|
254
|
-
audit(`READ_CORRUPT ${name} ${e.message}`)
|
|
255
|
-
throw e
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
function writeLayer(name, items) {
|
|
260
|
-
atomicWrite(paths[name], JSON.stringify({ version: 2, at: Date.now(), items }, null, 1))
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
// ============================================================
|
|
264
|
-
// 只读层缓存
|
|
265
|
-
//
|
|
266
|
-
// ★ 只给**只读路径**用:search / buildBlock / stats / list。
|
|
267
|
-
// 整理路径(loadDb → digest → saveDb)必须走 readLayer 直读——
|
|
268
|
-
// digest 会就地改条目(layer / text / supersededBy / sources),
|
|
269
|
-
// 如果改的是缓存里的对象,就成了"内存里有、磁盘上没有",下次读还拿到脏数据。
|
|
270
|
-
//
|
|
271
|
-
// ★ 失效判据是 mtime + size,和"每次都读"完全等价,只是省掉 JSON.parse:
|
|
272
|
-
// 对方一写盘(tmp + rename),mtime 必变,下一次读就重解析。
|
|
273
|
-
//
|
|
274
|
-
// ★ 防内存不足(三条闸门,触发任一条就退回直读,不影响正确性):
|
|
275
|
-
// 单文件 > cacheMaxFileBytes → 不缓存(大库只慢,不涨内存)
|
|
276
|
-
// 估算占用 > cacheMaxTotalBytes → LRU 淘汰到上限以内
|
|
277
|
-
// 堆已用 > cacheHeapLimitBytes → 清空缓存,本轮不写
|
|
278
|
-
// ============================================================
|
|
279
|
-
const layerCache = new Map() // name -> { sig, items, bytes }
|
|
280
|
-
let cacheBytes = 0
|
|
281
|
-
const cacheStat = { hits: 0, misses: 0, evicts: 0, skipped: 0 }
|
|
282
|
-
|
|
283
|
-
const cacheLimitFile = Number(cfg.cacheMaxFileBytes) || 64 * 1024 * 1024
|
|
284
|
-
const cacheLimitTotal = Number(cfg.cacheMaxTotalBytes) || 384 * 1024 * 1024
|
|
285
|
-
const cacheHeapLimit = Number(cfg.cacheHeapLimitBytes) || 1.2 * 1024 * 1024 * 1024
|
|
286
|
-
|
|
287
|
-
function layerSig(name) {
|
|
288
|
-
try { const st = statSync(paths[name]); return { sig: `${st.mtimeMs}:${st.size}`, size: st.size } } catch { return null }
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
function clearLayerCache() { layerCache.clear(); cacheBytes = 0 }
|
|
292
|
-
|
|
293
|
-
function readLayerCached(name) {
|
|
294
|
-
const meta = layerSig(name)
|
|
295
|
-
const hit = meta ? layerCache.get(name) : null
|
|
296
|
-
if (hit && hit.sig === meta.sig) {
|
|
297
|
-
cacheStat.hits++
|
|
298
|
-
layerCache.delete(name); layerCache.set(name, hit) // LRU:挪到队尾
|
|
299
|
-
return hit.items
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
cacheStat.misses++
|
|
303
|
-
// 解析本身仍可能失败(文件损坏)——和原来一样抛给调用方,不吞异常
|
|
304
|
-
const items = readLayer(name)
|
|
305
|
-
|
|
306
|
-
try {
|
|
307
|
-
if (!meta || meta.size > cacheLimitFile) { cacheStat.skipped++; return items }
|
|
308
|
-
// 解析后的对象大约是 JSON 文本的 2~3 倍
|
|
309
|
-
const bytes = meta.size * 3
|
|
310
|
-
const share = Number(cfg.cacheLayerShare?.[name]) || 0.1
|
|
311
|
-
const layerLimit = cacheLimitTotal * share
|
|
312
|
-
if (bytes > layerLimit) { cacheStat.skipped++; return items }
|
|
313
|
-
if (process.memoryUsage().heapUsed > cacheHeapLimit) {
|
|
314
|
-
clearLayerCache()
|
|
315
|
-
cacheStat.skipped++
|
|
316
|
-
return items
|
|
317
|
-
}
|
|
318
|
-
if (hit) { cacheBytes -= hit.bytes; layerCache.delete(name) }
|
|
319
|
-
layerCache.set(name, { sig: meta.sig, items, bytes })
|
|
320
|
-
cacheBytes += bytes
|
|
321
|
-
while (cacheBytes > cacheLimitTotal && layerCache.size > 1) {
|
|
322
|
-
const [k, v] = layerCache.entries().next().value
|
|
323
|
-
layerCache.delete(k)
|
|
324
|
-
cacheBytes -= v.bytes
|
|
325
|
-
cacheStat.evicts++
|
|
326
|
-
}
|
|
327
|
-
} catch { /* 缓存失败不影响读到的数据 */ }
|
|
328
|
-
return items
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
/**
|
|
332
|
-
* 层内 id → 条目索引(懒建,挂在缓存条目上,建一次一直用)。
|
|
333
|
-
* 返回 null 表示这层没进缓存(文件太大 / 堆太紧),调用方退回线性查找。
|
|
334
|
-
*/
|
|
335
|
-
function cachedIndex(name) {
|
|
336
|
-
const hit = layerCache.get(name)
|
|
337
|
-
const meta = layerSig(name)
|
|
338
|
-
if (!hit || !meta || hit.sig !== meta.sig) return null
|
|
339
|
-
if (!hit.byId) {
|
|
340
|
-
hit.byId = new Map()
|
|
341
|
-
for (const it of hit.items) hit.byId.set(it.id, it)
|
|
342
|
-
}
|
|
343
|
-
return hit.byId
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
/** 确保某层已进缓存并返回索引;没进缓存返回 null */
|
|
347
|
-
function ensureIndex(name) {
|
|
348
|
-
try { readLayerCached(name) } catch { return null }
|
|
349
|
-
return cachedIndex(name)
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
function tempPath(w = writer) { return join(tmpDir, `${w}.jsonl`) }
|
|
353
|
-
|
|
354
|
-
function readTempsAll() {
|
|
355
|
-
const out = []
|
|
356
|
-
let files = []
|
|
357
|
-
try { files = readdirSync(tmpDir) } catch { return { items: out, bad: 0 } }
|
|
358
|
-
let bad = 0
|
|
359
|
-
for (const f of files) {
|
|
360
|
-
if (!f.endsWith('.jsonl')) continue
|
|
361
|
-
const r = readJsonl(join(tmpDir, f))
|
|
362
|
-
out.push(...r.items)
|
|
363
|
-
bad += r.bad
|
|
364
|
-
}
|
|
365
|
-
return { items: out, bad }
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
/** 把当前所有 temp(含上次遗留的 claimed)改名认领。失败=对方正在写,下一轮再试 */
|
|
369
|
-
function claimTemps() {
|
|
370
|
-
const claimed = []
|
|
371
|
-
let files = []
|
|
372
|
-
try { files = readdirSync(tmpDir) } catch { return claimed }
|
|
373
|
-
const now = Date.now()
|
|
374
|
-
for (const f of files) {
|
|
375
|
-
if (!f.endsWith('.jsonl')) continue
|
|
376
|
-
const from = join(tmpDir, f)
|
|
377
|
-
const to = join(tmpDir, `${f.replace(/\.jsonl$/, '')}.claimed-${now}-${Math.random().toString(36).slice(2, 6)}.jsonl`)
|
|
378
|
-
try { renameSync(from, to); claimed.push(to) } catch { /* 被打开/正在写,跳过 */ }
|
|
379
|
-
}
|
|
380
|
-
return claimed
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
function claimSignals() {
|
|
384
|
-
const claimed = []
|
|
385
|
-
let files = []
|
|
386
|
-
try { files = readdirSync(sigDir) } catch { return claimed }
|
|
387
|
-
const now = Date.now()
|
|
388
|
-
for (const f of files) {
|
|
389
|
-
if (!f.endsWith('.jsonl')) continue
|
|
390
|
-
const from = join(sigDir, f)
|
|
391
|
-
const to = join(sigDir, `${f.replace(/\.jsonl$/, '')}.claimed-${now}-${Math.random().toString(36).slice(2, 6)}.jsonl`)
|
|
392
|
-
try { renameSync(from, to); claimed.push(to) } catch { /* 被打开,跳过 */ }
|
|
393
|
-
}
|
|
394
|
-
return claimed
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
// ============================================================
|
|
398
|
-
// 向量
|
|
399
|
-
// ============================================================
|
|
400
|
-
|
|
401
|
-
let vectors = new Map()
|
|
402
|
-
let vectorsDirty = false
|
|
403
|
-
let vecSig = ''
|
|
404
|
-
|
|
405
|
-
function vecFileSig() {
|
|
406
|
-
try { const st = statSync(vecPath); return `${st.mtimeMs}:${st.size}` } catch { return '' }
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
/** 别的进程可能刚写了新向量;文件变了就重读(不重读会一直用启动时那份) */
|
|
410
|
-
function maybeReloadVectors() {
|
|
411
|
-
if (vectorsDisabled) return
|
|
412
|
-
const s = vecFileSig()
|
|
413
|
-
if (s === vecSig) return
|
|
414
|
-
loadVectors()
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
function loadVectors() {
|
|
418
|
-
if (vectorsDisabled) return
|
|
419
|
-
vecSig = vecFileSig()
|
|
420
|
-
vectorsDirty = false
|
|
421
|
-
vectors = new Map()
|
|
422
|
-
let meta = null
|
|
423
|
-
try { meta = JSON.parse(readFileSync(vecMetaPath, 'utf8')) } catch { /* 老库没 meta */ }
|
|
424
|
-
if (meta && typeof meta.sig === 'string' && o.embedSignature && meta.sig !== o.embedSignature) {
|
|
425
|
-
// 嵌入模型换了:旧向量不可比较。数据不重要,直接不用(文件留着,不删)。
|
|
426
|
-
audit(`VEC_STALE ${meta.sig} → ${o.embedSignature},忽略旧向量`)
|
|
427
|
-
vectors = new Map()
|
|
428
|
-
return
|
|
429
|
-
}
|
|
430
|
-
let raw
|
|
431
|
-
try { raw = JSON.parse(readFileSync(vecPath, 'utf8')) } catch { return }
|
|
432
|
-
for (const [id, b64] of Object.entries(raw || {})) {
|
|
433
|
-
try {
|
|
434
|
-
const buf = Buffer.from(b64, 'base64')
|
|
435
|
-
vectors.set(id, new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4))
|
|
436
|
-
} catch { /* 单条坏了不连累别人 */ }
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
function saveVectors() {
|
|
441
|
-
if (vectorsDisabled || !vectorsDirty) return
|
|
442
|
-
const out = {}
|
|
443
|
-
for (const [id, v] of vectors) out[id] = Buffer.from(v.buffer, v.byteOffset, v.byteLength).toString('base64')
|
|
444
|
-
atomicWrite(vecPath, JSON.stringify(out))
|
|
445
|
-
const first = vectors.values().next().value
|
|
446
|
-
atomicWrite(vecMetaPath, JSON.stringify({
|
|
447
|
-
sig: o.embedSignature || '', dim: first ? first.length : 0, count: vectors.size, at: new Date().toISOString(),
|
|
448
|
-
}, null, 1))
|
|
449
|
-
vectorsDirty = false
|
|
450
|
-
vecSig = vecFileSig()
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
async function embedInto(items, chunk = 32) {
|
|
454
|
-
if (!embed || !items.length) return 0
|
|
455
|
-
// ★ 已经有向量的不重算。L2 溢出时这一批里带着「DL1 全部」,
|
|
456
|
-
// 它们早就 embed 过了;不过滤等于每次 L2 溢出都把整个 DL1 重算一遍(白花钱)。
|
|
457
|
-
const todo = items.filter((it) => it && it.id && !vectors.has(it.id))
|
|
458
|
-
if (!todo.length) return 0
|
|
459
|
-
items = todo
|
|
460
|
-
let n = 0
|
|
461
|
-
for (let i = 0; i < items.length; i += chunk) {
|
|
462
|
-
const slice = items.slice(i, i + chunk)
|
|
463
|
-
const texts = slice.map((it) => String(it.text || '').slice(0, 512))
|
|
464
|
-
bump('embed', slice.length)
|
|
465
|
-
try {
|
|
466
|
-
const res = await embed(texts)
|
|
467
|
-
for (let j = 0; j < slice.length; j++) {
|
|
468
|
-
if (res?.[j]) { vectors.set(slice[j].id, Float32Array.from(res[j])); n++; vectorsDirty = true }
|
|
469
|
-
}
|
|
470
|
-
} catch { /* 这一块失败,缺向量只影响语义检索,不挡落库 */ }
|
|
471
|
-
}
|
|
472
|
-
return n
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
async function embedOne(text) {
|
|
476
|
-
if (!embed) return null
|
|
477
|
-
bump('embed', 1)
|
|
478
|
-
try { const r = await embed([String(text || '').slice(0, 512)]); return r?.[0] ? Float32Array.from(r[0]) : null } catch { return null }
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
// ============================================================
|
|
482
|
-
// 写:只追加自己的 temp
|
|
483
|
-
// ============================================================
|
|
484
|
-
|
|
485
|
-
let turnCounter = 0
|
|
486
|
-
const newTurnId = () => `${writer}-${Date.now().toString(36)}-${(++turnCounter).toString(36)}-${Math.random().toString(36).slice(2, 5)}`
|
|
487
|
-
|
|
488
|
-
function writeTurn(rows = []) {
|
|
489
|
-
const list = (rows || []).filter((r) => r && String(r.text || '').trim())
|
|
490
|
-
if (!list.length) return { ok: false, error: '内容为空' }
|
|
491
|
-
const turnId = list[0].turnId || newTurnId()
|
|
492
|
-
const now = tick()
|
|
493
|
-
const items = list.map((r) => {
|
|
494
|
-
const it = makeItem({
|
|
495
|
-
text: String(r.text).trim(),
|
|
496
|
-
channel: r.channel || 'dialog',
|
|
497
|
-
speaker: r.speaker || '',
|
|
498
|
-
people: r.people || [],
|
|
499
|
-
scene: r.scene || '',
|
|
500
|
-
importance: Number.isFinite(Number(r.importance)) ? Number(r.importance) : 0.5,
|
|
501
|
-
kind: r.kind || 'raw',
|
|
502
|
-
at: r.at ?? now,
|
|
503
|
-
})
|
|
504
|
-
it.layer = 'L1'
|
|
505
|
-
it.turnId = turnId
|
|
506
|
-
it.origin = writer
|
|
507
|
-
if (r.impRough) it.impRough = true
|
|
508
|
-
return it
|
|
509
|
-
})
|
|
510
|
-
appendJsonl(tempPath(), items)
|
|
511
|
-
audit(`WRITE_TURN ${turnId} ${items.length} 条`)
|
|
512
|
-
return { ok: true, turnId, ids: items.map((it) => it.id), layer: 'L1' }
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
const write = (args = {}) => writeTurn([args])
|
|
516
|
-
const writeMany = (list = []) => writeTurn(list)
|
|
517
|
-
|
|
518
|
-
// ============================================================
|
|
519
|
-
// 读:L1 视图(真 L1 + 所有 temp),按 id 去重
|
|
520
|
-
// ============================================================
|
|
521
|
-
|
|
522
|
-
function dedupe(items) {
|
|
523
|
-
const seen = new Set()
|
|
524
|
-
const out = []
|
|
525
|
-
for (const it of items) {
|
|
526
|
-
if (!it || !it.id || seen.has(it.id)) continue
|
|
527
|
-
seen.add(it.id)
|
|
528
|
-
out.push(it)
|
|
529
|
-
}
|
|
530
|
-
return out
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
/**
|
|
534
|
-
* L1 视图 = 真 L1 + 所有 temp,按 id 去重。
|
|
535
|
-
*
|
|
536
|
-
* ★ 不再自己读 DL1/DL2/DL3 做"已下沉"比对(2026-11-02)。
|
|
537
|
-
* 原来每轮注入都要全量解析三个深层文件——DL1 上千条时是 10~30ms 的同步 IO,
|
|
538
|
-
* Node 单线程,这段时间整个进程是卡住的。而正常流程下 L1 根本不含深层条目:
|
|
539
|
-
* `saveDb` 特意"先写目标层、最后写 L1",`reconcile` 整理时还会再对一次账。
|
|
540
|
-
* 只有崩溃后才可能出现交叉,那时重复显示一条(不是丢),下一次整理就清掉。
|
|
541
|
-
*
|
|
542
|
-
* @param {{placed?: Set<string>}} [opts] 已确认在别的层里的 id(调用方手上有就传,
|
|
543
|
-
* 比如 buildBlock 已经读过 L2,顺手给出,不用再读一次)
|
|
544
|
-
*/
|
|
545
|
-
function readL1View({ placed = null } = {}) {
|
|
546
|
-
const temp = readTempsAll()
|
|
547
|
-
const base = (() => { try { return readLayerCached('L1') } catch { return [] } })()
|
|
548
|
-
const all = dedupe([...base, ...temp.items])
|
|
549
|
-
try { for (const it of readLayerCached('L2')) placed.add(it.id) } catch { /* 忽略 */ }
|
|
550
|
-
return { items: placed ? all.filter((it) => !placed.has(it.id)) : all, bad: temp.bad }
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
// ============================================================
|
|
554
|
-
// 整理:钥匙 + 合并 + 分流
|
|
555
|
-
// ============================================================
|
|
556
|
-
|
|
557
|
-
let lockHeld = false
|
|
558
|
-
let lockWriter = ''
|
|
559
|
-
let lockToken = ''
|
|
560
|
-
|
|
561
|
-
function readLock() {
|
|
562
|
-
try { return JSON.parse(readFileSync(lockPath, 'utf8')) } catch { return null }
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
function acquireLock() {
|
|
566
|
-
try {
|
|
567
|
-
const fd = openSync(lockPath, 'wx')
|
|
568
|
-
lockToken = `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`
|
|
569
|
-
try {
|
|
570
|
-
writeFileSync(fd, JSON.stringify({ pid: process.pid, writer, at: Date.now(), tab: lockToken, phase: 'start' }))
|
|
571
|
-
} finally { closeSync(fd) }
|
|
572
|
-
lockHeld = true
|
|
573
|
-
lockWriter = writer
|
|
574
|
-
return true
|
|
575
|
-
} catch (e) {
|
|
576
|
-
if (e.code !== 'EEXIST') { audit(`LOCK_FAIL ${e.message}`); return false }
|
|
577
|
-
let st = null
|
|
578
|
-
try { st = statSync(lockPath) } catch { return acquireLock() }
|
|
579
|
-
if (Date.now() - st.mtimeMs < cfg.staleLockMs) return false
|
|
580
|
-
// 心跳过期:把旧锁改名(原子),再抢一次
|
|
581
|
-
try { renameSync(lockPath, `${lockPath}.stale-${Date.now()}`) } catch { return false }
|
|
582
|
-
return acquireLock()
|
|
583
|
-
}
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
/**
|
|
587
|
-
* 持锁期间确认"这把钥匙还是我的"。
|
|
588
|
-
* 如果整理卡在模型调用里超过 staleLockMs,另一端会接管;接管后这边必须停手,
|
|
589
|
-
* 否则两个写者同时改深层文件,会互相覆盖。
|
|
590
|
-
*/
|
|
591
|
-
function checkLock() {
|
|
592
|
-
if (!lockHeld) throw new Error('没有钥匙')
|
|
593
|
-
const cur = readLock()
|
|
594
|
-
if (!cur || cur.tab !== lockToken) throw new Error('钥匙被接管,放弃本轮')
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
function heartbeat() {
|
|
598
|
-
if (!lockHeld) return
|
|
599
|
-
const cur = readLock()
|
|
600
|
-
if (cur && cur.tab && cur.tab !== lockToken) { lockHeld = false; return } // 已被接管
|
|
601
|
-
try { utimesSync(lockPath, new Date(), new Date()) } catch { /* 锁没了就下次再抢 */ }
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
function releaseLock() {
|
|
605
|
-
if (!lockHeld) return
|
|
606
|
-
const cur = readLock()
|
|
607
|
-
// 只删自己的锁;被接管后不能把新主人的锁删掉
|
|
608
|
-
if (!cur || !cur.tab || cur.tab === lockToken) {
|
|
609
|
-
try { unlinkSync(lockPath) } catch { /* 已没 */ }
|
|
610
|
-
}
|
|
611
|
-
lockHeld = false
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
function loadDb() {
|
|
615
|
-
const db = {}
|
|
616
|
-
for (const l of DATA_LAYERS) db[l] = readLayer(l, { strict: true })
|
|
617
|
-
return db
|
|
618
|
-
}
|
|
619
|
-
|
|
620
|
-
/**
|
|
621
|
-
* 每层内部去重 + 跨层去重(深层的 id 为准,L1/L2 里的副本丢掉)。
|
|
622
|
-
* 顺带保证"同一事件只存在一层"。
|
|
623
|
-
*/
|
|
624
|
-
function reconcile(db) {
|
|
625
|
-
// 返回"清掉了多少条"。调用方用它决定要不要落盘——不返回的话,
|
|
626
|
-
// 崩溃残留的交叉条目虽然被清出内存了,却可能因为"本轮没有别的变化"而写不回去。
|
|
627
|
-
let removed = 0
|
|
628
|
-
const deep = new Map()
|
|
629
|
-
for (const l of ['DL3', 'DL2', 'DL1']) {
|
|
630
|
-
const before = db[l].length
|
|
631
|
-
db[l] = dedupe(db[l])
|
|
632
|
-
removed += before - db[l].length
|
|
633
|
-
for (const it of db[l]) if (!deep.has(it.id)) deep.set(it.id, l)
|
|
634
|
-
}
|
|
635
|
-
const l1Before = db.L1.length
|
|
636
|
-
db.L1 = dedupe(db.L1).filter((it) => !deep.has(it.id))
|
|
637
|
-
const l2Before = db.L2.length
|
|
638
|
-
db.L2 = dedupe(db.L2).filter((it) => !deep.has(it.id))
|
|
639
|
-
// 已经进了 L2 的,也不能还留在 L1(崩在"L2 写完、L1 没改"时会这样)
|
|
640
|
-
const inL2 = new Set(db.L2.map((it) => it.id))
|
|
641
|
-
db.L1 = db.L1.filter((it) => !inL2.has(it.id))
|
|
642
|
-
removed += (l1Before - db.L1.length) + (l2Before - db.L2.length)
|
|
643
|
-
|
|
644
|
-
// 同一个 id 出现在两个深层:保留更靠后的(DL2 > DL1;DL3 只放摘要,理论上不会撞)
|
|
645
|
-
const seen = new Set()
|
|
646
|
-
for (const l of ['DL3', 'DL2', 'DL1']) {
|
|
647
|
-
db[l] = db[l].filter((it) => {
|
|
648
|
-
if (seen.has(it.id)) { audit(`RECONCILE_DUP ${it.id} ${l}`); removed++; return false }
|
|
649
|
-
seen.add(it.id)
|
|
650
|
-
return true
|
|
651
|
-
})
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
// DL2 不变式:每一条都必须被某条 DL3 摘要引用。
|
|
655
|
-
// 摘要被删/被合并后留下的孤儿,移到 DL1(快忘),不让它冒充"重要原文"。
|
|
656
|
-
const refs = new Set()
|
|
657
|
-
for (const s of db.DL3) for (const id of s.sources || []) refs.add(id)
|
|
658
|
-
const orphans = db.DL2.filter((it) => !refs.has(it.id))
|
|
659
|
-
if (orphans.length) {
|
|
660
|
-
db.DL2 = db.DL2.filter((it) => refs.has(it.id))
|
|
661
|
-
for (const it of orphans) {
|
|
662
|
-
if (db.DL1.some((x) => x.id === it.id)) continue
|
|
663
|
-
it.layer = 'DL1'
|
|
664
|
-
delete it.consolidatedAt
|
|
665
|
-
db.DL1.push(it)
|
|
666
|
-
}
|
|
667
|
-
audit(`RECONCILE_DL2_ORPHAN ${orphans.length} 条 → DL1(DL2 只放被引用的原文)`)
|
|
668
|
-
removed += orphans.length // 不算丢,只是换层;但文件确实变了,要写回
|
|
669
|
-
}
|
|
670
|
-
return removed
|
|
671
|
-
}
|
|
672
|
-
|
|
673
|
-
function sortDb(db) {
|
|
674
|
-
db.L1.sort((a, b) => (a.createdAt - b.createdAt) || cmpId(a.id, b.id))
|
|
675
|
-
db.L2.sort((a, b) => ((a.consolidatedAt ?? a.createdAt) - (b.consolidatedAt ?? b.createdAt)) || cmpId(a.id, b.id))
|
|
676
|
-
for (const l of ['DL1', 'DL2', 'DL3']) db[l].sort((a, b) => (a.createdAt - b.createdAt) || cmpId(a.id, b.id))
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
/**
|
|
680
|
-
* 按层写回。**先写目标层,最后写 L1**:
|
|
681
|
-
* 崩在中间时 L1 里还有原文,靠 id 去重不会重复、不会丢。
|
|
682
|
-
*/
|
|
683
|
-
function saveDb(db, { skip = [] } = {}) {
|
|
684
|
-
sortDb(db)
|
|
685
|
-
const skipSet = new Set(skip)
|
|
686
|
-
for (const l of ['L2', 'DL1', 'DL2', 'DL3', 'L1']) {
|
|
687
|
-
// 跳过只对"已经存在"的层生效:文件都还没有时得先建出来,
|
|
688
|
-
// 否则第一次整理遇到"内容没变"就少建一个文件,读侧会 ENOENT。
|
|
689
|
-
if (skipSet.has(l) && existsSync(paths[l])) continue
|
|
690
|
-
writeLayer(l, db[l])
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
function mergeTemps(db) {
|
|
695
|
-
const claimed = claimTemps()
|
|
696
|
-
if (!claimed.length) return { merged: 0, claimed }
|
|
697
|
-
const have = new Set(db.L1.map((it) => it.id))
|
|
698
|
-
for (const l of ['L2', 'DL1', 'DL2', 'DL3']) for (const it of db[l]) have.add(it.id)
|
|
699
|
-
let merged = 0
|
|
700
|
-
let bad = 0
|
|
701
|
-
for (const p of claimed) {
|
|
702
|
-
const r = readJsonl(p)
|
|
703
|
-
bad += r.bad
|
|
704
|
-
for (const it of r.items) {
|
|
705
|
-
if (!it || !it.id || have.has(it.id)) continue
|
|
706
|
-
if (!it.origin) it.origin = it.id.startsWith('m5_') ? '?' : writer
|
|
707
|
-
if (!it.turnId) it.turnId = it.id
|
|
708
|
-
it.layer = 'L1'
|
|
709
|
-
have.add(it.id)
|
|
710
|
-
db.L1.push(it)
|
|
711
|
-
merged++
|
|
712
|
-
}
|
|
713
|
-
}
|
|
714
|
-
if (bad) audit(`MERGE_TEMP_BAD ${bad} 行(写到一半崩溃留下的半行,已跳过)`)
|
|
715
|
-
return { merged, claimed }
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
function moveItem(db, it, target, now) {
|
|
719
|
-
if (it.layer === target) {
|
|
720
|
-
if (!db[target].some((x) => x.id === it.id)) db[target].push(it)
|
|
721
|
-
return
|
|
722
|
-
}
|
|
723
|
-
for (const l of DATA_LAYERS) db[l] = db[l].filter((x) => x.id !== it.id)
|
|
724
|
-
it.layer = target
|
|
725
|
-
if (target === 'L2') it.consolidatedAt = now
|
|
726
|
-
db[target].push(it)
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
function groupOf(it) {
|
|
730
|
-
return `${it.origin || '?'}\u0001${it.scene || ''}\u0001${it.channel || 'dialog'}`
|
|
731
|
-
}
|
|
732
|
-
|
|
733
|
-
function commonScene(items) {
|
|
734
|
-
const s = [...new Set(items.map((it) => it.scene || '').filter(Boolean))]
|
|
735
|
-
return s.length === 1 ? s[0] : ''
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
function mergeSummaries(target, src, now) {
|
|
739
|
-
if (String(src.text || '').length > String(target.text || '').length) target.text = src.text
|
|
740
|
-
const tFrom = target.spanFrom ?? target.createdAt
|
|
741
|
-
const tTo = target.spanTo ?? target.createdAt
|
|
742
|
-
const sFrom = src.spanFrom ?? src.createdAt
|
|
743
|
-
const sTo = src.spanTo ?? src.createdAt
|
|
744
|
-
target.spanFrom = Math.min(tFrom, sFrom)
|
|
745
|
-
target.spanTo = Math.max(tTo, sTo)
|
|
746
|
-
target.people = uniq([...(target.people || []), ...(src.people || [])]).slice(0, 10)
|
|
747
|
-
target.sources = uniq([...(target.sources || []), ...(src.sources || [])])
|
|
748
|
-
target.importance = Math.max(Number(target.importance) || 0, Number(src.importance) || 0)
|
|
749
|
-
target.heat = Math.max(heatOf(target, now, cfg), heatOf(src, now, cfg))
|
|
750
|
-
target.lastMentionedAt = now
|
|
751
|
-
target.repeatStrength = Math.max(target.repeatStrength || 1, src.repeatStrength || 1)
|
|
752
|
-
target.fusionCount = (target.fusionCount || 0) + 1
|
|
753
|
-
}
|
|
754
|
-
|
|
755
|
-
async function fuseSummary(db, sum, now) {
|
|
756
|
-
const sv = vectors.get(sum.id)
|
|
757
|
-
if (sv && typeof o.judgeRelation === 'function') {
|
|
758
|
-
const cands = []
|
|
759
|
-
for (const it of db.DL3) {
|
|
760
|
-
if (it.id === sum.id || it.kind !== 'summary' || it.supersededBy) continue
|
|
761
|
-
const v = vectors.get(it.id)
|
|
762
|
-
if (!v) continue
|
|
763
|
-
const sim = cosine(sv, v)
|
|
764
|
-
if (sim >= cfg.fusionPrefilter) cands.push({ it, sim })
|
|
765
|
-
}
|
|
766
|
-
if (cands.length) {
|
|
767
|
-
cands.sort((a, b) => b.sim - a.sim)
|
|
768
|
-
const top = cands[0]
|
|
769
|
-
bump('relation', 1)
|
|
770
|
-
let type = null
|
|
771
|
-
try { type = await o.judgeRelation(top.it.text, sum.text) } catch { type = null }
|
|
772
|
-
if (type === true) type = 'duplicate'
|
|
773
|
-
if (type === false) type = null
|
|
774
|
-
if (type === 'duplicate') {
|
|
775
|
-
mergeSummaries(top.it, sum, now)
|
|
776
|
-
vectors.delete(sum.id)
|
|
777
|
-
await embedInto([top.it])
|
|
778
|
-
l3Dirty = true
|
|
779
|
-
audit(`FUSE_DUP ${sum.id} → ${top.it.id}`)
|
|
780
|
-
return 'merged'
|
|
781
|
-
}
|
|
782
|
-
if (type === 'contradiction') {
|
|
783
|
-
sum.supersedes = top.it.id
|
|
784
|
-
top.it.supersededBy = sum.id
|
|
785
|
-
sum.heat = Math.max(Number(sum.heat) || 1, Number(top.it.heat) || 1)
|
|
786
|
-
sum.repeatStrength = Math.max(Number(sum.repeatStrength) || 1, Number(top.it.repeatStrength) || 1)
|
|
787
|
-
db.DL3.push(sum)
|
|
788
|
-
l3Dirty = true
|
|
789
|
-
audit(`SUPERSEDE ${top.it.id} ← ${sum.id}`)
|
|
790
|
-
return 'superseded'
|
|
791
|
-
}
|
|
792
|
-
if (type === 'update') {
|
|
793
|
-
// 进展:两条都留,但记下"新的接在旧的后面"。
|
|
794
|
-
// ★ memory5 有这个分支,memory6 最初漏了 —— 结果模型判 update 时
|
|
795
|
-
// 既不记 updates 链接、也不打日志,看起来像"融合完全没生效"。
|
|
796
|
-
sum.updates = top.it.id
|
|
797
|
-
db.DL3.push(sum)
|
|
798
|
-
l3Dirty = true
|
|
799
|
-
audit(`FUSE_UPDATE ${top.it.id} → ${sum.id}(进展,两条都保留)`)
|
|
800
|
-
return 'updated'
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
}
|
|
804
|
-
db.DL3.push(sum)
|
|
805
|
-
l3Dirty = true
|
|
806
|
-
return 'added'
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
/**
|
|
810
|
-
* 一批原文 → 摘要 + 路由。成功才搬;模型失败则整批回 L2(不丢)。
|
|
811
|
-
*
|
|
812
|
-
* ★ DL2 的唯一入口是"被摘要引用":`cited` → DL2。
|
|
813
|
-
* `+`(keep)永远只回 L2,绝不进 DL2——DL2 是 DL3 摘要的原文库,不是第二缓冲层。
|
|
814
|
-
*/
|
|
815
|
-
async function digest(db, batch, { secondChance = false } = {}) {
|
|
816
|
-
const groups = new Map()
|
|
817
|
-
for (const it of batch) {
|
|
818
|
-
const k = groupOf(it)
|
|
819
|
-
if (!groups.has(k)) groups.set(k, [])
|
|
820
|
-
groups.get(k).push(it)
|
|
821
|
-
}
|
|
822
|
-
const now = tick()
|
|
823
|
-
let made = 0
|
|
824
|
-
let keptTotal = 0 // 第二次整理:本轮最多留这么多 `+` 在 L2,其余进 DL1
|
|
825
|
-
for (const [key, group] of groups) {
|
|
826
|
-
const [origin, scene, channel] = key.split('\u0001')
|
|
827
|
-
const folks = uniq(group.flatMap((it) => (it.people?.length ? it.people : (it.speaker ? [it.speaker] : [])))).slice(0, 10)
|
|
828
|
-
const lines = group.map((it) => `${it.speaker ? `${it.speaker}:` : ''}${it.text}`)
|
|
829
|
-
const hint = [
|
|
830
|
-
scene ? `这一段来自场景「${scene}」,摘要不要和别的场景混在一起。` : '',
|
|
831
|
-
secondChance
|
|
832
|
-
? `这是第二次整理:+ 最多 ${cfg.l2KeepMax} 条,其余不要再标 +(它们会进流水账,这是正常的)。`
|
|
833
|
-
: '',
|
|
834
|
-
].filter(Boolean).join(' ')
|
|
835
|
-
let res = null
|
|
836
|
-
if (typeof o.summarizeWithSources === 'function') {
|
|
837
|
-
bump('summarize', group.length)
|
|
838
|
-
try {
|
|
839
|
-
res = await o.summarizeWithSources(lines, channel, folks, hint)
|
|
840
|
-
} catch { res = null }
|
|
841
|
-
}
|
|
842
|
-
if (!res || (!res.summaries?.length && !res.keep?.length)) {
|
|
843
|
-
// 无模型 / 失败:整批回 L2。下轮还有机会再压;绝不进 DL2(那里只放被引用的原文)。
|
|
844
|
-
for (const it of group) moveItem(db, it, 'L2', now)
|
|
845
|
-
audit(`DIGEST_FALLBACK ${channel} ${group.length} 条 → L2`)
|
|
846
|
-
continue
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
let sums = res.summaries || []
|
|
850
|
-
if (!secondChance) sums = sums.filter((s) => (Number(s.importance) || 0) >= 0.9)
|
|
851
|
-
const cited = new Set()
|
|
852
|
-
|
|
853
|
-
// 先把这一组的摘要全部造出来,再批量 embed、批量融合。
|
|
854
|
-
// 逐条 embed 是 N 次往返(10 条摘要 = 10 次),批量只要 1 次。
|
|
855
|
-
const planned = []
|
|
856
|
-
for (const s of sums) {
|
|
857
|
-
const srcs = (s.sources || []).map((i) => group[i - 1]).filter(Boolean)
|
|
858
|
-
if (!srcs.length) continue
|
|
859
|
-
const k = srcKey(srcs)
|
|
860
|
-
if (db.DL3.some((x) => x.srcKey === k)) { for (const it of srcs) cited.add(it.id); continue }
|
|
861
|
-
const sum = makeItem({
|
|
862
|
-
text: String(s.text || '').trim(),
|
|
863
|
-
channel,
|
|
864
|
-
kind: 'summary',
|
|
865
|
-
speaker: '',
|
|
866
|
-
people: folks,
|
|
867
|
-
scene: commonScene(srcs),
|
|
868
|
-
importance: Number(s.importance) || 0.7,
|
|
869
|
-
at: now,
|
|
870
|
-
})
|
|
871
|
-
sum.layer = 'DL3'
|
|
872
|
-
sum.spanFrom = Math.min(...srcs.map((it) => it.createdAt))
|
|
873
|
-
sum.spanTo = Math.max(...srcs.map((it) => it.createdAt))
|
|
874
|
-
sum.sources = srcs.map((it) => it.id)
|
|
875
|
-
sum.heat = 2
|
|
876
|
-
sum.origin = origin
|
|
877
|
-
sum.srcKey = k
|
|
878
|
-
planned.push({ sum, srcs })
|
|
879
|
-
}
|
|
880
|
-
// fuseSummary 要靠向量做预筛,所以向量必须先算完(一次批量)
|
|
881
|
-
if (planned.length) await embedInto(planned.map((p) => p.sum))
|
|
882
|
-
for (const { sum, srcs } of planned) {
|
|
883
|
-
await fuseSummary(db, sum, now)
|
|
884
|
-
for (const it of srcs) cited.add(it.id)
|
|
885
|
-
made++
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
const keep = new Set((res.keep || []).map((i) => group[i - 1]?.id).filter(Boolean))
|
|
889
|
-
for (const it of group) {
|
|
890
|
-
if (cited.has(it.id)) {
|
|
891
|
-
moveItem(db, it, 'DL2', now)
|
|
892
|
-
} else if (keep.has(it.id)) {
|
|
893
|
-
if (!secondChance) {
|
|
894
|
-
moveItem(db, it, 'L2', now)
|
|
895
|
-
} else if (keptTotal < cfg.l2KeepMax) {
|
|
896
|
-
// 第二次的 + 留在 L2(不是 DL2),但限量;给它新的 consolidatedAt,
|
|
897
|
-
// 排到 L2 队尾,下一轮不会又第一个被挑出来。
|
|
898
|
-
if (it.layer === 'L2') it.consolidatedAt = now
|
|
899
|
-
else moveItem(db, it, 'L2', now)
|
|
900
|
-
keptTotal++
|
|
901
|
-
} else {
|
|
902
|
-
moveItem(db, it, 'DL1', now)
|
|
903
|
-
}
|
|
904
|
-
} else {
|
|
905
|
-
moveItem(db, it, 'DL1', now)
|
|
906
|
-
}
|
|
907
|
-
}
|
|
908
|
-
await embedInto(group)
|
|
909
|
-
audit(`DIGEST ${origin}/${scene || '-'}/${channel} ${group.length} 条 → 摘要 ${made}(引用 ${cited.size})`)
|
|
910
|
-
}
|
|
911
|
-
return made
|
|
912
|
-
}
|
|
913
|
-
|
|
914
|
-
/** 一轮整理超时就放弃(钥匙别被一个卡住的模型带走) */
|
|
915
|
-
function checkTimeout(started) {
|
|
916
|
-
if (tick() - started > cfg.maxOrganizeMs) throw new Error('整理超时')
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
/**
|
|
920
|
-
* 按轮次取旧的一半(实现在模块顶部 pickTurnBatch,导出给单测)。
|
|
921
|
-
*/
|
|
922
|
-
const pickTurns = (items, cap, opts) => pickTurnBatch(items, cap, opts)
|
|
923
|
-
|
|
924
|
-
async function organizeL1(db, started) {
|
|
925
|
-
if (chars(db.L1) <= cfg.hotMaxChars && db.L1.length <= cfg.hotMaxItems) return 0
|
|
926
|
-
const batch = pickTurns(db.L1, cfg.hotMaxChars, {
|
|
927
|
-
orderKey: (it) => it.createdAt,
|
|
928
|
-
quietBefore: tick() - cfg.quietMs,
|
|
929
|
-
maxItems: cfg.hotMaxItems,
|
|
930
|
-
})
|
|
931
|
-
if (!batch.length) { audit('L1_FULL_NO_QUIET 都在聊,等安静了再整理'); return 0 }
|
|
932
|
-
checkTimeout(started)
|
|
933
|
-
await digest(db, batch, { secondChance: false })
|
|
934
|
-
return batch.length
|
|
935
|
-
}
|
|
936
|
-
|
|
937
|
-
async function organizeL2(db, started) {
|
|
938
|
-
if (chars(db.L2) <= cfg.consolidateMaxChars && db.L2.length <= cfg.consolidateMaxItems) return 0
|
|
939
|
-
const due = pickTurns(db.L2, cfg.consolidateMaxChars, {
|
|
940
|
-
orderKey: (it) => it.consolidatedAt ?? it.createdAt,
|
|
941
|
-
maxItems: cfg.consolidateMaxItems,
|
|
942
|
-
})
|
|
943
|
-
if (!due.length) return 0
|
|
944
|
-
checkTimeout(started)
|
|
945
|
-
// DL1 全部一起送进去(用户定:DL1 与 L2 旧半融合处理)
|
|
946
|
-
const batch = [...due, ...db.DL1]
|
|
947
|
-
await digest(db, batch, { secondChance: true })
|
|
948
|
-
return batch.length
|
|
949
|
-
}
|
|
950
|
-
|
|
951
|
-
// ============================================================
|
|
952
|
-
// 信号:attend / pin / forget
|
|
953
|
-
// ============================================================
|
|
954
|
-
|
|
955
|
-
const newOpId = () => `${writer}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`
|
|
956
|
-
|
|
957
|
-
function signal(op) {
|
|
958
|
-
const rec = { ...op, opId: op.opId || newOpId(), at: tick() }
|
|
959
|
-
appendJsonl(join(sigDir, `${writer}.jsonl`), [rec])
|
|
960
|
-
return rec
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
const attend = (ids) => {
|
|
964
|
-
const list = (Array.isArray(ids) ? ids : [ids]).filter(Boolean)
|
|
965
|
-
if (!list.length) return 0
|
|
966
|
-
signal({ op: 'attend', ids: list })
|
|
967
|
-
return list.length
|
|
968
|
-
}
|
|
969
|
-
const pin = (id, v = true) => { signal({ op: 'pin', ids: [String(id)], v: !!v }); return { ok: true, id, pinned: !!v } }
|
|
970
|
-
const forget = (id) => { signal({ op: 'forget', ids: [String(id)] }); return { ok: true, id, queued: true } }
|
|
971
|
-
|
|
972
|
-
function findAny(db, id) {
|
|
973
|
-
for (const l of ['DL3', 'DL2', 'DL1', 'L2', 'L1']) {
|
|
974
|
-
const it = db[l].find((x) => x.id === id)
|
|
975
|
-
if (it) return it
|
|
976
|
-
}
|
|
977
|
-
return null
|
|
978
|
-
}
|
|
979
|
-
|
|
980
|
-
function writeForgotten(it, now, retention) {
|
|
981
|
-
const rec = {
|
|
982
|
-
id: it.id,
|
|
983
|
-
layer: it.layer,
|
|
984
|
-
channel: it.channel,
|
|
985
|
-
text: String(it.text || '').slice(0, 240),
|
|
986
|
-
importance: it.importance,
|
|
987
|
-
created_local: new Date(it.createdAt).toLocaleString('zh-CN'),
|
|
988
|
-
forgotten_local: new Date(now).toLocaleString('zh-CN'),
|
|
989
|
-
lifespan_days: Math.round((now - it.createdAt) / DAY),
|
|
990
|
-
retention: retention === null ? null : Number(retention.toFixed(3)),
|
|
991
|
-
reason: '遗忘',
|
|
992
|
-
}
|
|
993
|
-
try { appendFileSync(gravePath, JSON.stringify(rec) + '\n', 'utf8'); return true } catch { return false }
|
|
994
|
-
}
|
|
995
|
-
|
|
996
|
-
function applySignals(db) {
|
|
997
|
-
const claimed = claimSignals()
|
|
998
|
-
if (!claimed.length) return { ops: 0, claimed }
|
|
999
|
-
let applied = { ops: [] }
|
|
1000
|
-
try { applied = JSON.parse(readFileSync(appliedPath, 'utf8')) } catch { /* 首次 */ }
|
|
1001
|
-
const done = new Set(applied.ops || [])
|
|
1002
|
-
let n = 0
|
|
1003
|
-
const now = tick()
|
|
1004
|
-
for (const p of claimed) {
|
|
1005
|
-
const r = readJsonl(p)
|
|
1006
|
-
for (const op of r.items) {
|
|
1007
|
-
if (!op || !op.opId || done.has(op.opId)) continue
|
|
1008
|
-
const ids = Array.isArray(op.ids) ? op.ids : []
|
|
1009
|
-
if (op.op === 'attend') {
|
|
1010
|
-
for (const id of ids) {
|
|
1011
|
-
const it = findAny(db, id)
|
|
1012
|
-
if (!it) continue
|
|
1013
|
-
touchHeat(it, now, cfg)
|
|
1014
|
-
it.recallCount = (Number(it.recallCount) || 0) + 1
|
|
1015
|
-
if (it.layer === 'DL1') moveItem(db, it, 'L2', now)
|
|
1016
|
-
if (it.layer === 'DL3') l3Dirty = true // 热度变了,L3 排序要跟着变
|
|
1017
|
-
n++
|
|
1018
|
-
}
|
|
1019
|
-
} else if (op.op === 'pin') {
|
|
1020
|
-
for (const id of ids) { const it = findAny(db, id); if (it) { it.pinned = !!op.v; n++ } }
|
|
1021
|
-
} else if (op.op === 'forget') {
|
|
1022
|
-
for (const id of ids) {
|
|
1023
|
-
const it = findAny(db, id)
|
|
1024
|
-
if (!it) continue
|
|
1025
|
-
if (!writeForgotten(it, now, null)) { audit(`FORGET_FAIL ${id} 台账写不进去,不删`); continue }
|
|
1026
|
-
if (it.layer === 'DL3') l3Dirty = true
|
|
1027
|
-
for (const l of DATA_LAYERS) db[l] = db[l].filter((x) => x.id !== id)
|
|
1028
|
-
vectors.delete(id); vectorsDirty = true
|
|
1029
|
-
n++
|
|
1030
|
-
}
|
|
1031
|
-
}
|
|
1032
|
-
done.add(op.opId)
|
|
1033
|
-
}
|
|
1034
|
-
}
|
|
1035
|
-
applied.ops = [...done].slice(-2000)
|
|
1036
|
-
// 落盘 / 记 applied / 删 claimed 由 organize 统一做(顺序:先存数据,再记账,再删源)
|
|
1037
|
-
return { ops: n, claimed, applied }
|
|
1038
|
-
}
|
|
1039
|
-
|
|
1040
|
-
// ============================================================
|
|
1041
|
-
// 遗忘 + L3 快照
|
|
1042
|
-
// ============================================================
|
|
1043
|
-
|
|
1044
|
-
function retentionOf(it, now) {
|
|
1045
|
-
const last = it.lastMentionedAt ?? it.createdAt
|
|
1046
|
-
const t = Math.max(0, (now - last) / DAY)
|
|
1047
|
-
return Math.exp(-t / halfLifeOf(it, cfg))
|
|
1048
|
-
}
|
|
1049
|
-
|
|
1050
|
-
function forgetDL1(db) {
|
|
1051
|
-
const now = tick()
|
|
1052
|
-
const keep = []
|
|
1053
|
-
let n = 0
|
|
1054
|
-
for (const it of db.DL1) {
|
|
1055
|
-
if (it.pinned) { keep.push(it); continue }
|
|
1056
|
-
const r = retentionOf(it, now)
|
|
1057
|
-
if (r >= cfg.forgetThreshold) { keep.push(it); continue }
|
|
1058
|
-
if (!writeForgotten(it, now, r)) { keep.push(it); continue } // 台账写不进去就不删
|
|
1059
|
-
vectors.delete(it.id); vectorsDirty = true
|
|
1060
|
-
n++
|
|
1061
|
-
}
|
|
1062
|
-
db.DL1 = keep
|
|
1063
|
-
if (n) audit(`FORGET ${n} 条(DL1 保持率低于 ${cfg.forgetThreshold})`)
|
|
1064
|
-
return n
|
|
1065
|
-
}
|
|
1066
|
-
|
|
1067
|
-
function scoreOf(it, now) {
|
|
1068
|
-
return (Number(it.importance) || 0) * heatOf(it, now, cfg)
|
|
1069
|
-
}
|
|
1070
|
-
|
|
1071
|
-
// L3 快照只在"内容真的变了"或"到刷新间隔"时重写:
|
|
1072
|
-
// 它排在注入块最前面,每 5 分钟重排一次会把后面的前缀缓存全打断。
|
|
1073
|
-
let l3Dirty = false
|
|
1074
|
-
let lastL3At = 0
|
|
1075
|
-
|
|
1076
|
-
/**
|
|
1077
|
-
* DL3 的版本指纹(mtime + size)。
|
|
1078
|
-
*
|
|
1079
|
-
* ★ 为什么需要:`l3Dirty` 是**进程内**变量。A 端整理出新摘要、刷新了 L3.json,
|
|
1080
|
-
* B 端拿钥匙时自己的 l3Dirty 是 false、lastL3At 也没到点 —— 它不会重写快照,
|
|
1081
|
-
* 新摘要要等 B 的下一个 viewRefreshMs(30 分钟)才进上下文。
|
|
1082
|
-
* 把"这份快照基于哪个版本的 DL3"写进 L3.json,跨进程才判断得出来过期没有。
|
|
1083
|
-
*/
|
|
1084
|
-
function dl3Version() {
|
|
1085
|
-
try { const st = statSync(paths.DL3); return `${st.mtimeMs}:${st.size}` } catch { return '0:0' }
|
|
1086
|
-
}
|
|
1087
|
-
/** DL3 文件多大(只用 stat,不解析)。注入判断"现算值不值得"时用 */
|
|
1088
|
-
function dl3Bytes() {
|
|
1089
|
-
try { return statSync(paths.DL3).size } catch { return 0 }
|
|
1090
|
-
}
|
|
1091
|
-
// 快照过期时现算的上限:小库(默认 1.5MB ≈ 三五千条摘要)现算只要几毫秒,
|
|
1092
|
-
// 大库读一次要几秒——注入是每轮都跑的,宁可看旧快照,等整理者下一轮刷新。
|
|
1093
|
-
const L3_INLINE_MAX = 1.5 * 1024 * 1024
|
|
1094
|
-
|
|
1095
|
-
/** 从 DL3 按 importance×heat 截前 N 条(纯计算,不写盘) */
|
|
1096
|
-
function computeL3(db = null, now = tick()) {
|
|
1097
|
-
const pool = (db && db.DL3) ? db.DL3 : (() => { try { return readLayerCached('DL3') } catch { return [] } })()
|
|
1098
|
-
const scored = pool
|
|
1099
|
-
.filter((it) => !it.supersededBy)
|
|
1100
|
-
.map((it) => ({ it, s: scoreOf(it, now) }))
|
|
1101
|
-
.sort((a, b) => b.s - a.s)
|
|
1102
|
-
const out = []
|
|
1103
|
-
let used = 0
|
|
1104
|
-
for (const { it } of scored) {
|
|
1105
|
-
const len = String(it.text || '').length
|
|
1106
|
-
if (used + len > cfg.longMaxChars) break
|
|
1107
|
-
out.push(it)
|
|
1108
|
-
used += len
|
|
1109
|
-
}
|
|
1110
|
-
return out
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
function refreshL3(db) {
|
|
1114
|
-
const now = tick()
|
|
1115
|
-
const out = computeL3(db, now)
|
|
1116
|
-
atomicWrite(paths.L3, JSON.stringify({ version: 3, at: now, dl3v: dl3Version(), items: out }, null, 1))
|
|
1117
|
-
lastL3At = now
|
|
1118
|
-
l3Dirty = false
|
|
1119
|
-
return out.length
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
/**
|
|
1123
|
-
* 读快照:只有「快照记的 DL3 版本 == 当前 DL3 版本」才算新鲜。
|
|
1124
|
-
* 过期(对方刚整理完)返回 null,调用方现算一份——不写盘,写盘是整理者的活。
|
|
1125
|
-
*/
|
|
1126
|
-
function readL3Snapshot() {
|
|
1127
|
-
try {
|
|
1128
|
-
const snap = JSON.parse(readFileSync(paths.L3, 'utf8'))
|
|
1129
|
-
const items = Array.isArray(snap?.items) ? snap.items : []
|
|
1130
|
-
if (snap && typeof snap.dl3v === 'string' && snap.dl3v === dl3Version()) return items
|
|
1131
|
-
// 快照过期:小库返回 null(让调用方现算,保证"对方刚写下一轮就能看见");
|
|
1132
|
-
// 大库直接给旧快照——读一次几十万条 DL3 要一两秒,注入扛不住。
|
|
1133
|
-
if (dl3Bytes() <= L3_INLINE_MAX) return null
|
|
1134
|
-
return items
|
|
1135
|
-
} catch { return null }
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
/** 快照是否落后于 DL3(整理者据此决定要不要重写,不受上面的"大库不现算"影响) */
|
|
1139
|
-
function l3SnapshotStale() {
|
|
1140
|
-
try {
|
|
1141
|
-
const snap = JSON.parse(readFileSync(paths.L3, 'utf8'))
|
|
1142
|
-
return snap?.dl3v !== dl3Version()
|
|
1143
|
-
} catch { return true }
|
|
1144
|
-
}
|
|
1145
|
-
|
|
1146
|
-
// ============================================================
|
|
1147
|
-
// 整理主流程
|
|
1148
|
-
// ============================================================
|
|
1149
|
-
|
|
1150
|
-
let organized = 0
|
|
1151
|
-
let busy = false
|
|
1152
|
-
|
|
1153
|
-
async function organize() {
|
|
1154
|
-
if (busy) return { ok: false, skipped: 'running' }
|
|
1155
|
-
if (!acquireLock()) return { ok: false, skipped: 'busy' }
|
|
1156
|
-
busy = true
|
|
1157
|
-
const started = tick()
|
|
1158
|
-
const hb = setInterval(heartbeat, 30 * 1000)
|
|
1159
|
-
let claimed = []
|
|
1160
|
-
try {
|
|
1161
|
-
const db = loadDb()
|
|
1162
|
-
// 整理时对账(注入路径不再做这件事,见 readL1View 的说明):
|
|
1163
|
-
// 清理崩溃留下的跨层交叉、DL2 孤儿。清掉的条目要能写回去,
|
|
1164
|
-
// 所以这里记下条数,最后并进 changed。
|
|
1165
|
-
const rec1 = reconcile(db)
|
|
1166
|
-
// DL3 的内容快照:用来判断"这轮到底动没动摘要",没动就别重写文件。
|
|
1167
|
-
// 重写会让 mtime 变、让 L3 快照被判过期,白白打断一次前缀缓存。
|
|
1168
|
-
const dl3Before = JSON.stringify(db.DL3)
|
|
1169
|
-
const m = mergeTemps(db)
|
|
1170
|
-
claimed = m.claimed
|
|
1171
|
-
maybeReloadVectors()
|
|
1172
|
-
// 先消费信号:attend 可能把 DL1 升回 L2,别让 L2 摘要先把这条吃掉。
|
|
1173
|
-
const sg = applySignals(db)
|
|
1174
|
-
// 信号可能刚删了摘要;再对一次账,DL2 孤儿同轮就归位到 DL1。
|
|
1175
|
-
const rec2 = reconcile(db)
|
|
1176
|
-
checkTimeout(started)
|
|
1177
|
-
const n1 = await organizeL1(db, started)
|
|
1178
|
-
checkTimeout(started)
|
|
1179
|
-
const n2 = await organizeL2(db, started)
|
|
1180
|
-
checkTimeout(started)
|
|
1181
|
-
const nf = forgetDL1(db)
|
|
1182
|
-
checkLock() // 卡太久可能被接管;不是我的钥匙就不许写
|
|
1183
|
-
// ★ 没变化就不落盘:turn/end 之后每轮都会调一次 organize,
|
|
1184
|
-
// 无条件写五个文件会把"每轮尝试整理"变成"每轮全量重写"。
|
|
1185
|
-
const changed = m.merged > 0 || (sg?.ops || 0) > 0 || n1 > 0 || n2 > 0 || nf > 0 ||
|
|
1186
|
-
rec1 > 0 || rec2 > 0
|
|
1187
|
-
if (changed) {
|
|
1188
|
-
const dl3Changed = JSON.stringify(db.DL3) !== dl3Before
|
|
1189
|
-
saveDb(db, { skip: dl3Changed ? [] : ['DL3'] })
|
|
1190
|
-
}
|
|
1191
|
-
// 先存数据,再记 applied,再删源文件(崩在中间最多重放一次,不会丢)
|
|
1192
|
-
if (sg?.applied) atomicWrite(appliedPath, JSON.stringify(sg.applied, null, 1))
|
|
1193
|
-
for (const p of claimed) { try { unlinkSync(p) } catch { /* 已没 */ } }
|
|
1194
|
-
for (const p of sg?.claimed || []) { try { unlinkSync(p) } catch { /* 已没 */ } }
|
|
1195
|
-
if (changed) saveVectors()
|
|
1196
|
-
// L3 快照:内容变了 / 快照过期 / 到了刷新间隔 才重写
|
|
1197
|
-
// (每 5 分钟无条件重排会打断前缀缓存,所以平时不动它)
|
|
1198
|
-
const l3Every = Number(cfg.viewRefreshMs) || 0
|
|
1199
|
-
const needL3 = l3Dirty || !l3Every || (tick() - lastL3At >= l3Every) || l3SnapshotStale()
|
|
1200
|
-
const l3 = needL3
|
|
1201
|
-
? refreshL3(db)
|
|
1202
|
-
: (() => { try { return readLayer('L3').length } catch { return 0 } })()
|
|
1203
|
-
organized++
|
|
1204
|
-
const state = {
|
|
1205
|
-
at: tick(), writer, elapsedMs: tick() - started,
|
|
1206
|
-
merged: m.merged, l1Moved: n1, l2Moved: n2, l3,
|
|
1207
|
-
count: { L1: db.L1.length, L2: db.L2.length, DL1: db.DL1.length, DL2: db.DL2.length, DL3: db.DL3.length },
|
|
1208
|
-
chars: { L1: chars(db.L1), L2: chars(db.L2), DL1: chars(db.DL1), DL2: chars(db.DL2), DL3: chars(db.DL3) },
|
|
1209
|
-
vectors: vectors.size,
|
|
1210
|
-
}
|
|
1211
|
-
atomicWrite(statePath, JSON.stringify(state, null, 1))
|
|
1212
|
-
audit(`ORGANIZE ok merged=${m.merged} L1=${n1} L2=${n2} L3=${l3} ${state.elapsedMs}ms`)
|
|
1213
|
-
return { ok: true, ...state }
|
|
1214
|
-
} catch (e) {
|
|
1215
|
-
audit(`ORGANIZE_FAIL ${e.message}`)
|
|
1216
|
-
return { ok: false, error: e.message }
|
|
1217
|
-
} finally {
|
|
1218
|
-
clearInterval(hb)
|
|
1219
|
-
busy = false
|
|
1220
|
-
releaseLock()
|
|
1221
|
-
}
|
|
1222
|
-
}
|
|
1223
|
-
|
|
1224
|
-
// ============================================================
|
|
1225
|
-
// 注入
|
|
1226
|
-
// ============================================================
|
|
1227
|
-
|
|
1228
|
-
function pack(list, limit) {
|
|
1229
|
-
const out = []
|
|
1230
|
-
let used = 0
|
|
1231
|
-
for (const it of list) {
|
|
1232
|
-
const l = lineOf(it)
|
|
1233
|
-
if (used + l.length + 1 > limit) {
|
|
1234
|
-
if (!out.length && limit > 40) {
|
|
1235
|
-
const cut = l.slice(0, Math.max(20, limit - 1))
|
|
1236
|
-
out.push(cut + '…')
|
|
1237
|
-
}
|
|
1238
|
-
break
|
|
1239
|
-
}
|
|
1240
|
-
out.push(l)
|
|
1241
|
-
used += l.length + 1
|
|
1242
|
-
}
|
|
1243
|
-
return out
|
|
1244
|
-
}
|
|
1245
|
-
|
|
1246
|
-
function lineOf(it) {
|
|
1247
|
-
const isSum = it.kind === 'summary' || !!it.spanFrom
|
|
1248
|
-
const tag = isSum && it.spanFrom
|
|
1249
|
-
? `${dayKey(it.spanFrom).slice(5)}~${dayKey(it.spanTo || it.createdAt).slice(5)} `
|
|
1250
|
-
: `${dayKey(it.createdAt).slice(5)} `
|
|
1251
|
-
const who = isSum
|
|
1252
|
-
? '[摘要] '
|
|
1253
|
-
: `[${it.origin || '?'}${it.speaker ? '·' + it.speaker : ''}] `
|
|
1254
|
-
const mark = it.supersededBy ? '(已作废)' : ''
|
|
1255
|
-
return `- ${mark}${tag}${who}${String(it.text || '').replace(/\s+/g, ' ').trim()}`
|
|
1256
|
-
}
|
|
1257
|
-
|
|
1258
|
-
function buildBlock(budget = 0) {
|
|
1259
|
-
// ★ 不再"读快照 → 空就现算",而是"版本对得上才用快照":
|
|
1260
|
-
// 对方刚整理完(DL3 变了、快照还没轮到它写)时,这里会现算一份,
|
|
1261
|
-
// 保证「对方写的东西,我下一轮注入就能看见」,不用等 30 分钟的刷新周期。
|
|
1262
|
-
// 现算不写盘——写快照是拿钥匙那一方的职责,否则又变成两个写者。
|
|
1263
|
-
let l3 = readL3Snapshot()
|
|
1264
|
-
if (l3 === null) l3 = computeL3(null)
|
|
1265
|
-
l3 = l3.filter((it) => !it.supersededBy)
|
|
1266
|
-
// L2 只读一次:既用来拼"最近沉淀的",顺手当"已下沉"的 id 集合给 L1 视图去重
|
|
1267
|
-
let l2raw = []
|
|
1268
|
-
try { l2raw = readLayerCached('L2') } catch { l2raw = [] }
|
|
1269
|
-
const l2 = l2raw.filter((it) => !it.supersededBy)
|
|
1270
|
-
const l1r = readL1View({ placed: new Set(l2raw.map((it) => it.id)) })
|
|
1271
|
-
const l1 = l1r.items.filter((it) => !it.supersededBy)
|
|
1272
|
-
|
|
1273
|
-
l2.sort((a, b) => ((a.consolidatedAt ?? a.createdAt) - (b.consolidatedAt ?? b.createdAt)) || cmpId(a.id, b.id))
|
|
1274
|
-
l1.sort((a, b) => (b.createdAt - a.createdAt) || cmpId(b.id, a.id))
|
|
1275
|
-
|
|
1276
|
-
const longLines = pack(l3, Infinity)
|
|
1277
|
-
const consLines = pack(l2, cfg.consolidateMaxChars)
|
|
1278
|
-
|
|
1279
|
-
// L1:按端配额。自己最多 hotQuotaOwn;剩下的给另一端(另一端空着时自己也只能到这条线)。
|
|
1280
|
-
const cap = cfg.hotMaxChars
|
|
1281
|
-
// 自己最多占 quotaOwn;另一端空着时,自己也只能到这条线(配额是"上限"不是"平分")
|
|
1282
|
-
const ownLimit = Math.max(1, Math.floor(cap * cfg.hotQuotaOwn))
|
|
1283
|
-
const mine = pack(l1.filter((it) => it.origin === writer), ownLimit)
|
|
1284
|
-
const mineUsed = mine.reduce((s, l) => s + l.length + 1, 0)
|
|
1285
|
-
const other = pack(l1.filter((it) => it.origin !== writer), Math.max(0, cap - mineUsed))
|
|
1286
|
-
|
|
1287
|
-
const parts = []
|
|
1288
|
-
if (longLines.length) parts.push('## 一直记得的', ...longLines)
|
|
1289
|
-
if (consLines.length) parts.push('## 最近沉淀的', ...consLines)
|
|
1290
|
-
if (mine.length || other.length) parts.push('## 刚发生的', ...mine, ...other)
|
|
1291
|
-
const shown = mine.length + other.length
|
|
1292
|
-
if (shown < l1.length) parts.push(`(即时层还有 ${l1.length - shown} 条未显示)`)
|
|
1293
|
-
const out = parts.join('\n')
|
|
1294
|
-
if (budget > 0 && out.length > budget) return out.slice(0, budget)
|
|
1295
|
-
return out
|
|
1296
|
-
}
|
|
1297
|
-
|
|
1298
|
-
// ============================================================
|
|
1299
|
-
// 检索
|
|
1300
|
-
// ============================================================
|
|
1301
|
-
|
|
1302
|
-
const STOP = new Set(['的', '了', '是', '在', '我', '你', '他', '她', '它', '和', '就', '都', '也', '吗', '呢', '啊', '这', '那', '什么', '怎么', '那事', '事情'])
|
|
1303
|
-
|
|
1304
|
-
function keywords(q) {
|
|
1305
|
-
const s = String(q || '').toLowerCase()
|
|
1306
|
-
const out = new Set()
|
|
1307
|
-
for (const w of s.split(/[\s,,。!?!?、;;::""''()()\[\]]+/)) {
|
|
1308
|
-
if (w.length >= 2 && !STOP.has(w)) out.add(w)
|
|
1309
|
-
}
|
|
1310
|
-
const zh = s.replace(/[^\u4e00-\u9fa5]/g, '')
|
|
1311
|
-
for (let i = 0; i < zh.length - 1; i++) {
|
|
1312
|
-
const bg = zh.slice(i, i + 2)
|
|
1313
|
-
if (!STOP.has(bg)) out.add(bg)
|
|
1314
|
-
}
|
|
1315
|
-
return out
|
|
1316
|
-
}
|
|
1317
|
-
|
|
1318
|
-
function parseTime(q, now) {
|
|
1319
|
-
const d0 = new Date(now)
|
|
1320
|
-
const startOfDay = (d) => new Date(d.getFullYear(), d.getMonth(), d.getDate()).getTime()
|
|
1321
|
-
const today = startOfDay(d0)
|
|
1322
|
-
const map = {
|
|
1323
|
-
刚刚: [now - 2 * 3600e3, now],
|
|
1324
|
-
今天: [today, today + DAY],
|
|
1325
|
-
昨天: [today - DAY, today],
|
|
1326
|
-
前天: [today - 2 * DAY, today - DAY],
|
|
1327
|
-
这周: [today - 6 * DAY, today + DAY],
|
|
1328
|
-
上周: [today - 13 * DAY, today - 6 * DAY],
|
|
1329
|
-
最近: [today - 7 * DAY, today + DAY],
|
|
1330
|
-
之前: [0, now],
|
|
1331
|
-
}
|
|
1332
|
-
for (const [k, v] of Object.entries(map)) {
|
|
1333
|
-
if (String(q).includes(k)) return { from: v[0], to: v[1], word: k }
|
|
1334
|
-
}
|
|
1335
|
-
return null
|
|
1336
|
-
}
|
|
1337
|
-
|
|
1338
|
-
const DETAIL_HINTS = /具体|原话|细节|怎么说的|怎么讲|当时|原原本本|哪一句|逐条|列一下|都做了|分别|多少|几号/
|
|
1339
|
-
const DEPTH_LAYERS = { summary: ['DL3'], detail: ['DL3', 'DL2'], full: ['DL3', 'DL2', 'DL1'] }
|
|
1340
|
-
const layerBoost = (l) => (l === 'DL3' ? 1 : l === 'DL2' ? 0.8 : 0.6)
|
|
1341
|
-
|
|
1342
|
-
async function search(query, opts = {}) {
|
|
1343
|
-
maybeReloadVectors()
|
|
1344
|
-
const q = String(query || '').trim()
|
|
1345
|
-
let depth = opts.depth || 'auto'
|
|
1346
|
-
if (depth === 'auto') depth = DETAIL_HINTS.test(q) ? 'detail' : 'summary'
|
|
1347
|
-
const limit = Number(opts.limit) > 0 ? Number(opts.limit) : 6
|
|
1348
|
-
const scene = opts.scene || null
|
|
1349
|
-
const now = tick()
|
|
1350
|
-
|
|
1351
|
-
const win = parseTime(q, now)
|
|
1352
|
-
const kw = keywords(win ? q.replace(win.word, '') : q)
|
|
1353
|
-
const qVec = await embedOne(q)
|
|
1354
|
-
|
|
1355
|
-
const layers = DEPTH_LAYERS[depth] || DEPTH_LAYERS.summary
|
|
1356
|
-
const pool = []
|
|
1357
|
-
for (const l of layers) {
|
|
1358
|
-
try { for (const it of readLayerCached(l)) pool.push(it) } catch { /* 单层坏了不影响别层 */ }
|
|
1359
|
-
}
|
|
1360
|
-
|
|
1361
|
-
const scored = []
|
|
1362
|
-
for (const it of pool) {
|
|
1363
|
-
if (it.supersededBy && !/以前|之前/.test(q)) continue
|
|
1364
|
-
if (scene && it.scene !== scene) continue
|
|
1365
|
-
const from = it.spanFrom ?? it.createdAt
|
|
1366
|
-
const to = it.spanTo ?? it.createdAt
|
|
1367
|
-
if (win && !(to >= win.from && from <= win.to)) continue
|
|
1368
|
-
|
|
1369
|
-
const body = [String(it.text || ''), it.speaker || '', (it.people || []).join(' '), it.scene || ''].join(' ')
|
|
1370
|
-
let hit = 0
|
|
1371
|
-
for (const k of kw) if (body.includes(k)) hit++
|
|
1372
|
-
const coverage = kw.size ? hit / kw.size : 0
|
|
1373
|
-
const v = vectors.get(it.id)
|
|
1374
|
-
const sem = qVec && v ? cosine(qVec, v) : 0
|
|
1375
|
-
if (kw.size && hit === 0 && sem < 0.2 && !win) continue
|
|
1376
|
-
|
|
1377
|
-
const timeScore = win ? 0.9 : 0.5
|
|
1378
|
-
const imp = Number(it.importance) || 0
|
|
1379
|
-
const score = qVec
|
|
1380
|
-
? 0.42 * sem + 0.22 * coverage + 0.20 * timeScore + 0.10 * imp + 0.06 * layerBoost(it.layer)
|
|
1381
|
-
: 0.55 * coverage + 0.25 * timeScore + 0.12 * imp + 0.08 * layerBoost(it.layer)
|
|
1382
|
-
scored.push({ it, score, sem, coverage })
|
|
1383
|
-
}
|
|
1384
|
-
|
|
1385
|
-
scored.sort((a, b) => b.score - a.score)
|
|
1386
|
-
let top = scored.slice(0, Math.max(limit * 2, limit))
|
|
1387
|
-
|
|
1388
|
-
// 精排:候选 > 1 才跑;失败就用原分
|
|
1389
|
-
let reranked = false
|
|
1390
|
-
if (opts.useRerank !== false && typeof o.rerank === 'function' && top.length > 1) {
|
|
1391
|
-
try {
|
|
1392
|
-
bump('rerank', top.length)
|
|
1393
|
-
const rs = await o.rerank(q, top.map((x) => x.it.text))
|
|
1394
|
-
// silicon.rerank 返回 [{ index, score }],不是裸数字;两种都兼容
|
|
1395
|
-
if (Array.isArray(rs) && rs.length) {
|
|
1396
|
-
const rel = new Map()
|
|
1397
|
-
rs.forEach((x, i) => {
|
|
1398
|
-
if (x && typeof x === 'object') rel.set(Number(x.index), Number(x.score) || 0)
|
|
1399
|
-
else rel.set(i, Number(x) || 0)
|
|
1400
|
-
})
|
|
1401
|
-
const minRel = Number(cfg.minRerank ?? 0.02)
|
|
1402
|
-
const ranked = top
|
|
1403
|
-
.map((x, i) => ({ ...x, rel: rel.has(i) ? rel.get(i) : 0 }))
|
|
1404
|
-
.filter((x) => x.rel >= minRel)
|
|
1405
|
-
.map((x) => ({ ...x, score: 0.55 * x.rel + 0.45 * x.score }))
|
|
1406
|
-
.sort((a, b) => b.score - a.score)
|
|
1407
|
-
// 精排把结果全滤光了,通常是模型/网络降级 → 保留原分,宁可多给几条
|
|
1408
|
-
if (ranked.length) { top = ranked; reranked = true }
|
|
1409
|
-
}
|
|
1410
|
-
} catch { /* 降级:用原分 */ }
|
|
1411
|
-
}
|
|
1412
|
-
|
|
1413
|
-
const hits = top.slice(0, limit).map((x) => ({
|
|
1414
|
-
id: x.it.id, text: x.it.text, layer: x.it.layer, channel: x.it.channel,
|
|
1415
|
-
when: dayKey(x.it.createdAt),
|
|
1416
|
-
scene: x.it.scene || '', createdAt: x.it.createdAt, spanFrom: x.it.spanFrom ?? null,
|
|
1417
|
-
importance: x.it.importance, score: Number(x.score.toFixed(4)),
|
|
1418
|
-
rel: x.rel ?? null, kind: x.it.kind, sources: x.it.sources || [],
|
|
1419
|
-
}))
|
|
1420
|
-
|
|
1421
|
-
// detail/full:顺着摘要的 sources 去 DL2/DL1 取原文(比拿查询扫 DL2 准)
|
|
1422
|
-
if (depth !== 'summary' && opts.followSources !== false) {
|
|
1423
|
-
// ★ 不再把 DL2/DL1 整层塞进 Map:10 万条时要 20 万次 set/has,每检索一次。
|
|
1424
|
-
// 改成"按需查索引"——索引挂在缓存条目上,建一次一直用。
|
|
1425
|
-
const idxD2 = ensureIndex('DL2')
|
|
1426
|
-
const idxD1 = ensureIndex('DL1')
|
|
1427
|
-
const findDeep = (id) => (idxD2 && idxD2.get(id)) || (idxD1 && idxD1.get(id)) || null
|
|
1428
|
-
const seen = new Set(hits.map((h) => h.id))
|
|
1429
|
-
const extra = []
|
|
1430
|
-
for (const h of hits) {
|
|
1431
|
-
if (h.kind !== 'summary' || !h.sources?.length) continue
|
|
1432
|
-
for (const id of h.sources) {
|
|
1433
|
-
if (seen.has(id)) continue
|
|
1434
|
-
const it = findDeep(id)
|
|
1435
|
-
if (!it) continue
|
|
1436
|
-
seen.add(id)
|
|
1437
|
-
extra.push({
|
|
1438
|
-
id: it.id, text: it.text, layer: it.layer, channel: it.channel,
|
|
1439
|
-
when: dayKey(it.createdAt), scene: it.scene || '',
|
|
1440
|
-
createdAt: it.createdAt, spanFrom: it.spanFrom ?? null, importance: it.importance,
|
|
1441
|
-
score: Number((h.score * 0.9).toFixed(4)), rel: h.rel ?? null, kind: it.kind, via: h.id,
|
|
1442
|
-
})
|
|
1443
|
-
}
|
|
1444
|
-
}
|
|
1445
|
-
hits.push(...extra.slice(0, 6))
|
|
1446
|
-
}
|
|
1447
|
-
|
|
1448
|
-
const relMax = hits.reduce((m, h) => Math.max(m, h.rel ?? h.score ?? 0), 0)
|
|
1449
|
-
const advice = !hits.length
|
|
1450
|
-
? '没有找到相关记忆。可以换 depth="full" 再搜一次,或确认这件事确实没记过。'
|
|
1451
|
-
: relMax >= 0.5
|
|
1452
|
-
? '相关度较高,可以直接用。'
|
|
1453
|
-
: relMax >= 0.1
|
|
1454
|
-
? '相关度中等,用之前自己核一下是不是真的在说这件事。'
|
|
1455
|
-
: '相关度偏低(可能只是字面擦边)。如果跟当前话题无关,换 depth="full" 再搜一次。'
|
|
1456
|
-
return { hits, confidence: Number(relMax.toFixed(3)), advice, depth, reranked, stages: layers }
|
|
1457
|
-
}
|
|
1458
|
-
|
|
1459
|
-
// ============================================================
|
|
1460
|
-
// 观测 / 维护
|
|
1461
|
-
// ============================================================
|
|
1462
|
-
|
|
1463
|
-
function stats() {
|
|
1464
|
-
const count = {}
|
|
1465
|
-
const charsBy = {}
|
|
1466
|
-
// L1 用"视图"统计(真 L1 + 所有 temp),否则整理前永远是 0
|
|
1467
|
-
try {
|
|
1468
|
-
const v = readL1View()
|
|
1469
|
-
count.L1 = v.items.length
|
|
1470
|
-
charsBy.L1 = chars(v.items)
|
|
1471
|
-
} catch { count.L1 = 0; charsBy.L1 = 0 }
|
|
1472
|
-
for (const l of ['L2', 'DL1', 'DL2', 'DL3']) {
|
|
1473
|
-
let items = []
|
|
1474
|
-
try { items = readLayerCached(l) } catch { /* 损坏的层按 0 展示,但 audit 里有记录 */ }
|
|
1475
|
-
count[l] = items.length
|
|
1476
|
-
charsBy[l] = chars(items)
|
|
1477
|
-
}
|
|
1478
|
-
let l3 = []
|
|
1479
|
-
try { l3 = readLayerCached('L3') } catch { /* 忽略 */ }
|
|
1480
|
-
count.L3 = l3.length
|
|
1481
|
-
charsBy.L3 = chars(l3)
|
|
1482
|
-
const temp = readTempsAll()
|
|
1483
|
-
let last = null
|
|
1484
|
-
try { last = JSON.parse(readFileSync(statePath, 'utf8')) } catch { /* 未跑过 */ }
|
|
1485
|
-
return {
|
|
1486
|
-
count, chars: charsBy,
|
|
1487
|
-
total: count.L1 + count.L2 + count.DL1 + count.DL2 + count.DL3,
|
|
1488
|
-
vectors: vectors.size, temp: temp.items.length, calls,
|
|
1489
|
-
lock: existsSync(lockPath), last,
|
|
1490
|
-
}
|
|
1491
|
-
}
|
|
1492
|
-
|
|
1493
|
-
function list(layer = null) {
|
|
1494
|
-
// ★ 一律返回数组副本:读的是缓存里的数组,直接给出去的话,
|
|
1495
|
-
// 调用方一个 sort/push 就把缓存改脏了(对象本身仍共享,只读字段可以用)。
|
|
1496
|
-
if (layer === 'L3') { try { return readLayerCached('L3').slice() } catch { return [] } }
|
|
1497
|
-
if (layer === 'L1') return readL1View().items.slice()
|
|
1498
|
-
if (layer) { try { return readLayerCached(layer).slice() } catch { return [] } }
|
|
1499
|
-
const out = [...readL1View().items]
|
|
1500
|
-
for (const l of ['L2', 'DL1', 'DL2', 'DL3']) { try { out.push(...readLayerCached(l)) } catch { /* 忽略 */ } }
|
|
1501
|
-
return out
|
|
1502
|
-
}
|
|
1503
|
-
|
|
1504
|
-
function graveyard({ limit = 50 } = {}) {
|
|
1505
|
-
let raw = ''
|
|
1506
|
-
try { raw = readFileSync(gravePath, 'utf8') } catch { return [] }
|
|
1507
|
-
return raw.split(/\r?\n/).filter(Boolean).slice(-limit).map((l) => { try { return JSON.parse(l) } catch { return null } }).filter(Boolean)
|
|
1508
|
-
}
|
|
1509
|
-
|
|
1510
|
-
function dumpVectors(p) {
|
|
1511
|
-
const out = {}
|
|
1512
|
-
for (const [id, v] of vectors) out[id] = Buffer.from(v.buffer, v.byteOffset, v.byteLength).toString('base64')
|
|
1513
|
-
writeFileSync(p, JSON.stringify(out), 'utf8')
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
let timer = null
|
|
1517
|
-
function start(opts = {}) {
|
|
1518
|
-
stop()
|
|
1519
|
-
const every = Number(opts.organizeEveryMs || cfg.organizeEveryMs) || 5 * 60 * 1000
|
|
1520
|
-
// 只有这一个定时器:整理里会按 viewRefreshMs 判断要不要重写 L3。
|
|
1521
|
-
// 不再单开一个 L3 定时器——那会在没拿钥匙时写 L3,等于第二个写者。
|
|
1522
|
-
timer = setInterval(() => { organize().catch(() => {}) }, every)
|
|
1523
|
-
if (timer.unref) timer.unref()
|
|
1524
|
-
// 启动补一轮(异步,不阻塞加载)
|
|
1525
|
-
Promise.resolve().then(() => organize()).catch(() => {})
|
|
1526
|
-
}
|
|
1527
|
-
function stop() {
|
|
1528
|
-
if (timer) clearInterval(timer)
|
|
1529
|
-
timer = null
|
|
1530
|
-
}
|
|
1531
|
-
|
|
1532
|
-
loadVectors()
|
|
1533
|
-
|
|
1534
|
-
return {
|
|
1535
|
-
// 写
|
|
1536
|
-
write, writeMany, writeTurn,
|
|
1537
|
-
// 读
|
|
1538
|
-
buildBlock, search, list, stats,
|
|
1539
|
-
// 维护
|
|
1540
|
-
organize, sweep: organize, start, stop,
|
|
1541
|
-
refreshL3, graveyard, dumpVectors,
|
|
1542
|
-
// 信号
|
|
1543
|
-
attend, pin, forget,
|
|
1544
|
-
// 兼容旧调用:split 布局没有"待摘要队列",整理统一由 organize() 做
|
|
1545
|
-
flushSummaries: async () => 0,
|
|
1546
|
-
pendingSummaries: () => 0,
|
|
1547
|
-
// 常量 / 调试
|
|
1548
|
-
LAYER, cfg, dir, writer, _paths: paths, _readLayer: readLayer,
|
|
1549
|
-
_readCached: readLayerCached,
|
|
1550
|
-
_cache: () => ({ size: layerCache.size, bytes: cacheBytes, ...cacheStat }),
|
|
1551
|
-
_clearCache: clearLayerCache,
|
|
1552
|
-
}
|
|
1553
|
-
}
|