dsh-memu-memory 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/README.md +19 -8
- package/README.zh.md +12 -6
- package/docs/contributing.md +16 -3
- package/docs/env-and-commands.md +8 -0
- package/docs/host-api.md +3 -1
- package/docs/memory-semantics.md +21 -2
- package/docs/troubleshooting.md +42 -3
- package/lib/activity-log.js +22 -5
- package/lib/backup.js +24 -2
- package/lib/binding-diagnostics.js +3 -3
- package/lib/client.js +1610 -117
- package/lib/consolidation-worker.js +68 -17
- package/lib/curator-evidence.js +131 -0
- package/lib/curator-policy.js +81 -0
- package/lib/curator.js +1299 -83
- package/lib/cycle.js +3 -0
- package/lib/embedding-download.js +39 -4
- package/lib/engine.js +6 -3
- package/lib/global-graph.js +282 -0
- package/lib/graph.js +190 -14
- package/lib/inbound.js +19 -0
- package/lib/index.js +33 -1
- package/lib/inject.js +277 -17
- package/lib/lexicon.js +1115 -0
- package/lib/local-vector-service.js +8 -6
- package/lib/managed-runtime.js +33 -2
- package/lib/memory.js +585 -76
- package/lib/memu.js +2 -2
- package/lib/migrate.js +83 -2
- package/lib/private-links.js +587 -0
- package/lib/private.js +128 -7
- package/lib/prompts/global-curator.md +76 -19
- package/lib/prompts/private-consolidation.coding.md +4 -1
- package/lib/prompts/private-consolidation.md +3 -1
- package/lib/prompts/private-consolidation.mixed.md +3 -1
- package/lib/prompts/private-consolidation.rp.md +3 -1
- package/lib/prompts.js +1 -1
- package/lib/rpc.js +622 -13
- package/lib/state.js +4 -1
- package/lib/text-retrieval.js +661 -0
- package/lib/text-similarity.js +124 -0
- package/lib/tools.js +230 -16
- package/lib/vector-migration.js +28 -3
- package/lib/vector-profile.js +24 -0
- package/lib/webui/standalone.css +17 -0
- package/lib/webui/standalone.js +2 -0
- package/package.json +3 -2
- package/scripts/acceptance-local.mjs +8 -5
- package/scripts/curator-pair-calibrate.mjs +259 -0
- package/scripts/lexicon-calibrate.mjs +230 -0
- package/scripts/lexicon-dedupe.mjs +318 -0
- package/scripts/replay.mjs +573 -0
- package/tests/absorb-review.test.mjs +353 -0
- package/tests/appendix-b.test.mjs +6 -2
- package/tests/appendix-b2.test.mjs +2 -2
- package/tests/client-export.test.mjs +520 -0
- package/tests/client-render.test.mjs +19 -3
- package/tests/consolidation-credential.test.mjs +16 -5
- package/tests/consolidation-feed.test.mjs +96 -0
- package/tests/consolidation-fork.test.mjs +144 -10
- package/tests/consolidation-snapshot.test.mjs +82 -4
- package/tests/curator-absorb.test.mjs +443 -0
- package/tests/curator-calibrate.test.mjs +173 -0
- package/tests/curator-clusters.test.mjs +281 -0
- package/tests/curator-commit.test.mjs +324 -0
- package/tests/curator-evidence.test.mjs +102 -0
- package/tests/curator-link.test.mjs +316 -0
- package/tests/curator-pairing.test.mjs +252 -0
- package/tests/curator-probe.test.mjs +15 -6
- package/tests/curator-review-mode.test.mjs +314 -0
- package/tests/disclosure.test.mjs +2 -1
- package/tests/fixtures/curator-pairs/labeled.v1.jsonl +90 -0
- package/tests/fixtures/lexicon-stub-hooks.mjs +15 -0
- package/tests/fixtures/lexicon-stub.mjs +284 -0
- package/tests/graph-cascade.test.mjs +223 -0
- package/tests/graph.test.mjs +113 -0
- package/tests/help-page.test.mjs +415 -0
- package/tests/hybrid-recall.test.mjs +32 -2
- package/tests/inbound.test.mjs +6 -3
- package/tests/inject-guide.test.mjs +28 -0
- package/tests/inject.test.mjs +157 -5
- package/tests/lexicon-calibrate.test.mjs +48 -0
- package/tests/lexicon-dedupe.test.mjs +171 -0
- package/tests/lexicon-frequency.test.mjs +179 -0
- package/tests/lexicon-inject.test.mjs +410 -0
- package/tests/lexicon-perf.test.mjs +436 -0
- package/tests/lexicon-ui.test.mjs +547 -0
- package/tests/lexicon.test.mjs +922 -0
- package/tests/managed-memory.test.mjs +55 -0
- package/tests/managed-runtime.test.mjs +32 -0
- package/tests/private-links.test.mjs +445 -0
- package/tests/prompts.test.mjs +35 -0
- package/tests/recall-dedup.test.mjs +446 -0
- package/tests/sidebar-beat.test.mjs +5 -1
- package/tests/tool-schema.test.mjs +1 -1
- package/tests/tools.test.mjs +112 -3
- package/tests/vector-migration.test.mjs +58 -1
- package/tests/webui-auth.test.mjs +348 -2
- package/tests/webui.test.mjs +191 -2
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
// 词典 revision-keyed 内存索引(§10 P1)的性能基准与等价性对拍。
|
|
2
|
+
//
|
|
3
|
+
// 两件事:
|
|
4
|
+
// 1. N=5000 词条时 matchLexiconHits 在 ~100 字用户消息上的耗时 p95 < 20ms。
|
|
5
|
+
// 这个数字是**本机回归上界,不是 SLA**:绝对值随机器/Node 版本浮动,断言的
|
|
6
|
+
// 意义是「索引化之后仍远低于注入路径的一次预算粒度」,防止有人把每次匹配
|
|
7
|
+
// 改回全量 parse / 全表重建桶。预热 5 轮后测 50 轮取 p95。
|
|
8
|
+
// 2. 索引化实现与「无缓存直接全量扫」的参照实现在 100 个随机 query 上逐字段
|
|
9
|
+
// 一致(matchLexiconHits / searchLexicon / visibleLexiconTerms 各对拍一组)。
|
|
10
|
+
// 参照实现是改造前算法的忠实拷贝(见本文件底部),刻意独立于 lib/lexicon.js
|
|
11
|
+
// 的内部实现,防止两边一起改错。
|
|
12
|
+
import test from 'node:test'
|
|
13
|
+
import assert from 'node:assert/strict'
|
|
14
|
+
import fs from 'node:fs'
|
|
15
|
+
import os from 'node:os'
|
|
16
|
+
import path from 'node:path'
|
|
17
|
+
import { performance } from 'node:perf_hooks'
|
|
18
|
+
|
|
19
|
+
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'dsh-memu-memory-lexicon-perf-'))
|
|
20
|
+
process.env.DSH_MEMU_HOME = home
|
|
21
|
+
|
|
22
|
+
const paths = await import('../lib/paths.js')
|
|
23
|
+
const state = await import('../lib/state.js')
|
|
24
|
+
const lexicon = await import('../lib/lexicon.js')
|
|
25
|
+
const inject = await import('../lib/inject.js')
|
|
26
|
+
const { estTokens } = await import('../lib/budget.js')
|
|
27
|
+
const { bigrams, dice } = await import('../lib/text-similarity.js')
|
|
28
|
+
const { contextVisible } = await import('../lib/context.js')
|
|
29
|
+
|
|
30
|
+
test.after(() => {
|
|
31
|
+
try { fs.rmSync(home, { recursive: true, force: true }) } catch {}
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
// ---- 程序化词条生成:term/aliases/examples 多样,status/scope/敏感度有分布 ----
|
|
35
|
+
|
|
36
|
+
const CJK = '糖刀月影雾灯茶雪夜星风花竹鹤泉溪桥柳岸莺燕蝶霜露雷霆云霞暮晓辰舟'.split('')
|
|
37
|
+
const LATIN = ['teatime', 'moonfall', 'embers', 'willow', 'cinder', 'harbor', 'lantern', 'quietus']
|
|
38
|
+
|
|
39
|
+
function mulberry32(seed) {
|
|
40
|
+
let a = seed >>> 0
|
|
41
|
+
return () => {
|
|
42
|
+
a = (a + 0x6d2b79f5) | 0
|
|
43
|
+
let t = Math.imul(a ^ (a >>> 15), 1 | a)
|
|
44
|
+
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
|
|
45
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** 生成 count 条词条的 terms 表(直接写盘,绕过 5000 次锁内 CAS——基准只关心读路径)。 */
|
|
50
|
+
function generateTerms(count, seed) {
|
|
51
|
+
const rand = mulberry32(seed)
|
|
52
|
+
const pick = () => CJK[Math.floor(rand() * CJK.length)]
|
|
53
|
+
const used = new Set()
|
|
54
|
+
const terms = {}
|
|
55
|
+
for (let i = 0; i < count; i++) {
|
|
56
|
+
let term
|
|
57
|
+
do {
|
|
58
|
+
if (i % 5 === 0) {
|
|
59
|
+
term = LATIN[i % LATIN.length] + '-' + i
|
|
60
|
+
} else {
|
|
61
|
+
// 30^4 ≈ 81 万组合空间:5000 条只有零星碰撞,do/while 消费 rand 重画即可。
|
|
62
|
+
term = pick() + pick() + pick() + pick()
|
|
63
|
+
}
|
|
64
|
+
} while (used.has(term))
|
|
65
|
+
used.add(term)
|
|
66
|
+
const normalizedTerm = lexicon.normalizeLexeme(term)
|
|
67
|
+
const aliases = []
|
|
68
|
+
const aliasCount = i % 4 === 0 ? 2 : i % 3 === 0 ? 1 : 0
|
|
69
|
+
for (let a = 0; a < aliasCount; a++) {
|
|
70
|
+
const alias = term.slice(0, 2) + CJK[(i + a * 9 + 3) % CJK.length] + (a ? '儿' : '子')
|
|
71
|
+
if (lexicon.normalizeLexeme(alias) !== normalizedTerm) aliases.push(alias)
|
|
72
|
+
}
|
|
73
|
+
const kind = ['term', 'slang', 'meme', 'trigger'][i % 4]
|
|
74
|
+
const status = i % 50 === 0 ? 'pending' : i % 47 === 0 ? 'rejected' : i % 97 === 0 ? 'archived' : 'active'
|
|
75
|
+
const meaning = '压测词条的含义说明,用来撑起锚点摘要与 B 通道的真实 token 成本测量。'.repeat(1) + `编号${i}。`
|
|
76
|
+
const context = i % 7 === 0 ? { worldId: `w${i % 3}` } : {}
|
|
77
|
+
terms[`lex-perf-${i}`] = {
|
|
78
|
+
term,
|
|
79
|
+
normalizedTerm,
|
|
80
|
+
kind,
|
|
81
|
+
meaning,
|
|
82
|
+
usage: i % 2 ? `谈${term}的时候用` : '',
|
|
83
|
+
aliases,
|
|
84
|
+
normalizedAliases: aliases.map(lexicon.normalizeLexeme).filter(Boolean),
|
|
85
|
+
examples: i % 3 === 0 ? [`这句话里有${term}`] : [],
|
|
86
|
+
context,
|
|
87
|
+
visibility: 'shared',
|
|
88
|
+
sensitivity: kind === 'trigger' ? 'sensitive' : 'normal',
|
|
89
|
+
strength: Math.round(((i * 0.6180339887) % 1) * 100) / 100,
|
|
90
|
+
confidence: 0.5 + (i % 5) / 10,
|
|
91
|
+
status,
|
|
92
|
+
source: 'user',
|
|
93
|
+
evidence: [],
|
|
94
|
+
fingerprint: `fp-${i}`,
|
|
95
|
+
createdAt: '2026-01-01T00:00:00.000Z',
|
|
96
|
+
updatedAt: '2026-01-02T00:00:00.000Z',
|
|
97
|
+
hitCounts: { candidate: 0, injection: 0, query: 0 },
|
|
98
|
+
lastInjectedAt: null,
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return terms
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function writePerfLibrary(libraryId, terms) {
|
|
105
|
+
paths.writeJson(path.join(home, 'lexicon', `${libraryId}.json`), {
|
|
106
|
+
version: 1,
|
|
107
|
+
revision: 1,
|
|
108
|
+
libraryId,
|
|
109
|
+
terms,
|
|
110
|
+
rejected: {},
|
|
111
|
+
})
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const perfLibrary = state.createLibrary({ name: 'lexicon-perf', cwd: path.join(home, 'perf') })
|
|
115
|
+
const PERF_TERMS = generateTerms(5000, 20261031)
|
|
116
|
+
writePerfLibrary(perfLibrary.id, PERF_TERMS)
|
|
117
|
+
const activeTerms = Object.values(PERF_TERMS).filter(entry => entry.status === 'active' && !entry.context.worldId)
|
|
118
|
+
const FILLER = '今晚的雾夜灯桥边茶铺霜露未散他忽然提起那些旧梗台词接得毫无痕迹我们笑作一团后来月影移过柳岸鹤鸣惊起溪桥下打盹的猫谁也没接话只是把茶续上灯花爆了一下'
|
|
119
|
+
// ~100 字消息:埋 2 个精确词面 + 1 个只差末字的 dice 变体,其余是共享前缀池的填充。
|
|
120
|
+
const PERF_MESSAGE = `${activeTerms[0].term}这句真是${activeTerms[1].term},${FILLER.slice(0, 30)}${activeTerms[2].term.slice(0, -1)}雪,${FILLER.slice(30)}`
|
|
121
|
+
|
|
122
|
+
// ---- 1) N=5k 基准:p95 < 20ms(回归上界,不是 SLA) ----
|
|
123
|
+
|
|
124
|
+
test('matchLexiconHits at N=5000: 50-round CPU batch under the regression bound', () => {
|
|
125
|
+
// 预热:首访触发建索引(一次全量 parse + 派生),其后是 stat 比对 + 缓存命中。
|
|
126
|
+
for (let i = 0; i < 5; i++) lexicon.matchLexiconHits(perfLibrary.id, PERF_MESSAGE, {})
|
|
127
|
+
// 断言用「50 轮 CPU 总量」而非墙钟或单次 CPU:npm test 并行跑十几个文件时机器
|
|
128
|
+
// 满载,墙钟被调度抖动顶爆(实测 2.15ms 单跑 → 满载 >20ms);而 Windows 的
|
|
129
|
+
// CPU 计时器有 ~16ms 粒度,单次采样会量化成 0/16ms 跳变。批总量把量化误差
|
|
130
|
+
// 摊薄且与并行负载无关(CPU 时间 = 真实计算量)。索引化实现 50 轮 CPU 约几毫秒;
|
|
131
|
+
// 若有人改回全量 parse/全表重建,单轮就要 100ms+,批总量轻松越界。墙钟只打印。
|
|
132
|
+
const wallSamples = []
|
|
133
|
+
const before = process.cpuUsage()
|
|
134
|
+
for (let i = 0; i < 50; i++) {
|
|
135
|
+
const wallStart = performance.now()
|
|
136
|
+
const hits = lexicon.matchLexiconHits(perfLibrary.id, PERF_MESSAGE, {})
|
|
137
|
+
wallSamples.push(performance.now() - wallStart)
|
|
138
|
+
assert.ok(Array.isArray(hits))
|
|
139
|
+
}
|
|
140
|
+
const used = process.cpuUsage(before)
|
|
141
|
+
const batchCpuMs = (used.user + used.system) / 1000
|
|
142
|
+
wallSamples.sort((a, b) => a - b)
|
|
143
|
+
const pick = (samples, q) => samples[Math.min(samples.length - 1, Math.max(0, Math.ceil(samples.length * q) - 1))]
|
|
144
|
+
const wallP95 = pick(wallSamples, 0.95)
|
|
145
|
+
const hits = lexicon.matchLexiconHits(perfLibrary.id, PERF_MESSAGE, {})
|
|
146
|
+
console.log(`[lexicon-perf] N=5000, message=${[...PERF_MESSAGE].length} chars, hits=${hits.length}; `
|
|
147
|
+
+ `cpu batch(50 rounds)=${batchCpuMs.toFixed(2)}ms avg=${(batchCpuMs / 50).toFixed(3)}ms/call `
|
|
148
|
+
+ `wall p95=${wallP95.toFixed(2)}ms `
|
|
149
|
+
+ `(CPU batch is the asserted metric — quantization-amortized and load-independent; regression upper bound, not an SLA)`)
|
|
150
|
+
assert.ok(hits.length >= 2, 'the message embeds exact terms, the pipeline must see them')
|
|
151
|
+
assert.ok(batchCpuMs < 1000, `50-round cpu batch ${batchCpuMs.toFixed(2)}ms must stay under the 1000ms regression upper bound (indexed implementation sits in single-digit ms)`)
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
// ---- 2) 等价性对拍:索引化实现 vs 无缓存全量扫参照实现 ----
|
|
155
|
+
|
|
156
|
+
const diffLibrary = state.createLibrary({ name: 'lexicon-diff', cwd: path.join(home, 'diff') })
|
|
157
|
+
const DIFF_TERMS = generateTerms(260, 777)
|
|
158
|
+
writePerfLibrary(diffLibrary.id, DIFF_TERMS)
|
|
159
|
+
// 参照实现用的快照一次性取好:match/visible 不写盘,search 的记账写盘只动
|
|
160
|
+
// hitCounts,对拍投影不包含它。
|
|
161
|
+
const DIFF_DATA = lexicon.loadLexicon(diffLibrary.id).terms
|
|
162
|
+
|
|
163
|
+
test('matchLexiconHits agrees with the naive full-scan reference on 100 random queries', () => {
|
|
164
|
+
const rand = mulberry32(424242)
|
|
165
|
+
const termList = Object.values(DIFF_TERMS)
|
|
166
|
+
const queries = []
|
|
167
|
+
for (let i = 0; i < 100; i++) {
|
|
168
|
+
const entry = termList[Math.floor(rand() * termList.length)]
|
|
169
|
+
const mode = i % 10
|
|
170
|
+
if (mode === 0) queries.push(entry.term)
|
|
171
|
+
else if (mode === 1) queries.push(`「${entry.term}」!!`)
|
|
172
|
+
else if (mode === 2) queries.push(entry.term.slice(1, -1)) // term 的中段子串
|
|
173
|
+
else if (mode === 3) queries.push(entry.term.slice(0, -1) + CJK[i % CJK.length]) // dice 变体
|
|
174
|
+
else if (mode === 4) queries.push(entry.normalizedAliases[0] || entry.term)
|
|
175
|
+
else if (mode === 5) queries.push(FILLER.slice(i % 10) + entry.term + FILLER.slice(0, i % 7))
|
|
176
|
+
else if (mode === 6) queries.push(FILLER.slice((i * 3) % 20, (i * 3) % 20 + 40)) // 无关句
|
|
177
|
+
else if (mode === 7) queries.push(entry.term[1]) // 单字:search 反向包含路径
|
|
178
|
+
else if (mode === 8) queries.push(entry.term + termList[(i + 1) % termList.length].term) // 双词连排
|
|
179
|
+
else queries.push(`zz${i}qq`) // latin 乱码:无命中
|
|
180
|
+
}
|
|
181
|
+
const contexts = [{}, { worldId: 'w1' }, { worldId: 'w0', characterId: 'c1' }]
|
|
182
|
+
let dicePaths = 0
|
|
183
|
+
for (let i = 0; i < queries.length; i++) {
|
|
184
|
+
const memoryContext = contexts[i % contexts.length]
|
|
185
|
+
const indexed = lexicon.matchLexiconHits(diffLibrary.id, queries[i], memoryContext)
|
|
186
|
+
const naive = naiveMatch(DIFF_DATA, queries[i], memoryContext)
|
|
187
|
+
const project = hits => hits.map(hit => ({ termId: hit.termId, term: hit.term, kindOfHit: hit.kindOfHit, score: hit.score }))
|
|
188
|
+
assert.deepEqual(project(indexed), project(naive), `query #${i}: ${JSON.stringify(queries[i])}`)
|
|
189
|
+
if (indexed.some(hit => hit.kindOfHit === 'dice')) dicePaths += 1
|
|
190
|
+
}
|
|
191
|
+
assert.ok(dicePaths >= 5, `the query mix must exercise the dice fallback (${dicePaths} did)`)
|
|
192
|
+
console.log(`[lexicon-perf] matchLexiconHits对拍: 100/100 identical to the naive full-scan reference (${dicePaths} queries hit the dice path)`)
|
|
193
|
+
})
|
|
194
|
+
|
|
195
|
+
test('searchLexicon and visibleLexiconTerms agree with the naive full-scan reference', () => {
|
|
196
|
+
const rand = mulberry32(909090)
|
|
197
|
+
const termList = Object.values(DIFF_TERMS)
|
|
198
|
+
for (let i = 0; i < 100; i++) {
|
|
199
|
+
const entry = termList[Math.floor(rand() * termList.length)]
|
|
200
|
+
const mode = i % 8
|
|
201
|
+
const query = mode === 0 ? entry.term
|
|
202
|
+
: mode === 1 ? `前缀${entry.term}后缀`
|
|
203
|
+
: mode === 2 ? entry.term.slice(0, 2)
|
|
204
|
+
: mode === 3 ? entry.term.slice(-2)
|
|
205
|
+
: mode === 4 ? (entry.normalizedAliases[0] || entry.term).slice(0, -1)
|
|
206
|
+
: mode === 5 ? entry.term[0]
|
|
207
|
+
: mode === 6 ? FILLER.slice(i % 11, (i % 11) + 12)
|
|
208
|
+
: `q${i}`
|
|
209
|
+
const memoryContext = i % 3 === 0 ? { worldId: 'w1' } : {}
|
|
210
|
+
// 生命周期 includeArchived(§2.1):false 只看 active,true 追加 archived;
|
|
211
|
+
// pending/rejected 两种模式都不出现。
|
|
212
|
+
const includeArchived = i % 4 === 0
|
|
213
|
+
const project = rows => rows.map(row => ({ termId: row.termId, term: row.term, matchedBy: row.matchedBy }))
|
|
214
|
+
assert.deepEqual(
|
|
215
|
+
project(lexicon.searchLexicon({ libraryId: diffLibrary.id, query, memoryContext }, { includeArchived })),
|
|
216
|
+
project(naiveSearch(DIFF_DATA, query, memoryContext, includeArchived)),
|
|
217
|
+
`search query #${i}: ${JSON.stringify(query)}`,
|
|
218
|
+
)
|
|
219
|
+
const projectVisible = rows => rows.map(row => row.termId)
|
|
220
|
+
assert.deepEqual(
|
|
221
|
+
projectVisible(lexicon.visibleLexiconTerms(diffLibrary.id, memoryContext, { includeArchived })),
|
|
222
|
+
projectVisible(naiveVisible(DIFF_DATA, memoryContext, includeArchived)),
|
|
223
|
+
`visible query #${i}`,
|
|
224
|
+
)
|
|
225
|
+
}
|
|
226
|
+
console.log('[lexicon-perf] searchLexicon/visibleLexiconTerms对拍: 100/100 identical to the naive full-scan reference')
|
|
227
|
+
})
|
|
228
|
+
|
|
229
|
+
// ---- 3) token 成本:锚点摘要(A 通道)与 B 通道各产出一次真实输出 ----
|
|
230
|
+
|
|
231
|
+
test('token cost of the A/B channel outputs on the 5k library (printed; anchor capped at 300)', () => {
|
|
232
|
+
const binding = state.ensureSession('lexicon-perf-session', { cwd: path.join(home, 'perf') })
|
|
233
|
+
const anchor = inject.buildLexiconAnchor(binding, binding.sessionId, 300)
|
|
234
|
+
const anchorTokens = estTokens(anchor.text)
|
|
235
|
+
const turn = inject.injectLexiconTurn({ binding, sessionId: binding.sessionId, userTexts: [PERF_MESSAGE], tokenBudget: 300 })
|
|
236
|
+
const turnTokens = estTokens(turn.text)
|
|
237
|
+
console.log(`[lexicon-perf] token cost on N=5000: anchor(A) estTokens=${anchorTokens} (${anchor.text.split('\n').length} lines), `
|
|
238
|
+
+ `B-channel estTokens=${turnTokens} (${turn.selected.length} of ${turn.hits.length} hits selected)`)
|
|
239
|
+
assert.ok(anchorTokens <= 300, `anchor section must fit the 300-token hard cap, got ${anchorTokens}`)
|
|
240
|
+
assert.ok(turnTokens <= 300, 'the B channel is budget-bounded by construction')
|
|
241
|
+
assert.ok(turn.selected.length >= 1, 'the message embeds real terms; budget 300 must fit at least one line')
|
|
242
|
+
})
|
|
243
|
+
|
|
244
|
+
// ---- 参照实现:改造前「无缓存全量扫」算法的忠实拷贝(lib/lexicon.js 旧 match/
|
|
245
|
+
// search/visible 逐行同构)。刻意复制而非 import:lib 内部已索引化,这份拷贝是
|
|
246
|
+
// 独立的第二实现,两边一致才说明索引没有改变语义。唯一的刻意差异是 sensitivity
|
|
247
|
+
//Rank 用统一后的定义(sensitivity === 'sensitive'),与当前 lib 保持一致。 ----
|
|
248
|
+
|
|
249
|
+
const LEXICON_DICE_THRESHOLD = lexicon.LEXICON_DICE_THRESHOLD
|
|
250
|
+
const LEXICON_MAX_BUCKETS = lexicon.LEXICON_MAX_BUCKETS
|
|
251
|
+
const LEXICON_MAX_CANDIDATES = lexicon.LEXICON_MAX_CANDIDATES
|
|
252
|
+
const LEXICON_MAX_TERMS = lexicon.LEXICON_MAX_TERMS
|
|
253
|
+
const LEXICON_MIN_INCLUDE_LEN = lexicon.LEXICON_MIN_INCLUDE_LEN
|
|
254
|
+
const BUCKET_WIDTH = 2
|
|
255
|
+
|
|
256
|
+
function naiveDisplayRank(a, b) {
|
|
257
|
+
return (Number(b.strength) || 0) - (Number(a.strength) || 0)
|
|
258
|
+
|| String(b.updatedAt || '').localeCompare(String(a.updatedAt || ''))
|
|
259
|
+
|| String(a.termId).localeCompare(String(b.termId))
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function naiveVisible(terms, memoryContext, includeArchived) {
|
|
263
|
+
const out = []
|
|
264
|
+
for (const [termId, entry] of Object.entries(terms)) {
|
|
265
|
+
const status = entry.status || 'active'
|
|
266
|
+
if (status !== 'active' && !(includeArchived && status === 'archived')) continue
|
|
267
|
+
if (!contextVisible(entry.context, memoryContext)) continue
|
|
268
|
+
out.push({ termId, ...entry })
|
|
269
|
+
}
|
|
270
|
+
out.sort((a, b) => naiveDisplayRank(a, b))
|
|
271
|
+
return out
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function naiveBucketKeysOf(entry) {
|
|
275
|
+
const keys = new Set()
|
|
276
|
+
for (const text of [entry.normalizedTerm, ...(entry.normalizedAliases || [])]) {
|
|
277
|
+
if (text && text.length >= BUCKET_WIDTH) keys.add(text.slice(0, BUCKET_WIDTH))
|
|
278
|
+
else if (text) keys.add(text)
|
|
279
|
+
}
|
|
280
|
+
return [...keys]
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
function naiveTextBuckets(normalizedText, maxBuckets = LEXICON_MAX_BUCKETS) {
|
|
284
|
+
const keys = []
|
|
285
|
+
const seen = new Set()
|
|
286
|
+
for (let i = 0; i + BUCKET_WIDTH <= normalizedText.length; i++) {
|
|
287
|
+
const key = normalizedText.slice(i, i + BUCKET_WIDTH)
|
|
288
|
+
if (seen.has(key)) continue
|
|
289
|
+
seen.add(key)
|
|
290
|
+
keys.push(key)
|
|
291
|
+
if (keys.length >= maxBuckets) break
|
|
292
|
+
}
|
|
293
|
+
return keys
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function naiveDiceFallback(normalizedText, candidates, { threshold = LEXICON_DICE_THRESHOLD, maxCandidates = LEXICON_MAX_CANDIDATES, maxTerms = LEXICON_MAX_TERMS } = {}) {
|
|
297
|
+
const textGrams = bigrams(normalizedText)
|
|
298
|
+
if (!textGrams.size) return []
|
|
299
|
+
const scored = []
|
|
300
|
+
for (const hit of candidates.slice(0, maxCandidates)) {
|
|
301
|
+
let best = 0
|
|
302
|
+
for (const surface of [hit.normalizedTerm, ...(hit.normalizedAliases || [])]) {
|
|
303
|
+
if (!surface || surface.length < BUCKET_WIDTH) continue
|
|
304
|
+
const key = surface.slice(0, BUCKET_WIDTH)
|
|
305
|
+
let pos = normalizedText.indexOf(key)
|
|
306
|
+
while (pos >= 0) {
|
|
307
|
+
const window = normalizedText.slice(pos, pos + surface.length)
|
|
308
|
+
const score = dice(bigrams(window), bigrams(surface))
|
|
309
|
+
if (score > best) best = score
|
|
310
|
+
if (best >= 1) break
|
|
311
|
+
pos = normalizedText.indexOf(key, pos + 1)
|
|
312
|
+
}
|
|
313
|
+
if (best >= threshold) break
|
|
314
|
+
}
|
|
315
|
+
if (best >= threshold) scored.push({ ...hit, kindOfHit: 'dice', score: best })
|
|
316
|
+
}
|
|
317
|
+
scored.sort((a, b) => (b.score - a.score)
|
|
318
|
+
|| (Number(b.strength) || 0) * (Number(b.confidence) || 0) - (Number(a.strength) || 0) * (Number(a.confidence) || 0)
|
|
319
|
+
|| String(a.termId).localeCompare(String(b.termId)))
|
|
320
|
+
return scored.slice(0, maxTerms)
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
function naiveHitOf(entry, termId, kindOfHit, score) {
|
|
324
|
+
return {
|
|
325
|
+
termId, term: entry.term, normalizedTerm: entry.normalizedTerm, kind: entry.kind,
|
|
326
|
+
meaning: entry.meaning, usage: entry.usage, sensitivity: entry.sensitivity,
|
|
327
|
+
strength: entry.strength, confidence: entry.confidence, context: entry.context,
|
|
328
|
+
updatedAt: entry.updatedAt, status: entry.status, kindOfHit, score,
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function naiveExactHits(terms, normalizedText, visibleIds) {
|
|
333
|
+
const hits = []
|
|
334
|
+
for (const termId of visibleIds) {
|
|
335
|
+
const entry = terms[termId]
|
|
336
|
+
if (normalizedText === entry.normalizedTerm) { hits.push(naiveHitOf(entry, termId, 'exact-term', 1)); continue }
|
|
337
|
+
if ((entry.normalizedAliases || []).includes(normalizedText)) { hits.push(naiveHitOf(entry, termId, 'exact-alias', 1)); continue }
|
|
338
|
+
const surfaces = [entry.normalizedTerm, ...(entry.normalizedAliases || [])]
|
|
339
|
+
let matched = null
|
|
340
|
+
for (const surface of surfaces) {
|
|
341
|
+
if (surface.length >= LEXICON_MIN_INCLUDE_LEN && normalizedText.includes(surface)) { matched = surface; break }
|
|
342
|
+
}
|
|
343
|
+
if (matched) hits.push(naiveHitOf(entry, termId, entry.normalizedTerm === matched ? 'exact-term' : 'exact-alias', 1))
|
|
344
|
+
}
|
|
345
|
+
return hits
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function naiveSensitivityRank(hit) {
|
|
349
|
+
return hit.sensitivity === 'sensitive' ? 1 : 0
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function naiveExactRank(kindOfHit) {
|
|
353
|
+
if (kindOfHit === 'exact-term') return 3
|
|
354
|
+
if (kindOfHit === 'exact-alias') return 2
|
|
355
|
+
return 1
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
function naiveScopeSpecificity(context) {
|
|
359
|
+
if (!context || typeof context !== 'object') return 0
|
|
360
|
+
return ['channelId', 'speakerId', 'characterId', 'worldId', 'branchId'].filter(key => context[key]).length
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function naiveStableRank(hits) {
|
|
364
|
+
return [...hits].sort((a, b) => {
|
|
365
|
+
const sa = naiveSensitivityRank(a), sb = naiveSensitivityRank(b)
|
|
366
|
+
if (sa !== sb) return sb - sa
|
|
367
|
+
const ea = naiveExactRank(a.kindOfHit), eb = naiveExactRank(b.kindOfHit)
|
|
368
|
+
if (ea !== eb) return eb - ea
|
|
369
|
+
if (ea === 1 && Math.abs((b.score || 0) - (a.score || 0)) > 1e-9) return (b.score || 0) - (a.score || 0)
|
|
370
|
+
const wa = (Number(a.strength) || 0) * (Number(a.confidence) || 0)
|
|
371
|
+
const wb = (Number(b.strength) || 0) * (Number(b.confidence) || 0)
|
|
372
|
+
if (Math.abs(wb - wa) > 1e-9) return wb - wa
|
|
373
|
+
const ga = naiveScopeSpecificity(a.context), gb = naiveScopeSpecificity(b.context)
|
|
374
|
+
if (ga !== gb) return gb - ga
|
|
375
|
+
const ta = String(a.updatedAt || ''), tb = String(b.updatedAt || '')
|
|
376
|
+
if (ta !== tb) return tb.localeCompare(ta)
|
|
377
|
+
return String(a.termId).localeCompare(String(b.termId))
|
|
378
|
+
})
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function naiveMatch(terms, userText, memoryContext) {
|
|
382
|
+
const normalizedText = lexicon.normalizeLexeme(userText)
|
|
383
|
+
if (!normalizedText) return []
|
|
384
|
+
const visibleIds = []
|
|
385
|
+
const byBucket = new Map()
|
|
386
|
+
for (const [termId, entry] of Object.entries(terms)) {
|
|
387
|
+
if ((entry.status || 'active') !== 'active') continue
|
|
388
|
+
if (!contextVisible(entry.context, memoryContext)) continue
|
|
389
|
+
visibleIds.push(termId)
|
|
390
|
+
for (const key of naiveBucketKeysOf(entry)) {
|
|
391
|
+
if (!byBucket.has(key)) byBucket.set(key, [])
|
|
392
|
+
byBucket.get(key).push(termId)
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
const exact = naiveExactHits(terms, normalizedText, visibleIds)
|
|
396
|
+
const exactIds = new Set(exact.map(hit => hit.termId))
|
|
397
|
+
const candidateIds = []
|
|
398
|
+
for (const key of naiveTextBuckets(normalizedText)) {
|
|
399
|
+
for (const termId of byBucket.get(key) || []) {
|
|
400
|
+
if (!exactIds.has(termId) && !candidateIds.includes(termId)) candidateIds.push(termId)
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
const candidates = candidateIds
|
|
404
|
+
.map(termId => naiveHitOf(terms[termId], termId, 'dice', 0))
|
|
405
|
+
.sort((a, b) => (Number(b.strength) || 0) * (Number(b.confidence) || 0) - (Number(a.strength) || 0) * (Number(a.strength) || 0))
|
|
406
|
+
const fuzzy = naiveDiceFallback(normalizedText, candidates)
|
|
407
|
+
return naiveStableRank([...exact, ...fuzzy])
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function naiveSearch(terms, query, memoryContext, includeArchived) {
|
|
411
|
+
const normalizedQuery = lexicon.normalizeLexeme(query)
|
|
412
|
+
const out = []
|
|
413
|
+
if (!normalizedQuery) return out
|
|
414
|
+
const matched = []
|
|
415
|
+
for (const [termId, entry] of Object.entries(terms)) {
|
|
416
|
+
const status = entry.status || 'active'
|
|
417
|
+
if (status !== 'active' && !(includeArchived && status === 'archived')) continue
|
|
418
|
+
if (!contextVisible(entry.context, memoryContext)) continue
|
|
419
|
+
let matchedBy = null
|
|
420
|
+
if (entry.normalizedTerm === normalizedQuery) matchedBy = 'term'
|
|
421
|
+
else if ((entry.normalizedAliases || []).includes(normalizedQuery)) matchedBy = 'alias'
|
|
422
|
+
else if (entry.normalizedTerm.length >= LEXICON_MIN_INCLUDE_LEN
|
|
423
|
+
&& (normalizedQuery.includes(entry.normalizedTerm) || entry.normalizedTerm.includes(normalizedQuery))) matchedBy = 'contains'
|
|
424
|
+
else if ((entry.normalizedAliases || []).some(alias => alias.length >= LEXICON_MIN_INCLUDE_LEN
|
|
425
|
+
&& (normalizedQuery.includes(alias) || alias.includes(normalizedQuery)))) matchedBy = 'alias-contains'
|
|
426
|
+
if (!matchedBy) continue
|
|
427
|
+
matched.push({ termId, ...entry, matchedBy })
|
|
428
|
+
}
|
|
429
|
+
matched.sort((a, b) => {
|
|
430
|
+
const ra = a.matchedBy === 'term' ? 3 : a.matchedBy === 'alias' ? 2 : 1
|
|
431
|
+
const rb = b.matchedBy === 'term' ? 3 : b.matchedBy === 'alias' ? 2 : 1
|
|
432
|
+
if (ra !== rb) return rb - ra
|
|
433
|
+
return String(a.termId).localeCompare(String(b.termId))
|
|
434
|
+
})
|
|
435
|
+
return matched
|
|
436
|
+
}
|