dsh-memu-memory 1.2.1 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/AGENTS.md +1 -1
  2. package/README.md +12 -5
  3. package/README.zh.md +4 -2
  4. package/docs/contributing.md +16 -3
  5. package/docs/host-api.md +3 -1
  6. package/docs/memory-semantics.md +21 -2
  7. package/docs/troubleshooting.md +42 -4
  8. package/lib/backup.js +24 -2
  9. package/lib/binding-diagnostics.js +2 -2
  10. package/lib/client.js +1361 -146
  11. package/lib/consolidation-worker.js +30 -8
  12. package/lib/curator-evidence.js +131 -0
  13. package/lib/curator-policy.js +81 -0
  14. package/lib/curator.js +1252 -91
  15. package/lib/cycle.js +3 -0
  16. package/lib/embedding-download.js +39 -4
  17. package/lib/engine.js +6 -3
  18. package/lib/global-graph.js +282 -0
  19. package/lib/graph.js +51 -9
  20. package/lib/inbound.js +19 -0
  21. package/lib/inject.js +277 -17
  22. package/lib/lexicon.js +1115 -0
  23. package/lib/local-vector-service.js +8 -6
  24. package/lib/managed-runtime.js +33 -2
  25. package/lib/memory.js +282 -17
  26. package/lib/memu.js +2 -2
  27. package/lib/migrate.js +83 -2
  28. package/lib/private-links.js +587 -0
  29. package/lib/private.js +32 -3
  30. package/lib/prompts/global-curator.md +76 -19
  31. package/lib/prompts/private-consolidation.coding.md +4 -1
  32. package/lib/prompts/private-consolidation.md +3 -1
  33. package/lib/prompts/private-consolidation.mixed.md +3 -1
  34. package/lib/prompts/private-consolidation.rp.md +3 -1
  35. package/lib/prompts.js +1 -1
  36. package/lib/rpc.js +577 -9
  37. package/lib/state.js +4 -1
  38. package/lib/text-retrieval.js +661 -0
  39. package/lib/text-similarity.js +36 -18
  40. package/lib/tools.js +168 -10
  41. package/lib/webui/standalone.css +17 -0
  42. package/lib/webui/standalone.js +2 -0
  43. package/package.json +1 -1
  44. package/scripts/acceptance-local.mjs +8 -5
  45. package/scripts/curator-pair-calibrate.mjs +259 -0
  46. package/scripts/lexicon-calibrate.mjs +230 -0
  47. package/scripts/lexicon-dedupe.mjs +318 -0
  48. package/tests/absorb-review.test.mjs +353 -0
  49. package/tests/appendix-b2.test.mjs +2 -2
  50. package/tests/client-export.test.mjs +520 -0
  51. package/tests/client-render.test.mjs +19 -3
  52. package/tests/consolidation-fork.test.mjs +14 -12
  53. package/tests/consolidation-snapshot.test.mjs +39 -0
  54. package/tests/curator-absorb.test.mjs +443 -0
  55. package/tests/curator-calibrate.test.mjs +173 -0
  56. package/tests/curator-clusters.test.mjs +281 -0
  57. package/tests/curator-commit.test.mjs +52 -5
  58. package/tests/curator-evidence.test.mjs +102 -0
  59. package/tests/curator-link.test.mjs +316 -0
  60. package/tests/curator-pairing.test.mjs +252 -0
  61. package/tests/curator-probe.test.mjs +15 -6
  62. package/tests/curator-review-mode.test.mjs +314 -0
  63. package/tests/disclosure.test.mjs +2 -1
  64. package/tests/fixtures/curator-pairs/labeled.v1.jsonl +90 -0
  65. package/tests/fixtures/lexicon-stub-hooks.mjs +15 -0
  66. package/tests/fixtures/lexicon-stub.mjs +284 -0
  67. package/tests/graph-cascade.test.mjs +35 -2
  68. package/tests/graph.test.mjs +96 -0
  69. package/tests/help-page.test.mjs +415 -0
  70. package/tests/inbound.test.mjs +6 -3
  71. package/tests/inject-guide.test.mjs +28 -0
  72. package/tests/inject.test.mjs +157 -5
  73. package/tests/lexicon-calibrate.test.mjs +48 -0
  74. package/tests/lexicon-dedupe.test.mjs +171 -0
  75. package/tests/lexicon-frequency.test.mjs +179 -0
  76. package/tests/lexicon-inject.test.mjs +410 -0
  77. package/tests/lexicon-perf.test.mjs +436 -0
  78. package/tests/lexicon-ui.test.mjs +547 -0
  79. package/tests/lexicon.test.mjs +922 -0
  80. package/tests/managed-runtime.test.mjs +32 -0
  81. package/tests/private-links.test.mjs +445 -0
  82. package/tests/prompts.test.mjs +35 -0
  83. package/tests/recall-dedup.test.mjs +33 -4
  84. package/tests/tool-schema.test.mjs +1 -1
  85. package/tests/tools.test.mjs +112 -3
  86. package/tests/webui-auth.test.mjs +348 -2
  87. package/tests/webui.test.mjs +188 -0
@@ -15,7 +15,7 @@ test.after(() => {
15
15
  try { fs.rmSync(home, { recursive: true, force: true }) } catch {}
16
16
  })
17
17
 
18
- test('anchor includes pinned private memory', () => {
18
+ test('anchor shows a pinned memory as a title card, without its body', () => {
19
19
  const cwd = path.join(home, 'project-anchor')
20
20
  const binding = state.ensureSession('inject-anchor', { cwd, agentPreset: 'rp' })
21
21
  const index = state.loadPrivateIndex(binding.libraryId)
@@ -32,11 +32,131 @@ test('anchor includes pinned private memory', () => {
32
32
  }
33
33
  state.savePrivateIndex(binding.libraryId, index)
34
34
  const anchor = inject.buildAnchor(binding, { ...state.getSettings(), anchorTokens: 700 })
35
+ assert.ok(anchor.text.includes('[mem-test]'), 'the id is the handle the model quotes back')
35
36
  assert.ok(anchor.text.includes('凉茶'))
36
- assert.ok(anchor.text.includes('今天他走的时候没看我'))
37
+ assert.equal(anchor.text.includes('今天他走的时候没看我'), false, 'the anchor is a directory; the body stays out')
37
38
  assert.ok(anchor.ids.includes('mem-test'))
38
39
  })
39
40
 
41
+ test('the rule is a preface: before the private header, and it names the recall path', () => {
42
+ const cwd = path.join(home, 'project-rule')
43
+ const binding = state.ensureSession('inject-rule', { cwd, agentPreset: 'rp' })
44
+ const index = state.loadPrivateIndex(binding.libraryId)
45
+ index.entries['mem-rule'] = {
46
+ track: 'memory',
47
+ layer: 'must_remember',
48
+ title: '规则之下的记忆',
49
+ content: '这段正文不该出现在锚点里。',
50
+ status: 'active',
51
+ pinSource: 'user_explicit',
52
+ createdAt: new Date().toISOString(),
53
+ updatedAt: new Date().toISOString(),
54
+ }
55
+ state.savePrivateIndex(binding.libraryId, index)
56
+ const anchor = inject.buildAnchor(binding, { ...state.getSettings(), anchorTokens: 700 })
57
+ const headerAt = anchor.text.indexOf('===== memory anchor =====')
58
+ const ruleAt = anchor.text.indexOf('[rule]')
59
+ const privateAt = anchor.text.indexOf('[private ·')
60
+ assert.ok(headerAt >= 0, 'the anchor starts with its header')
61
+ assert.ok(anchor.text.startsWith('===== memory anchor =====\n\n[rule]'), 'the header and rule are separated by a blank line')
62
+ assert.ok(ruleAt > headerAt && privateAt > ruleAt, 'the rule is read before the memories it governs')
63
+ assert.match(anchor.text, /\[rule\][^\n]*\n\n\[private ·/, 'the rule and the private section are separated by a blank line')
64
+ assert.ok(anchor.text.indexOf('[mem-rule]') > privateAt)
65
+ assert.ok(anchor.text.includes('标题即目录'), 'the rule says the anchor is a table of contents')
66
+ assert.ok(anchor.text.includes('memu_private_recall 按标题关键词查'), 'and how to get the original text back')
67
+ })
68
+
69
+ test('an empty title falls back to the first 40 characters of the content', () => {
70
+ const cwd = path.join(home, 'project-fallback')
71
+ const binding = state.ensureSession('inject-fallback', { cwd, agentPreset: 'rp' })
72
+ const content = '他答应过要在雨停之前回来,可是到了第二天早上门口还是空的,只有那把伞靠在墙边。'
73
+ const now = new Date().toISOString()
74
+ const index = state.loadPrivateIndex(binding.libraryId)
75
+ index.entries['mem-fallback'] = {
76
+ track: 'memory',
77
+ layer: 'long_term',
78
+ content,
79
+ status: 'active',
80
+ createdAt: now,
81
+ updatedAt: now,
82
+ }
83
+ state.savePrivateIndex(binding.libraryId, index)
84
+ const anchor = inject.buildAnchor(binding, { ...state.getSettings(), anchorTokens: 700 })
85
+ assert.ok(anchor.text.includes('[mem-fallback]'), 'the core segment is a title card too')
86
+ assert.ok(anchor.text.includes(content.slice(0, 40)), 'a card with no title is still searchable by its opening words')
87
+ assert.equal(anchor.text.includes('(untitled)'), false, 'content exists, so the row is never anonymous')
88
+ })
89
+
90
+ test('title-card lines keep source, disclosure, accuracy and the attachment count', () => {
91
+ const cwd = path.join(home, 'project-labels')
92
+ const binding = state.ensureSession('inject-labels', { cwd, agentPreset: 'rp' })
93
+ const now = new Date().toISOString()
94
+ const index = state.loadPrivateIndex(binding.libraryId)
95
+ index.entries['mem-labels'] = {
96
+ track: 'memory',
97
+ layer: 'must_remember',
98
+ title: '他编的那个故事',
99
+ content: '他说那是真的,但那只是他想出来的。',
100
+ source: 'imagined',
101
+ disclosure: 'never_say',
102
+ accuracy: 0.3,
103
+ confidence: 0.9,
104
+ attachments: [
105
+ { sha256: 'a'.repeat(64) },
106
+ { sha256: 'b'.repeat(64), caption: 'CAPTION-MUST-NOT-EXPAND' },
107
+ ],
108
+ status: 'active',
109
+ pinSource: 'self',
110
+ createdAt: now,
111
+ updatedAt: now,
112
+ }
113
+ state.savePrivateIndex(binding.libraryId, index)
114
+ const anchor = inject.buildAnchor(binding, { ...state.getSettings(), anchorTokens: 700 })
115
+ const line = anchor.text.split('\n').find((row) => row.includes('[mem-labels]'))
116
+ assert.ok(line, 'the title card exists')
117
+ assert.ok(line.includes('source=imagined'), 'an imagined memory must not read as a fact')
118
+ assert.ok(line.includes('记得,但别直接说出口'), 'never_say survives on a title card')
119
+ assert.ok(line.includes('细节未必准'), 'accuracy still rides on the line')
120
+ assert.ok(line.includes('附件2'), 'attachments are counted')
121
+ assert.equal(anchor.text.includes('CAPTION-MUST-NOT-EXPAND'), false, 'captions are expanded only by the full renderer')
122
+ })
123
+
124
+ test('with the same anchorTokens, title cards expose at least as many ids as full text', () => {
125
+ const cwd = path.join(home, 'project-compare')
126
+ const binding = state.ensureSession('inject-compare', { cwd, agentPreset: 'rp' })
127
+ const now = new Date().toISOString()
128
+ const index = state.loadPrivateIndex(binding.libraryId)
129
+ const items = []
130
+ const body = '这是一段用来占满预算的中文记忆正文,重复几遍让长度足够触发截断。'.repeat(3)
131
+ for (let i = 0; i < 16; i++) {
132
+ const id = `mem-cmp-${i}`
133
+ index.entries[id] = {
134
+ track: 'memory',
135
+ layer: 'must_remember',
136
+ title: `对比记忆 ${i}`,
137
+ content: `${body} BODY-${i}`,
138
+ status: 'active',
139
+ pinSource: 'self',
140
+ strength: 1 - i / 20,
141
+ createdAt: now,
142
+ updatedAt: new Date(Date.now() - i * 60000).toISOString(),
143
+ }
144
+ items.push([id, index.entries[id]])
145
+ }
146
+ state.savePrivateIndex(binding.libraryId, index)
147
+ const anchorTokens = 600
148
+ const anchor = inject.buildAnchor(binding, { ...state.getSettings(), anchorTokens })
149
+ // The old shape was the same head plus the body the card now omits. Pack it into the
150
+ // *whole* anchor budget (no section headers charged), which is more room than the
151
+ // full-text anchor ever actually got — so this comparison is deliberately unfair to
152
+ // the title cards.
153
+ const legacy = budget.packToBudget(items, anchorTokens, (entry, id) =>
154
+ `- [${id}] source=${entry.source || 'inferred'} · ${entry.title} · ${entry.layer || 'long_term'}`
155
+ + (entry.content ? `\n ${entry.content}` : ''))
156
+ assert.ok(anchor.ids.length >= legacy.ids.length,
157
+ `title cards exposed ${anchor.ids.length} ids, full text ${legacy.ids.length}`)
158
+ })
159
+
40
160
  test('pre-step inserts a plugin message before the user message', async () => {
41
161
  const cwd = path.join(home, 'project-step')
42
162
  const binding = state.ensureSession('inject-step', { cwd, agentPreset: 'coding' })
@@ -84,8 +204,10 @@ test('pre-step inserts a plugin message before the user message', async () => {
84
204
  assert.equal(out.messages[1].source.kind, 'user')
85
205
 
86
206
  const injected = out.messages[0].content[0].text
87
- assert.ok(injected.includes('mem-recall'), 'per-turn recall must surface the matching entry')
88
- assert.ok(injected.includes('tools/deploy.ps1'))
207
+ assert.ok(injected.includes('[mem-recall]'), 'per-turn recall must surface the matching entry by id')
208
+ assert.ok(injected.includes('Deploy convention'), 'the title is what reaches the prompt')
209
+ assert.equal(injected.includes('The deploy script lives in tools/deploy.ps1'), false,
210
+ 'the turn block is a title card; the body stays behind memu_private_recall')
89
211
  assert.ok(state.getSessionBinding('inject-step').injectedIds.includes('mem-recall'))
90
212
  assert.equal(state.getSessionBinding('inject-step').lastRecallError, undefined)
91
213
 
@@ -93,6 +215,35 @@ test('pre-step inserts a plugin message before the user message', async () => {
93
215
  assert.equal(skipped, decision)
94
216
  })
95
217
 
218
+ test('buildRecall injects the title card, not the body', async () => {
219
+ const cwd = path.join(home, 'project-recall-title')
220
+ const binding = state.ensureSession('inject-recall-title', { cwd, agentPreset: 'coding' })
221
+ const now = new Date().toISOString()
222
+ const index = state.loadPrivateIndex(binding.libraryId)
223
+ index.entries['mem-recall-title'] = {
224
+ track: 'memory',
225
+ layer: 'long_term',
226
+ title: '部署脚本的位置',
227
+ content: 'RECALL-BODY-ONLY 部署脚本放在 tools/deploy.ps1,需要 staging token。',
228
+ status: 'active',
229
+ confidence: 0.9,
230
+ createdAt: now,
231
+ updatedAt: now,
232
+ }
233
+ state.savePrivateIndex(binding.libraryId, index)
234
+ const exec = { agent: { session: { header: { id: 'inject-recall-title', cwd, agentPreset: 'coding' } } } }
235
+ const recall = await inject.buildRecall(binding, '部署脚本的位置', 1,
236
+ { ...state.getSettings(), relevanceThreshold: 0 }, exec, 'inject-recall-title')
237
+ assert.deepEqual(recall.errors, [])
238
+ assert.ok(recall.ids.includes('mem-recall-title'))
239
+ assert.ok(recall.text.includes('部署脚本的位置'), 'the title is the card')
240
+ assert.ok(recall.text.includes('[mem-recall-title]'), 'the id stays quotable')
241
+ assert.match(recall.text, /条目是标题卡/, 'the block tells the model how to use a title card')
242
+ assert.ok(recall.text.includes('memu_private_recall'), 'and which tool returns the body')
243
+ assert.equal(recall.text.includes('RECALL-BODY-ONLY'), false,
244
+ 'memu_private_recall still returns full text; the turn block does not')
245
+ })
246
+
96
247
  test('a failing recall is recorded instead of disappearing', async () => {
97
248
  const cwd = path.join(home, 'project-recall-error')
98
249
  const binding = state.ensureSession('inject-recall-error', { cwd, agentPreset: 'coding' })
@@ -215,7 +366,8 @@ test('per-turn mode reads the global layer every turn instead of on the cadence'
215
366
 
216
367
  const perTurn2 = await inject.buildRecall(binding, query, 2, { ...base, autoInject: 'per-turn' }, exec)
217
368
  assert.ok(perTurn2.ids.includes('mem-global-deploy'), 'per-turn recalls the global layer on a plain turn')
218
- assert.ok(perTurn2.text.includes('staging deploy token is rotated'))
369
+ assert.ok(perTurn2.text.includes('Staging token rotation'), 'the global title card arrives')
370
+ assert.equal(perTurn2.text.includes('rotated every Monday'), false, 'but not its body')
219
371
 
220
372
  const perTurn5 = await inject.buildRecall(binding, query, 5, { ...base, autoInject: 'per-turn' }, exec)
221
373
  assert.ok(perTurn5.ids.includes('mem-global-deploy'), 'and again on the next turn, not only every Nth turn')
@@ -0,0 +1,48 @@
1
+ // Dice 阈值标定脚手架(scripts/lexicon-calibrate.mjs)的轻量 smoke 测试:
2
+ // 只保证脚本独立运行不抛错、产出指标行;标定样本本身是给人看的报告,
3
+ // 不进 npm test 门禁断言(脚本不在此处做阈值回归)。
4
+ import test from 'node:test'
5
+ import assert from 'node:assert/strict'
6
+ import { spawnSync } from 'node:child_process'
7
+ import fs from 'node:fs'
8
+ import os from 'node:os'
9
+ import path from 'node:path'
10
+ import { fileURLToPath } from 'node:url'
11
+
12
+ const here = path.dirname(fileURLToPath(import.meta.url))
13
+ const scriptPath = path.join(here, '..', 'scripts', 'lexicon-calibrate.mjs')
14
+ const home = fs.mkdtempSync(path.join(os.tmpdir(), 'dsh-memu-memory-lexicon-calibrate-smoke-'))
15
+ process.env.DSH_MEMU_HOME = home
16
+
17
+ test.after(() => {
18
+ try { fs.rmSync(home, { recursive: true, force: true }) } catch {}
19
+ })
20
+
21
+ function runCalibrate(args) {
22
+ return spawnSync(process.execPath, [scriptPath, ...args], {
23
+ encoding: 'utf8',
24
+ timeout: 120_000,
25
+ env: { ...process.env, DSH_MEMU_HOME: fs.mkdtempSync(path.join(os.tmpdir(), 'dsh-memu-memory-lexicon-calibrate-run-')) },
26
+ })
27
+ }
28
+
29
+ test('lexicon-calibrate.mjs runs standalone, never touches the caller data dir, and prints the 0.60 metrics', () => {
30
+ const run = runCalibrate([])
31
+ assert.equal(run.status, 0, `calibrate exits 0. stderr: ${run.stderr}`)
32
+ assert.match(run.stdout, /只出报告,不自动改 LEXICON_DICE_THRESHOLD/)
33
+ assert.match(run.stdout, /正例 16 对,负例 16 对/)
34
+ assert.match(run.stdout, /当前 0\.60 下的指标:precision=/)
35
+ // 汇总表覆盖默认阈值扫描 0.55..0.75。
36
+ for (const threshold of ['0.55', '0.60', '0.65', '0.70', '0.75']) {
37
+ assert.ok(run.stdout.includes(threshold), `sweep table contains ${threshold}`)
38
+ }
39
+ })
40
+
41
+ test('lexicon-calibrate.mjs accepts a --threshold list and still exits cleanly', () => {
42
+ const run = runCalibrate(['--threshold=0.5,0.62,0.9'])
43
+ assert.equal(run.status, 0, `stderr: ${run.stderr}`)
44
+ assert.match(run.stdout, /当前 0\.50 下的指标/, 'the primary threshold is the first given value')
45
+ for (const threshold of ['0.50', '0.62', '0.90']) {
46
+ assert.ok(run.stdout.includes(threshold), `sweep table contains ${threshold}`)
47
+ }
48
+ })
@@ -0,0 +1,171 @@
1
+ // 历史重复词条迁移工具(scripts/lexicon-dedupe.mjs)的端到端测试:
2
+ // 通过子进程跑真实 CLI(dry-run / --apply 两条路径),fixture 直接写
3
+ // data/lexicon/<id>.json,覆盖:可合并重复组、active+sensitive 跳过、
4
+ // 跨作用域同词不合并、pending revision 草稿不当作重复。
5
+ import test from 'node:test'
6
+ import assert from 'node:assert/strict'
7
+ import { spawnSync } from 'node:child_process'
8
+ import fs from 'node:fs'
9
+ import os from 'node:os'
10
+ import path from 'node:path'
11
+ import { fileURLToPath } from 'node:url'
12
+
13
+ const here = path.dirname(fileURLToPath(import.meta.url))
14
+ const scriptPath = path.join(here, '..', 'scripts', 'lexicon-dedupe.mjs')
15
+
16
+ const home = fs.mkdtempSync(path.join(os.tmpdir(), 'dsh-memu-memory-lexicon-dedupe-'))
17
+ process.env.DSH_MEMU_HOME = home
18
+
19
+ const state = await import('../lib/state.js')
20
+ const paths = await import('../lib/paths.js')
21
+ const lexicon = await import('../lib/lexicon.js')
22
+
23
+ test.after(() => {
24
+ try { fs.rmSync(home, { recursive: true, force: true }) } catch {}
25
+ })
26
+
27
+ function runCli(args) {
28
+ return spawnSync(process.execPath, [scriptPath, ...args], {
29
+ encoding: 'utf8',
30
+ env: { ...process.env, DSH_MEMU_HOME: home },
31
+ timeout: 60_000,
32
+ })
33
+ }
34
+
35
+ function entryOf(overrides, termId, extra = {}) {
36
+ return {
37
+ term: overrides.term,
38
+ normalizedTerm: lexicon.normalizeLexeme(overrides.term),
39
+ normalizedAliases: (overrides.aliases || []).map(a => lexicon.normalizeLexeme(a)),
40
+ kind: 'term',
41
+ meaning: overrides.meaning || '',
42
+ usage: overrides.usage || '',
43
+ aliases: overrides.aliases || [],
44
+ examples: overrides.examples || [],
45
+ context: overrides.context || {},
46
+ knownBy: overrides.knownBy,
47
+ visibility: 'shared',
48
+ sensitivity: overrides.sensitivity || 'normal',
49
+ strength: overrides.strength ?? 0.5,
50
+ confidence: overrides.confidence ?? 0.7,
51
+ status: overrides.status || 'active',
52
+ source: 'user',
53
+ evidence: overrides.evidence || [],
54
+ fingerprint: `fp-${termId}`,
55
+ createdAt: overrides.createdAt || '2026-01-01T00:00:00.000Z',
56
+ updatedAt: overrides.updatedAt || '2026-01-01T00:00:00.000Z',
57
+ hitCounts: overrides.hitCounts || { candidate: 0, injection: 0, query: 0 },
58
+ lastInjectedAt: null,
59
+ ...extra,
60
+ }
61
+ }
62
+
63
+ // ---- fixture:一本含四种形态的词典 ----
64
+ const library = state.createLibrary({ name: 'dedupe-fixture', cwd: path.join(home, 'dedupe') })
65
+ const lexFile = path.join(home, 'lexicon', `${library.id}.json`)
66
+
67
+ const A1 = 'lex-dup-a1', A2 = 'lex-dup-a2', A3 = 'lex-dup-a3', DRAFT = 'lex-dup-draft'
68
+ const S1 = 'lex-dup-s1', S2 = 'lex-dup-s2'
69
+ const C1 = 'lex-dup-c1', C2 = 'lex-dup-c2'
70
+
71
+ const fixture = {
72
+ version: 1,
73
+ revision: 7,
74
+ libraryId: library.id,
75
+ rejected: {
76
+ 'fp-lex-dup-a3': { termId: A3, reason: '历史驳回记录', at: '2026-03-01T00:00:00.000Z' },
77
+ },
78
+ terms: {
79
+ // 组 A:同词同作用域三条(active 旧 / active 新 / rejected)+ 一条 active 的待审修订草稿。
80
+ [A1]: entryOf({ term: '糖刀', meaning: '短义', aliases: ['糖里藏刀'], strength: 0.3, status: 'active', updatedAt: '2026-01-01T00:00:00.000Z' }, A1),
81
+ [A2]: entryOf({ term: '糖刀', meaning: '较长的保留条目含义', aliases: [], examples: ['保留条目的例句'], strength: 0.9, status: 'active', updatedAt: '2026-02-01T00:00:00.000Z' }, A2),
82
+ [A3]: entryOf({
83
+ term: '糖刀',
84
+ meaning: '被驳回条目里最长的含义文本,合并时应当胜出成为保留条目的含义',
85
+ aliases: ['刀里藏糖'],
86
+ examples: ['驳回条目的例句'],
87
+ strength: 0.5,
88
+ status: 'rejected',
89
+ evidence: [{ kind: 'message', id: 'm-1', quote: '这条留言是证据' }],
90
+ hitCounts: { candidate: 4, injection: 1, query: 2 },
91
+ updatedAt: '2026-03-01T00:00:00.000Z',
92
+ }, A3),
93
+ // 待审修订草稿(revisionOf 指向 A2):不是重复,必须原样保留。
94
+ [DRAFT]: entryOf({ term: '糖刀', meaning: '草稿含义', status: 'pending', updatedAt: '2026-04-01T00:00:00.000Z' }, DRAFT, { revisionOf: A2, previousRevision: 6 }),
95
+ // 组 S:active+sensitive 与其 pending 重复 → 整组跳过。
96
+ [S1]: entryOf({ term: '红牌', meaning: '敏感触发词', kind: 'trigger', sensitivity: 'sensitive', status: 'active' }, S1),
97
+ [S2]: entryOf({ term: '红牌', meaning: '重复的敏感触发词', kind: 'trigger', sensitivity: 'sensitive', status: 'pending' }, S2),
98
+ // 组 C:同词跨作用域 → 不是重复。
99
+ [C1]: entryOf({ term: '月引', meaning: 'w1 的月引', context: { worldId: 'w1' } }, C1),
100
+ [C2]: entryOf({ term: '月引', meaning: 'w2 的月引', context: { worldId: 'w2' } }, C2),
101
+ },
102
+ }
103
+ paths.writeJson(lexFile, fixture)
104
+
105
+ const readStore = () => JSON.parse(fs.readFileSync(lexFile, 'utf8'))
106
+
107
+ test('dry-run reports duplicate groups without touching a single byte', () => {
108
+ const before = fs.readFileSync(lexFile)
109
+ const run = runCli([])
110
+ assert.equal(run.status, 0, `dry-run exits 0: ${run.stderr}`)
111
+ const out = run.stdout
112
+ assert.match(out, /dry-run/)
113
+ assert.match(out, /重复组 2 组/, '组 A 与组 S 是重复组;组 C 跨作用域不算')
114
+ assert.match(out, new RegExp(`保留 ${A2}`), '组 A 保留 active 且 updatedAt 新的 A2')
115
+ assert.match(out, new RegExp(`并入并删除: ${A1}, ${A3}`))
116
+ assert.match(out, /active\+sensitive/, '组 S 因 active+sensitive 被跳过')
117
+ assert.match(out, /mergedFrom 记入: \[lex-dup-a1, lex-dup-a3\]/)
118
+ assert.deepEqual(fs.readFileSync(lexFile), before, 'dry-run must not modify the file')
119
+ })
120
+
121
+ test('--apply merges duplicate groups, keeps sensitive/cross-scope entries and the revision draft', () => {
122
+ const run = runCli(['--apply'])
123
+ assert.equal(run.status, 0, `apply exits 0: ${run.stderr}`)
124
+ const store = readStore()
125
+
126
+ // 组 A:只剩保留条目 + 待审草稿。
127
+ assert.equal(store.terms[A1], undefined, 'A1 merged away')
128
+ assert.equal(store.terms[A3], undefined, 'A3 merged away')
129
+ const kept = store.terms[A2]
130
+ assert.ok(kept, 'A2 kept')
131
+ assert.deepEqual(kept.mergedFrom, [A1, A3], 'mergedFrom records the removed ids for audit')
132
+ assert.equal(kept.meaning, fixture.terms[A3].meaning, 'meaning takes the longest across the group')
133
+ assert.deepEqual([...kept.aliases].sort(), ['刀里藏糖', '糖里藏刀'], 'aliases are the deduplicated union')
134
+ assert.deepEqual(kept.examples.sort(), ['保留条目的例句', '驳回条目的例句'].sort())
135
+ assert.equal(kept.strength, 0.9, 'strength takes the max')
136
+ assert.equal(kept.status, 'active')
137
+ assert.equal(kept.evidence.length, 1, 'evidence merged from the rejected member')
138
+ assert.deepEqual(kept.hitCounts, { candidate: 4, injection: 1, query: 2 }, 'hit counts sum across the group')
139
+ // 拒绝指纹改指保留条目。
140
+ assert.equal(store.rejected['fp-lex-dup-a3'].termId, A2, 'the rejected fingerprint now points at the kept entry')
141
+ // 草稿、敏感组、跨作用域都原样。
142
+ assert.equal(store.terms[DRAFT].status, 'pending', 'the active term\'s revision draft is not a duplicate')
143
+ assert.equal(store.terms[DRAFT].revisionOf, A2)
144
+ assert.ok(store.terms[S1] && store.terms[S2], 'the active+sensitive group is untouched')
145
+ assert.equal(store.terms[S1].meaning, '敏感触发词')
146
+ assert.ok(store.terms[C1] && store.terms[C2], 'cross-scope same word is not a duplicate')
147
+ assert.ok(store.revision > 7, 'the revision advanced')
148
+ // 结果仍然是一本可严格读取的词典:8 条 − 2 条被并 = 6 条。
149
+ const loaded = lexicon.loadLexicon(library.id)
150
+ assert.ok(loaded.terms[A2])
151
+ assert.equal(Object.keys(loaded.terms).length, 6)
152
+ })
153
+
154
+ test('the merged library still enforces write-side uniqueness (no re-creation of duplicates)', () => {
155
+ // 非敏感字段(usage)原地更新:撞上的是保留条目 A2,绝不会新建重复词条。
156
+ const keptMeaning = readStore().terms[A2].meaning
157
+ const upsert = lexicon.upsertLexiconTerm({ libraryId: library.id, term: '糖刀', kind: 'term', meaning: keptMeaning, usage: '再次直接写入' })
158
+ assert.equal(upsert.created, false)
159
+ assert.equal(upsert.term.termId, A2, 'the kept entry is the collision target, exactly like a never-duplicated term')
160
+ })
161
+
162
+ test('--library narrows the scan; unsafe ids are a typed usage error', () => {
163
+ const run = runCli(['--library', library.id])
164
+ assert.equal(run.status, 0)
165
+ assert.match(run.stdout, /重复组 1 组/, 'only the skipped active+sensitive group remains')
166
+ assert.match(run.stdout, /active\+sensitive/, 'reported as skipped, never touched')
167
+ assert.match(run.stdout, /dry-run 未写入/)
168
+ const bad = runCli(['--library', '../escape'])
169
+ assert.equal(bad.status, 2, 'bad --library is a usage error, not a crash')
170
+ assert.match(bad.stderr, /invalid library id/)
171
+ })
@@ -0,0 +1,179 @@
1
+ // 词典频率先验(施工稿 §6.3 包 C):Wilson 下界、effective 因子进入 stableRank、
2
+ // hitCounts 写入 clamp、以及生命周期(active/archived)对频率证据的影响。所有断言
3
+ // 只依赖落盘数据,不依赖当前时间——排序的确定性在这里与 lexicon-inject 各钉一遍。
4
+ // 面板/卡片流(§2.4)的 UI 断言在 tests/lexicon-ui.test.mjs(G2b 重写),本文件只管
5
+ // 数据面:当前版本没有待审队列,所以这里也不再钉 pending 文案。
6
+ import test from 'node:test'
7
+ import assert from 'node:assert/strict'
8
+ import fs from 'node:fs'
9
+ import os from 'node:os'
10
+ import path from 'node:path'
11
+
12
+ const home = fs.mkdtempSync(path.join(os.tmpdir(), 'dsh-memu-memory-lexicon-frequency-'))
13
+ process.env.DSH_MEMU_HOME = home
14
+
15
+ const paths = await import('../lib/paths.js')
16
+ const state = await import('../lib/state.js')
17
+ const lexicon = await import('../lib/lexicon.js')
18
+
19
+ test.after(() => {
20
+ try { state.flushState() } catch {}
21
+ try { fs.rmSync(home, { recursive: true, force: true }) } catch {}
22
+ })
23
+
24
+ let serial = 0
25
+ function setupLibrary(name, { terms = [] } = {}) {
26
+ serial += 1
27
+ const binding = state.ensureSession(`freq-${name}-${serial}`, { cwd: path.join(home, `project-${name}-${serial}`), agentPreset: 'rp' })
28
+ for (const args of terms) lexicon.upsertLexiconTerm({ libraryId: binding.libraryId, ...args })
29
+ return binding
30
+ }
31
+
32
+ function lexFile(libraryId) {
33
+ return path.join(home, 'lexicon', `${libraryId}.json`)
34
+ }
35
+
36
+ /** 直改磁盘上的词条字段(hitCounts/strength/updatedAt),靠 mtime 失效重建索引。 */
37
+ function patchTerm(libraryId, term, patch) {
38
+ const lex = lexicon.loadLexicon(libraryId)
39
+ const normalized = lexicon.normalizeLexeme(term)
40
+ const entry = Object.values(lex.terms).find(item => item.normalizedTerm === normalized)
41
+ assert.ok(entry, `term ${term} must exist`)
42
+ Object.assign(entry, patch)
43
+ paths.writeJson(lexFile(libraryId), lex)
44
+ }
45
+
46
+ function termIdOf(libraryId, term) {
47
+ const normalized = lexicon.normalizeLexeme(term)
48
+ const lex = lexicon.loadLexicon(libraryId)
49
+ return Object.entries(lex.terms).find(([, entry]) => entry.normalizedTerm === normalized)?.[0] || null
50
+ }
51
+
52
+ // ---- 1-2. Wilson 下界公式 ----
53
+
54
+ test('usageLowerBound: n=0 is 0, success is clamped to n, output stays in [0,1]', () => {
55
+ assert.equal(lexicon.usageLowerBound({ candidate: 0, injection: 0, query: 0 }), 0, 'never-candidated terms carry no frequency evidence')
56
+ assert.equal(lexicon.usageLowerBound(undefined), 0)
57
+ assert.equal(lexicon.usageLowerBound(null), 0)
58
+ assert.equal(lexicon.usageLowerBound({}), 0)
59
+ assert.equal(lexicon.usageLowerBound({ candidate: -5, injection: 3 }), 0, 'negative denominator is treated as zero')
60
+ // success 被 min(n, ...) 钳住:脏数据 injection > candidate 也不会把 p 抬过 1。
61
+ const bounded = lexicon.usageLowerBound({ candidate: 10, injection: 50, query: 0 })
62
+ const maxed = lexicon.usageLowerBound({ candidate: 10, injection: 10, query: 0 })
63
+ assert.equal(bounded, maxed, 'success above n is clamped to n')
64
+ for (const value of [bounded, maxed]) assert.ok(value >= 0 && value <= 1, 'the bound stays in [0,1]')
65
+ // 1/1 的 Wilson 下界 ≈ 0.207:一次偶发命中拿不到多少先验。
66
+ const single = lexicon.usageLowerBound({ candidate: 1, injection: 1, query: 0 })
67
+ assert.ok(Math.abs(single - 0.207) < 0.01, `1/1 lower bound should be ~0.207, got ${single}`)
68
+ })
69
+
70
+ test('usageLowerBound: 10/9 ranks above 10/1 — a high miss rate is punished', () => {
71
+ const good = lexicon.usageLowerBound({ candidate: 10, injection: 9, query: 0 })
72
+ const bad = lexicon.usageLowerBound({ candidate: 10, injection: 1, query: 0 })
73
+ assert.ok(good > bad, `9/10 (${good}) must beat 1/10 (${bad})`)
74
+ // 大样本的稳定命中率高于小样本的同比率(Wilson 下界的意义所在)。
75
+ const big = lexicon.usageLowerBound({ candidate: 120, injection: 100, query: 0 })
76
+ const small = lexicon.usageLowerBound({ candidate: 10, injection: 8, query: 0 })
77
+ assert.ok(big > small, '100/120 must exceed 8/10 despite the same ratio')
78
+ })
79
+
80
+ // ---- 3. 高 strength 零计数仍压过低 strength 高计数 ----
81
+
82
+ test('effective factor: high strength with zero hits still outranks low strength with heavy usage', () => {
83
+ const binding = setupLibrary('strength-first', { terms: [
84
+ { term: '强词先', kind: 'term', meaning: '强权重零计数' },
85
+ { term: '弱词后', kind: 'term', meaning: '弱权重高计数' },
86
+ ] })
87
+ const lib = binding.libraryId
88
+ patchTerm(lib, '强词先', { strength: 0.9, confidence: 0.8, updatedAt: '2026-01-01T00:00:00.000Z' })
89
+ patchTerm(lib, '弱词后', { strength: 0.2, confidence: 0.8, updatedAt: '2026-01-01T00:00:00.000Z',
90
+ hitCounts: { candidate: 1000, injection: 1000, query: 0 } })
91
+ const hits = lexicon.matchLexiconHits(lib, '强词先和弱词后都出现在这句话里', binding.memoryContext)
92
+ assert.equal(hits.length, 2)
93
+ assert.equal(hits[0].term, '强词先', 'a bounded [0.85,1.00] prior must never overturn strength')
94
+ assert.equal(hits[0].hitCounts.candidate, 0, 'the winner indeed has zero usage')
95
+ })
96
+
97
+ // ---- 4. 同 strength/confidence,注入率高的排前面 ----
98
+
99
+ test('effective factor: same strength and confidence, the heavily injected term ranks first', () => {
100
+ const binding = setupLibrary('usage-first', { terms: [
101
+ { term: '高频入', kind: 'term', meaning: '十次候选十次注入' },
102
+ { term: '低频入', kind: 'term', meaning: '十次候选一次注入' },
103
+ ] })
104
+ const lib = binding.libraryId
105
+ patchTerm(lib, '高频入', { strength: 0.5, confidence: 0.5, updatedAt: '2026-01-01T00:00:00.000Z',
106
+ hitCounts: { candidate: 10, injection: 10, query: 0 } })
107
+ patchTerm(lib, '低频入', { strength: 0.5, confidence: 0.5, updatedAt: '2026-01-01T00:00:00.000Z',
108
+ hitCounts: { candidate: 10, injection: 1, query: 0 } })
109
+ const hits = lexicon.matchLexiconHits(lib, '高频入和低频入都出现在这句话里', binding.memoryContext)
110
+ assert.equal(hits.length, 2)
111
+ assert.equal(hits[0].term, '高频入', 'Wilson lower bound lifts the term that really gets injected')
112
+ assert.equal(hits[1].term, '低频入')
113
+ })
114
+
115
+ // ---- 5. 排序稳定:纯函数,与当前时间/调用次数无关 ----
116
+
117
+ test('ranking is a pure function of the stored data, stable across repeated calls', () => {
118
+ const binding = setupLibrary('stable', { terms: [
119
+ { term: '稳定甲', kind: 'term', meaning: '一' },
120
+ { term: '稳定乙', kind: 'term', meaning: '二' },
121
+ { term: '稳定丙', kind: 'term', meaning: '三' },
122
+ ] })
123
+ const lib = binding.libraryId
124
+ patchTerm(lib, '稳定甲', { strength: 0.6, confidence: 0.6, updatedAt: '2026-01-02T00:00:00.000Z',
125
+ hitCounts: { candidate: 4, injection: 1, query: 0 } })
126
+ patchTerm(lib, '稳定乙', { strength: 0.6, confidence: 0.6, updatedAt: '2026-01-01T00:00:00.000Z',
127
+ hitCounts: { candidate: 4, injection: 4, query: 0 } })
128
+ patchTerm(lib, '稳定丙', { strength: 0.6, confidence: 0.6, updatedAt: '2026-01-01T00:00:00.000Z',
129
+ hitCounts: { candidate: 4, injection: 4, query: 0 } })
130
+ const text = '稳定甲稳定乙稳定丙同时出现'
131
+ const first = lexicon.matchLexiconHits(lib, text, binding.memoryContext).map(hit => hit.term)
132
+ assert.equal(first.length, 3)
133
+ const second = lexicon.matchLexiconHits(lib, text, binding.memoryContext).map(hit => hit.term)
134
+ assert.deepEqual(second, first, 'the order never depends on the clock or iteration order')
135
+ // 稳定乙/丙同分同 updatedAt:并列对按 termId 字典序,稳定甲(低注入率)垫底。
136
+ const idA = termIdOf(lib, '稳定乙')
137
+ const idB = termIdOf(lib, '稳定丙')
138
+ const tiePair = idA.localeCompare(idB) < 0 ? ['稳定乙', '稳定丙'] : ['稳定丙', '稳定乙']
139
+ assert.deepEqual(first.slice(0, 2), tiePair, 'the tied pair falls back to the lexicographic termId')
140
+ assert.equal(first[2], '稳定甲', 'the weak injection rate stays behind on the same strength')
141
+ })
142
+
143
+ // ---- 6. hitCounts 写入 clamp 到 1,000,000 ----
144
+
145
+ test('lexiconRecordHits clamps hitCounts at HIT_COUNT_CAP and keeps the prior bounded', () => {
146
+ const binding = setupLibrary('clamp', { terms: [{ term: '爆表词', kind: 'term', meaning: '计数封顶' }] })
147
+ const lib = binding.libraryId
148
+ const id = termIdOf(lib, '爆表词')
149
+ assert.ok(id)
150
+ patchTerm(lib, '爆表词', { hitCounts: { candidate: 5_000_000, injection: 2_000_000, query: 3_000_000 } })
151
+ const { counts } = lexicon.lexiconRecordHits(lib, [id], 'candidate')
152
+ assert.equal(counts[id].candidate, lexicon.HIT_COUNT_CAP, 'candidate is clamped to 1,000,000 even after increment')
153
+ assert.equal(counts[id].injection, lexicon.HIT_COUNT_CAP, 'pre-existing out-of-range values are pulled back in')
154
+ assert.equal(counts[id].query, lexicon.HIT_COUNT_CAP)
155
+ const lb = lexicon.usageLowerBound(counts[id])
156
+ assert.ok(lb >= 0 && lb <= 1, `the factor input stays finite and bounded, got ${lb}`)
157
+ assert.ok(lexicon.usageLowerBound({ candidate: lexicon.HIT_COUNT_CAP, injection: lexicon.HIT_COUNT_CAP }) > 0.9999, 'a perfect huge sample bounds just under 1 (Wilson never quite reaches it)')
158
+ // 落盘值同样被封顶:下一次读取不会看到越界计数。
159
+ const stored = Object.values(lexicon.loadLexicon(lib).terms).find(entry => entry.termId === id || entry.term === '爆表词')
160
+ assert.equal(stored.hitCounts.candidate, lexicon.HIT_COUNT_CAP)
161
+ })
162
+
163
+ // ---- 7. 生命周期 × 频率先验(§2.1):归档词条不再参与匹配/注入 ----
164
+
165
+ test('archived terms stop accruing frequency evidence and keep their stored counts', () => {
166
+ const binding = setupLibrary('archived', { terms: [{ term: '归档频率词', kind: 'term', meaning: '归档后不再候选' }] })
167
+ const lib = binding.libraryId
168
+ const id = termIdOf(lib, '归档频率词')
169
+ lexicon.lexiconRecordHit(lib, id, 'candidate')
170
+ lexicon.lexiconRecordHit(lib, id, 'injection')
171
+ lexicon.archiveLexiconTerm({ libraryId: lib, termId: id })
172
+ // 归档后不再进匹配:没有 candidate/injection 可累积。
173
+ assert.deepEqual(lexicon.matchLexiconHits(lib, '归档频率词出现在这句话里', binding.memoryContext), [])
174
+ const stored = lexicon.loadLexicon(lib).terms[id]
175
+ assert.deepEqual(stored.hitCounts, { candidate: 1, injection: 1, query: 0 }, 'the audit counts survive archiving unchanged')
176
+ // 恢复后重新参与匹配(计数是历史,不是可见性开关)。
177
+ lexicon.restoreLexiconTerm({ libraryId: lib, termId: id })
178
+ assert.equal(lexicon.matchLexiconHits(lib, '归档频率词出现在这句话里', binding.memoryContext).length, 1)
179
+ })