dsh-memu-memory 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/README.md +19 -8
- package/README.zh.md +12 -6
- package/docs/contributing.md +16 -3
- package/docs/env-and-commands.md +8 -0
- package/docs/host-api.md +3 -1
- package/docs/memory-semantics.md +21 -2
- package/docs/troubleshooting.md +42 -3
- package/lib/activity-log.js +22 -5
- package/lib/backup.js +24 -2
- package/lib/binding-diagnostics.js +3 -3
- package/lib/client.js +1610 -117
- package/lib/consolidation-worker.js +68 -17
- package/lib/curator-evidence.js +131 -0
- package/lib/curator-policy.js +81 -0
- package/lib/curator.js +1299 -83
- package/lib/cycle.js +3 -0
- package/lib/embedding-download.js +39 -4
- package/lib/engine.js +6 -3
- package/lib/global-graph.js +282 -0
- package/lib/graph.js +190 -14
- package/lib/inbound.js +19 -0
- package/lib/index.js +33 -1
- package/lib/inject.js +277 -17
- package/lib/lexicon.js +1115 -0
- package/lib/local-vector-service.js +8 -6
- package/lib/managed-runtime.js +33 -2
- package/lib/memory.js +585 -76
- package/lib/memu.js +2 -2
- package/lib/migrate.js +83 -2
- package/lib/private-links.js +587 -0
- package/lib/private.js +128 -7
- package/lib/prompts/global-curator.md +76 -19
- package/lib/prompts/private-consolidation.coding.md +4 -1
- package/lib/prompts/private-consolidation.md +3 -1
- package/lib/prompts/private-consolidation.mixed.md +3 -1
- package/lib/prompts/private-consolidation.rp.md +3 -1
- package/lib/prompts.js +1 -1
- package/lib/rpc.js +622 -13
- package/lib/state.js +4 -1
- package/lib/text-retrieval.js +661 -0
- package/lib/text-similarity.js +124 -0
- package/lib/tools.js +230 -16
- package/lib/vector-migration.js +28 -3
- package/lib/vector-profile.js +24 -0
- package/lib/webui/standalone.css +17 -0
- package/lib/webui/standalone.js +2 -0
- package/package.json +3 -2
- package/scripts/acceptance-local.mjs +8 -5
- package/scripts/curator-pair-calibrate.mjs +259 -0
- package/scripts/lexicon-calibrate.mjs +230 -0
- package/scripts/lexicon-dedupe.mjs +318 -0
- package/scripts/replay.mjs +573 -0
- package/tests/absorb-review.test.mjs +353 -0
- package/tests/appendix-b.test.mjs +6 -2
- package/tests/appendix-b2.test.mjs +2 -2
- package/tests/client-export.test.mjs +520 -0
- package/tests/client-render.test.mjs +19 -3
- package/tests/consolidation-credential.test.mjs +16 -5
- package/tests/consolidation-feed.test.mjs +96 -0
- package/tests/consolidation-fork.test.mjs +144 -10
- package/tests/consolidation-snapshot.test.mjs +82 -4
- package/tests/curator-absorb.test.mjs +443 -0
- package/tests/curator-calibrate.test.mjs +173 -0
- package/tests/curator-clusters.test.mjs +281 -0
- package/tests/curator-commit.test.mjs +324 -0
- package/tests/curator-evidence.test.mjs +102 -0
- package/tests/curator-link.test.mjs +316 -0
- package/tests/curator-pairing.test.mjs +252 -0
- package/tests/curator-probe.test.mjs +15 -6
- package/tests/curator-review-mode.test.mjs +314 -0
- package/tests/disclosure.test.mjs +2 -1
- package/tests/fixtures/curator-pairs/labeled.v1.jsonl +90 -0
- package/tests/fixtures/lexicon-stub-hooks.mjs +15 -0
- package/tests/fixtures/lexicon-stub.mjs +284 -0
- package/tests/graph-cascade.test.mjs +223 -0
- package/tests/graph.test.mjs +113 -0
- package/tests/help-page.test.mjs +415 -0
- package/tests/hybrid-recall.test.mjs +32 -2
- package/tests/inbound.test.mjs +6 -3
- package/tests/inject-guide.test.mjs +28 -0
- package/tests/inject.test.mjs +157 -5
- package/tests/lexicon-calibrate.test.mjs +48 -0
- package/tests/lexicon-dedupe.test.mjs +171 -0
- package/tests/lexicon-frequency.test.mjs +179 -0
- package/tests/lexicon-inject.test.mjs +410 -0
- package/tests/lexicon-perf.test.mjs +436 -0
- package/tests/lexicon-ui.test.mjs +547 -0
- package/tests/lexicon.test.mjs +922 -0
- package/tests/managed-memory.test.mjs +55 -0
- package/tests/managed-runtime.test.mjs +32 -0
- package/tests/private-links.test.mjs +445 -0
- package/tests/prompts.test.mjs +35 -0
- package/tests/recall-dedup.test.mjs +446 -0
- package/tests/sidebar-beat.test.mjs +5 -1
- package/tests/tool-schema.test.mjs +1 -1
- package/tests/tools.test.mjs +112 -3
- package/tests/vector-migration.test.mjs +58 -1
- package/tests/webui-auth.test.mjs +348 -2
- package/tests/webui.test.mjs +191 -2
|
@@ -0,0 +1,587 @@
|
|
|
1
|
+
// Private auto-association + entity hubs (algorithm-first, zero extra model tokens).
|
|
2
|
+
//
|
|
3
|
+
// Trigger: after a private consolidation settles, the created/updated entry ids are
|
|
4
|
+
// handed to this module. It never calls an LLM. Two layers live here:
|
|
5
|
+
//
|
|
6
|
+
// * suggestPrivateLinks — PURE. Reads an index snapshot (+ the library's entity
|
|
7
|
+
// list) and returns what it would connect. No disk writes, no activity log.
|
|
8
|
+
// * autoLinkPrivateMemories — APPLIES the high-confidence half: memory↔memory
|
|
9
|
+
// relation links and entity hubs. Uncertain pairs are returned as candidates
|
|
10
|
+
// for the current consolidation worker to adjudicate (it already has the
|
|
11
|
+
// transcript in hand; starting another consolidation would re-read everything).
|
|
12
|
+
//
|
|
13
|
+
// The high-precision signals:
|
|
14
|
+
// * both entries are eligible (active, not expired, not automatic-secret),
|
|
15
|
+
// visible to the conversation, share the same fact context and the same track;
|
|
16
|
+
// * their titles share a RARE token (df <= max(2, ceil(0.15·N))): a Latin token
|
|
17
|
+
// of >= 3 chars, or a CJK bigram. These are the "name-like" tokens;
|
|
18
|
+
// * pairScore < 0.82 — at/above that band it is a duplicate/merge problem, not
|
|
19
|
+
// a link; findConflict must find no contradiction; an existing relation/person
|
|
20
|
+
// link in either direction means it is already handled (idempotent);
|
|
21
|
+
// * a name-like shared token whose entity kind is resolved (person cue, or a
|
|
22
|
+
// Latin token with no cue → thing) becomes a hub: >= 2 memories sharing the
|
|
23
|
+
// stable name are linked to one entity, and the memory pair is auto-linked.
|
|
24
|
+
// A CJK token with no person cue has an uncertain kind (null) and a token
|
|
25
|
+
// that contradicts an existing entity kind is a mismatch: neither creates a
|
|
26
|
+
// hub or a memory link; the pair becomes one uncertain candidate with a
|
|
27
|
+
// suggestedEntity for this consolidation's worker.
|
|
28
|
+
//
|
|
29
|
+
// Non-name-like shared tokens never auto-link. They surface as uncertain
|
|
30
|
+
// candidates only when 0.55 <= pairScore < 0.82. A pair appears in exactly one
|
|
31
|
+
// output: auto-linked pairs are never duplicated into candidates, and uncertain
|
|
32
|
+
// pairs are never auto-linked.
|
|
33
|
+
//
|
|
34
|
+
// Links are metadata only: they never touch the vector store, never delete
|
|
35
|
+
// memory, and can be removed from the relationship graph. A failed association
|
|
36
|
+
// pass must never fail the consolidation job, so every write is individually
|
|
37
|
+
// caught and only a warning is logged; the top-level index load is caught too.
|
|
38
|
+
|
|
39
|
+
import { sameFactContext } from './context.js'
|
|
40
|
+
import { createEntity, linkEntries, listEntities } from './graph.js'
|
|
41
|
+
import { findConflict, isMemoryEligible, visibleToSession } from './memory.js'
|
|
42
|
+
import { pairFeatures, pairScore, tokenize } from './text-retrieval.js'
|
|
43
|
+
import { loadPrivateIndex } from './state.js'
|
|
44
|
+
import { record } from './activity-log.js'
|
|
45
|
+
|
|
46
|
+
const DEFAULT_PER_ENTRY = 2
|
|
47
|
+
const DEFAULT_TOTAL = 5
|
|
48
|
+
const DEFAULT_MAX_CANDIDATES = 8
|
|
49
|
+
const DEFAULT_MAX_ENTITIES = 2
|
|
50
|
+
const SCORE_CANDIDATE_MIN = 0.55
|
|
51
|
+
const SCORE_DUPLICATE = 0.82
|
|
52
|
+
const DF_RATIO = 0.15
|
|
53
|
+
const LATIN_NAME_MIN_LENGTH = 3
|
|
54
|
+
const ENTITY_CONFIDENCE = 0.65
|
|
55
|
+
// findConflict compares the same five identity keys; grouping hubs by the same
|
|
56
|
+
// key keeps "not across context" true for the entity side too.
|
|
57
|
+
const FACT_KEYS = ['channelId', 'speakerId', 'characterId', 'worldId', 'branchId']
|
|
58
|
+
const BLOCKING_LINK_KINDS = new Set(['relation', 'person'])
|
|
59
|
+
const CJK_RE = /^[\u3400-\u9fff\uf900-\ufaff]+$/
|
|
60
|
+
const LATIN_RE = /[a-z]/
|
|
61
|
+
const EMPTY_SET = new Set()
|
|
62
|
+
|
|
63
|
+
// Title tokens that carry no identifying power on their own. The draft list is
|
|
64
|
+
// kept and expanded with ordinary bigrams/words so a shared "名字"/"关于"/"the"
|
|
65
|
+
// never looks like a stable name. Bigrams touching a function character are
|
|
66
|
+
// dropped separately by isStopToken (字的/的由/…).
|
|
67
|
+
const STOPWORDS = new Set([
|
|
68
|
+
'名字', '由来', '记忆', '关于', '什么', '怎么', '一个', '这个', '那个', '我们', '你们', '他们',
|
|
69
|
+
'她们', '自己', '现在', '已经', '只是', '就是', '时候', '因为', '所以', '但是', '如果', '然后',
|
|
70
|
+
'还是', '可以', '不是', '没有', '内容', '信息', '角色', '设定', '故事', '对话', '用户', '一些',
|
|
71
|
+
'进行', '今天', '昨天', '明天', '之前', '之后', '一起', '知道', '觉得', '喜欢', '开始', '结束',
|
|
72
|
+
'东西', '事情', '问题', '感觉', '看到', '听到', '打算', '准备', '应该', '需要', '可能', '真的',
|
|
73
|
+
'非常', '特别', '有点', '有些', '一样', '这样', '那样', '这里', '那里', '哪里', '第一', '第二',
|
|
74
|
+
'于是', '而且', '并且', '或者', '虽然', '不过', '对于', '为了', '通过', '以及', '还有', '记录',
|
|
75
|
+
'笔记', '计划', '项目', '会议', '工作', '生活', '世界', '地方', '时间', '目前', '最近', '以后',
|
|
76
|
+
'以前', '当时', '那时', '这时', '不会', '不能', '不要',
|
|
77
|
+
'the', 'and', 'for', 'with', 'that', 'this', 'from', 'have', 'will', 'your', 'you', 'name',
|
|
78
|
+
'memory', 'user', 'role', 'story', 'about', 'into', 'when', 'what', 'which', 'were', 'was',
|
|
79
|
+
'are', 'is', 'an', 'of', 'to', 'in', 'on', 'at', 'by', 'as', 'or', 'be', 'it', 'its', 'not',
|
|
80
|
+
'but', 'they', 'them', 'their', 'we', 'our', 'me', 'my', 'he', 'she', 'his', 'her', 'him',
|
|
81
|
+
'has', 'had', 'do', 'does', 'did', 'can', 'could', 'would', 'should', 'there', 'here', 'then',
|
|
82
|
+
'than', 'so', 'if', 'all', 'any', 'each', 'other', 'more', 'most', 'some', 'such', 'no', 'only',
|
|
83
|
+
'own', 'same', 'too', 'very', 'just', 'also', 'how', 'why', 'who', 'whom', 'whose', 'out', 'up',
|
|
84
|
+
'down', 'over', 'under', 'again', 'further', 'once', 'during', 'before', 'after', 'above',
|
|
85
|
+
'below', 'between', 'both', 'few', 'many', 'much', 'every', 'off', 'project', 'note', 'notes',
|
|
86
|
+
'meeting', 'update', 'info', 'detail', 'details', 'thing', 'things', 'stuff', 'today',
|
|
87
|
+
'yesterday', 'tomorrow', 'day', 'night',
|
|
88
|
+
])
|
|
89
|
+
|
|
90
|
+
// A CJK bigram containing a particle/pronoun is grammatical glue, not a name.
|
|
91
|
+
// Kept intentionally narrow: characters that legitimately appear inside names
|
|
92
|
+
// (上/下/中/一/…) are not listed.
|
|
93
|
+
const CJK_FUNCTION_CHARS = new Set([
|
|
94
|
+
'的', '了', '是', '在', '和', '与', '也', '就', '都', '很', '又', '还', '你', '我', '他', '她',
|
|
95
|
+
'它', '们', '这', '那', '哪', '什', '么', '吗', '呢', '吧', '啊', '呀', '哦', '嗯', '之', '其',
|
|
96
|
+
'而', '且', '但', '却', '把', '被', '让', '给', '对', '从', '向', '于', '为', '及', '或', '并',
|
|
97
|
+
'着', '过', '得', '地', '有', '没', '不', '要', '会', '能', '可', '以', '说', '去', '来',
|
|
98
|
+
])
|
|
99
|
+
|
|
100
|
+
// Person cues. Deliberately conservative: a pronoun or "角色"/"名字" near the
|
|
101
|
+
// name is the cheap signal that the token is a person and not a thing. English
|
|
102
|
+
// cues are word-bounded so "the" never matches "he".
|
|
103
|
+
const PERSON_CUE_RE = /猫娘|角色|人物|名字|人设|称呼|登场|姓名|男主|女主|男配|女配|她|他|\bcp\b|\bcharacter\b|\bperson\b|\bshe\b|\bhe\b|\bher\b|\bhis\b|\bhim\b/i
|
|
104
|
+
|
|
105
|
+
function entryOf(value) {
|
|
106
|
+
// Accept both plain entries and [id, entry] pairs (Object.entries output).
|
|
107
|
+
return Array.isArray(value) ? value[1] : value
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function hasCjk(token) {
|
|
111
|
+
return typeof token === 'string' && CJK_RE.test(token)
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function isLatinNameLike(token) {
|
|
115
|
+
return typeof token === 'string' && token.length >= LATIN_NAME_MIN_LENGTH && LATIN_RE.test(token) && !hasCjk(token)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function isNameLikeToken(token) {
|
|
119
|
+
if (hasCjk(token)) return token.length >= 2
|
|
120
|
+
return isLatinNameLike(token)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function isStopToken(token) {
|
|
124
|
+
if (STOPWORDS.has(token)) return true
|
|
125
|
+
if (hasCjk(token)) {
|
|
126
|
+
for (const char of token) if (CJK_FUNCTION_CHARS.has(char)) return true
|
|
127
|
+
}
|
|
128
|
+
return false
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Case/NFKC/whitespace/punctuation-insensitive entity-name key. `Yuike`,
|
|
133
|
+
* `yuike`, `Yuike!` and `yuike` all collapse to `yuike`, so the algorithm
|
|
134
|
+
* never creates a second entity for a name it has already seen.
|
|
135
|
+
*/
|
|
136
|
+
export function normalizeEntityName(name) {
|
|
137
|
+
return String(name || '')
|
|
138
|
+
.normalize('NFKC')
|
|
139
|
+
.toLowerCase()
|
|
140
|
+
.replace(/[\s\p{P}\p{S}\p{Cf}]+/gu, '')
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function titleTokens(entry) {
|
|
144
|
+
const out = new Set()
|
|
145
|
+
for (const token of tokenize(entry?.title || '')) {
|
|
146
|
+
if (token.length >= 2 && !isStopToken(token)) out.add(token)
|
|
147
|
+
}
|
|
148
|
+
return out
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function factKey(entry) {
|
|
152
|
+
const context = entry?.context || {}
|
|
153
|
+
return FACT_KEYS.map(key => String(context[key] || '')).join('\u0000')
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function trackOf(entry) {
|
|
157
|
+
return entry?.track || 'memory'
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function escapeRegExp(value) {
|
|
161
|
+
return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function displayNameOfToken(token, titles) {
|
|
165
|
+
let pattern
|
|
166
|
+
try { pattern = new RegExp(escapeRegExp(token), 'i') } catch { return token }
|
|
167
|
+
for (const title of titles) {
|
|
168
|
+
const match = String(title || '').normalize('NFKC').match(pattern)
|
|
169
|
+
if (match) return match[0]
|
|
170
|
+
}
|
|
171
|
+
return token
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Decide whether a shared name-like token names a person or a thing.
|
|
176
|
+
* * any involved title/content carries a person cue → person;
|
|
177
|
+
* * a Latin name-like token without a cue → thing (conservative, never org/place);
|
|
178
|
+
* * a CJK token without a cue → null: the type is unresolved and the caller
|
|
179
|
+
* must not auto-create an entity for it.
|
|
180
|
+
*/
|
|
181
|
+
export function inferEntityKind(entries, token) {
|
|
182
|
+
const texts = (Array.isArray(entries) ? entries : [])
|
|
183
|
+
.map(entryOf)
|
|
184
|
+
.map(entry => `${entry?.title || ''}\n${entry?.content || ''}`)
|
|
185
|
+
.join('\n')
|
|
186
|
+
if (PERSON_CUE_RE.test(texts)) return 'person'
|
|
187
|
+
if (isLatinNameLike(token)) return 'thing'
|
|
188
|
+
return null
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function resolveEntity(entities, token, kind) {
|
|
192
|
+
const key = normalizeEntityName(token)
|
|
193
|
+
if (!key) return { entityId: null, conflict: false }
|
|
194
|
+
let mismatch = null
|
|
195
|
+
for (const entity of Array.isArray(entities) ? entities : []) {
|
|
196
|
+
if (!entity || !entity.id) continue
|
|
197
|
+
const names = [entity.name, ...(Array.isArray(entity.aliases) ? entity.aliases : [])]
|
|
198
|
+
if (!names.some(name => normalizeEntityName(name) === key)) continue
|
|
199
|
+
if (kind !== null && entity.kind === kind) return { entityId: entity.id, conflict: false }
|
|
200
|
+
if (!mismatch) mismatch = entity
|
|
201
|
+
}
|
|
202
|
+
if (mismatch) return { entityId: null, conflict: true, existingKind: mismatch.kind || null }
|
|
203
|
+
return { entityId: null, conflict: false }
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function hasBlockingLink(entry, targetId) {
|
|
207
|
+
return (entry?.links || []).some(link => link?.to === targetId && BLOCKING_LINK_KINDS.has(link?.kind))
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function pairKey(a, b) {
|
|
211
|
+
return a < b ? `${a}\u0000${b}` : `${b}\u0000${a}`
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function collapse(text) {
|
|
215
|
+
return String(text || '').replace(/\s+/g, ' ').trim()
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function clip(text, max) {
|
|
219
|
+
return String(text || '').slice(0, max)
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// Candidate title/preview describe the memory the new/updated entry points AT,
|
|
223
|
+
// i.e. the other side of the pair.
|
|
224
|
+
function describeOther(entry) {
|
|
225
|
+
const title = collapse(entry?.title || '') || collapse(entry?.content || '')
|
|
226
|
+
return { title: clip(title, 80), preview: clip(collapse(entry?.content || ''), 120) }
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function linkConfidence(score) {
|
|
230
|
+
// Name-like links live in the 0.6–0.78 band from the spec.
|
|
231
|
+
return Math.max(0.6, Math.min(0.78, 0.55 + score * 0.3))
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function capNumber(value, fallback) {
|
|
235
|
+
const number = Number(value)
|
|
236
|
+
return Number.isFinite(number) ? Math.max(0, Math.floor(number)) : fallback
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function compareTokenDf(a, b, df) {
|
|
240
|
+
return (df.get(a) || 0) - (df.get(b) || 0) || b.length - a.length || a.localeCompare(b)
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Pure candidate suggestion. Never writes.
|
|
245
|
+
*
|
|
246
|
+
* Universe: eligible entries (active/not secret/visible/same library) whose
|
|
247
|
+
* pair shares a fact context and track, where at least one endpoint is in
|
|
248
|
+
* `ids`. Returns
|
|
249
|
+
* { autoLinks: [{ from, to, score, token, nameLike, confidence }],
|
|
250
|
+
* entityHubs: [{ token, name, kind, entityId?, memoryIds }],
|
|
251
|
+
* candidates: [{ from, to, score, sharedToken?, title, preview, suggestedEntity? }] }
|
|
252
|
+
*/
|
|
253
|
+
export function suggestPrivateLinks({
|
|
254
|
+
index,
|
|
255
|
+
entities = [],
|
|
256
|
+
ids,
|
|
257
|
+
sessionId,
|
|
258
|
+
perEntry = DEFAULT_PER_ENTRY,
|
|
259
|
+
total = DEFAULT_TOTAL,
|
|
260
|
+
maxCandidates = DEFAULT_MAX_CANDIDATES,
|
|
261
|
+
maxEntities = DEFAULT_MAX_ENTITIES,
|
|
262
|
+
} = {}) {
|
|
263
|
+
const entries = index?.entries || {}
|
|
264
|
+
const eligibleList = Object.entries(entries)
|
|
265
|
+
.filter(([, entry]) => entry && isMemoryEligible(entry, { automatic: true })
|
|
266
|
+
&& entry.visibility !== 'secret'
|
|
267
|
+
&& (!sessionId || visibleToSession(entry, sessionId)))
|
|
268
|
+
.map(([id, entry]) => ({ id: String(id), entry }))
|
|
269
|
+
const empty = { autoLinks: [], entityHubs: [], candidates: [] }
|
|
270
|
+
if (eligibleList.length < 2) return empty
|
|
271
|
+
const eligible = new Map(eligibleList.map(item => [item.id, item.entry]))
|
|
272
|
+
const wanted = []
|
|
273
|
+
const wantedSet = new Set()
|
|
274
|
+
for (const raw of Array.isArray(ids) ? ids : []) {
|
|
275
|
+
const id = String(raw)
|
|
276
|
+
if (wantedSet.has(id) || !eligible.has(id)) continue
|
|
277
|
+
wantedSet.add(id)
|
|
278
|
+
wanted.push(id)
|
|
279
|
+
}
|
|
280
|
+
if (!wanted.length) return empty
|
|
281
|
+
|
|
282
|
+
const tokensById = new Map()
|
|
283
|
+
const df = new Map()
|
|
284
|
+
for (const { id, entry } of eligibleList) {
|
|
285
|
+
const set = titleTokens(entry)
|
|
286
|
+
tokensById.set(id, set)
|
|
287
|
+
for (const token of set) df.set(token, (df.get(token) || 0) + 1)
|
|
288
|
+
}
|
|
289
|
+
const dfCap = Math.max(2, Math.ceil(eligibleList.length * DF_RATIO))
|
|
290
|
+
const isRare = token => (df.get(token) || 0) <= dfCap
|
|
291
|
+
|
|
292
|
+
// Entity-hub groups: one rare name-like token within one fact context + track.
|
|
293
|
+
// Rarity bounds the group, context/track bounds who may be connected.
|
|
294
|
+
const groups = new Map()
|
|
295
|
+
for (const item of eligibleList) {
|
|
296
|
+
for (const token of tokensById.get(item.id) || EMPTY_SET) {
|
|
297
|
+
if (!isNameLikeToken(token) || !isRare(token)) continue
|
|
298
|
+
const key = `${token}\u0000${factKey(item.entry)}\u0000${trackOf(item.entry)}`
|
|
299
|
+
if (!groups.has(key)) groups.set(key, { token, entries: [] })
|
|
300
|
+
groups.get(key).entries.push(item)
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
const analyses = new Map()
|
|
304
|
+
for (const [key, group] of groups) {
|
|
305
|
+
if (group.entries.length < 2) continue
|
|
306
|
+
if (!group.entries.some(item => wantedSet.has(item.id))) continue
|
|
307
|
+
const kind = inferEntityKind(group.entries.map(item => item.entry), group.token)
|
|
308
|
+
const name = displayNameOfToken(group.token, group.entries.map(item => item.entry?.title || ''))
|
|
309
|
+
const resolution = resolveEntity(entities, group.token, kind)
|
|
310
|
+
analyses.set(key, {
|
|
311
|
+
token: group.token,
|
|
312
|
+
kind,
|
|
313
|
+
name,
|
|
314
|
+
entries: group.entries,
|
|
315
|
+
entityId: resolution.entityId,
|
|
316
|
+
conflict: resolution.conflict,
|
|
317
|
+
})
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
const autoLinks = []
|
|
321
|
+
const candidates = []
|
|
322
|
+
const seenPairs = new Set()
|
|
323
|
+
const wantedOrder = new Map(wanted.map((id, indexInIds) => [id, indexInIds]))
|
|
324
|
+
for (const fromId of wanted) {
|
|
325
|
+
const a = eligible.get(fromId)
|
|
326
|
+
const aTokens = tokensById.get(fromId) || EMPTY_SET
|
|
327
|
+
for (const other of eligibleList) {
|
|
328
|
+
if (other.id === fromId) continue
|
|
329
|
+
const key = pairKey(fromId, other.id)
|
|
330
|
+
if (seenPairs.has(key)) continue
|
|
331
|
+
const otherOrder = wantedOrder.get(other.id)
|
|
332
|
+
// Both endpoints created/updated: the earlier id is the source of the
|
|
333
|
+
// link; the later one will not evaluate the same unordered pair again.
|
|
334
|
+
if (otherOrder !== undefined && otherOrder < wantedOrder.get(fromId)) continue
|
|
335
|
+
seenPairs.add(key)
|
|
336
|
+
const b = other.entry
|
|
337
|
+
if (trackOf(a) !== trackOf(b)) continue
|
|
338
|
+
if (!sameFactContext(a.context, b.context)) continue
|
|
339
|
+
if (hasBlockingLink(a, other.id) || hasBlockingLink(b, fromId)) continue
|
|
340
|
+
const score = pairScore(pairFeatures(a, b))
|
|
341
|
+
if (!Number.isFinite(score) || score >= SCORE_DUPLICATE) continue
|
|
342
|
+
const sharedRare = [...aTokens]
|
|
343
|
+
.filter(token => (tokensById.get(other.id) || EMPTY_SET).has(token) && isRare(token))
|
|
344
|
+
.sort((x, y) => compareTokenDf(x, y, df))
|
|
345
|
+
if (!sharedRare.length && score < SCORE_CANDIDATE_MIN) continue
|
|
346
|
+
if (findConflict({ [other.id]: b }, a)) continue
|
|
347
|
+
const nameTokens = sharedRare.filter(isNameLikeToken)
|
|
348
|
+
let resolved = null
|
|
349
|
+
let unresolved = null
|
|
350
|
+
for (const token of nameTokens) {
|
|
351
|
+
const analysis = analyses.get(`${token}\u0000${factKey(a)}\u0000${trackOf(a)}`)
|
|
352
|
+
if (!analysis) continue
|
|
353
|
+
if (analysis.kind !== null && !analysis.conflict) {
|
|
354
|
+
if (!resolved) resolved = analysis
|
|
355
|
+
} else if (!unresolved) {
|
|
356
|
+
unresolved = analysis
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
const base = { from: fromId, to: other.id, score }
|
|
360
|
+
if (resolved) {
|
|
361
|
+
// A name-like pair may still be linked even when a hub is created for
|
|
362
|
+
// it; a resolved pair must not be duplicated into candidates.
|
|
363
|
+
autoLinks.push({ ...base, token: resolved.token, nameLike: true, confidence: linkConfidence(score) })
|
|
364
|
+
continue
|
|
365
|
+
}
|
|
366
|
+
if (nameTokens.length) {
|
|
367
|
+
// Name-like token, but the entity type is unresolved (CJK without a
|
|
368
|
+
// person cue) or conflicts with an existing entity kind: nothing is
|
|
369
|
+
// linked automatically; the worker decides, so this is one uncertain
|
|
370
|
+
// candidate carrying a suggested entity.
|
|
371
|
+
const analysis = unresolved || analyses.get(`${nameTokens[0]}\u0000${factKey(a)}\u0000${trackOf(a)}`) || null
|
|
372
|
+
const otherText = describeOther(b)
|
|
373
|
+
const candidate = {
|
|
374
|
+
...base,
|
|
375
|
+
sharedToken: analysis?.token || nameTokens[0],
|
|
376
|
+
title: otherText.title,
|
|
377
|
+
preview: otherText.preview,
|
|
378
|
+
}
|
|
379
|
+
if (analysis) candidate.suggestedEntity = { name: analysis.name, kind: analysis.kind }
|
|
380
|
+
candidates.push(candidate)
|
|
381
|
+
continue
|
|
382
|
+
}
|
|
383
|
+
if (score >= SCORE_CANDIDATE_MIN) {
|
|
384
|
+
const otherText = describeOther(b)
|
|
385
|
+
const candidate = { ...base, title: otherText.title, preview: otherText.preview }
|
|
386
|
+
if (sharedRare.length) candidate.sharedToken = sharedRare[0]
|
|
387
|
+
candidates.push(candidate)
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
autoLinks.sort((a, b) => b.score - a.score
|
|
393
|
+
|| a.from.localeCompare(b.from)
|
|
394
|
+
|| a.to.localeCompare(b.to))
|
|
395
|
+
const perEntryCap = capNumber(perEntry, DEFAULT_PER_ENTRY)
|
|
396
|
+
const totalCap = capNumber(total, DEFAULT_TOTAL)
|
|
397
|
+
const perFrom = new Map()
|
|
398
|
+
const chosenLinks = []
|
|
399
|
+
for (const link of autoLinks) {
|
|
400
|
+
if (chosenLinks.length >= totalCap) break
|
|
401
|
+
const count = perFrom.get(link.from) || 0
|
|
402
|
+
if (count >= perEntryCap) continue
|
|
403
|
+
perFrom.set(link.from, count + 1)
|
|
404
|
+
chosenLinks.push(link)
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
const entityHubs = [...analyses.values()]
|
|
408
|
+
.filter(analysis => analysis.kind !== null && !analysis.conflict)
|
|
409
|
+
.sort((a, b) => b.entries.length - a.entries.length
|
|
410
|
+
|| compareTokenDf(a.token, b.token, df))
|
|
411
|
+
.slice(0, capNumber(maxEntities, DEFAULT_MAX_ENTITIES))
|
|
412
|
+
.map(analysis => {
|
|
413
|
+
const hub = { token: analysis.token, name: analysis.name, kind: analysis.kind }
|
|
414
|
+
if (analysis.entityId) hub.entityId = analysis.entityId
|
|
415
|
+
hub.memoryIds = analysis.entries.map(item => item.id)
|
|
416
|
+
return hub
|
|
417
|
+
})
|
|
418
|
+
|
|
419
|
+
candidates.sort((a, b) => b.score - a.score
|
|
420
|
+
|| a.from.localeCompare(b.from)
|
|
421
|
+
|| a.to.localeCompare(b.to))
|
|
422
|
+
return {
|
|
423
|
+
autoLinks: chosenLinks,
|
|
424
|
+
entityHubs,
|
|
425
|
+
candidates: candidates.slice(0, capNumber(maxCandidates, DEFAULT_MAX_CANDIDATES)),
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
function autoLinkNote(link) {
|
|
430
|
+
return `自动关联:两条记忆共享稳定称谓「${link.token}」;内容相似度 ${link.score.toFixed(2)}。算法自动建立,可在关系图移除。`
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function hubNote(hub) {
|
|
434
|
+
return `自动实体枢纽:多条记忆共享稳定称谓「${hub.token}」。`
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function execFor(sessionId) {
|
|
438
|
+
return sessionId ? { agent: { session: { header: { id: sessionId } } } } : undefined
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* Apply the high-confidence part of suggestPrivateLinks and return the uncertain
|
|
443
|
+
* candidates untouched. `ids` are the entries created/updated by the finished
|
|
444
|
+
* consolidation; an entry that is no longer eligible is ignored by the pure pass.
|
|
445
|
+
*
|
|
446
|
+
* Return counts: `linked` = memory↔memory relation links written, `entities` =
|
|
447
|
+
* memory→entity links written, `suggested` = number of candidates handed to the
|
|
448
|
+
* worker. A consolidation must never fail because of this module: the index load
|
|
449
|
+
* and the suggestion pass are caught at the top, every individual write is caught
|
|
450
|
+
* and logged at warn level.
|
|
451
|
+
*/
|
|
452
|
+
export async function autoLinkPrivateMemories({
|
|
453
|
+
libraryId,
|
|
454
|
+
sessionId,
|
|
455
|
+
ids,
|
|
456
|
+
perEntry = DEFAULT_PER_ENTRY,
|
|
457
|
+
total = DEFAULT_TOTAL,
|
|
458
|
+
maxCandidates = DEFAULT_MAX_CANDIDATES,
|
|
459
|
+
maxEntities = DEFAULT_MAX_ENTITIES,
|
|
460
|
+
} = {}) {
|
|
461
|
+
const result = { linked: 0, entities: 0, suggested: 0, candidates: [], autoLinked: [] }
|
|
462
|
+
if (!libraryId) return result
|
|
463
|
+
const idList = (Array.isArray(ids) ? ids : []).map(id => String(id)).filter(Boolean)
|
|
464
|
+
if (!idList.length) return result
|
|
465
|
+
|
|
466
|
+
let index
|
|
467
|
+
let entityList = []
|
|
468
|
+
try {
|
|
469
|
+
index = loadPrivateIndex(libraryId)
|
|
470
|
+
// No session filter: reuse must see every entity in the library, including
|
|
471
|
+
// names that only appear in conversations this session cannot read.
|
|
472
|
+
entityList = listEntities({ libraryId }).items
|
|
473
|
+
} catch (error) {
|
|
474
|
+
record({
|
|
475
|
+
kind: 'write', scope: 'private', sessionId, libraryId, ok: false, level: 'warn',
|
|
476
|
+
message: `私有自动关联跳过:${String(error?.message || error)}`,
|
|
477
|
+
detail: { op: 'auto-link', phase: 'load', error: String(error?.message || error) },
|
|
478
|
+
refs: [],
|
|
479
|
+
})
|
|
480
|
+
return result
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
let suggestion
|
|
484
|
+
try {
|
|
485
|
+
suggestion = suggestPrivateLinks({ index, entities: entityList, ids: idList, sessionId, perEntry, total, maxCandidates, maxEntities })
|
|
486
|
+
} catch (error) {
|
|
487
|
+
record({
|
|
488
|
+
kind: 'write', scope: 'private', sessionId, libraryId, ok: false, level: 'warn',
|
|
489
|
+
message: `私有自动关联候选计算失败:${String(error?.message || error)}`,
|
|
490
|
+
detail: { op: 'auto-link', phase: 'suggest', error: String(error?.message || error) },
|
|
491
|
+
refs: [],
|
|
492
|
+
})
|
|
493
|
+
return result
|
|
494
|
+
}
|
|
495
|
+
result.suggested = suggestion.candidates.length
|
|
496
|
+
result.candidates = suggestion.candidates
|
|
497
|
+
|
|
498
|
+
const exec = execFor(sessionId)
|
|
499
|
+
for (const link of suggestion.autoLinks) {
|
|
500
|
+
try {
|
|
501
|
+
linkEntries({
|
|
502
|
+
exec,
|
|
503
|
+
libraryId,
|
|
504
|
+
fromId: link.from,
|
|
505
|
+
toId: link.to,
|
|
506
|
+
kind: 'relation',
|
|
507
|
+
note: autoLinkNote(link),
|
|
508
|
+
confidence: link.confidence,
|
|
509
|
+
source: 'auto',
|
|
510
|
+
})
|
|
511
|
+
result.linked += 1
|
|
512
|
+
result.autoLinked.push({ from: link.from, to: link.to, kind: 'relation' })
|
|
513
|
+
record({
|
|
514
|
+
kind: 'write', scope: 'private', sessionId, libraryId, ok: true,
|
|
515
|
+
message: `自动关联 ${link.from} → ${link.to}(「${link.token}」)`,
|
|
516
|
+
detail: {
|
|
517
|
+
op: 'auto-link', from: link.from, to: link.to, kind: 'relation',
|
|
518
|
+
score: link.score, token: link.token, confidence: link.confidence, source: 'auto',
|
|
519
|
+
},
|
|
520
|
+
refs: [link.from, link.to],
|
|
521
|
+
})
|
|
522
|
+
} catch (error) {
|
|
523
|
+
record({
|
|
524
|
+
kind: 'write', scope: 'private', sessionId, libraryId, ok: false, level: 'warn',
|
|
525
|
+
message: `自动关联跳过 ${link.from} → ${link.to}:${String(error?.message || error)}`,
|
|
526
|
+
detail: { op: 'auto-link', from: link.from, to: link.to, error: String(error?.message || error) },
|
|
527
|
+
refs: [link.from, link.to],
|
|
528
|
+
})
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
for (const hub of suggestion.entityHubs) {
|
|
533
|
+
let entity = null
|
|
534
|
+
try {
|
|
535
|
+
entity = hub.entityId
|
|
536
|
+
? { id: hub.entityId, kind: hub.kind }
|
|
537
|
+
: createEntity({ exec, libraryId, name: hub.name, kind: hub.kind })
|
|
538
|
+
} catch (error) {
|
|
539
|
+
record({
|
|
540
|
+
kind: 'write', scope: 'private', sessionId, libraryId, ok: false, level: 'warn',
|
|
541
|
+
message: `自动实体枢纽跳过「${hub.name}」:${String(error?.message || error)}`,
|
|
542
|
+
detail: { op: 'auto-entity', name: hub.name, kind: hub.kind, error: String(error?.message || error) },
|
|
543
|
+
refs: hub.memoryIds,
|
|
544
|
+
})
|
|
545
|
+
continue
|
|
546
|
+
}
|
|
547
|
+
for (const memoryId of hub.memoryIds) {
|
|
548
|
+
try {
|
|
549
|
+
const fresh = loadPrivateIndex(libraryId).entries[memoryId]
|
|
550
|
+
if (!fresh) throw new Error('memory is gone')
|
|
551
|
+
if ((fresh.links || []).some(link => link?.to === entity.id)) continue
|
|
552
|
+
linkEntries({
|
|
553
|
+
exec,
|
|
554
|
+
libraryId,
|
|
555
|
+
fromId: memoryId,
|
|
556
|
+
toId: entity.id,
|
|
557
|
+
kind: hub.kind === 'person' ? 'person' : 'relation',
|
|
558
|
+
note: hubNote(hub),
|
|
559
|
+
confidence: ENTITY_CONFIDENCE,
|
|
560
|
+
source: 'auto',
|
|
561
|
+
})
|
|
562
|
+
result.entities += 1
|
|
563
|
+
result.autoLinked.push({ from: memoryId, to: entity.id, kind: hub.kind === 'person' ? 'person' : 'relation' })
|
|
564
|
+
record({
|
|
565
|
+
kind: 'write', scope: 'private', sessionId, libraryId, ok: true,
|
|
566
|
+
message: `自动实体枢纽 ${memoryId} → ${entity.id}(「${hub.name}」)`,
|
|
567
|
+
detail: {
|
|
568
|
+
op: 'auto-entity', memory: memoryId, entity: entity.id, name: hub.name,
|
|
569
|
+
kind: hub.kind, token: hub.token, confidence: ENTITY_CONFIDENCE, source: 'auto',
|
|
570
|
+
},
|
|
571
|
+
refs: [memoryId, entity.id],
|
|
572
|
+
})
|
|
573
|
+
} catch (error) {
|
|
574
|
+
record({
|
|
575
|
+
kind: 'write', scope: 'private', sessionId, libraryId, ok: false, level: 'warn',
|
|
576
|
+
message: `自动实体枢纽跳过 ${memoryId} → 「${hub.name}」:${String(error?.message || error)}`,
|
|
577
|
+
detail: { op: 'auto-entity', memory: memoryId, entity: entity?.id || null, name: hub.name, error: String(error?.message || error) },
|
|
578
|
+
refs: [memoryId],
|
|
579
|
+
})
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
return result
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
export default { suggestPrivateLinks, autoLinkPrivateMemories, normalizeEntityName, inferEntityKind }
|