@restregular/dsh-memory 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -58
- package/cordis.patch.yml +6 -8
- package/icon.svg +1 -0
- package/lib/client.js +859 -0
- package/lib/index.js +72 -0
- package/lib/inject/apply.js +80 -80
- package/lib/inject/inject.js +127 -127
- package/lib/panel.js +353 -0
- package/lib/provider/apply.js +133 -128
- package/lib/provider/embed-install.js +1144 -1002
- package/lib/provider/embed-server.js +308 -294
- package/lib/provider/embed.js +103 -103
- package/lib/provider/hardware.js +225 -0
- package/lib/provider/index.js +1271 -907
- package/lib/provider/models.js +253 -0
- package/lib/provider/recommend.js +187 -0
- package/lib/provider/rerank-cache.js +62 -62
- package/lib/provider/rerank.js +169 -169
- package/lib/provider/scoring.js +143 -143
- package/lib/provider/secrets.js +55 -55
- package/lib/provider/store.js +491 -491
- package/lib/tools/apply.js +104 -104
- package/lib/tools/index.js +224 -224
- package/package.json +17 -6
package/lib/provider/rerank.js
CHANGED
|
@@ -1,169 +1,169 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* LLM-as-judge rerank stage (M2). Pure logic: the caller injects a one-shot
|
|
3
|
-
* LLM invoke function, so unit tests run against a fake and the cordis wiring
|
|
4
|
-
* (apply.ts) adapts `ctx.llm` into it. Every failure path degrades to "no
|
|
5
|
-
* rerank" — retrieval must never depend on this stage succeeding.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
export const defaultRerankConfig = {
|
|
23
|
-
enabled: true,
|
|
24
|
-
provider: '',
|
|
25
|
-
model: '',
|
|
26
|
-
timeoutMs: 1500,
|
|
27
|
-
dailyBudgetTokens: 200_000,
|
|
28
|
-
weight: 0.6,
|
|
29
|
-
topN: 10,
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
/** One-shot LLM invocation the host provides (adapter sees GenerateOptions). */
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
/** Rough token estimate; CJK-heavy text runs ~1.5-2 chars per token. */
|
|
49
|
-
export function estimateTokens(text ) {
|
|
50
|
-
return Math.ceil(text.length / 2)
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
/** Daily budget bucket keyed by local date; resets when the date rolls over. */
|
|
54
|
-
export class DailyBudget {
|
|
55
|
-
day = ''
|
|
56
|
-
used = 0
|
|
57
|
-
limit
|
|
58
|
-
|
|
59
|
-
constructor(limit ) {
|
|
60
|
-
this.limit = limit
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
/** Try to reserve n tokens; returns false when the day's budget is spent. */
|
|
64
|
-
trySpend(n ) {
|
|
65
|
-
this.rollIfNeeded()
|
|
66
|
-
if (this.used + n > this.limit) return false
|
|
67
|
-
this.used += n
|
|
68
|
-
return true
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
get usedToday() {
|
|
72
|
-
this.rollIfNeeded()
|
|
73
|
-
return this.used
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
rollIfNeeded() {
|
|
77
|
-
const today = new Date().toISOString().slice(0, 10)
|
|
78
|
-
if (today !== this.day) {
|
|
79
|
-
this.day = today
|
|
80
|
-
this.used = 0
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
export function buildRerankPrompt(query , candidates ) {
|
|
86
|
-
const lines = candidates.map((c, i) => `${i + 1}. [${c.id}] ${c.content.slice(0, 500)}`)
|
|
87
|
-
return {
|
|
88
|
-
system: '你是检索重排序器。根据与查询的相关性给每个候选记忆打分。只输出 JSON 数组,格式 [[id, 分数], ...],分数为 0 到 1 的小数。不要输出其他内容。',
|
|
89
|
-
prompt: `查询:${query}\n\n候选记忆:\n${lines.join('\n')}`,
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
/** Parse `[[id, score], ...]` out of the model output; tolerates prose around it. */
|
|
94
|
-
export function parseRerankOutput(text , validIds ) {
|
|
95
|
-
const out = new Map ()
|
|
96
|
-
const start = text.indexOf('[')
|
|
97
|
-
const end = text.lastIndexOf(']')
|
|
98
|
-
if (start < 0 || end <= start) return out
|
|
99
|
-
let parsed
|
|
100
|
-
try {
|
|
101
|
-
parsed = JSON.parse(text.slice(start, end + 1))
|
|
102
|
-
} catch {
|
|
103
|
-
return out
|
|
104
|
-
}
|
|
105
|
-
if (!Array.isArray(parsed)) return out
|
|
106
|
-
for (const pair of parsed) {
|
|
107
|
-
if (!Array.isArray(pair) || pair.length < 2) continue
|
|
108
|
-
const [id, score] = pair
|
|
109
|
-
if (typeof id !== 'string' || !validIds.has(id)) continue
|
|
110
|
-
const n = typeof score === 'number' ? score : Number(score)
|
|
111
|
-
if (!Number.isFinite(n)) continue
|
|
112
|
-
out.set(id, Math.min(1, Math.max(0, n)))
|
|
113
|
-
}
|
|
114
|
-
return out
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
* Run the rerank stage. Returns null on any skip/failure — callers fall back
|
|
119
|
-
* to the recall order untouched.
|
|
120
|
-
*/
|
|
121
|
-
export async function rerank(opts
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
) {
|
|
129
|
-
const { invoke, cfg, query, candidates, budget } = opts
|
|
130
|
-
if (!cfg.enabled || !invoke) return null
|
|
131
|
-
if (candidates.length < 2) return null // not worth a model call
|
|
132
|
-
|
|
133
|
-
const batch = candidates.slice(0, cfg.topN)
|
|
134
|
-
const { system, prompt } = buildRerankPrompt(query, batch)
|
|
135
|
-
|
|
136
|
-
const est = estimateTokens(system) + estimateTokens(prompt) + 100
|
|
137
|
-
if (!budget.trySpend(est)) return null // budget exhausted → recall order stands
|
|
138
|
-
|
|
139
|
-
const timeout = AbortSignal.timeout(cfg.timeoutMs)
|
|
140
|
-
const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout
|
|
141
|
-
let text
|
|
142
|
-
try {
|
|
143
|
-
text = await invoke({ system, prompt, signal })
|
|
144
|
-
} catch {
|
|
145
|
-
return null // timeout / transport / adapter error → graceful degradation
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
const validIds = new Set(batch.map((c) => c.id))
|
|
149
|
-
const scores = parseRerankOutput(text, validIds)
|
|
150
|
-
if (scores.size === 0) return null // unusable output → recall order stands
|
|
151
|
-
return { scores, tokensSpent: est + estimateTokens(text) }
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/** Blend judge scores into recall scores. Unscored candidates keep recall. */
|
|
155
|
-
export function blendScores(
|
|
156
|
-
candidates ,
|
|
157
|
-
outcome ,
|
|
158
|
-
weight ,
|
|
159
|
-
) {
|
|
160
|
-
const w = Math.min(1, Math.max(0, weight))
|
|
161
|
-
return candidates.map((c) => {
|
|
162
|
-
const llm = outcome.scores.get(c.id)
|
|
163
|
-
const score = llm === undefined ? c.recallScore : (1 - w) * c.recallScore + w * llm
|
|
164
|
-
return { id: c.id, score }
|
|
165
|
-
})
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
//# sourceURL=
|
|
1
|
+
/**
|
|
2
|
+
* LLM-as-judge rerank stage (M2). Pure logic: the caller injects a one-shot
|
|
3
|
+
* LLM invoke function, so unit tests run against a fake and the cordis wiring
|
|
4
|
+
* (apply.ts) adapts `ctx.llm` into it. Every failure path degrades to "no
|
|
5
|
+
* rerank" — retrieval must never depend on this stage succeeding.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
export const defaultRerankConfig = {
|
|
23
|
+
enabled: true,
|
|
24
|
+
provider: '',
|
|
25
|
+
model: '',
|
|
26
|
+
timeoutMs: 1500,
|
|
27
|
+
dailyBudgetTokens: 200_000,
|
|
28
|
+
weight: 0.6,
|
|
29
|
+
topN: 10,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** One-shot LLM invocation the host provides (adapter sees GenerateOptions). */
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
/** Rough token estimate; CJK-heavy text runs ~1.5-2 chars per token. */
|
|
49
|
+
export function estimateTokens(text ) {
|
|
50
|
+
return Math.ceil(text.length / 2)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Daily budget bucket keyed by local date; resets when the date rolls over. */
|
|
54
|
+
export class DailyBudget {
|
|
55
|
+
day = ''
|
|
56
|
+
used = 0
|
|
57
|
+
limit
|
|
58
|
+
|
|
59
|
+
constructor(limit ) {
|
|
60
|
+
this.limit = limit
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Try to reserve n tokens; returns false when the day's budget is spent. */
|
|
64
|
+
trySpend(n ) {
|
|
65
|
+
this.rollIfNeeded()
|
|
66
|
+
if (this.used + n > this.limit) return false
|
|
67
|
+
this.used += n
|
|
68
|
+
return true
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
get usedToday() {
|
|
72
|
+
this.rollIfNeeded()
|
|
73
|
+
return this.used
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
rollIfNeeded() {
|
|
77
|
+
const today = new Date().toISOString().slice(0, 10)
|
|
78
|
+
if (today !== this.day) {
|
|
79
|
+
this.day = today
|
|
80
|
+
this.used = 0
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function buildRerankPrompt(query , candidates ) {
|
|
86
|
+
const lines = candidates.map((c, i) => `${i + 1}. [${c.id}] ${c.content.slice(0, 500)}`)
|
|
87
|
+
return {
|
|
88
|
+
system: '你是检索重排序器。根据与查询的相关性给每个候选记忆打分。只输出 JSON 数组,格式 [[id, 分数], ...],分数为 0 到 1 的小数。不要输出其他内容。',
|
|
89
|
+
prompt: `查询:${query}\n\n候选记忆:\n${lines.join('\n')}`,
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Parse `[[id, score], ...]` out of the model output; tolerates prose around it. */
|
|
94
|
+
export function parseRerankOutput(text , validIds ) {
|
|
95
|
+
const out = new Map ()
|
|
96
|
+
const start = text.indexOf('[')
|
|
97
|
+
const end = text.lastIndexOf(']')
|
|
98
|
+
if (start < 0 || end <= start) return out
|
|
99
|
+
let parsed
|
|
100
|
+
try {
|
|
101
|
+
parsed = JSON.parse(text.slice(start, end + 1))
|
|
102
|
+
} catch {
|
|
103
|
+
return out
|
|
104
|
+
}
|
|
105
|
+
if (!Array.isArray(parsed)) return out
|
|
106
|
+
for (const pair of parsed) {
|
|
107
|
+
if (!Array.isArray(pair) || pair.length < 2) continue
|
|
108
|
+
const [id, score] = pair
|
|
109
|
+
if (typeof id !== 'string' || !validIds.has(id)) continue
|
|
110
|
+
const n = typeof score === 'number' ? score : Number(score)
|
|
111
|
+
if (!Number.isFinite(n)) continue
|
|
112
|
+
out.set(id, Math.min(1, Math.max(0, n)))
|
|
113
|
+
}
|
|
114
|
+
return out
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Run the rerank stage. Returns null on any skip/failure — callers fall back
|
|
119
|
+
* to the recall order untouched.
|
|
120
|
+
*/
|
|
121
|
+
export async function rerank(opts
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
) {
|
|
129
|
+
const { invoke, cfg, query, candidates, budget } = opts
|
|
130
|
+
if (!cfg.enabled || !invoke) return null
|
|
131
|
+
if (candidates.length < 2) return null // not worth a model call
|
|
132
|
+
|
|
133
|
+
const batch = candidates.slice(0, cfg.topN)
|
|
134
|
+
const { system, prompt } = buildRerankPrompt(query, batch)
|
|
135
|
+
|
|
136
|
+
const est = estimateTokens(system) + estimateTokens(prompt) + 100
|
|
137
|
+
if (!budget.trySpend(est)) return null // budget exhausted → recall order stands
|
|
138
|
+
|
|
139
|
+
const timeout = AbortSignal.timeout(cfg.timeoutMs)
|
|
140
|
+
const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout
|
|
141
|
+
let text
|
|
142
|
+
try {
|
|
143
|
+
text = await invoke({ system, prompt, signal })
|
|
144
|
+
} catch {
|
|
145
|
+
return null // timeout / transport / adapter error → graceful degradation
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const validIds = new Set(batch.map((c) => c.id))
|
|
149
|
+
const scores = parseRerankOutput(text, validIds)
|
|
150
|
+
if (scores.size === 0) return null // unusable output → recall order stands
|
|
151
|
+
return { scores, tokensSpent: est + estimateTokens(text) }
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Blend judge scores into recall scores. Unscored candidates keep recall. */
|
|
155
|
+
export function blendScores(
|
|
156
|
+
candidates ,
|
|
157
|
+
outcome ,
|
|
158
|
+
weight ,
|
|
159
|
+
) {
|
|
160
|
+
const w = Math.min(1, Math.max(0, weight))
|
|
161
|
+
return candidates.map((c) => {
|
|
162
|
+
const llm = outcome.scores.get(c.id)
|
|
163
|
+
const score = llm === undefined ? c.recallScore : (1 - w) * c.recallScore + w * llm
|
|
164
|
+
return { id: c.id, score }
|
|
165
|
+
})
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
//# sourceURL=source/memory-local/src/rerank.ts
|
package/lib/provider/scoring.js
CHANGED
|
@@ -1,145 +1,145 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Lexical scoring — pure functions, no I/O.
|
|
3
|
-
*
|
|
4
|
-
* Tokenization handles mixed CJK/Latin text: Latin words plus CJK unigrams
|
|
5
|
-
* and bigrams (bigrams matter — single Chinese characters are too ambiguous).
|
|
6
|
-
*/
|
|
1
|
+
/**
|
|
2
|
+
* Lexical scoring — pure functions, no I/O.
|
|
3
|
+
*
|
|
4
|
+
* Tokenization handles mixed CJK/Latin text: Latin words plus CJK unigrams
|
|
5
|
+
* and bigrams (bigrams matter — single Chinese characters are too ambiguous).
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
export function tokenize(text ) {
|
|
16
|
+
const lower = text.toLowerCase()
|
|
17
|
+
const tokens = []
|
|
18
|
+
for (const m of lower.matchAll(/[a-z0-9][a-z0-9_+\-.]{1,}/g)) tokens.push(m[0])
|
|
19
|
+
for (const run of lower.matchAll(/[\u3400-\u4dbf\u4e00-\u9fff]+/g)) {
|
|
20
|
+
const chars = [...run[0]]
|
|
21
|
+
for (const c of chars) tokens.push(c)
|
|
22
|
+
for (let i = 0; i + 1 < chars.length; i++) tokens.push(chars[i] + chars[i + 1] )
|
|
23
|
+
}
|
|
24
|
+
return tokens
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function buildStats(tokenSets ) {
|
|
28
|
+
const df = new Map ()
|
|
29
|
+
let n = 0
|
|
30
|
+
for (const set of tokenSets) {
|
|
31
|
+
n++
|
|
32
|
+
for (const t of set) df.set(t, (df.get(t) ?? 0) + 1)
|
|
33
|
+
}
|
|
34
|
+
return { df, n }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** IDF-weighted overlap of query tokens present in the record, normalized to 0..1. */
|
|
38
|
+
export function lexicalScore(queryTokens , recordTokens , stats ) {
|
|
39
|
+
const unique = [...new Set(queryTokens)]
|
|
40
|
+
if (unique.length === 0 || stats.n === 0) return 0
|
|
41
|
+
let hit = 0
|
|
42
|
+
let total = 0
|
|
43
|
+
for (const t of unique) {
|
|
44
|
+
const idf = Math.log(1 + stats.n / (1 + (stats.df.get(t) ?? 0)))
|
|
45
|
+
total += idf
|
|
46
|
+
if (recordTokens.has(t)) hit += idf
|
|
47
|
+
}
|
|
48
|
+
return total === 0 ? 0 : hit / total
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* True when the query shares at least one STRONG token with the record —
|
|
53
|
+
* a Latin word or a CJK bigram. Single CJK characters are too ambiguous to
|
|
54
|
+
* stand alone: a query mentioning 完 shouldn't match every record containing
|
|
55
|
+
* 完. Matches carried only by weak tokens must clear a higher score floor
|
|
56
|
+
* (0.25: the measured floor of real hits) instead of the permissive one.
|
|
57
|
+
*/
|
|
58
|
+
export function hasStrongHit(queryTokens , recordTokens ) {
|
|
59
|
+
for (const t of queryTokens) {
|
|
60
|
+
if (t.length >= 2 && recordTokens.has(t)) return true
|
|
61
|
+
}
|
|
62
|
+
return false
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Exponential recency decay with configurable half-life. */
|
|
66
|
+
export function recencyFactor(lastAccessedAt , now , halfLifeDays ) {
|
|
67
|
+
const ageDays = Math.max(0, now - lastAccessedAt) / 86_400_000
|
|
68
|
+
return Math.pow(0.5, ageDays / Math.max(halfLifeDays, 0.001))
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
export function recordScore(
|
|
79
|
+
lexical ,
|
|
80
|
+
importance ,
|
|
81
|
+
accessCount ,
|
|
82
|
+
lastAccessedAt ,
|
|
83
|
+
opts ,
|
|
84
|
+
) {
|
|
85
|
+
const recency = recencyFactor(lastAccessedAt, opts.now, opts.halfLifeDays)
|
|
86
|
+
const importanceFactor = 0.5 + 0.5 * Math.min(1, Math.max(0, importance))
|
|
87
|
+
const accessBoost = 1 + 0.1 * Math.log(1 + accessCount)
|
|
88
|
+
const penalty = opts.superseded ? 0.3 : 1
|
|
89
|
+
return lexical * recency * importanceFactor * accessBoost * penalty
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Cosine similarity for unit vectors == dot product. Normalizes defensively. */
|
|
93
|
+
export function cosine(a , b ) {
|
|
94
|
+
const n = Math.min(a.length, b.length)
|
|
95
|
+
let dot = 0
|
|
96
|
+
let na = 0
|
|
97
|
+
let nb = 0
|
|
98
|
+
for (let i = 0; i < n; i++) {
|
|
99
|
+
dot += a[i] * b[i]
|
|
100
|
+
na += a[i] * a[i]
|
|
101
|
+
nb += b[i] * b[i]
|
|
102
|
+
}
|
|
103
|
+
if (na === 0 || nb === 0) return 0
|
|
104
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb))
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Normalize a vector in place; returns it. Zero vectors are left as-is. */
|
|
108
|
+
export function normalizeInPlace(v ) {
|
|
109
|
+
let sum = 0
|
|
110
|
+
for (let i = 0; i < v.length; i++) sum += v[i] * v[i]
|
|
111
|
+
const norm = Math.sqrt(sum)
|
|
112
|
+
if (norm > 0) for (let i = 0; i < v.length; i++) v[i] = v[i] / norm
|
|
113
|
+
return v
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** LRU cache for rerank/embedding dedupe. */
|
|
117
|
+
export class LruSet {
|
|
118
|
+
map = new Map ()
|
|
119
|
+
capacity
|
|
120
|
+
|
|
121
|
+
constructor(capacity ) {
|
|
122
|
+
this.capacity = capacity
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
get(key ) {
|
|
126
|
+
const v = this.map.get(key)
|
|
127
|
+
if (v !== undefined) {
|
|
128
|
+
this.map.delete(key)
|
|
129
|
+
this.map.set(key, v)
|
|
130
|
+
}
|
|
131
|
+
return v
|
|
132
|
+
}
|
|
133
|
+
set(key , value ) {
|
|
134
|
+
if (this.map.has(key)) this.map.delete(key)
|
|
135
|
+
this.map.set(key, value)
|
|
136
|
+
while (this.map.size > this.capacity) {
|
|
137
|
+
const oldest = this.map.keys().next().value
|
|
138
|
+
if (oldest === undefined) break
|
|
139
|
+
this.map.delete(oldest)
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
7
143
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
144
|
|
|
15
|
-
|
|
16
|
-
const lower = text.toLowerCase()
|
|
17
|
-
const tokens = []
|
|
18
|
-
for (const m of lower.matchAll(/[a-z0-9][a-z0-9_+\-.]{1,}/g)) tokens.push(m[0])
|
|
19
|
-
for (const run of lower.matchAll(/[\u3400-\u4dbf\u4e00-\u9fff]+/g)) {
|
|
20
|
-
const chars = [...run[0]]
|
|
21
|
-
for (const c of chars) tokens.push(c)
|
|
22
|
-
for (let i = 0; i + 1 < chars.length; i++) tokens.push(chars[i] + chars[i + 1] )
|
|
23
|
-
}
|
|
24
|
-
return tokens
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export function buildStats(tokenSets ) {
|
|
28
|
-
const df = new Map ()
|
|
29
|
-
let n = 0
|
|
30
|
-
for (const set of tokenSets) {
|
|
31
|
-
n++
|
|
32
|
-
for (const t of set) df.set(t, (df.get(t) ?? 0) + 1)
|
|
33
|
-
}
|
|
34
|
-
return { df, n }
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/** IDF-weighted overlap of query tokens present in the record, normalized to 0..1. */
|
|
38
|
-
export function lexicalScore(queryTokens , recordTokens , stats ) {
|
|
39
|
-
const unique = [...new Set(queryTokens)]
|
|
40
|
-
if (unique.length === 0 || stats.n === 0) return 0
|
|
41
|
-
let hit = 0
|
|
42
|
-
let total = 0
|
|
43
|
-
for (const t of unique) {
|
|
44
|
-
const idf = Math.log(1 + stats.n / (1 + (stats.df.get(t) ?? 0)))
|
|
45
|
-
total += idf
|
|
46
|
-
if (recordTokens.has(t)) hit += idf
|
|
47
|
-
}
|
|
48
|
-
return total === 0 ? 0 : hit / total
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
/**
|
|
52
|
-
* True when the query shares at least one STRONG token with the record —
|
|
53
|
-
* a Latin word or a CJK bigram. Single CJK characters are too ambiguous to
|
|
54
|
-
* stand alone: a query mentioning 完 shouldn't match every record containing
|
|
55
|
-
* 完. Matches carried only by weak tokens must clear a higher score floor
|
|
56
|
-
* (0.25: the measured floor of real hits) instead of the permissive one.
|
|
57
|
-
*/
|
|
58
|
-
export function hasStrongHit(queryTokens , recordTokens ) {
|
|
59
|
-
for (const t of queryTokens) {
|
|
60
|
-
if (t.length >= 2 && recordTokens.has(t)) return true
|
|
61
|
-
}
|
|
62
|
-
return false
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
/** Exponential recency decay with configurable half-life. */
|
|
66
|
-
export function recencyFactor(lastAccessedAt , now , halfLifeDays ) {
|
|
67
|
-
const ageDays = Math.max(0, now - lastAccessedAt) / 86_400_000
|
|
68
|
-
return Math.pow(0.5, ageDays / Math.max(halfLifeDays, 0.001))
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
export function recordScore(
|
|
79
|
-
lexical ,
|
|
80
|
-
importance ,
|
|
81
|
-
accessCount ,
|
|
82
|
-
lastAccessedAt ,
|
|
83
|
-
opts ,
|
|
84
|
-
) {
|
|
85
|
-
const recency = recencyFactor(lastAccessedAt, opts.now, opts.halfLifeDays)
|
|
86
|
-
const importanceFactor = 0.5 + 0.5 * Math.min(1, Math.max(0, importance))
|
|
87
|
-
const accessBoost = 1 + 0.1 * Math.log(1 + accessCount)
|
|
88
|
-
const penalty = opts.superseded ? 0.3 : 1
|
|
89
|
-
return lexical * recency * importanceFactor * accessBoost * penalty
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
/** Cosine similarity for unit vectors == dot product. Normalizes defensively. */
|
|
93
|
-
export function cosine(a , b ) {
|
|
94
|
-
const n = Math.min(a.length, b.length)
|
|
95
|
-
let dot = 0
|
|
96
|
-
let na = 0
|
|
97
|
-
let nb = 0
|
|
98
|
-
for (let i = 0; i < n; i++) {
|
|
99
|
-
dot += a[i] * b[i]
|
|
100
|
-
na += a[i] * a[i]
|
|
101
|
-
nb += b[i] * b[i]
|
|
102
|
-
}
|
|
103
|
-
if (na === 0 || nb === 0) return 0
|
|
104
|
-
return dot / (Math.sqrt(na) * Math.sqrt(nb))
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/** Normalize a vector in place; returns it. Zero vectors are left as-is. */
|
|
108
|
-
export function normalizeInPlace(v ) {
|
|
109
|
-
let sum = 0
|
|
110
|
-
for (let i = 0; i < v.length; i++) sum += v[i] * v[i]
|
|
111
|
-
const norm = Math.sqrt(sum)
|
|
112
|
-
if (norm > 0) for (let i = 0; i < v.length; i++) v[i] = v[i] / norm
|
|
113
|
-
return v
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
/** LRU cache for rerank/embedding dedupe. */
|
|
117
|
-
export class LruSet {
|
|
118
|
-
map = new Map ()
|
|
119
|
-
capacity
|
|
120
|
-
|
|
121
|
-
constructor(capacity ) {
|
|
122
|
-
this.capacity = capacity
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
get(key ) {
|
|
126
|
-
const v = this.map.get(key)
|
|
127
|
-
if (v !== undefined) {
|
|
128
|
-
this.map.delete(key)
|
|
129
|
-
this.map.set(key, v)
|
|
130
|
-
}
|
|
131
|
-
return v
|
|
132
|
-
}
|
|
133
|
-
set(key , value ) {
|
|
134
|
-
if (this.map.has(key)) this.map.delete(key)
|
|
135
|
-
this.map.set(key, value)
|
|
136
|
-
while (this.map.size > this.capacity) {
|
|
137
|
-
const oldest = this.map.keys().next().value
|
|
138
|
-
if (oldest === undefined) break
|
|
139
|
-
this.map.delete(oldest)
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
//# sourceURL=D:\.deepseek\memory-plugin\packages\memory-local\src\scoring.ts
|
|
145
|
+
//# sourceURL=source/memory-local/src/scoring.ts
|