dsh-plugin-discover 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/LICENSE +26 -0
- package/README.md +210 -3
- package/README.zh.md +209 -0
- package/client.js +477 -0
- package/cordis.patch.yml +3 -0
- package/icon.svg +14 -0
- package/index.js +347 -0
- package/lib/bm25.js +190 -0
- package/lib/build-worker.js +32 -0
- package/lib/build.js +145 -0
- package/lib/cli.js +523 -0
- package/lib/compat.js +157 -0
- package/lib/filters.js +105 -0
- package/lib/model-client.js +163 -0
- package/lib/normalize.js +369 -0
- package/lib/paths.js +49 -0
- package/lib/quality.js +61 -0
- package/lib/readme-clean.js +128 -0
- package/lib/readme-source.js +380 -0
- package/lib/report.js +244 -0
- package/lib/runtime.js +247 -0
- package/lib/search.js +447 -0
- package/lib/store.js +184 -0
- package/lib/tokenize.js +141 -0
- package/lib/vector-store.js +142 -0
- package/lib/vector.js +592 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +67 -6
package/lib/vector.js
ADDED
|
@@ -0,0 +1,592 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Optional vector rerank over the BM25 top-50.
|
|
3
|
+
*
|
|
4
|
+
* The model lives in a child process (the optional model package); this module
|
|
5
|
+
* owns the fp16 caches, README excerpts, cosine similarity, and the RRF merge
|
|
6
|
+
* with the lexical order.
|
|
7
|
+
*
|
|
8
|
+
* The ranking fuses up to three ranked lists with weighted RRF; weights are
|
|
9
|
+
* mode-specific (RRF_WEIGHTS): fields mode BM25 1 : fields 0.35, readme mode
|
|
10
|
+
* BM25 1 : fields 0.3 : README 0.4. `fields` (balance) has only the first two
|
|
11
|
+
* lists; `readme` (high) computes both vector lists. A list a record is
|
|
12
|
+
* missing contributes nothing, so a record without a README vector still
|
|
13
|
+
* ranks through BM25 and its field vector.
|
|
14
|
+
*
|
|
15
|
+
* Modes:
|
|
16
|
+
* off — no vectors, no model
|
|
17
|
+
* fields — name/title/topics/capability/description vectors
|
|
18
|
+
* readme — field vectors plus README excerpt vectors
|
|
19
|
+
*/
|
|
20
|
+
import { ModelClient, ModelUnavailableError, resolveModelServer } from './model-client.js'
|
|
21
|
+
import { VectorStore, decodeVector, hashText } from './vector-store.js'
|
|
22
|
+
import { cleanReadmeText, README_CLEAN_VERSION } from './readme-clean.js'
|
|
23
|
+
import {
|
|
24
|
+
README_CONCURRENCY,
|
|
25
|
+
README_RATE_PER_SECOND,
|
|
26
|
+
README_RATE_BURST,
|
|
27
|
+
README_REQUEST_TIMEOUT_MS,
|
|
28
|
+
README_TOTAL_BUDGET_MS,
|
|
29
|
+
NEGATIVE_CACHE_TTL_MS,
|
|
30
|
+
RateLimiter,
|
|
31
|
+
githubGuard,
|
|
32
|
+
githubNotice,
|
|
33
|
+
readmeTargets,
|
|
34
|
+
fetchGithubReadme,
|
|
35
|
+
fetchNpmReadme,
|
|
36
|
+
resolveNpmVersion,
|
|
37
|
+
} from './readme-source.js'
|
|
38
|
+
|
|
39
|
+
const RRF_K = 60
|
|
40
|
+
const CACHE_TTL_MS = 90 * 86_400_000
|
|
41
|
+
const README_TOKEN_LIMIT = 384
|
|
42
|
+
const BATCH_SIZE = 64
|
|
43
|
+
|
|
44
|
+
/** Identifies the ranking algorithm in code, reports, and diagnostics. */
|
|
45
|
+
export const ALGORITHM_VERSION = 'v1.1'
|
|
46
|
+
|
|
47
|
+
/** gate2 default: max query-to-candidate cosine must exceed this to count as semantic evidence. */
|
|
48
|
+
export const GATE2_MAX_SIMILARITY = 0.85
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Weighted-RRF weights by mode (retuned 2026-10-25 on the real-usage set):
|
|
52
|
+
* the field list is weak on near-duplicate clusters, the README list carries
|
|
53
|
+
* high mode. A zero weight disables that list's contribution.
|
|
54
|
+
*/
|
|
55
|
+
export const RRF_WEIGHTS = Object.freeze({
|
|
56
|
+
fields: Object.freeze({ bm25: 1, fields: 0.35, readme: 0 }),
|
|
57
|
+
readme: Object.freeze({ bm25: 1, fields: 0.3, readme: 0.4 }),
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
let sharedClient = null
|
|
61
|
+
|
|
62
|
+
function getSharedClient(serverPath) {
|
|
63
|
+
if (sharedClient && sharedClient.serverPath === serverPath && !sharedClient.closed) return sharedClient
|
|
64
|
+
sharedClient = new ModelClient(serverPath)
|
|
65
|
+
return sharedClient
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Stop the idle model child (plugin unload or process shutdown). */
|
|
69
|
+
export function disposeVectorRuntime() {
|
|
70
|
+
try { sharedClient?.dispose() } catch {}
|
|
71
|
+
sharedClient = null
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function vectorRuntimeAvailable(indexDir) {
|
|
75
|
+
return typeof indexDir === 'string' && resolveModelServer(indexDir) !== null
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function fieldsText(record) {
|
|
79
|
+
return [
|
|
80
|
+
record.name,
|
|
81
|
+
record.title,
|
|
82
|
+
(record.topics ?? []).join(' '),
|
|
83
|
+
(record.capability ?? []).join(' '),
|
|
84
|
+
record.description,
|
|
85
|
+
].filter(value => typeof value === 'string' && value.trim() !== '').join('\n')
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function fieldKey(record) {
|
|
89
|
+
const contentHash = typeof record.contentHash === 'string' && record.contentHash !== ''
|
|
90
|
+
? record.contentHash
|
|
91
|
+
: hashText(fieldsText(record))
|
|
92
|
+
return `${record.id}:${contentHash}`
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function fresh(entry, now, ttl = CACHE_TTL_MS) {
|
|
96
|
+
return Boolean(entry && typeof entry.at === 'number' && now - entry.at < ttl)
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function dot(a, b) {
|
|
100
|
+
const length = Math.min(a.length, b.length)
|
|
101
|
+
let sum = 0
|
|
102
|
+
for (let index = 0; index < length; index += 1) sum += a[index] * b[index]
|
|
103
|
+
return sum
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Weighted reciprocal-rank fusion. Ranks are 1-based; a missing list
|
|
108
|
+
* contributes nothing. This keeps a record without a README vector eligible
|
|
109
|
+
* through its BM25 and field-vector ranks instead of dropping it.
|
|
110
|
+
*/
|
|
111
|
+
export function weightedRrf(ranks, weights) {
|
|
112
|
+
let score = 0
|
|
113
|
+
for (const list of ['bm25', 'fields', 'readme']) {
|
|
114
|
+
const rank = ranks?.[list]
|
|
115
|
+
const weight = weights?.[list] ?? 0
|
|
116
|
+
if (!Number.isFinite(rank) || rank < 1 || weight <= 0) continue
|
|
117
|
+
score += weight / (RRF_K + rank)
|
|
118
|
+
}
|
|
119
|
+
return score
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function topTwoAverage(similarities) {
|
|
123
|
+
if (similarities.length === 0) return 0
|
|
124
|
+
const sorted = [...similarities].sort((a, b) => b - a)
|
|
125
|
+
return sorted.length >= 2 ? (sorted[0] + sorted[1]) / 2 : sorted[0]
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async function mapLimit(items, limit, worker) {
|
|
129
|
+
const results = new Array(items.length)
|
|
130
|
+
let next = 0
|
|
131
|
+
async function run(workerIndex) {
|
|
132
|
+
while (next < items.length) {
|
|
133
|
+
const index = next
|
|
134
|
+
next += 1
|
|
135
|
+
results[index] = await worker(items[index], index, workerIndex)
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
const workers = Array.from({ length: Math.min(limit, items.length) }, (_, index) => run(index))
|
|
139
|
+
await Promise.all(workers)
|
|
140
|
+
return results
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function excerptSource(entry, key) {
|
|
144
|
+
if (typeof entry?.source === 'string' && entry.source !== '') return entry.source
|
|
145
|
+
return key.startsWith('npm:') ? 'npm-cdn' : 'github-raw'
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function finishExcerpt(fetched, key, client, store, now) {
|
|
149
|
+
try {
|
|
150
|
+
const cleaned = cleanReadmeText(fetched.text)
|
|
151
|
+
if (cleaned.trim() === '') return null
|
|
152
|
+
const truncated = await client.truncate(cleaned, README_TOKEN_LIMIT)
|
|
153
|
+
if (typeof truncated.text !== 'string' || truncated.text.trim() === '') return null
|
|
154
|
+
const excerpt = {
|
|
155
|
+
text: truncated.text,
|
|
156
|
+
hash: hashText(truncated.text),
|
|
157
|
+
etag: fetched.etag ?? null,
|
|
158
|
+
fetchedAt: now,
|
|
159
|
+
source: fetched.source,
|
|
160
|
+
cleanVersion: README_CLEAN_VERSION,
|
|
161
|
+
...(fetched.provider ? { provider: fetched.provider } : {}),
|
|
162
|
+
}
|
|
163
|
+
store.setExcerpt(key, excerpt)
|
|
164
|
+
return { ...excerpt, at: Date.now() }
|
|
165
|
+
} catch {
|
|
166
|
+
return null
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async function readmeExcerpt(record, client, store, now, {
|
|
171
|
+
fetchImpl,
|
|
172
|
+
limiter,
|
|
173
|
+
githubDeadline,
|
|
174
|
+
requestTimeoutMs = README_REQUEST_TIMEOUT_MS,
|
|
175
|
+
}) {
|
|
176
|
+
const { github, npm } = readmeTargets(record)
|
|
177
|
+
const githubKey = github?.key ?? ''
|
|
178
|
+
const githubCached = githubKey ? store.getExcerpt(githubKey) : null
|
|
179
|
+
const githubCurrent = Boolean(
|
|
180
|
+
githubCached && !githubCached.noReadme
|
|
181
|
+
&& githubCached.cleanVersion === README_CLEAN_VERSION
|
|
182
|
+
&& fresh(githubCached, now),
|
|
183
|
+
)
|
|
184
|
+
if (githubCurrent) {
|
|
185
|
+
return { ...githubCached, source: excerptSource(githubCached, githubKey) }
|
|
186
|
+
}
|
|
187
|
+
const githubNegative = Boolean(
|
|
188
|
+
githubCached?.noReadme && fresh(githubCached, now, NEGATIVE_CACHE_TTL_MS),
|
|
189
|
+
)
|
|
190
|
+
let stale = githubCached && !githubCached.noReadme
|
|
191
|
+
&& githubCached.cleanVersion === README_CLEAN_VERSION
|
|
192
|
+
? githubCached
|
|
193
|
+
: null
|
|
194
|
+
|
|
195
|
+
let npmVersion = npm?.version ?? null
|
|
196
|
+
let npmKey = npmVersion ? `npm:${npm.name}@${npmVersion}` : ''
|
|
197
|
+
let npmCached = npmKey ? store.getExcerpt(npmKey) : null
|
|
198
|
+
const npmNegative = Boolean(
|
|
199
|
+
npmCached?.noReadme && fresh(npmCached, now, NEGATIVE_CACHE_TTL_MS),
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
if (github && !githubNegative && Date.now() < githubDeadline && githubGuard.canUse()) {
|
|
203
|
+
const revalidate = githubCached && !githubCached.noReadme
|
|
204
|
+
&& githubCached.cleanVersion === README_CLEAN_VERSION
|
|
205
|
+
? githubCached
|
|
206
|
+
: null
|
|
207
|
+
const result = await fetchGithubReadme(github, {
|
|
208
|
+
fetchImpl,
|
|
209
|
+
limiter,
|
|
210
|
+
timeoutMs: requestTimeoutMs,
|
|
211
|
+
etag: revalidate?.etag ?? null,
|
|
212
|
+
})
|
|
213
|
+
if (result.notModified && revalidate) {
|
|
214
|
+
githubGuard.recordSuccess()
|
|
215
|
+
store.setExcerpt(githubKey, { ...revalidate, fetchedAt: now, source: 'github-raw' })
|
|
216
|
+
return { ...revalidate, at: Date.now(), fetchedAt: now, source: 'github-raw' }
|
|
217
|
+
}
|
|
218
|
+
if (result.status === 200 && typeof result.text === 'string') {
|
|
219
|
+
githubGuard.recordSuccess()
|
|
220
|
+
const excerpt = await finishExcerpt(result, githubKey, client, store, now)
|
|
221
|
+
if (excerpt) return excerpt
|
|
222
|
+
} else if (result.status === 404 || result.empty) {
|
|
223
|
+
githubGuard.recordSuccess()
|
|
224
|
+
store.setExcerpt(githubKey, { noReadme: true, source: 'github-raw' })
|
|
225
|
+
} else if (result.rateLimited) {
|
|
226
|
+
githubGuard.recordSuccess()
|
|
227
|
+
githubGuard.recordRateLimit(Date.now(), result.retryAfterMs ?? 0)
|
|
228
|
+
} else if (result.network || result.transport) {
|
|
229
|
+
githubGuard.recordTransportFailure(result.timeout ? 'timeout' : 'transport')
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
if (npm && !npmNegative) {
|
|
234
|
+
if (!npmVersion) {
|
|
235
|
+
npmVersion = await resolveNpmVersion(npm.name, {
|
|
236
|
+
fetchImpl,
|
|
237
|
+
limiter,
|
|
238
|
+
timeoutMs: requestTimeoutMs,
|
|
239
|
+
now: () => Date.now(),
|
|
240
|
+
})
|
|
241
|
+
if (npmVersion) {
|
|
242
|
+
npmKey = `npm:${npm.name}@${npmVersion}`
|
|
243
|
+
npmCached = store.getExcerpt(npmKey)
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
if (npmVersion) {
|
|
247
|
+
if (npmCached && !npmCached.noReadme && npmCached.cleanVersion === README_CLEAN_VERSION) {
|
|
248
|
+
if (fresh(npmCached, now)) {
|
|
249
|
+
return { ...npmCached, source: excerptSource(npmCached, npmKey) }
|
|
250
|
+
}
|
|
251
|
+
stale ??= npmCached
|
|
252
|
+
}
|
|
253
|
+
if (npmCached?.noReadme && fresh(npmCached, now, NEGATIVE_CACHE_TTL_MS)) return stale
|
|
254
|
+
const result = await fetchNpmReadme(npm.name, npmVersion, {
|
|
255
|
+
fetchImpl,
|
|
256
|
+
limiter,
|
|
257
|
+
timeoutMs: requestTimeoutMs,
|
|
258
|
+
etag: npmCached && npmCached.cleanVersion === README_CLEAN_VERSION ? npmCached.etag : null,
|
|
259
|
+
})
|
|
260
|
+
if (result.notModified && npmCached && npmCached.cleanVersion === README_CLEAN_VERSION) {
|
|
261
|
+
store.setExcerpt(npmKey, { ...npmCached, fetchedAt: now, source: 'npm-cdn' })
|
|
262
|
+
return { ...npmCached, at: Date.now(), fetchedAt: now, source: 'npm-cdn' }
|
|
263
|
+
}
|
|
264
|
+
if (result.status === 200 && typeof result.text === 'string') {
|
|
265
|
+
const excerpt = await finishExcerpt(result, npmKey, client, store, now)
|
|
266
|
+
if (excerpt) return excerpt
|
|
267
|
+
} else if (result.status === 404 || result.empty) {
|
|
268
|
+
store.setExcerpt(npmKey, { noReadme: true, source: 'npm-cdn' })
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return stale
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
async function fillFields({ records, candidates, client, store, vectors, now }) {
|
|
276
|
+
const missing = []
|
|
277
|
+
for (const entry of candidates) {
|
|
278
|
+
const record = records[entry.doc]
|
|
279
|
+
const key = fieldKey(record)
|
|
280
|
+
const cached = store.getField(key)
|
|
281
|
+
if (fresh(cached, now)) {
|
|
282
|
+
vectors.set(entry.doc, decodeVector(cached.v))
|
|
283
|
+
continue
|
|
284
|
+
}
|
|
285
|
+
missing.push({ entry, record, key, text: fieldsText(record) })
|
|
286
|
+
}
|
|
287
|
+
for (let offset = 0; offset < missing.length; offset += BATCH_SIZE) {
|
|
288
|
+
const batch = missing.slice(offset, offset + BATCH_SIZE)
|
|
289
|
+
const started = Date.now()
|
|
290
|
+
const result = await client.embed(batch.map(item => item.text), 'passage')
|
|
291
|
+
if (process.env.DSH_VECTOR_DEBUG) {
|
|
292
|
+
const chars = batch.reduce((sum, item) => sum + item.text.length, 0)
|
|
293
|
+
console.error(`[vector] fields batch ${batch.length} texts, ${chars} chars, ${Date.now() - started} ms`)
|
|
294
|
+
}
|
|
295
|
+
for (let index = 0; index < batch.length; index += 1) {
|
|
296
|
+
const vector = result[index]
|
|
297
|
+
if (!vector) continue
|
|
298
|
+
vectors.set(batch[index].entry.doc, vector)
|
|
299
|
+
store.setField(batch[index].key, vector, batch[index].record.contentHash ?? null)
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
async function fillReadme({
|
|
305
|
+
records,
|
|
306
|
+
candidates,
|
|
307
|
+
client,
|
|
308
|
+
store,
|
|
309
|
+
vectors,
|
|
310
|
+
now,
|
|
311
|
+
fetchImpl,
|
|
312
|
+
readmeOptions = null,
|
|
313
|
+
readmeTotalBudgetMs = README_TOTAL_BUDGET_MS,
|
|
314
|
+
}) {
|
|
315
|
+
githubGuard.beginRound(Date.now())
|
|
316
|
+
const concurrency = Math.max(1, Math.round(
|
|
317
|
+
Number(readmeOptions?.concurrency) || README_CONCURRENCY,
|
|
318
|
+
))
|
|
319
|
+
const requestTimeoutMs = Math.max(1000,
|
|
320
|
+
Number(readmeOptions?.requestTimeoutMs) || README_REQUEST_TIMEOUT_MS)
|
|
321
|
+
const totalBudgetMs = readmeOptions?.totalBudgetMs !== undefined && readmeOptions?.totalBudgetMs !== null
|
|
322
|
+
? Math.max(0, Number(readmeOptions.totalBudgetMs))
|
|
323
|
+
: readmeTotalBudgetMs
|
|
324
|
+
const githubDeadline = Date.now() + totalBudgetMs
|
|
325
|
+
const limiters = Array.from({ length: concurrency }, () => new RateLimiter({
|
|
326
|
+
rate: README_RATE_PER_SECOND,
|
|
327
|
+
burst: README_RATE_BURST,
|
|
328
|
+
}))
|
|
329
|
+
const tasks = candidates.map(entry => async workerIndex => {
|
|
330
|
+
const record = records[entry.doc]
|
|
331
|
+
const excerpt = await readmeExcerpt(record, client, store, now, {
|
|
332
|
+
fetchImpl,
|
|
333
|
+
limiter: limiters[workerIndex],
|
|
334
|
+
githubDeadline,
|
|
335
|
+
requestTimeoutMs,
|
|
336
|
+
})
|
|
337
|
+
if (!excerpt) return null
|
|
338
|
+
return { entry, record, excerpt, key: `${record.id}:${excerpt.hash}` }
|
|
339
|
+
})
|
|
340
|
+
const resolved = (await mapLimit(
|
|
341
|
+
tasks,
|
|
342
|
+
concurrency,
|
|
343
|
+
(task, _index, workerIndex) => task(workerIndex),
|
|
344
|
+
)).filter(Boolean)
|
|
345
|
+
const readmeSources = { github: 0, npm: 0, fields: 0 }
|
|
346
|
+
for (const item of resolved) {
|
|
347
|
+
if (item.excerpt.source === 'npm-cdn') readmeSources.npm += 1
|
|
348
|
+
else readmeSources.github += 1
|
|
349
|
+
}
|
|
350
|
+
const missing = []
|
|
351
|
+
let readmeCovered = 0
|
|
352
|
+
for (const item of resolved) {
|
|
353
|
+
const cached = store.getReadme(item.key)
|
|
354
|
+
if (fresh(cached, now)) {
|
|
355
|
+
vectors.set(item.entry.doc, decodeVector(cached.v))
|
|
356
|
+
readmeCovered += 1
|
|
357
|
+
} else {
|
|
358
|
+
missing.push(item)
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
for (let offset = 0; offset < missing.length; offset += BATCH_SIZE) {
|
|
362
|
+
const batch = missing.slice(offset, offset + BATCH_SIZE)
|
|
363
|
+
const result = await client.embed(batch.map(item => item.excerpt.text), 'passage')
|
|
364
|
+
for (let index = 0; index < batch.length; index += 1) {
|
|
365
|
+
const vector = result[index]
|
|
366
|
+
if (!vector) continue
|
|
367
|
+
vectors.set(batch[index].entry.doc, vector)
|
|
368
|
+
store.setReadme(batch[index].key, vector, batch[index].excerpt.hash)
|
|
369
|
+
readmeCovered += 1
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// No field fallback here: the field list stays separate, so a record
|
|
374
|
+
// without a README vector still contributes through its field vector. The
|
|
375
|
+
// counter is coverage diagnostics only.
|
|
376
|
+
const readmeFallback = Math.max(0, candidates.length - readmeCovered)
|
|
377
|
+
readmeSources.fields = readmeFallback
|
|
378
|
+
const circuit = githubGuard.status()
|
|
379
|
+
return {
|
|
380
|
+
readmeCovered,
|
|
381
|
+
readmeFallback,
|
|
382
|
+
readmeSources,
|
|
383
|
+
...(circuit ? { circuit, notice: githubNotice(circuit.reason) } : {}),
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function degraded(mode, reason, candidates, error) {
|
|
388
|
+
const notice = degradationNotice(mode, reason, error)
|
|
389
|
+
return {
|
|
390
|
+
mode,
|
|
391
|
+
algorithm: ALGORITHM_VERSION,
|
|
392
|
+
covered: 0,
|
|
393
|
+
degraded: reason,
|
|
394
|
+
...(error ? { error } : {}),
|
|
395
|
+
...(notice ? { notice } : {}),
|
|
396
|
+
candidates,
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/** English notice so the session LLM can warn the user about the fallback. */
|
|
401
|
+
function degradationNotice(mode, reason, error) {
|
|
402
|
+
if (reason === 'runtime-missing') {
|
|
403
|
+
return `Vector rerank (mode "${mode}") was skipped: the local embedding model could not be started${error ? ` (${error})` : ''}. Results are lexical (BM25) only. Warn the user that vector enhancement degraded; suggest checking that the optional model package (dsh-plugin-discover-model) is installed and runnable, or setting vector.mode to "off".`
|
|
404
|
+
}
|
|
405
|
+
if (reason === 'embedding-failed') {
|
|
406
|
+
return `Vector rerank (mode "${mode}") failed while embedding: ${error ?? 'unknown error'}. Results are lexical (BM25) only. Warn the user that vector enhancement degraded for this query.`
|
|
407
|
+
}
|
|
408
|
+
return null
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
const MODEL_ATTEMPTS = 3
|
|
412
|
+
const MODEL_RETRY_DELAYS_MS = [0, 800, 2000]
|
|
413
|
+
|
|
414
|
+
function sleep(ms) {
|
|
415
|
+
return new Promise(resolve => setTimeout(resolve, ms))
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Rerank the grouped BM25 top-50 with weighted RRF over the BM25, field-vector,
|
|
420
|
+
* and — in readme mode — README-vector lists (weights: RRF_WEIGHTS by mode).
|
|
421
|
+
* @param {object[]} candidates grouped candidates in lexical order
|
|
422
|
+
* @param {object[]} routes prepared query routes (original + rewrites)
|
|
423
|
+
* @param {object} options
|
|
424
|
+
*/
|
|
425
|
+
export async function vectorRerank(candidates, routes, {
|
|
426
|
+
mode = 'off',
|
|
427
|
+
gate2Threshold = GATE2_MAX_SIMILARITY,
|
|
428
|
+
indexDir = null,
|
|
429
|
+
records = [],
|
|
430
|
+
now = Date.now(),
|
|
431
|
+
clientFactory = null,
|
|
432
|
+
fetchImpl = null,
|
|
433
|
+
serverPath = null,
|
|
434
|
+
disposeClient = true,
|
|
435
|
+
readmeOptions = null,
|
|
436
|
+
readmeTotalBudgetMs = README_TOTAL_BUDGET_MS,
|
|
437
|
+
} = {}) {
|
|
438
|
+
if (mode !== 'fields' && mode !== 'readme') return null
|
|
439
|
+
const selectedGate2Threshold = Number.isFinite(Number(gate2Threshold))
|
|
440
|
+
? Number(gate2Threshold)
|
|
441
|
+
: GATE2_MAX_SIMILARITY
|
|
442
|
+
if (typeof indexDir !== 'string' || indexDir === '') return degraded(mode, 'index-dir-missing', candidates, null)
|
|
443
|
+
const resolvedServer = serverPath ?? resolveModelServer(indexDir)
|
|
444
|
+
if (!resolvedServer) return degraded(mode, 'runtime-missing', candidates, 'model package is not installed')
|
|
445
|
+
const store = new VectorStore(indexDir)
|
|
446
|
+
const fetchReader = fetchImpl ?? globalThis.fetch
|
|
447
|
+
|
|
448
|
+
async function attempt(client) {
|
|
449
|
+
const texts = routes
|
|
450
|
+
.map(route => (typeof route.text === 'string' && route.text !== '' ? route.text : (route.terms ?? []).join(' ')))
|
|
451
|
+
.filter(text => text !== '')
|
|
452
|
+
if (texts.length === 0) return degraded(mode, 'no-query-vectors', candidates, null)
|
|
453
|
+
const queryVectors = await client.embed(texts, 'query')
|
|
454
|
+
if (!queryVectors.length) return degraded(mode, 'no-query-vectors', candidates, null)
|
|
455
|
+
|
|
456
|
+
const fieldVectors = new Map()
|
|
457
|
+
const readmeVectors = new Map()
|
|
458
|
+
let readmeState = null
|
|
459
|
+
// Field vectors are cheap and always computed; the README list is an
|
|
460
|
+
// additional, independent list in `readme` mode.
|
|
461
|
+
await fillFields({ records, candidates, client, store, vectors: fieldVectors, now })
|
|
462
|
+
if (mode === 'readme') {
|
|
463
|
+
readmeState = await fillReadme({
|
|
464
|
+
records, candidates, client, store, vectors: readmeVectors, now, fetchImpl: fetchReader,
|
|
465
|
+
readmeOptions,
|
|
466
|
+
readmeTotalBudgetMs,
|
|
467
|
+
})
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
const scored = candidates.map((entry, bm25Rank) => {
|
|
471
|
+
const fieldVector = fieldVectors.get(entry.doc) ?? null
|
|
472
|
+
const readmeVector = readmeVectors.get(entry.doc) ?? null
|
|
473
|
+
const fieldSimilarities = fieldVector
|
|
474
|
+
? queryVectors.map(query => dot(fieldVector, query))
|
|
475
|
+
: []
|
|
476
|
+
const readmeSimilarities = readmeVector
|
|
477
|
+
? queryVectors.map(query => dot(readmeVector, query))
|
|
478
|
+
: []
|
|
479
|
+
const fieldSimilarity = fieldSimilarities.length ? topTwoAverage(fieldSimilarities) : null
|
|
480
|
+
const readmeSimilarity = readmeSimilarities.length ? topTwoAverage(readmeSimilarities) : null
|
|
481
|
+
const fieldMax = fieldSimilarities.length ? Math.max(...fieldSimilarities) : null
|
|
482
|
+
const readmeMax = readmeSimilarities.length ? Math.max(...readmeSimilarities) : null
|
|
483
|
+
const gate2MaxSimilarity = mode === 'readme'
|
|
484
|
+
? Math.max(fieldMax ?? -Infinity, readmeMax ?? -Infinity)
|
|
485
|
+
: (fieldMax ?? -Infinity)
|
|
486
|
+
return {
|
|
487
|
+
entry,
|
|
488
|
+
bm25Rank,
|
|
489
|
+
fieldVector,
|
|
490
|
+
readmeVector,
|
|
491
|
+
fieldSimilarity,
|
|
492
|
+
readmeSimilarity,
|
|
493
|
+
fieldMaxSimilarity: fieldMax,
|
|
494
|
+
readmeMaxSimilarity: readmeMax,
|
|
495
|
+
gate2MaxSimilarity,
|
|
496
|
+
}
|
|
497
|
+
})
|
|
498
|
+
const fieldOrder = scored
|
|
499
|
+
.filter(item => item.fieldVector)
|
|
500
|
+
.sort((a, b) => b.fieldSimilarity - a.fieldSimilarity
|
|
501
|
+
|| a.bm25Rank - b.bm25Rank
|
|
502
|
+
|| a.entry.doc - b.entry.doc)
|
|
503
|
+
const fieldRank = new Map(fieldOrder.map((item, index) => [item.entry.doc, index + 1]))
|
|
504
|
+
const readmeOrder = scored
|
|
505
|
+
.filter(item => item.readmeVector)
|
|
506
|
+
.sort((a, b) => b.readmeSimilarity - a.readmeSimilarity
|
|
507
|
+
|| a.bm25Rank - b.bm25Rank
|
|
508
|
+
|| a.entry.doc - b.entry.doc)
|
|
509
|
+
const readmeRank = new Map(readmeOrder.map((item, index) => [item.entry.doc, index + 1]))
|
|
510
|
+
|
|
511
|
+
const weights = RRF_WEIGHTS[mode]
|
|
512
|
+
for (const item of scored) {
|
|
513
|
+
const fieldRankValue = fieldRank.get(item.entry.doc)
|
|
514
|
+
const readmeRankValue = readmeRank.get(item.entry.doc)
|
|
515
|
+
const gate2Max = Number.isFinite(item.gate2MaxSimilarity) ? item.gate2MaxSimilarity : null
|
|
516
|
+
item.entry.gate2MaxSimilarity = gate2Max
|
|
517
|
+
item.entry.gate2Pass = gate2Max !== null && gate2Max > selectedGate2Threshold
|
|
518
|
+
item.entry.fieldMaxSimilarity = item.fieldMaxSimilarity
|
|
519
|
+
item.entry.readmeMaxSimilarity = item.readmeMaxSimilarity
|
|
520
|
+
// Ranks are 1-based; a missing list contributes nothing.
|
|
521
|
+
item.entry.score = weightedRrf({
|
|
522
|
+
bm25: item.bm25Rank + 1,
|
|
523
|
+
fields: fieldRankValue,
|
|
524
|
+
readme: readmeRankValue,
|
|
525
|
+
}, weights)
|
|
526
|
+
item.entry.vectorSimilarity = mode === 'readme'
|
|
527
|
+
? (item.readmeSimilarity ?? item.fieldSimilarity)
|
|
528
|
+
: item.fieldSimilarity
|
|
529
|
+
item.entry.fieldSimilarity = item.fieldSimilarity
|
|
530
|
+
item.entry.readmeSimilarity = item.readmeSimilarity
|
|
531
|
+
item.entry.fieldRank = fieldRankValue ?? null
|
|
532
|
+
item.entry.readmeRank = readmeRankValue ?? null
|
|
533
|
+
}
|
|
534
|
+
const ordered = scored
|
|
535
|
+
.slice()
|
|
536
|
+
.sort((a, b) => b.entry.score - a.entry.score
|
|
537
|
+
|| a.bm25Rank - b.bm25Rank
|
|
538
|
+
|| a.entry.doc - b.entry.doc)
|
|
539
|
+
.map(item => item.entry)
|
|
540
|
+
const covered = scored.filter(item => item.fieldVector || item.readmeVector).length
|
|
541
|
+
if (covered === 0) return degraded(mode, 'no-vectors', candidates, null)
|
|
542
|
+
const gate2Values = scored
|
|
543
|
+
.map(item => item.gate2MaxSimilarity)
|
|
544
|
+
.filter(value => Number.isFinite(value))
|
|
545
|
+
const gate2MaxSimilarity = gate2Values.length ? Math.max(...gate2Values) : null
|
|
546
|
+
const gate2 = {
|
|
547
|
+
threshold: selectedGate2Threshold,
|
|
548
|
+
maxSimilarity: gate2MaxSimilarity,
|
|
549
|
+
passed: gate2MaxSimilarity !== null && gate2MaxSimilarity > selectedGate2Threshold,
|
|
550
|
+
}
|
|
551
|
+
return {
|
|
552
|
+
mode,
|
|
553
|
+
algorithm: ALGORITHM_VERSION,
|
|
554
|
+
covered,
|
|
555
|
+
fieldCovered: fieldOrder.length,
|
|
556
|
+
gate2,
|
|
557
|
+
candidates: ordered,
|
|
558
|
+
...(mode === 'readme' ? readmeState : {}),
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
// Retry with a fresh model process so a transient spawn failure does not
|
|
563
|
+
// silently degrade to lexical-only results; caches make retries cheap.
|
|
564
|
+
let lastError = null
|
|
565
|
+
try {
|
|
566
|
+
for (let round = 0; round < MODEL_ATTEMPTS; round += 1) {
|
|
567
|
+
if (MODEL_RETRY_DELAYS_MS[round] > 0) await sleep(MODEL_RETRY_DELAYS_MS[round])
|
|
568
|
+
const ephemeral = Boolean(clientFactory) || disposeClient
|
|
569
|
+
const client = clientFactory
|
|
570
|
+
? clientFactory(resolvedServer)
|
|
571
|
+
: (ephemeral ? new ModelClient(resolvedServer) : getSharedClient(resolvedServer))
|
|
572
|
+
try {
|
|
573
|
+
return await attempt(client)
|
|
574
|
+
} catch (error) {
|
|
575
|
+
if (!(error instanceof ModelUnavailableError)) {
|
|
576
|
+
return degraded(mode, 'embedding-failed', candidates, error instanceof Error ? error.message : String(error))
|
|
577
|
+
}
|
|
578
|
+
lastError = error
|
|
579
|
+
try { client.dispose?.() } catch {}
|
|
580
|
+
if (!ephemeral && sharedClient === client) sharedClient = null
|
|
581
|
+
} finally {
|
|
582
|
+
if (ephemeral) {
|
|
583
|
+
try { client.dispose?.() } catch {}
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
const message = lastError instanceof Error ? lastError.message : String(lastError ?? 'model unavailable')
|
|
588
|
+
return degraded(mode, 'runtime-missing', candidates, message)
|
|
589
|
+
} finally {
|
|
590
|
+
try { store.flush() } catch {}
|
|
591
|
+
}
|
|
592
|
+
}
|
package/locale/en.json
ADDED
package/locale/zh.json
ADDED
package/package.json
CHANGED
|
@@ -1,6 +1,67 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "dsh-plugin-discover",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
1
|
+
{
|
|
2
|
+
"name": "dsh-plugin-discover",
|
|
3
|
+
"version": "1.0.1",
|
|
4
|
+
"description": "Find DeepSeek Harness plugins with a local, model-free index: Chinese natural-language search, quality filters, deterministic ranking, and install targets — never installs anything.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./index.js",
|
|
9
|
+
"./client": "./client.js",
|
|
10
|
+
"./package.json": "./package.json",
|
|
11
|
+
"./locale/*.json": "./locale/*.json",
|
|
12
|
+
"./icon": "./icon.svg"
|
|
13
|
+
},
|
|
14
|
+
"bin": {
|
|
15
|
+
"dsh-plugin-discover": "lib/cli.js"
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"index.js",
|
|
19
|
+
"client.js",
|
|
20
|
+
"lib",
|
|
21
|
+
"cordis.patch.yml",
|
|
22
|
+
"locale/*.json",
|
|
23
|
+
"icon.svg",
|
|
24
|
+
"README.md",
|
|
25
|
+
"README.zh.md",
|
|
26
|
+
"CHANGELOG.md",
|
|
27
|
+
"LICENSE"
|
|
28
|
+
],
|
|
29
|
+
"scripts": {
|
|
30
|
+
"test": "node --test --test-isolation=none test/*.test.js"
|
|
31
|
+
},
|
|
32
|
+
"keywords": [
|
|
33
|
+
"deepseek-harness",
|
|
34
|
+
"dsh",
|
|
35
|
+
"dsh-plugin",
|
|
36
|
+
"plugin-discovery",
|
|
37
|
+
"search",
|
|
38
|
+
"bm25",
|
|
39
|
+
"chinese",
|
|
40
|
+
"local-first",
|
|
41
|
+
"offline"
|
|
42
|
+
],
|
|
43
|
+
"license": "MIT",
|
|
44
|
+
"engines": {
|
|
45
|
+
"node": ">=22.19.0"
|
|
46
|
+
},
|
|
47
|
+
"dsh": {
|
|
48
|
+
"bundle": {
|
|
49
|
+
"patch": "./cordis.patch.yml"
|
|
50
|
+
},
|
|
51
|
+
"client": {
|
|
52
|
+
"platform": "web",
|
|
53
|
+
"immediately": true,
|
|
54
|
+
"inject": [
|
|
55
|
+
"@deepseek-ai/dsh-client-locale",
|
|
56
|
+
"@deepseek-ai/dsh-client-ui-plugin-manager"
|
|
57
|
+
]
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
"peerDependencies": {
|
|
61
|
+
"@deepseek-ai/cordis": "^4.0.5",
|
|
62
|
+
"@deepseek-ai/dsh-tools": "^0.2.1-rc.2"
|
|
63
|
+
},
|
|
64
|
+
"dependencies": {
|
|
65
|
+
"@deepseek-ai/schemastery": "^3.18.1"
|
|
66
|
+
}
|
|
67
|
+
}
|