dsh-metabolism 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -8
- package/cordis.patch.yml +1 -0
- package/engine.js +931 -59
- package/index.js +11 -0
- package/package.json +12 -2
- package/recall.js +268 -36
- package//345/220/221/351/207/217.js +1 -1
- package//345/256/210/345/215/253.js +111 -0
- package//346/221/230/350/246/201/345/235/227.js +53 -0
- package//346/227/266/351/227/264/347/272/277.js +61 -0
- package//346/265/256/347/216/260.js +128 -0
- package//350/264/264/346/235/241.js +229 -0
- package//351/207/215/346/216/222.js +69 -0
- package//351/252/250/346/236/266.js +302 -0
- package//351/252/250/346/236/266/345/267/245/345/205/267.js +473 -0
package/index.js
CHANGED
|
@@ -178,6 +178,11 @@ async function archive(root, payload) {
|
|
|
178
178
|
const times = events.map((event) => Number(event.time)).filter((time) => Number.isFinite(time))
|
|
179
179
|
const startedAt = times.length > 0 ? Math.min(...times) : endedAt
|
|
180
180
|
const finishedAt = times.length > 0 ? Math.max(...times) : endedAt
|
|
181
|
+
// seq 区间写进摘要头:以后从茅房捞原文,不用先猜它在第几条
|
|
182
|
+
const seqs = events.map((event) => Number(event.seq)).filter((seq) => Number.isFinite(seq))
|
|
183
|
+
const seqFrom = seqs.length > 0 ? Math.min(...seqs) : null
|
|
184
|
+
const seqTo = seqs.length > 0 ? Math.max(...seqs) : null
|
|
185
|
+
const seqLine = seqFrom === null || seqTo === null ? '(没记到)' : `${seqFrom}–${seqTo}`
|
|
181
186
|
|
|
182
187
|
const directory = join(root, dayKey(endedAt))
|
|
183
188
|
await mkdir(directory, { recursive: true })
|
|
@@ -190,6 +195,10 @@ async function archive(root, payload) {
|
|
|
190
195
|
events.map((event) => JSON.stringify({ seq: event.seq, type: event.type, time: event.time, data: event.data })).join('\n') + '\n')
|
|
191
196
|
|
|
192
197
|
const summaryName = `${rawName.slice(0, -'.jsonl'.length)}.summary.md`
|
|
198
|
+
// 召回命令要用最终文件名,所以放在 rawName 之后才算
|
|
199
|
+
const recallLine = seqFrom === null || seqTo === null
|
|
200
|
+
? '(没有 seq 区间,只能按关键词捞)'
|
|
201
|
+
: `metabolism_recall(file=${rawName}, fromSeq=${seqFrom}, toSeq=${seqTo})`
|
|
193
202
|
await writeFile(
|
|
194
203
|
join(directory, summaryName),
|
|
195
204
|
[
|
|
@@ -197,7 +206,9 @@ async function archive(root, payload) {
|
|
|
197
206
|
'',
|
|
198
207
|
`- 压缩编号:${compactionId}`,
|
|
199
208
|
`- 覆盖时间:${clockText(startedAt)} – ${clockText(finishedAt)}`,
|
|
209
|
+
`- seq 区间:${seqLine}`,
|
|
200
210
|
`- 原文:${rawName}`,
|
|
211
|
+
`- 召回:${recallLine}`,
|
|
201
212
|
`- 事件条数:${events.length}`,
|
|
202
213
|
`- 折算 tokens:${shadowedTokenCount}`,
|
|
203
214
|
`- 摘要模型:${payload.summaryModel || '(未记录)'}`,
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-metabolism",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"type": "module",
|
|
5
|
-
"description": "
|
|
5
|
+
"description": "上下文代谢:压缩时把原文留在本地,窗口里换一块有味道的摘要;七天后折索引,够放就不折,要细节再翻回原处。",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"author": "长命 (Changming) — 第一作者; 久安 (Jiuan)",
|
|
8
8
|
"repository": {
|
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
".": "./index.js",
|
|
18
18
|
"./engine": "./engine.js",
|
|
19
19
|
"./recall": "./recall.js",
|
|
20
|
+
"./骨架": "./骨架.js",
|
|
21
|
+
"./骨架工具": "./骨架工具.js",
|
|
20
22
|
"./locale/*.json": "./locale/*.json",
|
|
21
23
|
"./icon": "./icon.svg"
|
|
22
24
|
},
|
|
@@ -24,7 +26,15 @@
|
|
|
24
26
|
"index.js",
|
|
25
27
|
"engine.js",
|
|
26
28
|
"recall.js",
|
|
29
|
+
"骨架.js",
|
|
30
|
+
"骨架工具.js",
|
|
27
31
|
"向量.js",
|
|
32
|
+
"重排.js",
|
|
33
|
+
"浮现.js",
|
|
34
|
+
"贴条.js",
|
|
35
|
+
"守卫.js",
|
|
36
|
+
"摘要块.js",
|
|
37
|
+
"时间线.js",
|
|
28
38
|
"cordis.patch.yml",
|
|
29
39
|
"locale/*.json",
|
|
30
40
|
"docs",
|
package/recall.js
CHANGED
|
@@ -16,17 +16,24 @@
|
|
|
16
16
|
*
|
|
17
17
|
* @module dsh-metabolism/recall
|
|
18
18
|
*/
|
|
19
|
-
import { existsSync, readdirSync, readFileSync, statSync, writeFileSync, mkdirSync } from 'node:fs'
|
|
19
|
+
import { appendFileSync, existsSync, readdirSync, readFileSync, statSync, writeFileSync, mkdirSync } from 'node:fs'
|
|
20
20
|
import { join, resolve, dirname } from 'node:path'
|
|
21
21
|
import { homedir } from 'node:os'
|
|
22
22
|
import { cosineTop, embedTexts, readVectors, vectorsReady, MODEL as VECTOR_MODEL } from './向量.js'
|
|
23
|
+
import { rerankScores } from './重排.js'
|
|
24
|
+
// 骨架工具跟着召回一起挂:同一个 tools 服务,省一条补丁条目(2026-11-19)。
|
|
25
|
+
import { apply as skeletonApply } from './骨架工具.js'
|
|
26
|
+
// 浮现也一起挂:它只要有事件口子,不需要 tools(默认关,开关里打开才动)。
|
|
27
|
+
import { apply as morphApply } from './浮现.js'
|
|
28
|
+
// 贴条(每轮最多一条):同样只要有事件口子(默认关)。
|
|
29
|
+
import { apply as attachApply } from './贴条.js'
|
|
23
30
|
|
|
24
31
|
/** 词面与语义各占多少(合起来排位用倒数名次,不看绝对分)。 */
|
|
25
32
|
const RRF_LEXICAL = 1
|
|
26
|
-
const RRF_VECTOR =
|
|
33
|
+
const RRF_VECTOR = Number(process.env.METAB_RRF_VECTOR ?? localOverrides().rrfVector ?? 1)
|
|
27
34
|
const RRF_K = 60
|
|
28
35
|
/** 只在向量那一路出现的,余弦低于这个就不放进来(避免尾巴上的杂项)。 */
|
|
29
|
-
const VECTOR_ONLY_MIN_COS = 0
|
|
36
|
+
const VECTOR_ONLY_MIN_COS = Number(localOverrides().cosMin ?? 0)
|
|
30
37
|
|
|
31
38
|
/** 常见虚词二元组:这些命中不算数。 */
|
|
32
39
|
const STOP_BIGRAMS = new Set([
|
|
@@ -42,6 +49,24 @@ const STOP_CHARS = new Set('的了是在和就都也还把被让给跟吗呢吧
|
|
|
42
49
|
/** 一条归档里最多收多少字(超长的工具输出只留开头,够检索)。 */
|
|
43
50
|
const MAX_UNIT_CHARS = 20000
|
|
44
51
|
|
|
52
|
+
/**
|
|
53
|
+
* 长单元切成小块(2026-11-20)。一整段工具输出或一份评测输出当成一条,
|
|
54
|
+
* 里面能对上几十个词,会把真答案整个挤下去——线上跑分只有 5/30 就是它干的。
|
|
55
|
+
* 500 字一块、相邻重叠 100 字,保证跨块的句子还能连上。
|
|
56
|
+
*/
|
|
57
|
+
const CHUNK_CHARS = Math.max(200, Number(localOverrides().chunkChars) || 500)
|
|
58
|
+
const CHUNK_OVERLAP = Math.max(0, Number(localOverrides().chunkOverlap) || 100)
|
|
59
|
+
|
|
60
|
+
function chunksOf(text) {
|
|
61
|
+
if (text.length <= CHUNK_CHARS) return [text]
|
|
62
|
+
const out = []
|
|
63
|
+
for (let at = 0; at < text.length; at += CHUNK_CHARS - CHUNK_OVERLAP) {
|
|
64
|
+
out.push(text.slice(at, at + CHUNK_CHARS))
|
|
65
|
+
if (at + CHUNK_CHARS >= text.length) break
|
|
66
|
+
}
|
|
67
|
+
return out
|
|
68
|
+
}
|
|
69
|
+
|
|
45
70
|
/** 把一句话切成检索用的词:拉丁词按空格,中文按二元组。 */
|
|
46
71
|
export function termsOf(text) {
|
|
47
72
|
const terms = new Set()
|
|
@@ -255,6 +280,7 @@ export function buildIndex(archiveRoot, indexDir, log = () => {}) {
|
|
|
255
280
|
if (!existsSync(paths.dir)) mkdirSync(paths.dir, { recursive: true })
|
|
256
281
|
const seen = new Set()
|
|
257
282
|
const lines = []
|
|
283
|
+
const files = {}
|
|
258
284
|
for (const entry of filesInRange(archiveRoot, null)) {
|
|
259
285
|
let units = []
|
|
260
286
|
try {
|
|
@@ -263,24 +289,33 @@ export function buildIndex(archiveRoot, indexDir, log = () => {}) {
|
|
|
263
289
|
log(`读 ${entry.file} 失败:${error?.message ?? error}`)
|
|
264
290
|
continue
|
|
265
291
|
}
|
|
292
|
+
try {
|
|
293
|
+
const stat = statSync(entry.path)
|
|
294
|
+
files[`${entry.day}/${entry.file}`] = { size: stat.size, mtime: stat.mtimeMs }
|
|
295
|
+
} catch {
|
|
296
|
+
// 记不上就记不上,下次全量再来
|
|
297
|
+
}
|
|
266
298
|
for (const unit of units) {
|
|
267
|
-
const
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
299
|
+
for (const piece of chunksOf(unit.text)) {
|
|
300
|
+
if (seen.has(piece)) continue
|
|
301
|
+
seen.add(piece)
|
|
302
|
+
const text = piece.length > MAX_UNIT_CHARS ? piece.slice(0, MAX_UNIT_CHARS) : piece
|
|
303
|
+
lines.push(JSON.stringify({
|
|
304
|
+
i: lines.length,
|
|
305
|
+
d: dayOf(unit.time, entry.day),
|
|
306
|
+
f: entry.file,
|
|
307
|
+
s: unit.seq,
|
|
308
|
+
t: unit.time,
|
|
309
|
+
r: unit.role,
|
|
310
|
+
n: piece.length,
|
|
311
|
+
x: text,
|
|
312
|
+
}))
|
|
313
|
+
}
|
|
281
314
|
}
|
|
282
315
|
}
|
|
283
316
|
writeFileSync(paths.file, lines.join('\n'), 'utf8')
|
|
317
|
+
// 全量建完顺手记一份清单,下次就能走增量
|
|
318
|
+
writeFileSync(manifestPath(indexDir), JSON.stringify({ version: 1, nextId: lines.length, files }), 'utf8')
|
|
284
319
|
return { units: lines.length }
|
|
285
320
|
}
|
|
286
321
|
|
|
@@ -288,7 +323,113 @@ export function buildIndex(archiveRoot, indexDir, log = () => {}) {
|
|
|
288
323
|
function indexStale(archiveRoot, indexDir) {
|
|
289
324
|
const paths = indexPaths(indexDir)
|
|
290
325
|
if (!existsSync(paths.file)) return true
|
|
291
|
-
|
|
326
|
+
const manifest = readManifest(indexDir)
|
|
327
|
+
if (manifest === null) return newestArchiveMtime(archiveRoot) > statSync(paths.file).mtimeMs
|
|
328
|
+
const known = manifest.files ?? {}
|
|
329
|
+
for (const entry of filesInRange(archiveRoot, null)) {
|
|
330
|
+
const key = `${entry.day}/${entry.file}`
|
|
331
|
+
let stat
|
|
332
|
+
try {
|
|
333
|
+
stat = statSync(entry.path)
|
|
334
|
+
} catch {
|
|
335
|
+
continue
|
|
336
|
+
}
|
|
337
|
+
const old = known[key]
|
|
338
|
+
if (old === undefined || old.size !== stat.size) return true
|
|
339
|
+
}
|
|
340
|
+
return false
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
const MANIFEST_FILE = '已收录.json'
|
|
344
|
+
|
|
345
|
+
function manifestPath(indexDir) {
|
|
346
|
+
return join(indexDir, MANIFEST_FILE)
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function readManifest(indexDir) {
|
|
350
|
+
try {
|
|
351
|
+
const parsed = JSON.parse(readFileSync(manifestPath(indexDir), 'utf8'))
|
|
352
|
+
if (parsed === null || typeof parsed !== 'object') return null
|
|
353
|
+
if (parsed.files === undefined) return null
|
|
354
|
+
return parsed
|
|
355
|
+
} catch {
|
|
356
|
+
return null
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* 增量:只把「新落地的归档文件」追加进索引,不重扫全库。
|
|
362
|
+
*
|
|
363
|
+
* 老大 2026-11-20 要的:茅房天天涨,每来一个新文件就把 212MB 重读一遍太蠢。
|
|
364
|
+
* 没有清单或索引不存在时返回 null,调用方退回全量 buildIndex。
|
|
365
|
+
* 去重只在这一次新增的批内做(跨文件的完全重复留给全量重建清)。
|
|
366
|
+
*/
|
|
367
|
+
export function appendToIndex(archiveRoot, indexDir, log = () => {}) {
|
|
368
|
+
const paths = indexPaths(indexDir)
|
|
369
|
+
if (!existsSync(paths.file)) return null
|
|
370
|
+
const manifest = readManifest(indexDir)
|
|
371
|
+
if (manifest === null) return null
|
|
372
|
+
|
|
373
|
+
const known = manifest.files ?? {}
|
|
374
|
+
const todo = []
|
|
375
|
+
const present = new Set()
|
|
376
|
+
for (const entry of filesInRange(archiveRoot, null)) {
|
|
377
|
+
const key = `${entry.day}/${entry.file}`
|
|
378
|
+
present.add(key)
|
|
379
|
+
let stat
|
|
380
|
+
try {
|
|
381
|
+
stat = statSync(entry.path)
|
|
382
|
+
} catch {
|
|
383
|
+
continue
|
|
384
|
+
}
|
|
385
|
+
const old = known[key]
|
|
386
|
+
if (old === undefined || old.size !== stat.size) todo.push({ entry, stat, key })
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// 已经消失的文件从清单里清掉(索引里留着的那几条不动,宁可多不可少)
|
|
390
|
+
const files = {}
|
|
391
|
+
for (const [key, value] of Object.entries(known)) if (present.has(key)) files[key] = value
|
|
392
|
+
for (const item of todo) files[item.key] = { size: item.stat.size, mtime: item.stat.mtimeMs }
|
|
393
|
+
|
|
394
|
+
let nextId = Number(manifest.nextId) || 0
|
|
395
|
+
const lines = []
|
|
396
|
+
const seen = new Set()
|
|
397
|
+
for (const { entry } of todo) {
|
|
398
|
+
let units = []
|
|
399
|
+
try {
|
|
400
|
+
units = unitsOfFile(entry.path, entry.day, entry.file)
|
|
401
|
+
} catch (error) {
|
|
402
|
+
log(`读 ${entry.file} 失败:${error?.message ?? error}`)
|
|
403
|
+
continue
|
|
404
|
+
}
|
|
405
|
+
for (const unit of units) {
|
|
406
|
+
for (const piece of chunksOf(unit.text)) {
|
|
407
|
+
if (seen.has(piece)) continue
|
|
408
|
+
seen.add(piece)
|
|
409
|
+
const text = piece.length > MAX_UNIT_CHARS ? piece.slice(0, MAX_UNIT_CHARS) : piece
|
|
410
|
+
lines.push(JSON.stringify({
|
|
411
|
+
i: nextId,
|
|
412
|
+
d: dayOf(unit.time, entry.day),
|
|
413
|
+
f: entry.file,
|
|
414
|
+
s: unit.seq,
|
|
415
|
+
t: unit.time,
|
|
416
|
+
r: unit.role,
|
|
417
|
+
n: piece.length,
|
|
418
|
+
x: text,
|
|
419
|
+
}))
|
|
420
|
+
nextId += 1
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
if (lines.length > 0) {
|
|
426
|
+
const current = readFileSync(paths.file)
|
|
427
|
+
const needsBreak = current.length > 0 && current[current.length - 1] !== 0x0a
|
|
428
|
+
appendFileSync(paths.file, `${needsBreak ? '\n' : ''}${lines.join('\n')}\n`, 'utf8')
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
writeFileSync(manifestPath(indexDir), JSON.stringify({ version: 1, nextId, files }), 'utf8')
|
|
432
|
+
return { added: todo.length, units: lines.length }
|
|
292
433
|
}
|
|
293
434
|
|
|
294
435
|
let cache = null
|
|
@@ -342,22 +483,34 @@ function searchIndex(query, options, archiveRoot, indexDir, log) {
|
|
|
342
483
|
const limit = Math.max(1, Math.min(400, Number(options.pool ?? options.limit) || 5))
|
|
343
484
|
let range = options.from !== undefined || options.to !== undefined
|
|
344
485
|
? { from: options.from, to: options.to }
|
|
345
|
-
: parseQueryTime(String(query))
|
|
486
|
+
: parseQueryTime(String(query), options.now === undefined ? new Date() : new Date(options.now))
|
|
346
487
|
if (range === null && Number(options.days) > 0) range = { from: daysAgoDate(Number(options.days)) }
|
|
347
488
|
const terms = termsOf(query)
|
|
348
489
|
if (terms.length === 0) return { terms, range, scanned: 0, files: 0, hits: [] }
|
|
490
|
+
// 自指过滤(2026-11-20):问句本身被归档过(题目文件、以前的提问),检索会把它自己翻出来,
|
|
491
|
+
// 看着像命中其实是在复述问题。问句够长(≥10 字)且原样出现在条目里就跳过。
|
|
492
|
+
const rawQuery = String(query).trim()
|
|
493
|
+
const skipSelfRef = rawQuery.length >= 10
|
|
494
|
+
// 块边界会把问句截断,整句对不上,所以再拿开头十二个字对一次。
|
|
495
|
+
const selfPrefix = skipSelfRef ? rawQuery.slice(0, 16) : ''
|
|
349
496
|
|
|
350
497
|
if (indexStale(archiveRoot, indexDir) && !rebuilding && options.noRebuild !== true) {
|
|
351
498
|
rebuilding = true
|
|
352
|
-
//
|
|
499
|
+
// 索引旧了:先用旧的答,后台补进去(重建期间不接新活)。
|
|
500
|
+
// 先走增量:只读新落地的归档文件;清单缺失或没底子才退回全量。
|
|
353
501
|
setTimeout(() => {
|
|
354
502
|
try {
|
|
355
503
|
const started = Date.now()
|
|
356
|
-
const
|
|
504
|
+
const added = appendToIndex(archiveRoot, indexDir, log)
|
|
357
505
|
cache = null
|
|
358
|
-
|
|
506
|
+
if (added === null) {
|
|
507
|
+
const built = buildIndex(archiveRoot, indexDir, log)
|
|
508
|
+
log(`索引已全量重建:${built.units} 条,用时 ${Math.round((Date.now() - started) / 1000)}s`)
|
|
509
|
+
} else {
|
|
510
|
+
log(`索引已增量更新:追加 ${added.units} 条(${added.added} 个新文件),用时 ${Math.round((Date.now() - started) / 1000)}s`)
|
|
511
|
+
}
|
|
359
512
|
} catch (error) {
|
|
360
|
-
log(
|
|
513
|
+
log(`索引更新失败:${error?.message ?? error}`)
|
|
361
514
|
} finally {
|
|
362
515
|
rebuilding = false
|
|
363
516
|
}
|
|
@@ -372,6 +525,12 @@ function searchIndex(query, options, archiveRoot, indexDir, log) {
|
|
|
372
525
|
const wantsNumber = NUMERIC_Q.test(String(query))
|
|
373
526
|
const K1 = 1.2
|
|
374
527
|
const B = 0.75
|
|
528
|
+
/** 工具结果权重:默认 0.3。设 1 就不压,设 0 等于一律不看工具输出。 */
|
|
529
|
+
const TOOL_WEIGHT = options.toolWeight === undefined
|
|
530
|
+
? Math.max(0, Number(localOverrides().toolWeight ?? 0.3))
|
|
531
|
+
: Math.max(0, Number(options.toolWeight))
|
|
532
|
+
/** 词频封顶:一条里同一个词最多按 8 次算。设 1 就退回「只算有没有出现」的老口径。 */
|
|
533
|
+
const TF_CAP = Math.max(1, Number(process.env.METAB_TF_CAP) || Number(localOverrides().tfCap) || 8)
|
|
375
534
|
|
|
376
535
|
const hits = []
|
|
377
536
|
let scanned = 0
|
|
@@ -388,12 +547,18 @@ function searchIndex(query, options, archiveRoot, indexDir, log) {
|
|
|
388
547
|
}
|
|
389
548
|
scanned += 1
|
|
390
549
|
const text = lower[index]
|
|
550
|
+
if (skipSelfRef && (unit.x.includes(rawQuery) || unit.x.includes(selfPrefix))) continue
|
|
551
|
+
const lengthNorm = 1 - B + (B * text.length) / avgLen
|
|
391
552
|
let got = 0
|
|
392
553
|
let rareGot = 0
|
|
393
554
|
const matched = []
|
|
394
555
|
for (let k = 0; k < terms.length; k += 1) {
|
|
395
|
-
|
|
396
|
-
|
|
556
|
+
const at = text.indexOf(terms[k])
|
|
557
|
+
if (at < 0) continue
|
|
558
|
+
// 词频:同一个词出现几次就算几次(封顶 TF_CAP,免得一条工具输出靠堆词霸榜)
|
|
559
|
+
let tf = 0
|
|
560
|
+
for (let pos = at; pos >= 0 && tf < TF_CAP; pos = text.indexOf(terms[k], pos + terms[k].length)) tf += 1
|
|
561
|
+
got += weights[k] * ((tf * (K1 + 1)) / (tf + K1 * lengthNorm))
|
|
397
562
|
matched.push(terms[k])
|
|
398
563
|
if (rare[k]) rareGot += weights[k]
|
|
399
564
|
}
|
|
@@ -424,10 +589,11 @@ function searchIndex(query, options, archiveRoot, indexDir, log) {
|
|
|
424
589
|
})
|
|
425
590
|
continue
|
|
426
591
|
}
|
|
427
|
-
const norm = 1 / (1 + K1 * (1 - B + (B * text.length) / avgLen))
|
|
428
|
-
const bm25 = got * (1 + K1) * norm
|
|
429
592
|
const rareCoverage = rareTotal > 0 ? rareGot / rareTotal : 1
|
|
430
|
-
let score =
|
|
593
|
+
let score = got * rareCoverage
|
|
594
|
+
// 工具结果降权(2026-11-20 量出来的):前十条里 275 条是工具输出,真答案全被挤下去。
|
|
595
|
+
// 全部剔掉后三十道题前五全中,所以按权重压着用,不删——万一答案真在工具输出里还捞得到。
|
|
596
|
+
if (unit.r === '工具') score *= TOOL_WEIGHT
|
|
431
597
|
if (wantsNumber && NUMERIC_DOC.test(unit.x)) score *= 1.35
|
|
432
598
|
score *= recencyFactor(unit.t, options)
|
|
433
599
|
hits.push({
|
|
@@ -452,10 +618,16 @@ export function searchArchive(archiveRoot, query, options = {}) {
|
|
|
452
618
|
const limit = Math.max(1, Math.min(20, Number(options.limit) || 5))
|
|
453
619
|
let range = options.from !== undefined || options.to !== undefined
|
|
454
620
|
? { from: options.from, to: options.to }
|
|
455
|
-
: parseQueryTime(String(query))
|
|
621
|
+
: parseQueryTime(String(query), options.now === undefined ? new Date() : new Date(options.now))
|
|
456
622
|
if (range === null && Number(options.days) > 0) range = { from: daysAgoDate(Number(options.days)) }
|
|
457
623
|
const terms = termsOf(query)
|
|
458
624
|
if (terms.length === 0) return { terms, range, scanned: 0, files: 0, hits: [] }
|
|
625
|
+
// 自指过滤(2026-11-20):问句本身被归档过(题目文件、以前的提问),检索会把它自己翻出来,
|
|
626
|
+
// 看着像命中其实是在复述问题。问句够长(≥10 字)且原样出现在条目里就跳过。
|
|
627
|
+
const rawQuery = String(query).trim()
|
|
628
|
+
const skipSelfRef = rawQuery.length >= 10
|
|
629
|
+
// 块边界会把问句截断,整句对不上,所以再拿开头十二个字对一次。
|
|
630
|
+
const selfPrefix = skipSelfRef ? rawQuery.slice(0, 16) : ''
|
|
459
631
|
|
|
460
632
|
const files = filesInRange(archiveRoot, range)
|
|
461
633
|
const hits = []
|
|
@@ -464,6 +636,7 @@ export function searchArchive(archiveRoot, query, options = {}) {
|
|
|
464
636
|
const units = unitsOfFile(entry.path, entry.day, entry.file)
|
|
465
637
|
scanned += units.length
|
|
466
638
|
for (const unit of units) {
|
|
639
|
+
if (skipSelfRef && (unit.text.includes(rawQuery) || unit.text.includes(selfPrefix))) continue
|
|
467
640
|
const lowered = unit.text.toLowerCase()
|
|
468
641
|
const matched = terms.filter((term) => lowered.includes(term))
|
|
469
642
|
if (matched.length === 0) continue
|
|
@@ -516,6 +689,9 @@ async function fuseSemantic(lexical, query, options, dir, archiveRoot, log) {
|
|
|
516
689
|
const inRange = (row) => {
|
|
517
690
|
const unit = units[row]
|
|
518
691
|
if (unit === undefined) return false
|
|
692
|
+
// 工具结果不进向量那一路(2026-11-20 量出来的):词面那路已经把它压到 0.3,
|
|
693
|
+
// 向量这路要是不管,它又会从后面爬回前排,实测前五反而从 24 掉到 17。
|
|
694
|
+
if (unit.r === '工具') return false
|
|
519
695
|
if (excluded !== null && excluded.has(unit.f)) return false
|
|
520
696
|
if (range === null || range === undefined) return true
|
|
521
697
|
if (range.from !== undefined && unit.d < range.from) return false
|
|
@@ -534,7 +710,7 @@ async function fuseSemantic(lexical, query, options, dir, archiveRoot, log) {
|
|
|
534
710
|
}
|
|
535
711
|
}
|
|
536
712
|
|
|
537
|
-
const pinned = lexical.hits.slice(0, Math.max(0, Math.min(5, Number(options.pinLexical ??
|
|
713
|
+
const pinned = lexical.hits.slice(0, Math.max(0, Math.min(5, Number(options.pinLexical ?? localOverrides().pinLexical ?? 0)))).filter((hit) => hit.index !== undefined)
|
|
538
714
|
const pinnedRows = new Set(pinned.map((hit) => hit.index))
|
|
539
715
|
|
|
540
716
|
const ranked = new Map()
|
|
@@ -598,14 +774,48 @@ export async function search(archiveRoot, query, options = {}, indexDir = undefi
|
|
|
598
774
|
log(`走索引失败,退回现扫:${error?.message ?? error}`)
|
|
599
775
|
return searchArchive(archiveRoot, query, options)
|
|
600
776
|
}
|
|
601
|
-
|
|
602
|
-
|
|
777
|
+
const limit = Math.max(1, Math.min(20, Number(options.limit) || 5))
|
|
778
|
+
// 要重排就得先把候选捞宽一点:只留 5 条再重排,等于让它在 5 条里换位子,前五永远不变(2026-11-20 踩过)。
|
|
779
|
+
const wantRerank = options.rerank === true || localOverrides().rerank === true
|
|
780
|
+
const poolSize = wantRerank ? Math.max(limit, Number(options.rerankTopN ?? localOverrides().rerankTopN ?? 10)) : limit
|
|
781
|
+
let merged = { ...lexical, semantic: false, hits: lexical.hits.slice(0, poolSize) }
|
|
782
|
+
if (options.semantic === true) {
|
|
783
|
+
try {
|
|
784
|
+
merged = await fuseSemantic(lexical, query, options, dir, archiveRoot, log)
|
|
785
|
+
} catch (error) {
|
|
786
|
+
log(`向量那步没成,只按词面答:${error?.message ?? error}`)
|
|
787
|
+
}
|
|
603
788
|
}
|
|
789
|
+
const out = await applyRerank(merged, query, options, archiveRoot, log)
|
|
790
|
+
return { ...out, hits: out.hits.slice(0, limit) }
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
/**
|
|
794
|
+
* 重排(2026-11-20):只动前 rerankTopN 条,拿问题和条目拼起来逐条打分,再按分数重排。
|
|
795
|
+
* 实测本机一条 0.7 秒,所以默认只排十条,且失败就按原顺序返回,不能让重排把检索拖死。
|
|
796
|
+
*/
|
|
797
|
+
async function applyRerank(merged, query, options, archiveRoot, log) {
|
|
798
|
+
const cfg = localOverrides()
|
|
799
|
+
if (options.rerank !== true && cfg.rerank !== true) return merged
|
|
800
|
+
const topN = Math.max(1, Math.min(30, Number(options.rerankTopN ?? cfg.rerankTopN ?? 10)))
|
|
801
|
+
const head = merged.hits.slice(0, topN)
|
|
802
|
+
if (head.length < 2) return merged
|
|
604
803
|
try {
|
|
605
|
-
|
|
804
|
+
const started = Date.now()
|
|
805
|
+
const scores = await rerankScores(String(query), head.map((hit) => hit.text), resolve(archiveRoot, '..', '..'), {
|
|
806
|
+
model: options.rerankModel ?? cfg.rerankModel,
|
|
807
|
+
log,
|
|
808
|
+
})
|
|
809
|
+
if (!Array.isArray(scores) || scores.length !== head.length) return merged
|
|
810
|
+
const ranked = head
|
|
811
|
+
.map((hit, index) => ({ hit, score: Number(scores[index]) }))
|
|
812
|
+
.sort((left, right) => right.score - left.score)
|
|
813
|
+
.map((item) => ({ ...item.hit, rerank: item.score }))
|
|
814
|
+
log(`重排:前 ${head.length} 条重打分,用时 ${Math.round((Date.now() - started) / 1000)}s`)
|
|
815
|
+
return { ...merged, rerank: true, hits: [...ranked, ...merged.hits.slice(topN)] }
|
|
606
816
|
} catch (error) {
|
|
607
|
-
log(
|
|
608
|
-
return
|
|
817
|
+
log(`重排没成,按原顺序答:${error?.message ?? error}`)
|
|
818
|
+
return merged
|
|
609
819
|
}
|
|
610
820
|
}
|
|
611
821
|
|
|
@@ -684,7 +894,8 @@ export function expandSegment(archiveRoot, options = {}) {
|
|
|
684
894
|
}
|
|
685
895
|
|
|
686
896
|
export const name = 'metabolism-recall'
|
|
687
|
-
|
|
897
|
+
// llm 是骨架工具回填时用来写卡的(不传就报 cannot get property "llm" without inject)
|
|
898
|
+
export const inject = ['tools', 'llm']
|
|
688
899
|
|
|
689
900
|
const PARAMETERS = {
|
|
690
901
|
type: 'object',
|
|
@@ -802,4 +1013,25 @@ export function apply(ctx, config) {
|
|
|
802
1013
|
return
|
|
803
1014
|
}
|
|
804
1015
|
ctx.tools.register(definition)
|
|
1016
|
+
|
|
1017
|
+
// 骨架工具同批挂上;它自己挂了也不影响召回
|
|
1018
|
+
try {
|
|
1019
|
+
skeletonApply(ctx, { archiveRoot })
|
|
1020
|
+
} catch (error) {
|
|
1021
|
+
log(`骨架工具没挂上:${error?.message ?? error}`)
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
// 浮现:默认关,开关.json 里 "浮现": true 才注入
|
|
1025
|
+
try {
|
|
1026
|
+
morphApply(ctx, { archiveRoot })
|
|
1027
|
+
} catch (error) {
|
|
1028
|
+
log(`浮现没挂上:${error?.message ?? error}`)
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
// 贴条:默认关,开关.json 里 "每轮一条": true 才动
|
|
1032
|
+
try {
|
|
1033
|
+
attachApply(ctx, { archiveRoot })
|
|
1034
|
+
} catch (error) {
|
|
1035
|
+
log(`贴条没挂上:${error?.message ?? error}`)
|
|
1036
|
+
}
|
|
805
1037
|
}
|
|
@@ -63,7 +63,7 @@ export async function embedTexts(texts, workspace, options = {}) {
|
|
|
63
63
|
const prefix = options.query === true ? QUERY_PREFIX : ''
|
|
64
64
|
for (let start = 0; start < texts.length; start += batch) {
|
|
65
65
|
const slice = texts.slice(start, start + batch).map((text) => prefix + String(text).slice(0, MAX_CHARS))
|
|
66
|
-
const result = await extract(slice, { pooling: 'mean', normalize: true })
|
|
66
|
+
const result = await extract(slice, { pooling: options.pooling ?? 'mean', normalize: true })
|
|
67
67
|
const flat = result.data
|
|
68
68
|
const dims = result.dims
|
|
69
69
|
for (let row = 0; row < slice.length; row += 1) {
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 守卫:模型生成的卡落盘前过闸。设计稿 v3.1 第四节。
|
|
3
|
+
*
|
|
4
|
+
* 方向修正(2026-11-19 实测后):设计稿写的是「原文有的高信号,卡里必须逐字存活」,
|
|
5
|
+
* 实测不可行——摘要本来就不会抄全部路径,按那个跑,十张卡九张被退回。
|
|
6
|
+
* 真正会出事的是反过来的情况:卡里写了某个路径/编号,原文里根本没有(编的),
|
|
7
|
+
* 或者写错了一个字符(对不上)。所以改成「以卡为准,回原文核对」。
|
|
8
|
+
*
|
|
9
|
+
* 三档处置:
|
|
10
|
+
* - 对得上的:放行,记一行。
|
|
11
|
+
* - 少量对不上(不超过一半):把那几行原话删掉,字段里带的也清成留白——宁可少说。
|
|
12
|
+
* - 一半以上对不上:这张卡整条不用,退回只写指针。
|
|
13
|
+
*
|
|
14
|
+
* 第四条老规矩照旧:判不了就留白,这里不编。
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** 最多核对几条,免得一张大卡拖慢压缩。 */
|
|
18
|
+
const MAX_CHECK = 60
|
|
19
|
+
|
|
20
|
+
/** 抓高信号串:路径、URL、UUID、全大写错误码、key=value、三位以上数字。 */
|
|
21
|
+
export function extractSignals(text) {
|
|
22
|
+
if (typeof text !== 'string' || text.length === 0) return []
|
|
23
|
+
const found = new Set()
|
|
24
|
+
const rules = [
|
|
25
|
+
/[A-Za-z]:\\[^\s,。;、)】"'<>|*?]+/g, // Windows 路径
|
|
26
|
+
/https?:\/\/[^\s,。;)】"'<>]+/g, // URL
|
|
27
|
+
/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, // UUID
|
|
28
|
+
/\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b/g, // 带下划线的全大写(EPERM_ACCESS_DENIED)
|
|
29
|
+
/\b[\w.-]+=[^\s,。;、)】"']{2,}/g, // key=value
|
|
30
|
+
/\d{3,}/g, // 三位以上数字
|
|
31
|
+
]
|
|
32
|
+
for (const rule of rules) {
|
|
33
|
+
for (const hit of text.match(rule) ?? []) {
|
|
34
|
+
const clean = hit.replace(/[.,;:)\]]+$/, '')
|
|
35
|
+
if (clean.length >= 4 && clean.length <= 200) found.add(clean)
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return [...found].slice(0, MAX_CHECK)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** 把一张卡里的所有文字拼起来。 */
|
|
42
|
+
function cardText(card) {
|
|
43
|
+
return [
|
|
44
|
+
card.doing,
|
|
45
|
+
...(Array.isArray(card.requirements) ? card.requirements : []),
|
|
46
|
+
...(Array.isArray(card.done) ? card.done : []),
|
|
47
|
+
card.next,
|
|
48
|
+
...(Array.isArray(card.quotes) ? card.quotes : []),
|
|
49
|
+
...(Array.isArray(card.keywords) ? card.keywords : []),
|
|
50
|
+
]
|
|
51
|
+
.filter((line) => typeof line === 'string' && line.length > 0)
|
|
52
|
+
.join('\n')
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** 这一行里带了不该带的串,就整行换掉。 */
|
|
56
|
+
function scrub(line, bad, placeholder) {
|
|
57
|
+
const text = String(line ?? '')
|
|
58
|
+
if (text.length === 0) return text
|
|
59
|
+
return bad.some((signal) => text.includes(signal)) ? placeholder : text
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* 过闸。返回 { card, notes, dropped, missing }。
|
|
64
|
+
* dropped 为 true 表示这张卡不能用,调用方应退回只写指针。
|
|
65
|
+
*/
|
|
66
|
+
export function guardCard(card, sample, options = {}) {
|
|
67
|
+
const notes = []
|
|
68
|
+
if (card === null || card === undefined) {
|
|
69
|
+
return { card: null, notes, dropped: false, missing: [] }
|
|
70
|
+
}
|
|
71
|
+
const said = extractSignals(cardText(card))
|
|
72
|
+
if (said.length === 0) {
|
|
73
|
+
notes.push('卡里没有可核对的高信号串')
|
|
74
|
+
return { card, notes, dropped: false, missing: [] }
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// 原文样本作白名单。样本本身是摘录,可能不全,所以只在「对不上」时才动手,
|
|
78
|
+
// 且要超过一半才整条退回——宁可漏掉几条,也不要把好卡判死。
|
|
79
|
+
const source = new Set(extractSignals(sample))
|
|
80
|
+
if (source.size === 0) {
|
|
81
|
+
notes.push('没有原文样本,跳过核对')
|
|
82
|
+
return { card, notes, dropped: false, missing: [] }
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const wrong = said.filter((signal) => !source.has(signal))
|
|
86
|
+
if (wrong.length === 0) {
|
|
87
|
+
notes.push(`卡里 ${said.length} 条高信号全部对得上原文`)
|
|
88
|
+
return { card, notes, dropped: false, missing: [] }
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const ratio = wrong.length / said.length
|
|
92
|
+
const dropRatio = Number(options.dropRatio) > 0 ? Number(options.dropRatio) : 0.5
|
|
93
|
+
if (ratio > dropRatio) {
|
|
94
|
+
notes.push(`卡里 ${wrong.length}/${said.length} 条高信号原文里没有(${Math.round(ratio * 100)}%),整条退回指针`)
|
|
95
|
+
return { card: null, notes, dropped: true, missing: wrong }
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// 少量对不上:把带错的行清掉,剩下的照用
|
|
99
|
+
const blank = '(原文对不上,已留白)'
|
|
100
|
+
card.quotes = (Array.isArray(card.quotes) ? card.quotes : []).filter(
|
|
101
|
+
(line) => !wrong.some((signal) => String(line).includes(signal)),
|
|
102
|
+
)
|
|
103
|
+
card.doing = scrub(card.doing, wrong, blank)
|
|
104
|
+
card.next = scrub(card.next, wrong, blank)
|
|
105
|
+
card.requirements = (Array.isArray(card.requirements) ? card.requirements : [])
|
|
106
|
+
.filter((line) => !wrong.some((signal) => String(line).includes(signal)))
|
|
107
|
+
card.done = (Array.isArray(card.done) ? card.done : [])
|
|
108
|
+
.filter((line) => !wrong.some((signal) => String(line).includes(signal)))
|
|
109
|
+
notes.push(`卡里 ${wrong.length}/${said.length} 条高信号原文里没有,已清掉那几行:${wrong.slice(0, 5).join('、')}`)
|
|
110
|
+
return { card, notes, dropped: false, missing: wrong }
|
|
111
|
+
}
|