dsh-shared-kb 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,194 @@
1
+ /**
2
+ * 检索:全文 + 向量 + 融合排序。
3
+ *
4
+ * 为什么必须混合(hybrid):
5
+ *
6
+ * - 纯向量检索对「标准号、型号、条款号、专有名词缩写」不敏感。语义相近
7
+ * 不代表字面匹配:搜 "GB/T 19963.2-2024" 时,向量只认「标准文件」这个
8
+ * 语义,会把一堆不相关标准排前面;全文检索则精准命中。
9
+ * - 纯全文检索对「同义改写」无能为力:用户问「风机并网要做哪些检测」,
10
+ * 原文写「风电机组接入电力系统的试验项目」,没有共同 token。
11
+ *
12
+ * 融合方式:RRF(Reciprocal Rank Fusion)。它只看排名不看分数,
13
+ * 因此不需要把 BM25 的「越小越好」与余弦的「越大越好」硬凑到同一量纲,
14
+ * 也不会因为某一路的分数尺度不同而让另一路失效。常数 k 取 60(原论文推荐)。
15
+ *
16
+ * 额外加一层「字面加分」:命中查询里的罕见 token 时给一个小的固定加分,
17
+ * 用于处理 RRF 排序后仍然并列的情况(比如两段文字都语义相关,但只有一段
18
+ * 真的出现了那个型号)。
19
+ */
20
+ import { buildMatchQuery, hasQueryTokens, makeSnippet, tokenize } from "../indexer/tokenize.js";
21
+
22
+ export const DEFAULT_LIMIT = 20;
23
+ export const RRF_K = 60;
24
+ /** 语义一路的最低分(余弦)。低于它认为「不相关」,不参与融合,避免噪声被 RRF 抬上来。 */
25
+ export const MIN_VECTOR_SCORE = 0.12;
26
+
27
+ /**
28
+ * 融合两路结果。
29
+ *
30
+ * @param {{docId, part, score, text?, title?, heading?, page?}[]} fullText
31
+ * @param {{docId, part, score, text?, title?, heading?, page?}[]} vector
32
+ * @param {{k?: number, weights?: {fullText?: number, vector?: number}, query?: string}} [options]
33
+ */
34
+ export function fuseResults(fullText, vector, options = {}) {
35
+ const k = Number(options.k) || RRF_K;
36
+ const wF = options.weights?.fullText ?? 1;
37
+ const wV = options.weights?.vector ?? 1;
38
+ const queryTokens = options.query ? new Set(tokenize(options.query, { bigram: false })) : null;
39
+
40
+ /** key = docId#part */
41
+ const merged = new Map();
42
+
43
+ const ingest = (rows, weight, channel) => {
44
+ rows.forEach((row, rank) => {
45
+ const key = `${row.docId}#${row.part}`;
46
+ const existing =
47
+ merged.get(key) ??
48
+ {
49
+ docId: row.docId,
50
+ part: row.part,
51
+ text: row.text ?? "",
52
+ title: row.title ?? "",
53
+ heading: row.heading ?? "",
54
+ page: row.page ?? null,
55
+ channels: {},
56
+ rrf: 0,
57
+ rawScore: 0,
58
+ };
59
+ existing.rrf += weight * (1 / (k + rank + 1));
60
+ existing.channels[channel] = { rank: rank + 1, score: row.score };
61
+ if (row.text) existing.text = row.text;
62
+ if (row.title) existing.title = row.title;
63
+ if (row.heading) existing.heading = row.heading;
64
+ if (row.page != null) existing.page = row.page;
65
+ if (!Number.isNaN(Number(row.score))) existing.rawScore += Number(row.score);
66
+ merged.set(key, existing);
67
+ });
68
+ };
69
+
70
+ ingest(fullText, wF, "fullText");
71
+ ingest(vector, wV, "vector");
72
+
73
+ const out = [...merged.values()].map((row) => {
74
+ let bonus = 0;
75
+ if (queryTokens) {
76
+ const haystack = `${row.title}\n${row.heading}\n${row.text}`;
77
+ const lower = haystack.toLowerCase();
78
+ let hits = 0;
79
+ for (const token of queryTokens) {
80
+ if (token.length >= 2 && lower.includes(token)) hits += 1;
81
+ }
82
+ // 命中越多加分越多,但上限压住,不让它盖过 RRF
83
+ bonus = Math.min(0.0009, hits * 0.00015);
84
+ }
85
+ return { ...row, score: row.rrf + bonus };
86
+ });
87
+
88
+ out.sort((a, b) => b.score - a.score || String(a.docId).localeCompare(String(b.docId)) || a.part - b.part);
89
+ return out;
90
+ }
91
+
92
+ /**
93
+ * 混合检索。
94
+ *
95
+ * @param {object} params
96
+ * @param {import("../indexer/sqlite-index.js").IndexHandle} params.index
97
+ * @param {string} params.query
98
+ * @param {{provider: any}|null} [params.embedder]
99
+ * @param {{mode?: "hybrid"|"fulltext"|"semantic", limit?: number, docIds?: string[], minVectorScore?: number}} [params.options]
100
+ */
101
+ export async function search(params) {
102
+ const { index, query, embedder = null } = params;
103
+ const options = params.options ?? {};
104
+ const mode = ["hybrid", "fulltext", "semantic"].includes(options.mode) ? options.mode : "hybrid";
105
+ const limit = Math.max(1, Math.min(100, Math.round(Number(options.limit) || DEFAULT_LIMIT)));
106
+ const docIds = Array.isArray(options.docIds) && options.docIds.length > 0 ? options.docIds : null;
107
+ const minVectorScore = Number.isFinite(Number(options.minVectorScore))
108
+ ? Number(options.minVectorScore)
109
+ : MIN_VECTOR_SCORE;
110
+
111
+ const diagnostics = { mode, usedVector: false, vectorNote: null, fullTextHits: 0, vectorHits: 0 };
112
+ let fullText = [];
113
+ let vector = [];
114
+
115
+ if (mode !== "semantic") {
116
+ if (!hasQueryTokens(query)) {
117
+ return { hits: [], diagnostics: { ...diagnostics, note: "查询没有有效 token(纯标点或空串)" } };
118
+ }
119
+ const expr = buildMatchQuery(query, { mode: "or" });
120
+ fullText = index.searchFullText(expr, { limit: limit * 4, docIds });
121
+ diagnostics.fullTextHits = fullText.length;
122
+ }
123
+
124
+ if (mode !== "fulltext") {
125
+ if (!embedder) {
126
+ diagnostics.vectorNote = "未配置语义 provider,本次只用全文检索";
127
+ } else {
128
+ try {
129
+ const embedOne = embedder.embedQuery ? (text) => embedder.embedQuery(text) : async (text) => (await embedder.embed([text]))[0];
130
+ const queryVector = await embedOne(query);
131
+ if (queryVector && queryVector.length > 0) {
132
+ const raw = index.searchVector(queryVector, { limit: limit * 4, docIds });
133
+ vector = raw.filter((row) => row.score >= minVectorScore);
134
+ diagnostics.usedVector = true;
135
+ diagnostics.vectorHits = vector.length;
136
+ if (raw.length > 0 && vector.length === 0) {
137
+ diagnostics.vectorNote = `向量最高分 ${raw[0].score.toFixed(3)} 低于阈值 ${minVectorScore},本次视为无语义命中`;
138
+ }
139
+ } else {
140
+ diagnostics.vectorNote = "查询向量为空";
141
+ }
142
+ } catch (error) {
143
+ diagnostics.vectorNote = `语义检索失败,已回退全文:${String(error?.message ?? error)}`;
144
+ }
145
+ }
146
+ }
147
+
148
+ const fused = fuseResults(fullText, vector, { query });
149
+ const hits = fused.slice(0, limit).map((row) => ({
150
+ docId: row.docId,
151
+ part: row.part,
152
+ title: row.title,
153
+ heading: row.heading,
154
+ page: row.page,
155
+ score: Number(row.score.toFixed(6)),
156
+ channels: row.channels,
157
+ snippet: makeSnippet(row.text, query),
158
+ }));
159
+
160
+ return { hits, diagnostics };
161
+ }
162
+
163
+ /**
164
+ * 把命中按文档聚合:同一篇文档的多个块合成一条,取最好的块做代表。
165
+ * 这是检索结果给人看时的默认视图(用户关心「哪份资料相关」,不是「哪一块」)。
166
+ */
167
+ export function groupByDoc(hits, { perDoc = 1 } = {}) {
168
+ const map = new Map();
169
+ for (const hit of hits) {
170
+ const bucket = map.get(hit.docId) ?? { docId: hit.docId, title: hit.title, best: null, parts: [], score: 0 };
171
+ bucket.parts.push(hit.part);
172
+ bucket.score = Math.max(bucket.score, hit.score);
173
+ if (!bucket.best || hit.score > bucket.best.score) bucket.best = hit;
174
+ if (!bucket.title && hit.title) bucket.title = hit.title;
175
+ map.set(hit.docId, bucket);
176
+ }
177
+ return [...map.values()]
178
+ .map((bucket) => ({
179
+ docId: bucket.docId,
180
+ title: bucket.title,
181
+ score: bucket.score,
182
+ matchedParts: bucket.parts.length,
183
+ parts: bucket.parts.sort((a, b) => a - b).slice(0, 20),
184
+ top: bucket.best
185
+ ? {
186
+ part: bucket.best.part,
187
+ heading: bucket.best.heading,
188
+ page: bucket.best.page,
189
+ snippet: bucket.best.snippet,
190
+ }
191
+ : null,
192
+ }))
193
+ .sort((a, b) => b.score - a.score);
194
+ }