@sparkelf/dsh-knowledge-base 0.1.28 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +616 -578
- package/lib/client.js.map +1 -1
- package/lib/index.js +33 -9
- package/lib/types/ftsIndex.d.ts +12 -0
- package/package.json +1 -1
package/lib/index.js
CHANGED
|
@@ -9053,6 +9053,31 @@ async function syncIndexUnlocked(options) {
|
|
|
9053
9053
|
* few unreadable descendants cannot starve the result; the returned list is still
|
|
9054
9054
|
* capped at `topK`.
|
|
9055
9055
|
*/
|
|
9056
|
+
/**
|
|
9057
|
+
* 把一段连续中文切成长度 3 的滑动窗口(去重,最多 `limit` 个,默认 12)。
|
|
9058
|
+
*
|
|
9059
|
+
* trigram FTS 只能匹配长度 ≥3 的片段:整句中文当 phrase 查通常 0 命中,
|
|
9060
|
+
* 而窗口化之后「加油站」这样的 trigram 就能命中索引,再按 chunk 合并回来。
|
|
9061
|
+
*
|
|
9062
|
+
* 成本上限(有意为之,别去掉):只有在「整句 phrase 查询返回 0 行」时才会走到这里,
|
|
9063
|
+
* 且最多展开 `limit` 个窗口 → 最多多跑 `limit` 次 FTS 查询;每个窗口仍受原来的
|
|
9064
|
+
* `candidateWindow`(topK*4)限制,所以单次检索的 SQL 次数与返回量都是常数级的。
|
|
9065
|
+
* 12 个窗口足够覆盖一句 14 字左右的中文提问里最有信息量的前缀片段;再长也不会增加开销。
|
|
9066
|
+
*/
|
|
9067
|
+
function trigramWindows(text, limit = 12) {
|
|
9068
|
+
const chars = [...text].filter((char) => /[\p{Script=Han}A-Za-z0-9]/u.test(char));
|
|
9069
|
+
if (chars.length < 3) return [];
|
|
9070
|
+
const windows = [];
|
|
9071
|
+
const seen = /* @__PURE__ */ new Set();
|
|
9072
|
+
for (let i = 0; i + 3 <= chars.length; i += 1) {
|
|
9073
|
+
const window = chars.slice(i, i + 3).join("");
|
|
9074
|
+
if (seen.has(window)) continue;
|
|
9075
|
+
seen.add(window);
|
|
9076
|
+
windows.push(window);
|
|
9077
|
+
if (windows.length >= limit) break;
|
|
9078
|
+
}
|
|
9079
|
+
return windows;
|
|
9080
|
+
}
|
|
9056
9081
|
async function searchIndex(options) {
|
|
9057
9082
|
return withIndexLock(() => withBusyRetry(() => searchIndexUnlocked(options)));
|
|
9058
9083
|
}
|
|
@@ -9089,13 +9114,11 @@ async function searchIndexUnlocked(options) {
|
|
|
9089
9114
|
ensureChunkColumns(db);
|
|
9090
9115
|
const clauses = ["d.scope = 'global'"];
|
|
9091
9116
|
const params = [];
|
|
9092
|
-
|
|
9093
|
-
|
|
9094
|
-
|
|
9095
|
-
|
|
9096
|
-
|
|
9097
|
-
params.push(...options.caller.departmentCodes);
|
|
9098
|
-
}
|
|
9117
|
+
clauses.push("(d.scope = 'personal' AND d.owner_id = ?)");
|
|
9118
|
+
params.push(options.caller.callerId);
|
|
9119
|
+
if (options.caller.departmentCodes.length > 0) {
|
|
9120
|
+
clauses.push(`(d.scope = 'department' AND d.department_code IN (${options.caller.departmentCodes.map(() => "?").join(",")}))`);
|
|
9121
|
+
params.push(...options.caller.departmentCodes);
|
|
9099
9122
|
}
|
|
9100
9123
|
const narrowed = options.scope === "personal" || options.scope === "department" || options.scope === "global" ? options.scope : void 0;
|
|
9101
9124
|
const scopeClause = narrowed === void 0 ? "" : " AND d.scope = ?";
|
|
@@ -9133,7 +9156,8 @@ async function searchIndexUnlocked(options) {
|
|
|
9133
9156
|
};
|
|
9134
9157
|
const rows = runTerm(query);
|
|
9135
9158
|
const tokens = query.split(/\s+/u).map((token) => token.trim()).filter((token) => token !== "");
|
|
9136
|
-
const
|
|
9159
|
+
const cjkWindows = tokens.length === 1 && rows.length === 0 ? trigramWindows(query) : [];
|
|
9160
|
+
const relaxed = tokens.length > 1 && rows.length < 2 || cjkWindows.length > 0;
|
|
9137
9161
|
const merged = /* @__PURE__ */ new Map();
|
|
9138
9162
|
const addRows = (candidates) => {
|
|
9139
9163
|
for (const row of candidates) {
|
|
@@ -9142,7 +9166,7 @@ async function searchIndexUnlocked(options) {
|
|
|
9142
9166
|
if (existing === void 0 || row.score < existing.score) merged.set(key, row);
|
|
9143
9167
|
}
|
|
9144
9168
|
};
|
|
9145
|
-
const queries = relaxed ? tokens.slice(0, 8) : [query];
|
|
9169
|
+
const queries = relaxed ? cjkWindows.length > 0 ? cjkWindows : tokens.slice(0, 8) : [query];
|
|
9146
9170
|
if (relaxed) for (const token of queries) addRows(runTerm(token));
|
|
9147
9171
|
else addRows(rows);
|
|
9148
9172
|
const ordered = [...merged.values()].sort((a, b) => {
|
package/lib/types/ftsIndex.d.ts
CHANGED
|
@@ -287,6 +287,18 @@ export declare function indexStats(indexPath: string): Promise<{
|
|
|
287
287
|
* few unreadable descendants cannot starve the result; the returned list is still
|
|
288
288
|
* capped at `topK`.
|
|
289
289
|
*/
|
|
290
|
+
/**
|
|
291
|
+
* 把一段连续中文切成长度 3 的滑动窗口(去重,最多 `limit` 个,默认 12)。
|
|
292
|
+
*
|
|
293
|
+
* trigram FTS 只能匹配长度 ≥3 的片段:整句中文当 phrase 查通常 0 命中,
|
|
294
|
+
* 而窗口化之后「加油站」这样的 trigram 就能命中索引,再按 chunk 合并回来。
|
|
295
|
+
*
|
|
296
|
+
* 成本上限(有意为之,别去掉):只有在「整句 phrase 查询返回 0 行」时才会走到这里,
|
|
297
|
+
* 且最多展开 `limit` 个窗口 → 最多多跑 `limit` 次 FTS 查询;每个窗口仍受原来的
|
|
298
|
+
* `candidateWindow`(topK*4)限制,所以单次检索的 SQL 次数与返回量都是常数级的。
|
|
299
|
+
* 12 个窗口足够覆盖一句 14 字左右的中文提问里最有信息量的前缀片段;再长也不会增加开销。
|
|
300
|
+
*/
|
|
301
|
+
export declare function trigramWindows(text: string, limit?: number): string[];
|
|
290
302
|
export declare function searchIndex(options: {
|
|
291
303
|
indexPath: string;
|
|
292
304
|
query: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sparkelf/dsh-knowledge-base",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.30",
|
|
4
4
|
"description": "DataOps-backed knowledge base for DeepSeek Harness: a left-sidebar destination, Settings configuration, and index tooling over a mounted NAS view.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./lib/index.js",
|