dsh-free-search 0.8.6 → 0.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +93 -1
- package/lib/domain_filter.js +100 -0
- package/lib/index.js +38 -8
- package/lib/research.js +5 -0
- package/package.json +3 -2
package/lib/client.js
CHANGED
|
@@ -51,6 +51,13 @@ window.__ModuleLoader__.load({
|
|
|
51
51
|
".dshfs-select{color-scheme:light dark}",
|
|
52
52
|
".dshfs-select option, .dshfs-select optgroup{background-color:#ffffff;color:#1d1d1f}",
|
|
53
53
|
"@media (prefers-color-scheme: dark){.dshfs-select{color-scheme:dark}.dshfs-select option, .dshfs-select optgroup{background-color:#1e1f24;color:#f5f5f7}}",
|
|
54
|
+
".dshfs-textarea{border:1px solid rgba(0,0,0,.1);font:inherit;font-variant-numeric:tabular-nums;color:inherit;background:rgba(0,0,0,.03);border-radius:10px;padding:8px 12px;min-height:96px;height:auto;font-size:14px;line-height:1.5;transition:all 160ms cubic-bezier(.4,0,.2,1);width:100%;box-sizing:border-box;resize:vertical;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,\"Liberation Mono\",monospace}",
|
|
55
|
+
"@media (prefers-color-scheme: dark){.dshfs-textarea{border:1px solid rgba(255,255,255,.1);background:rgba(255,255,255,.05)}}",
|
|
56
|
+
".dshfs-textarea:hover:not(:disabled){border-color:rgba(0,0,0,.2);background:rgba(0,0,0,.05)}",
|
|
57
|
+
"@media (prefers-color-scheme: dark){.dshfs-textarea:hover:not(:disabled){border-color:rgba(255,255,255,.2);background:rgba(255,255,255,.08)}}",
|
|
58
|
+
".dshfs-textarea:focus-visible{outline:none;border-color:#0071e3;box-shadow:0 0 0 3px rgba(0,113,227,.25);background:#ffffff}",
|
|
59
|
+
"@media (prefers-color-scheme: dark){.dshfs-textarea:focus-visible{border-color:#0a84ff;box-shadow:0 0 0 3px rgba(10,132,255,.25);background:#1c1c1e}}",
|
|
60
|
+
".dshfs-textarea:disabled{opacity:.5;cursor:default}",
|
|
54
61
|
".dshfs-ttl{width:100px}",
|
|
55
62
|
".dshfs-fieldRow{display:flex;align-items:center;gap:12px;flex-wrap:wrap}",
|
|
56
63
|
".dshfs-keyStorage{width:auto;min-width:200px}",
|
|
@@ -60,7 +67,7 @@ window.__ModuleLoader__.load({
|
|
|
60
67
|
".dshfs-platform{display:flex;align-items:center;gap:6px;color:inherit;font-size:14px;cursor:pointer}",
|
|
61
68
|
".dshfs-platform input{accent-color:#0071e3;width:16px;height:16px}",
|
|
62
69
|
".dshfs-priorityList{display:flex;flex-direction:column;gap:6px}",
|
|
63
|
-
".dshfs-priorityRow{display:flex;align-items:center;gap:8px;padding:6px 8px;border:1px solid rgba(0,0,0,.08);border-radius:10px;background:rgba(0,0,0,.02)}",
|
|
70
|
+
".dshfs-priorityRow{display:flex;align-items:center;gap:8px;padding:6px 8px;border:1px solid rgba(0,0,0,.08);border-radius:10px;background:rgba(0,0,0,.02);box-sizing:border-box;min-height:42px}",
|
|
64
71
|
"@media (prefers-color-scheme: dark){.dshfs-priorityRow{border-color:rgba(255,255,255,.09);background:rgba(255,255,255,.035)}}",
|
|
65
72
|
".dshfs-priorityRowDisabled{opacity:.58}",
|
|
66
73
|
".dshfs-priorityIndex{width:26px;flex:none;color:#6e6e73;font-size:12px;font-variant-numeric:tabular-nums;text-align:right}",
|
|
@@ -204,6 +211,9 @@ window.__ModuleLoader__.load({
|
|
|
204
211
|
platformHint: "为 agent 的 platform_search 工具启用平台。禁用的平台会被跳过。",
|
|
205
212
|
cacheTtl: "结果缓存时长(分钟)",
|
|
206
213
|
cacheTtlHint: "0 关闭缓存,最长 5 分钟。缩短可加快时效,延长可防限流、省额度。",
|
|
214
|
+
blockedDomainsLabel: "域名黑名单(blockedDomains)",
|
|
215
|
+
blockedDomainsHint: "一行一个域名,命中的站点及其所有子域都会从搜索结果里剔除(csdn.net 覆盖 blog.csdn.net)。留空=不过滤。可以直接粘贴完整网址,scheme、路径和端口会被自动去掉。",
|
|
216
|
+
blockedDomainsWarn: (corrected, dupes, dropped) => `已忽略 ${dupes} 个重复项、${dropped} 行无法识别;${corrected} 行已自动修正为裸域名。规则对下一次搜索生效,旧结果缓存会同步失效。`,
|
|
207
217
|
unavailable: "设置不可用 —— free-search 桥接未暴露。",
|
|
208
218
|
saveFailed: "保存失败",
|
|
209
219
|
saveFailedDetail: (d) => `保存失败:${d}`,
|
|
@@ -310,6 +320,9 @@ window.__ModuleLoader__.load({
|
|
|
310
320
|
platformHint: "Enable platforms for the agent's platform_search tool. Disabled platforms are skipped.",
|
|
311
321
|
cacheTtl: "Result cache TTL (minutes)",
|
|
312
322
|
cacheTtlHint: "0 disables caching, max 5 minutes. Lower = fresher results, higher = less rate-limiting / fewer credits used.",
|
|
323
|
+
blockedDomainsLabel: "Domain blocklist (blockedDomains)",
|
|
324
|
+
blockedDomainsHint: "One domain per line. A matched site and all of its subdomains are dropped from search results (csdn.net covers blog.csdn.net). Leave empty to disable. Full URLs are accepted — scheme, path and port are stripped automatically.",
|
|
325
|
+
blockedDomainsWarn: (corrected, dupes, dropped) => `Ignored ${dupes} duplicate(s) and ${dropped} unrecognized line(s); corrected ${corrected} line(s) to bare domains. Rules apply to the next search, and the result cache is invalidated accordingly.`,
|
|
313
326
|
unavailable: "Settings unavailable — the free-search bridge is not exposed.",
|
|
314
327
|
saveFailed: "save failed",
|
|
315
328
|
saveFailedDetail: (d) => `save failed: ${d}`,
|
|
@@ -414,6 +427,45 @@ window.__ModuleLoader__.load({
|
|
|
414
427
|
if (!Array.isArray(value)) return FAILURE_CLASSES.map((cls) => cls.id);
|
|
415
428
|
return [...new Set(value.filter((id) => FAILURE_CLASSES.some((cls) => cls.id === id)))];
|
|
416
429
|
};
|
|
430
|
+
// 域名黑名单:本地解析(显示 + 保存)。
|
|
431
|
+
// 注意:清洗规则必须与 lib/domain_filter.js 的 normalizeDomainList 保持一致——
|
|
432
|
+
// 服务端也会规范化一次(用户可直接编辑 YAML 绕过 UI),两边结果相同才不会有"存进去对、读回来错"。
|
|
433
|
+
// 这里额外返回统计量,用于字段下方的非阻断提示。
|
|
434
|
+
const parseBlockedDomains = (text) => {
|
|
435
|
+
const lines = String(text ?? "")
|
|
436
|
+
.split("\n")
|
|
437
|
+
.map((line) => line.trim())
|
|
438
|
+
.filter(Boolean);
|
|
439
|
+
const stats = { corrected: 0, dupes: 0, dropped: 0 };
|
|
440
|
+
const seen = new Set();
|
|
441
|
+
const domains = [];
|
|
442
|
+
for (const line of lines) {
|
|
443
|
+
let s = line.toLowerCase();
|
|
444
|
+
const before = s;
|
|
445
|
+
s = s.replace(/^[a-z][a-z0-9+.-]*:\/\//, "");
|
|
446
|
+
const cut = s.search(/[/?#]/);
|
|
447
|
+
if (cut >= 0) s = s.slice(0, cut);
|
|
448
|
+
if (!s) {
|
|
449
|
+
stats.dropped++;
|
|
450
|
+
continue;
|
|
451
|
+
}
|
|
452
|
+
const colon = s.lastIndexOf(":");
|
|
453
|
+
if (colon >= 0 && /^\d+$/.test(s.slice(colon + 1))) s = s.slice(0, colon);
|
|
454
|
+
s = s.replace(/^www\./, "").replace(/\.+$/, "");
|
|
455
|
+
if (!s || !/^[a-z0-9][a-z0-9.-]*$/.test(s) || s.startsWith(".")) {
|
|
456
|
+
stats.dropped++;
|
|
457
|
+
continue;
|
|
458
|
+
}
|
|
459
|
+
if (before !== s) stats.corrected++;
|
|
460
|
+
if (seen.has(s)) {
|
|
461
|
+
stats.dupes++;
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
464
|
+
seen.add(s);
|
|
465
|
+
domains.push(s);
|
|
466
|
+
}
|
|
467
|
+
return { domains, stats };
|
|
468
|
+
};
|
|
417
469
|
|
|
418
470
|
async function bridgeDescribe() {
|
|
419
471
|
const response = await fetch(`${BRIDGE_PREFIX}/describe`, {
|
|
@@ -520,6 +572,7 @@ window.__ModuleLoader__.load({
|
|
|
520
572
|
const [zhPool, setZhPool] = react.useState(DEFAULT_ZH_POOL);
|
|
521
573
|
const [enPool, setEnPool] = react.useState(DEFAULT_EN_POOL);
|
|
522
574
|
const [fallbackOn, setFallbackOn] = react.useState(FAILURE_CLASSES.map((cls) => cls.id));
|
|
575
|
+
const [blockedDomains, setBlockedDomains] = react.useState("");
|
|
523
576
|
const [safeSearch, setSafeSearch] = react.useState("off");
|
|
524
577
|
const [bingMarket, setBingMarket] = react.useState("zh-CN");
|
|
525
578
|
const [anysearchKey, setAnysearchKey] = react.useState("");
|
|
@@ -624,6 +677,8 @@ window.__ModuleLoader__.load({
|
|
|
624
677
|
setZhPool(normalizePoolIds(v.zhPool, DEFAULT_ZH_POOL));
|
|
625
678
|
setEnPool(normalizePoolIds(v.enPool, DEFAULT_EN_POOL));
|
|
626
679
|
setFallbackOn(normalizeFallbackOn(v.fallbackOn));
|
|
680
|
+
// 黑名单以文本形式回显(一行一个);配置里存的是数组
|
|
681
|
+
setBlockedDomains((Array.isArray(v.blockedDomains) ? v.blockedDomains : []).join("\n"));
|
|
627
682
|
setSafeSearch(v.safeSearch === "strict" || v.safeSearch === "moderate" ? v.safeSearch : "off");
|
|
628
683
|
setBingMarket(v.bingMarket === undefined ? "zh-CN" : v.bingMarket);
|
|
629
684
|
setLang(v.lang === "en" ? "en" : "zh");
|
|
@@ -763,6 +818,7 @@ window.__ModuleLoader__.load({
|
|
|
763
818
|
ops.push({ op: "set", path: ["zhPool"], value: normalizePoolIds(zhPool, DEFAULT_ZH_POOL) });
|
|
764
819
|
ops.push({ op: "set", path: ["enPool"], value: normalizePoolIds(enPool, DEFAULT_EN_POOL) });
|
|
765
820
|
ops.push({ op: "set", path: ["fallbackOn"], value: normalizeFallbackOn(fallbackOn) });
|
|
821
|
+
ops.push({ op: "set", path: ["blockedDomains"], value: parseBlockedDomains(blockedDomains).domains });
|
|
766
822
|
ops.push({ op: "set", path: ["lang"], value: lang });
|
|
767
823
|
ops.push({ op: "set", path: ["keyStorage"], value: keyStorage });
|
|
768
824
|
ops.push({ op: "set", path: ["safeSearch"], value: safeSearch });
|
|
@@ -1340,6 +1396,42 @@ react_jsx_runtime.jsx("div", {
|
|
|
1340
1396
|
}),
|
|
1341
1397
|
],
|
|
1342
1398
|
}),
|
|
1399
|
+
react_jsx_runtime.jsx("div", {
|
|
1400
|
+
className: "dshfs-field",
|
|
1401
|
+
children: [
|
|
1402
|
+
react_jsx_runtime.jsx("div", {
|
|
1403
|
+
className: "dshfs-label",
|
|
1404
|
+
children: t.blockedDomainsLabel,
|
|
1405
|
+
}),
|
|
1406
|
+
react_jsx_runtime.jsx("textarea", {
|
|
1407
|
+
className: "dshfs-textarea",
|
|
1408
|
+
rows: 5,
|
|
1409
|
+
placeholder: "csdn.net\n51cto.com",
|
|
1410
|
+
value: blockedDomains,
|
|
1411
|
+
spellCheck: false,
|
|
1412
|
+
disabled: !ready || saving,
|
|
1413
|
+
onChange: (e) => {
|
|
1414
|
+
setBlockedDomains(e.target.value);
|
|
1415
|
+
setDirty(true);
|
|
1416
|
+
setFailed(false);
|
|
1417
|
+
setSaveError("");
|
|
1418
|
+
},
|
|
1419
|
+
}),
|
|
1420
|
+
react_jsx_runtime.jsx("p", {
|
|
1421
|
+
className: "dshfs-hint",
|
|
1422
|
+
children: t.blockedDomainsHint,
|
|
1423
|
+
}),
|
|
1424
|
+
(() => {
|
|
1425
|
+
// 轻量提示:从不阻断保存,只告诉用户哪些行被自动处理了
|
|
1426
|
+
const { stats } = parseBlockedDomains(blockedDomains);
|
|
1427
|
+
if (stats.corrected === 0 && stats.dupes === 0 && stats.dropped === 0) return null;
|
|
1428
|
+
return react_jsx_runtime.jsx("p", {
|
|
1429
|
+
className: "dshfs-hint",
|
|
1430
|
+
children: t.blockedDomainsWarn(stats.corrected, stats.dupes, stats.dropped),
|
|
1431
|
+
});
|
|
1432
|
+
})(),
|
|
1433
|
+
],
|
|
1434
|
+
}),
|
|
1343
1435
|
react_jsx_runtime.jsx("div", {
|
|
1344
1436
|
className: "dshfs-field",
|
|
1345
1437
|
children: [
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// #72 域名黑名单:在结果进入上下文前过滤内容农场(CSDN 等)。
|
|
2
|
+
// 匹配语义(issue #72 约定):
|
|
3
|
+
// - 域名相等或其子域命中(csdn.net 覆盖 blog.csdn.net)
|
|
4
|
+
// - 忽略大小写;www. 前缀归一;无 scheme / 无路径;
|
|
5
|
+
// - 不做正则(notcsdn.net 不得被 csdn.net 命中)
|
|
6
|
+
// 生效方式:provider.search 入口按当次请求的 blockedDomains 设置"活动黑名单",
|
|
7
|
+
// uniqueSources 在收敛(去重 + limit)之前过滤——被屏蔽条目让位给后续候选,
|
|
8
|
+
// 因此 maxResults 仍能填满,且 truncated 只在真正触及上限时为 true。
|
|
9
|
+
|
|
10
|
+
let active = [];
|
|
11
|
+
/**
|
|
12
|
+
* 配置清洗:把用户随手填的域名/URL 归一成裸域名规则。
|
|
13
|
+
* 处理 trim、scheme、路径/query/hash、端口、www.、大小写、去重(保序)、丢弃空值与非法行。
|
|
14
|
+
*
|
|
15
|
+
* 为什么要服务端兜底:blockedDomains 是 .volatile() 配置,用户可以直接编辑 YAML 绕过 UI,
|
|
16
|
+
* 所以规范化必须在 setActiveBlockedDomains 里做,而不是只做在设置页。
|
|
17
|
+
* isDomainBlocked 的匹配语义不受影响——规则只是被清洗成更可靠的等价形式。
|
|
18
|
+
*/
|
|
19
|
+
export function normalizeDomainList(list) {
|
|
20
|
+
if (!Array.isArray(list)) return [];
|
|
21
|
+
const out = [];
|
|
22
|
+
const seen = new Set();
|
|
23
|
+
for (const raw of list) {
|
|
24
|
+
let s = String(raw ?? "").trim().toLowerCase();
|
|
25
|
+
if (!s) continue;
|
|
26
|
+
// 去 scheme(http:// https:// 或其它的 protocol://)
|
|
27
|
+
s = s.replace(/^[a-z][a-z0-9+.-]*:\/\//, "");
|
|
28
|
+
// 去路径 / query / hash
|
|
29
|
+
const cut = s.search(/[/?#]/);
|
|
30
|
+
if (cut >= 0) s = s.slice(0, cut);
|
|
31
|
+
if (!s) continue;
|
|
32
|
+
// 去端口(仅当冒号后是纯数字)
|
|
33
|
+
const colon = s.lastIndexOf(":");
|
|
34
|
+
if (colon >= 0 && /^\d+$/.test(s.slice(colon + 1))) s = s.slice(0, colon);
|
|
35
|
+
// 去 www. 前缀与尾随点号
|
|
36
|
+
s = s.replace(/^www\./, "").replace(/\.+$/, "");
|
|
37
|
+
// 只接受"看起来像域名"的行:空格、下划线、中文等直接丢弃(它们永远匹配不到 host)
|
|
38
|
+
if (!s || !/^[a-z0-9][a-z0-9.-]*$/.test(s) || s.startsWith(".")) continue;
|
|
39
|
+
if (seen.has(s)) continue;
|
|
40
|
+
seen.add(s);
|
|
41
|
+
out.push(s);
|
|
42
|
+
}
|
|
43
|
+
return out;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** 设置本次搜索生效的黑名单(provider.search / research 入口调用)。空数组 = 不过滤。 */
|
|
47
|
+
export function setActiveBlockedDomains(list) {
|
|
48
|
+
active = normalizeDomainList(list);
|
|
49
|
+
}
|
|
50
|
+
export function getActiveBlockedDomains() {
|
|
51
|
+
return [...active];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function normalizeHost(host) {
|
|
55
|
+
return String(host ?? "").toLowerCase().trim().replace(/^www\./, "");
|
|
56
|
+
}
|
|
57
|
+
function normalizeRule(domain) {
|
|
58
|
+
return String(domain ?? "").toLowerCase().trim().replace(/^www\./, "").replace(/\/+$/, "");
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** 单个 URL 是否命中黑名单;URL 无法解析时返回 false(不误伤)。 */
|
|
62
|
+
export function isDomainBlocked(url, blocked = active) {
|
|
63
|
+
if (!blocked || blocked.length === 0) return false;
|
|
64
|
+
let host;
|
|
65
|
+
try {
|
|
66
|
+
host = normalizeHost(new URL(String(url)).hostname);
|
|
67
|
+
} catch {
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
if (!host) return false;
|
|
71
|
+
return blocked.some((raw) => {
|
|
72
|
+
const rule = normalizeRule(raw);
|
|
73
|
+
if (!rule) return false;
|
|
74
|
+
return host === rule || host.endsWith("." + rule);
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** 列表过滤:结果条目统一为 { url } 形态,命中黑名的整条丢弃(保持原顺序=让位语义)。 */
|
|
79
|
+
export function filterBlockedDomains(items, blocked = active) {
|
|
80
|
+
if (!blocked || blocked.length === 0) return items;
|
|
81
|
+
return items.filter((item) => !isDomainBlocked(item?.url, blocked));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* 结果级收口:引擎返回的 { sources } 在交给上层(provider 返回值 / multi 合并 /
|
|
86
|
+
* runEngineTest / 工具 execute)之前统一过滤。
|
|
87
|
+
*
|
|
88
|
+
* 为什么需要它:uniqueSources 只是 40+ 个引擎里 29 个的"收敛去重"helper,
|
|
89
|
+
* searchAnysearch / searchPlatform 等 11 个引擎并不经过它(各自直接 return sources)。
|
|
90
|
+
* 只在 uniqueSources 里过滤会漏掉这些引擎——#72 首版就是这么漏的。
|
|
91
|
+
* 因此过滤必须落在"结果离开引擎"的所有咽喉点上,而不是某一个 helper 里。
|
|
92
|
+
*
|
|
93
|
+
* 无黑名单 / 无命中时原样返回(零分配),调用方可以直接重新赋值。
|
|
94
|
+
*/
|
|
95
|
+
export function applyBlockedFilterToResult(result) {
|
|
96
|
+
if (!result || !Array.isArray(result.sources)) return result;
|
|
97
|
+
const sources = filterBlockedDomains(result.sources);
|
|
98
|
+
if (sources.length === result.sources.length) return result;
|
|
99
|
+
return { ...result, sources };
|
|
100
|
+
}
|
package/lib/index.js
CHANGED
|
@@ -10,6 +10,7 @@ import { registerResearchTool } from "./research.js";
|
|
|
10
10
|
import { registerDomainSearchTool } from "./vertical.js";
|
|
11
11
|
import { createFetchProvider } from "./fetch_provider.js";
|
|
12
12
|
import { fetchWeather, sanitizeWeatherResult, WEATHER_OUTPUT_SCHEMA } from "./weather.js";
|
|
13
|
+
import { setActiveBlockedDomains, getActiveBlockedDomains, isDomainBlocked, applyBlockedFilterToResult } from "./domain_filter.js";
|
|
13
14
|
|
|
14
15
|
// 兼容 guard(#55):Node 会优先解析插件目录下的嵌套依赖,若那里残留旧的
|
|
15
16
|
// @deepseek-ai/schemastery(如 node_modules/dsh-free-search/node_modules/... -> 3.18.1),
|
|
@@ -361,10 +362,12 @@ const CACHE_MAX_ENTRIES = 50;
|
|
|
361
362
|
// fallback 条目(实际引擎 ≠ 首选引擎时)TTL = 配置 TTL 的 1/5(默认 5 分钟 → 60s):
|
|
362
363
|
// 首选引擎恢复后最多 1 分钟即可拿到新结果,避免回退结果被完整 TTL 钉死;首选成功条目仍用完整 TTL。
|
|
363
364
|
|
|
364
|
-
function buildCacheKey(query, maxResults, timeRangeLabel, preferred, disabledEngines = [], fallbackOrder = []) {
|
|
365
|
+
function buildCacheKey(query, maxResults, timeRangeLabel, preferred, disabledEngines = [], fallbackOrder = [], blockedDomains = []) {
|
|
365
366
|
const disabledKey = normalizeDisabledEngines(disabledEngines).slice().sort().join(",");
|
|
366
367
|
const orderKey = normalizeFallbackOrder(fallbackOrder).join(",");
|
|
367
|
-
|
|
368
|
+
// #72:黑名单参与 key——改动 blockedDomains 后旧缓存自然失效,不会 5 分钟内仍命中旧结果
|
|
369
|
+
const blockedKey = (Array.isArray(blockedDomains) ? blockedDomains : []).map((d) => String(d).toLowerCase().trim().replace(/^www\./, "")).slice().sort().join(",");
|
|
370
|
+
return [query ?? "", maxResults ?? 5, timeRangeLabel ?? "", preferred, disabledKey, orderKey, blockedKey].join("\u0000");
|
|
368
371
|
}
|
|
369
372
|
//#endregion
|
|
370
373
|
|
|
@@ -425,10 +428,14 @@ function extractDdgUrl(rel) {
|
|
|
425
428
|
return rel;
|
|
426
429
|
}
|
|
427
430
|
|
|
431
|
+
// #72:收敛前过滤域名黑名单——先剔除被屏蔽条目,再去重 + 截断,
|
|
432
|
+
// 因此后面的候选会自动补位(用户得到的是"满员的好结果",而不是变短的结果)。
|
|
428
433
|
function uniqueSources(sources, limit) {
|
|
434
|
+
const blocked = getActiveBlockedDomains();
|
|
435
|
+
const kept = blocked.length ? sources.filter((s) => !isDomainBlocked(s.url, blocked)) : sources;
|
|
429
436
|
const seen = new Set();
|
|
430
437
|
const out = [];
|
|
431
|
-
for (const s of
|
|
438
|
+
for (const s of kept) {
|
|
432
439
|
if (s.url && !seen.has(s.url)) {
|
|
433
440
|
seen.add(s.url);
|
|
434
441
|
out.push(s);
|
|
@@ -3073,6 +3080,8 @@ const Config = z.object({
|
|
|
3073
3080
|
disabledEngines: z.array(z.string()).default([]).volatile(),
|
|
3074
3081
|
fallbackOrder: z.array(z.string()).default([]).volatile(),
|
|
3075
3082
|
fallbackOn: z.array(z.string()).default(DEFAULT_FALLBACK_ON).volatile(),
|
|
3083
|
+
// #72 域名黑名单(默认空=不过滤;子域匹配;provider.search 入口置活动值,uniqueSources 收敛前过滤)
|
|
3084
|
+
blockedDomains: z.array(z.string()).default([]).volatile(),
|
|
3076
3085
|
cache: z.boolean().default(true).volatile(), // 单 query 结果缓存开关(防限流/省额度)
|
|
3077
3086
|
cacheTtl: z.number().default(5).volatile(), // 缓存时长(分钟),0-5 可配置(使用处再 clamp)
|
|
3078
3087
|
keyStorage: z.string().default("credentials").volatile(), // key 存储位置:credentials(凭据中心)| settings(设置页)
|
|
@@ -3320,6 +3329,8 @@ function apply(ctx, config) {
|
|
|
3320
3329
|
throw new Error("query is required");
|
|
3321
3330
|
}
|
|
3322
3331
|
const cfg = current();
|
|
3332
|
+
// #72:本次请求生效的域名黑名单(模块级活动值;各结果咽喉点过滤读取它)
|
|
3333
|
+
setActiveBlockedDomains(cfg.blockedDomains);
|
|
3323
3334
|
const disabledEngines = normalizeDisabledEngines(cfg.disabledEngines);
|
|
3324
3335
|
const fallbackOnSet = new Set(normalizeFallbackOn(cfg.fallbackOn));
|
|
3325
3336
|
const disabledSet = new Set(disabledEngines);
|
|
@@ -3340,7 +3351,7 @@ function apply(ctx, config) {
|
|
|
3340
3351
|
const cacheTtlMs = (Math.min(Math.max(Number(cfg.cacheTtl) ?? 5, 0), 5)) * 60 * 1000;
|
|
3341
3352
|
const cacheEnabled = cfg.cache !== false && cacheTtlMs > 0;
|
|
3342
3353
|
const cacheKey = cacheEnabled
|
|
3343
|
-
? buildCacheKey(request.query, request.maxResults, timeRangeLabel, preferred, disabledEngines, fallbackOrder)
|
|
3354
|
+
? buildCacheKey(request.query, request.maxResults, timeRangeLabel, preferred, disabledEngines, fallbackOrder, cfg.blockedDomains)
|
|
3344
3355
|
: null;
|
|
3345
3356
|
if (cacheKey !== null) {
|
|
3346
3357
|
const hit = searchCache.get(cacheKey);
|
|
@@ -3348,7 +3359,8 @@ function apply(ctx, config) {
|
|
|
3348
3359
|
if (signal?.aborted) throw new Error("search aborted");
|
|
3349
3360
|
searchCache.delete(cacheKey);
|
|
3350
3361
|
searchCache.set(cacheKey, hit);
|
|
3351
|
-
|
|
3362
|
+
// #72:缓存命中同样过一遍黑名单——首版修复前写入的缓存里可能残留屏蔽域名
|
|
3363
|
+
return { ...applyBlockedFilterToResult({ ...hit.value, sources: hit.value.sources?.slice() }), _cache: "hit" };
|
|
3352
3364
|
}
|
|
3353
3365
|
if (hit) searchCache.delete(cacheKey);
|
|
3354
3366
|
}
|
|
@@ -3625,6 +3637,12 @@ function apply(ctx, config) {
|
|
|
3625
3637
|
continue;
|
|
3626
3638
|
}
|
|
3627
3639
|
|
|
3640
|
+
if (result.sources.length > 0) {
|
|
3641
|
+
// #72 结果级收口:无论走哪个引擎(其中 11 个不经过 uniqueSources),
|
|
3642
|
+
// 在写缓存与返回之前统一按黑名单过滤;全被屏蔽时视为该引擎 0 结果,继续下一个引擎。
|
|
3643
|
+
result = applyBlockedFilterToResult(result);
|
|
3644
|
+
}
|
|
3645
|
+
|
|
3628
3646
|
if (result.sources.length > 0) {
|
|
3629
3647
|
usedEngine = engine;
|
|
3630
3648
|
clearCooldown(engine);
|
|
@@ -3794,6 +3812,9 @@ function apply(ctx, config) {
|
|
|
3794
3812
|
// 测试工具:让 agent 逐个测试所有搜索引擎,报告可用性
|
|
3795
3813
|
const runEngineTest = async (engine, query, timeRange, maxResults = 2) => {
|
|
3796
3814
|
const cfg = current();
|
|
3815
|
+
// #72:runEngineTest 也被 multi_search 工具 / 设置页 bridge 直接调用,
|
|
3816
|
+
// 不经过 provider.search,所以这里必须自行设置活动黑名单
|
|
3817
|
+
setActiveBlockedDomains(cfg.blockedDomains);
|
|
3797
3818
|
const q = query || "DeepSeek Harness";
|
|
3798
3819
|
const n = Math.min(Math.max(Number(maxResults) || 2, 1), 20);
|
|
3799
3820
|
const tr = parseTimeRange(timeRange);
|
|
@@ -3928,7 +3949,8 @@ function apply(ctx, config) {
|
|
|
3928
3949
|
}
|
|
3929
3950
|
return {
|
|
3930
3951
|
ok: true,
|
|
3931
|
-
|
|
3952
|
+
// #72:成功结果出引擎前统一过滤,覆盖不经过 uniqueSources 的 11 个引擎
|
|
3953
|
+
sources: applyBlockedFilterToResult({ sources: result.sources ?? [] }).sources.map((s) =>
|
|
3932
3954
|
s.snippet ? { ...s, snippet: cleanSnippet(s.snippet, snippetCap(engine)) } : s
|
|
3933
3955
|
),
|
|
3934
3956
|
truncated: result.truncated ?? false,
|
|
@@ -3942,6 +3964,8 @@ function apply(ctx, config) {
|
|
|
3942
3964
|
|
|
3943
3965
|
runMultiSearch = async ({ query, maxResults, engines, timeRange } = {}) => {
|
|
3944
3966
|
if (!query || !String(query).trim()) throw new Error("query is required");
|
|
3967
|
+
// #72:multi_search 工具直连本函数、不经过 provider.search,自行设置活动黑名单
|
|
3968
|
+
setActiveBlockedDomains(current().blockedDomains);
|
|
3945
3969
|
const q = String(query).trim();
|
|
3946
3970
|
const limit = Math.min(Math.max(Number(maxResults) || 8, 1), 20);
|
|
3947
3971
|
const disabled = new Set(normalizeDisabledEngines(current().disabledEngines));
|
|
@@ -4006,8 +4030,13 @@ function apply(ctx, config) {
|
|
|
4006
4030
|
|
|
4007
4031
|
const urlMap = new Map();
|
|
4008
4032
|
let itemSeq = 0;
|
|
4033
|
+
// #72:合并前过滤——引擎里未被 uniqueSources 覆盖的结果(如 anysearch)也在这里被挡住
|
|
4034
|
+
const blockedRaw = getActiveBlockedDomains();
|
|
4009
4035
|
for (const { engine, sources } of rawResults) {
|
|
4010
|
-
|
|
4036
|
+
const keptSources = blockedRaw.length
|
|
4037
|
+
? sources.filter((s) => !isDomainBlocked(s?.url, blockedRaw))
|
|
4038
|
+
: sources;
|
|
4039
|
+
for (const source of keptSources) {
|
|
4011
4040
|
if (!source || !source.url) continue;
|
|
4012
4041
|
let normUrl = String(source.url).trim();
|
|
4013
4042
|
try {
|
|
@@ -4269,7 +4298,8 @@ function apply(ctx, config) {
|
|
|
4269
4298
|
}
|
|
4270
4299
|
}
|
|
4271
4300
|
// lossless JSON 不允许 undefined 字段:剔除缺失字段
|
|
4272
|
-
|
|
4301
|
+
// #72:platform_search 直连 searchPlatform(不经 provider.search/runEngineTest),在此收口过滤
|
|
4302
|
+
const sources = applyBlockedFilterToResult({ sources: result.sources ?? [] }).sources.map((s) => {
|
|
4273
4303
|
const source = {};
|
|
4274
4304
|
if (s.url !== undefined && s.url !== null && s.url !== "") source.url = s.url;
|
|
4275
4305
|
if (s.title !== undefined && s.title !== null && s.title !== "") source.title = String(s.title);
|
package/lib/research.js
CHANGED
|
@@ -7,6 +7,7 @@ import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
|
7
7
|
import { getFresh, put as cachePut, search as cacheSearch } from "./cache.js";
|
|
8
8
|
import { rerank, embed, cosineMatrix, decide, judgeQueryIntent } from "./rerank.js";
|
|
9
9
|
import { httpsJson, htmlToText, callExa, callFirecrawl, callCrawl4ai } from "./fetch_upstream.js";
|
|
10
|
+
import { isDomainBlocked } from "./domain_filter.js";
|
|
10
11
|
|
|
11
12
|
// 模型组件的受控替身(测试/评审用;抓取层的替身在 fetch_upstream.js 的 CONTROLLED_STUBS)
|
|
12
13
|
export const CONTROLLED_STUBS = { rerank: null, embed: null, decide: null, judgeQueryIntent: null };
|
|
@@ -227,6 +228,8 @@ export function buildClusters(sources) {
|
|
|
227
228
|
let clusterSeq = 0;
|
|
228
229
|
for (const s of sources) {
|
|
229
230
|
if (!s || typeof s.url !== "string") continue;
|
|
231
|
+
// #72:黑名单域在候选构建阶段就剔除——items / 簇 / 正文三层都不会再出现它
|
|
232
|
+
if (isDomainBlocked(s.url)) continue;
|
|
230
233
|
const key = normDedupeKey(s.url);
|
|
231
234
|
if (seenKeys.has(key)) continue; // 同键(www/m/mobile 归一)只留首个原始 URL
|
|
232
235
|
seenKeys.add(key);
|
|
@@ -677,6 +680,8 @@ function parseUpdatedFromHtml(html) {
|
|
|
677
680
|
/** 单 URL 抓取(含缓存两分与四层回退;每次网络尝试都过预算 gate)。成功返回:
|
|
678
681
|
* { content, tier, truncated, fetchedAt(ISO,缓存命中=真实原抓时间), updatedAt(可取得时), cache } */
|
|
679
682
|
async function fetchTiered(ctx, url, title, state, budget, intent) {
|
|
683
|
+
// #72:黑名单站点直接不抓(缓存也不读);不算失败,不写 note——这是用户主动配置的过滤
|
|
684
|
+
if (isDomainBlocked(url)) return null;
|
|
680
685
|
const now = Date.now();
|
|
681
686
|
const cached = getFresh(url, DOC_CACHE_MAX_AGE_MS);
|
|
682
687
|
if (cached && (intent.freshness === "none" || canUseCache(cached.fetched_at, intent.freshness, now)[0])) {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-free-search",
|
|
3
|
-
"version": "0.8.
|
|
4
|
-
"description": "Free web search for DeepSeek Harness: 24 engines (Bing/DuckDuckGo/AnySearch/SearXNG/Exa/Tavily/Keenable/Firecrawl/Parallel keyless; Zhihu global/site via MCP and Perplexity/SerpBase/DeepSeek/You.com/Baidu/Kimi/Aliyun/Doubao with key; OpenAI/Gemini/Claude model built-in search, explicit-only) + auto smart routing + global engine enable/disable & fallback priority + editable zh/en language pools + multi mode (parallel full language-pool merge) + multi_search cross-source tool + failure-aware fallback (quota/auth cooldown, transient retry, fallbackOn) + time filtering + platform search + web_fetch, with web settings UI.",
|
|
3
|
+
"version": "0.8.7",
|
|
4
|
+
"description": "Free web search for DeepSeek Harness: 24 engines (Bing/DuckDuckGo/AnySearch/SearXNG/Exa/Tavily/Keenable/Firecrawl/Parallel keyless; Zhihu global/site via MCP and Perplexity/SerpBase/DeepSeek/You.com/Baidu/Kimi/Aliyun/Doubao with key; OpenAI/Gemini/Claude model built-in search, explicit-only) + auto smart routing + global engine enable/disable & fallback priority + editable zh/en language pools + multi mode (parallel full language-pool merge) + multi_search cross-source tool + failure-aware fallback (quota/auth cooldown, transient retry, fallbackOn) + time filtering + platform search + domain blocklist (blockedDomains) + web_fetch, with web settings UI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
7
7
|
"exports": {
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"lib/fetch_provider.js",
|
|
21
21
|
"lib/fetch_upstream.js",
|
|
22
22
|
"lib/weather.js",
|
|
23
|
+
"lib/domain_filter.js",
|
|
23
24
|
"cordis.patch.yml"
|
|
24
25
|
],
|
|
25
26
|
"keywords": [
|