@240xu/dsh-websearch 2.0.6 → 2.1.0

package/README.md

@@ -73,6 +73,13 @@ pnpm install
 | `numResults` | 8 | 每次搜索返回结果数（1-50） |
 | `concurrency` | 6 | 并发后端数（1-10） |
 | `backendTimeoutMs` | 30000 | 单后端超时毫秒 |
+| `recency` | any | 时间范围过滤：day / week / month / year（映射到各后端原生参数） |
+| `language` | 空 | 搜索语言，如 en、zh-CN（SearXNG/Brave/Serper 支持） |
+| `safeSearch` | true | 安全搜索开关（SearXNG/Brave 支持） |
+| `dedupStrategy` | url | url 仅按链接去重；url+title 额外合并同标题的转载镜像 |
+| `rerank` | false | 按查询词相关性重排序（确定性；并列时保持后端优先级） |
+
+**v2.1 结果策略**：设置面板新增「结果策略」分区。时间/语言过滤按各后端能力自动映射——Brave `freshness/country/search_lang`、Tavily `time_range`、Serper `tbs/hl/gl`（Google 日期语法）、SearXNG `time_range/language`、DDG `df`；不支持的后端自动忽略对应维度。内部超时以真实原因失败（backend "<id>" timed out after Nms），不再被静默降级为取消；Mojeek 的 API Key 改走 Authorization 头，不再出现在 URL 中。
 
 **11 个后端开关**：`enableExa` / `enableParallel` / `enableDdg` / `enableSearxng`（默认开）；`enableBrave` / `enableTavily` / `enableSerper` / `enableMojeek` / `enableDeepseek` / `enableAnthropic` / `enableOpenai`（默认关）
 
@@ -98,7 +105,9 @@ pnpm install
 - **One provider, no ambiguity**: a single `registeredSearchProvider({id:"unified"})` — the `dsh-web` seam's selection rule picks it unambiguously, and `search()` caps `maxResults` itself.
 - **`Promise.allSettled` fan-out**: every enabled + available backend fires concurrently; a single backend failure is demoted to a soft `null` so the rest still contribute — only when ALL fail does the provider throw `WEB_PROVIDER_ERROR`.
 - **Per-backend abort demotion**: a single backend aborting becomes soft-null; the provider only rethrows `WEB_ABORTED` when the caller's own `AbortSignal` fires.
-- **URL dedup + merge**: results across backends are merged and deduped by case-insensitive URL; missing title/snippet/publishedAt from one backend get filled in from another (e.g. Exa's title + Parallel's excerpt).
+- **Dedup strategies**: `url` keeps the historical URL-key merge with cross-backend field fill-in; `url+title` additionally collapses same-story syndicated mirrors (title Jaccard >= 0.9, CJK-aware tokenizer) while still enriching the kept entry.
+- **Deterministic rerank (optional)**: query-term overlap scoring (title x3 / snippet x2 / URL x1); stable ties preserve fan-out priority — same input always yields the same order.
+- **Honest timeouts**: each backend runs under an internal abort controller merged into the caller signal; when only the internal timer fires, the failure is reclassified as WEB_PROVIDER_ERROR ("backend <id> timed out") instead of being masked as a user cancellation.
 - **Concurrency & Timeout Control**: `concurrency` (default 6) limits simultaneous calls; `backendTimeoutMs` (default 30s) caps each backend.
 - **Streamable-http MCP, custom handshake**: `lib/util/mcp-client.js` implements `initialize → notifications/initialized → tools/call` with `Mcp-Session-Id` header caching against Exa and Parallel — no dependency on the full MCP SDK.
 - **hooks.recordRequest/recordOutcome bridge**: each backend routes its request/outcome to `lib/util/log.js` which records `web/unified-search-backend-request` / `…backend-outcome` events onto the active DSH session via `ctx.get("agents")?.currentInitiator()?.session.append(...)` (mirrors `dsh-web-search-deepseek`).

package/lib/backends/anthropic-like.js

@@ -19,7 +19,7 @@ import { throwIfSearchAborted, searchAborted, isAbortError } from "../util/abort
 import { WebError } from "@deepseek-ai/dsh-web";
 
 /** Attribution header; bumped with the package version. */
-const UA = "dsh-websearch/2.0";
+const UA = "dsh-websearch/2.1";
 
 /**
  * Factory: create an Anthropic-like Messages backend.

package/lib/backends/brave.js

@@ -7,6 +7,7 @@
 
 import { WebError } from "@deepseek-ai/dsh-web";
 import { throwIfSearchAborted, searchAborted, isAbortError } from "../util/abort.js";
+import { applyBraveFilters, normalizeFilters, safeSearchParams } from "../util/filters.js";
 
 const BRAVE_ENDPOINT = "https://api.search.brave.com/res/v1/web/search";
 
@@ -33,6 +34,9 @@ export const braveBackend = {
     const url = new URL(baseURL);
     url.searchParams.set("q", request.query);
     url.searchParams.set("count", String(count));
+    applyBraveFilters(url, normalizeFilters(request.filters));
+    const safe = safeSearchParams("brave", backendOpts.safeSearch);
+    if (safe) url.searchParams.set("safesearch", safe);
 
     hooks?.recordRequest?.({
       endpoint: baseURL,
@@ -47,7 +51,7 @@ export const braveBackend = {
         headers: {
           "X-Subscription-Token": apiKey,
           "Accept": "application/json",
-          "User-Agent": "dsh-websearch/2.0",
+          "User-Agent": "dsh-websearch/2.1",
         },
         signal,
       });

package/lib/backends/ddg.js

@@ -6,6 +6,7 @@
 // are richer, DDG is the zero-dependency safety-net.
 
 import { searchAborted, isAbortError } from "../util/abort.js";
+import { applyDdgFilters, normalizeFilters } from "../util/filters.js";
 import { WebError } from "@deepseek-ai/dsh-web";
 
 export const DDG_ENDPOINT = "https://html.duckduckgo.com/html/";
@@ -26,7 +27,9 @@ export const ddgBackend = {
    */
   async search(request, signal, options, hooks = {}) {
     throwIfSearchAborted(signal);
-    const body = "q=" + encodeURIComponent(request.query) + "&b=" + Math.random().toString().slice(2);
+    const params = new URLSearchParams({ q: request.query, b: Math.random().toString().slice(2) });
+    applyDdgFilters(params, normalizeFilters(request.filters));
+    const body = params.toString();
     hooks.recordRequest?.({ endpoint: DDG_ENDPOINT, toolName: "html", body: { q: request.query }, headers: {} });
 
     let text;
@@ -37,7 +40,7 @@ export const ddgBackend = {
         headers: {
           "Content-Type": "application/x-www-form-urlencoded",
           "Accept": "text/html",
-          "User-Agent": "dsh-websearch/2.0",
+          "User-Agent": "dsh-websearch/2.1",
         },
         body,
         ...(signal !== undefined ? { signal } : {}),

package/lib/backends/exa.js

@@ -45,16 +45,15 @@ export const exaBackend = {
       );
     }
 
-    hooks.recordOutcome?.({ status: "ok" });
-    const sources = parseExaResult(result, request.maxResults ?? 8);
-
     if (result?.isError === true) {
-      throw new WebError(
-        `exa returned an MCP error: ${String(result.content?.[0]?.text ?? "unknown")}`,
-        "WEB_PROVIDER_ERROR",
-      );
+      const message = String(result.content?.[0]?.text ?? "unknown");
+      hooks.recordOutcome?.({ status: "error", message });
+      throw new WebError(`exa returned an MCP error: ${message}`, "WEB_PROVIDER_ERROR");
     }
 
+    const sources = parseExaResult(result, request.maxResults ?? 8);
+    hooks.recordOutcome?.({ status: "ok", count: sources.length });
+
     return { sources, content: undefined, truncated: false };
   },
 };

package/lib/backends/mojeek.js

@@ -35,11 +35,13 @@ export const mojeekBackend = {
     url.searchParams.set("q", request.query);
     url.searchParams.set("fmt", "json");
     url.searchParams.set("count", String(count));
-    url.searchParams.set("api_key", apiKey);
 
+    // Auth via Authorization header — keeps the API key out of URLs (which
+    // leak into logs, history and referer headers).
     const headers = {
       "Accept": "application/json",
-      "User-Agent": "dsh-websearch/2.0",
+      "Authorization": "Bearer " + apiKey,
+      "User-Agent": "dsh-websearch/2.1",
     };
 
     hooks?.recordRequest?.({

package/lib/backends/openai.js

@@ -13,7 +13,7 @@
 import { throwIfSearchAborted, searchAborted, isAbortError } from "../util/abort.js";
 import { WebError } from "@deepseek-ai/dsh-web";
 
-const UA = "dsh-websearch/2.0";
+const UA = "dsh-websearch/2.1";
 
 export const openaiBackend = {
   id: "openai",

package/lib/backends/searxng.js

@@ -8,6 +8,7 @@
 
 import { WebError } from "@deepseek-ai/dsh-web";
 import { throwIfSearchAborted, searchAborted, isAbortError } from "../util/abort.js";
+import { applySearxngFilters, normalizeFilters, safeSearchParams } from "../util/filters.js";
 
 const DEFAULT_SEARXNG_URL = "https://searx.be";
 
@@ -33,13 +34,17 @@ export const searxngBackend = {
     url.searchParams.set("q", request.query);
     url.searchParams.set("format", "json");
     url.searchParams.set("categories", "general");
-    url.searchParams.set("language", "en");
-    url.searchParams.set("safesearch", "1");
+    // Language: settings/request filter wins, else historical "en" default.
+    const filters = normalizeFilters(request.filters);
+    url.searchParams.set("language", filters.lang ?? "en");
+    applySearxngFilters(url, filters); // time_range (+ language override)
+    // Safe-search defaults ON (preserves the previous hardcoded behavior).
+    url.searchParams.set("safesearch", safeSearchParams("searxng", backendOpts.safeSearch ?? true) ?? "1");
     url.searchParams.set("pageno", "1");
 
     const headers = {
       "Accept": "application/json",
-      "User-Agent": "dsh-websearch/2.0",
+      "User-Agent": "dsh-websearch/2.1",
     };
     
     if (apiKey) {

package/lib/backends/serper.js

@@ -7,6 +7,7 @@
 
 import { WebError } from "@deepseek-ai/dsh-web";
 import { throwIfSearchAborted, searchAborted, isAbortError } from "../util/abort.js";
+import { applySerperFilters, normalizeFilters } from "../util/filters.js";
 
 const SERPER_ENDPOINT = "https://google.serper.dev/search";
 
@@ -35,6 +36,7 @@ export const serperBackend = {
       q: request.query,
       num: count,
     };
+    applySerperFilters(body, normalizeFilters(request.filters));
 
     hooks?.recordRequest?.({
       endpoint: baseURL,
@@ -50,7 +52,7 @@ export const serperBackend = {
           "X-API-KEY": apiKey,
           "Content-Type": "application/json",
           "Accept": "application/json",
-          "User-Agent": "dsh-websearch/2.0",
+          "User-Agent": "dsh-websearch/2.1",
         },
         body: JSON.stringify(body),
         signal,

package/lib/backends/tavily.js

@@ -7,6 +7,7 @@
 
 import { WebError } from "@deepseek-ai/dsh-web";
 import { throwIfSearchAborted, searchAborted, isAbortError } from "../util/abort.js";
+import { applyTavilyFilters, normalizeFilters } from "../util/filters.js";
 
 const TAVILY_ENDPOINT = "https://api.tavily.com/search";
 
@@ -39,6 +40,7 @@ export const tavilyBackend = {
       include_answer: true,
       include_raw_content: false,
     };
+    applyTavilyFilters(body, normalizeFilters(request.filters));
 
     hooks?.recordRequest?.({
       endpoint: baseURL,
@@ -54,7 +56,7 @@ export const tavilyBackend = {
           "Authorization": `Bearer ${apiKey}`,
           "Content-Type": "application/json",
           "Accept": "application/json",
-          "User-Agent": "dsh-websearch/2.0",
+          "User-Agent": "dsh-websearch/2.1",
         },
         body: JSON.stringify(body),
         signal,

package/lib/client.js

@@ -25,6 +25,13 @@ window.__ModuleLoader__.load({
 			{ id: "openai", keyEnv: "OPENAI_API_KEY", label: "OpenAI" },
 		];
 		const GLOBAL_FIELDS = ["numResults", "concurrency", "backendTimeoutMs"];
+		/** Enum-valued settings (validated client-side, saved as strings). */
+		const SELECT_FIELDS = {
+			recency: ["any", "day", "week", "month", "year"],
+			dedupStrategy: ["url", "url+title"],
+		};
+		/** Boolean settings staged as "true"/"false" strings by the selects. */
+		const BOOLEAN_FIELDS = ["rerank", "safeSearch"];
 		const KEYLESS = BACKENDS.filter((b) => !b.keyEnv);
 		const KEYED = BACKENDS.filter((b) => b.keyEnv);
 
@@ -65,6 +72,22 @@ window.__ModuleLoader__.load({
 			togglesTitle: "启用后端",
 			globalsTitle: "全局参数",
 			keysTitle: "API 密钥（保存后写入凭证库，立即生效）",
+			shapingTitle: "结果策略",
+			recencyLabel: "时间范围",
+			langLabel: "搜索语言",
+			safeSearchLabel: "安全搜索",
+			dedupLabel: "去重策略",
+			rerankLabel: "相关性重排",
+			on: "开",
+			off: "关",
+			recencyAny: "不限",
+			recencyDay: "一天内",
+			recencyWeek: "一周内",
+			recencyMonth: "一月内",
+			recencyYear: "一年内",
+			dedupUrl: "按链接",
+			dedupTitle: "链接 + 标题（合并转载）",
+			langPlaceholder: "如 en、zh-CN，留空为默认",
 		};
 		const en = {
 			nav: "Web Search",
@@ -92,6 +115,22 @@ window.__ModuleLoader__.load({
 			togglesTitle: "Enabled backends",
 			globalsTitle: "Global parameters",
 			keysTitle: "API keys (saved to the credential store, effective immediately)",
+			shapingTitle: "Result shaping",
+			recencyLabel: "Recency",
+			langLabel: "Language",
+			safeSearchLabel: "Safe search",
+			dedupLabel: "Dedup strategy",
+			rerankLabel: "Relevance rerank",
+			on: "On",
+			off: "Off",
+			recencyAny: "Any time",
+			recencyDay: "Past day",
+			recencyWeek: "Past week",
+			recencyMonth: "Past month",
+			recencyYear: "Past year",
+			dedupUrl: "By URL",
+			dedupTitle: "URL + title (merge syndication)",
+			langPlaceholder: "e.g. en, zh-CN; blank = backend default",
 		};
 
 		//#region styles
@@ -202,6 +241,18 @@ window.__ModuleLoader__.load({
 					for (const [field, staged] of this.staged) {
 						// Empty draft clears the override so the schema default applies again.
 						if (staged.clear || staged.text === "") { await this.scope.unset(field); doneFields.push(field); continue; }
+						if (SELECT_FIELDS[field]) {
+							if (!SELECT_FIELDS[field].includes(staged.text)) { this.failed = true; continue; }
+							await this.scope.set(field, staged.text);
+							doneFields.push(field);
+							continue;
+						}
+						if (BOOLEAN_FIELDS.includes(field)) {
+							if (staged.text !== "true" && staged.text !== "false") { this.failed = true; continue; }
+							await this.scope.set(field, staged.text === "true");
+							doneFields.push(field);
+							continue;
+						}
 						if (GLOBAL_FIELDS.includes(field)) {
 							const num = Number(staged.text);
 							// Mirror the server schema bounds client-side so bad input is
@@ -272,6 +323,21 @@ window.__ModuleLoader__.load({
 			] });
 		}
 
+		/** Labeled <select> for enum / boolean settings. */
+		function SelectField(props) {
+			return react_jsx_runtime.jsxs("div", { children: [
+				react_jsx_runtime.jsx("div", { className: "wsc-fieldHead", children: react_jsx_runtime.jsx("span", { className: "wsc-label", children: props.label }) }),
+				react_jsx_runtime.jsx("select", {
+					className: "wsc-input",
+					value: props.value,
+					onChange: (e) => props.onChange(e.target.value),
+					children: props.options.map((o) =>
+						react_jsx_runtime.jsx("option", { value: o.value, children: o.label }, o.value)
+					),
+				}),
+			] });
+		}
+
 		/** The card root. */
 		function UnifiedSearchSection(props) {
 			const state = props.useUnifiedSearchCard((snapshot) => snapshot);
@@ -288,6 +354,23 @@ window.__ModuleLoader__.load({
 			const keyedBackends = BACKENDS.filter((b) => b.keyEnv);
 			const keysReady = keyedBackends.filter((b) => state.keyState[b.keyEnv]?.configured).length;
 
+			const recencyOptions = [
+				{ value: "any", label: props.t("recencyAny") },
+				{ value: "day", label: props.t("recencyDay") },
+				{ value: "week", label: props.t("recencyWeek") },
+				{ value: "month", label: props.t("recencyMonth") },
+				{ value: "year", label: props.t("recencyYear") },
+			];
+			const dedupOptions = [
+				{ value: "url", label: props.t("dedupUrl") },
+				{ value: "url+title", label: props.t("dedupTitle") },
+			];
+			const boolOptions = [
+				{ value: "true", label: props.t("on") },
+				{ value: "false", label: props.t("off") },
+			];
+			const selVal = (f, dflt) => state.stagedText(f) ?? String(state.value[f] ?? dflt);
+
 			return react_jsx_runtime.jsxs("div", { className: "wsc-card", children: [
 				react_jsx_runtime.jsx("div", { style: { display: "flex", gap: 14, alignItems: "center", padding: "10px 12px", marginBottom: 10, borderRadius: 8, background: "rgba(127,127,127,.08)", fontSize: 13 }, children: [
 					react_jsx_runtime.jsxs("span", { children: [props.t("statusEnabled"), ": ", react_jsx_runtime.jsx("b", { children: enabledCount + "/" + BACKENDS.length })] }),
@@ -327,6 +410,42 @@ window.__ModuleLoader__.load({
 						react_jsx_runtime.jsx("input", { className: "wsc-input", value: state.stagedText("backendTimeoutMs") ?? String(state.value.backendTimeoutMs ?? ""), onChange: e => card.edit("backendTimeoutMs", e.target.value), inputMode: "numeric" }),
 					] }),
 				] }),
+				react_jsx_runtime.jsx("h4", { className: "wsc-sectionTitle", children: props.t("shapingTitle") }),
+				react_jsx_runtime.jsxs("div", { className: "wsc-grid", children: [
+					react_jsx_runtime.jsx(SelectField, {
+						label: props.t("recencyLabel"),
+						value: selVal("recency", "any"),
+						options: recencyOptions,
+						onChange: (v) => card.edit("recency", v),
+					}),
+					react_jsx_runtime.jsxs("div", { children: [
+						react_jsx_runtime.jsx("div", { className: "wsc-fieldHead", children: react_jsx_runtime.jsx("span", { className: "wsc-label", children: props.t("langLabel") }) }),
+						react_jsx_runtime.jsx("input", {
+							className: "wsc-input",
+							placeholder: props.t("langPlaceholder"),
+							value: state.stagedText("language") ?? String(state.value.language ?? ""),
+							onChange: e => card.edit("language", e.target.value),
+						}),
+					] }),
+					react_jsx_runtime.jsx(SelectField, {
+						label: props.t("dedupLabel"),
+						value: selVal("dedupStrategy", "url"),
+						options: dedupOptions,
+						onChange: (v) => card.edit("dedupStrategy", v),
+					}),
+					react_jsx_runtime.jsx(SelectField, {
+						label: props.t("rerankLabel"),
+						value: selVal("rerank", "false"),
+						options: boolOptions,
+						onChange: (v) => card.edit("rerank", v),
+					}),
+					react_jsx_runtime.jsx(SelectField, {
+						label: props.t("safeSearchLabel"),
+						value: selVal("safeSearch", "true"),
+						options: boolOptions,
+						onChange: (v) => card.edit("safeSearch", v),
+					}),
+				] }),
 				react_jsx_runtime.jsx("h4", { className: "wsc-sectionTitle", children: props.t("keysTitle") }),
 				react_jsx_runtime.jsx("div", { className: "wsc-grid", children: BACKENDS.filter((b) => b.keyEnv).map((b) =>
 					react_jsx_runtime.jsxs("div", { children: [

package/lib/index.js

@@ -130,6 +130,18 @@ export const Config = z.object({
   backendTimeoutMs: z.number().step(1000).min(1000).max(120000).default(30000)
     .description("单后端最大等待时间（毫秒）"),
 
+  // v2.1 result shaping
+  recency: z.union(["any", "day", "week", "month", "year"]).default("any")
+    .description("时间范围过滤（映射到各后端原生参数；any=不过滤）"),
+  language: z.string().default("")
+    .description("搜索语言（如 en、zh-CN；留空用各后端默认）"),
+  safeSearch: z.boolean().default(true)
+    .description("安全搜索（SearXNG / Brave 支持开关，其余后端忽略）"),
+  dedupStrategy: z.union(["url", "url+title"]).default("url")
+    .description("去重策略：url 仅按链接；url+title 额外合并同标题的转载镜像"),
+  rerank: z.boolean().default(false)
+    .description("按查询词相关性重排序（确定性、并列时保持后端优先级）"),
+
   // Enable toggles (one per backend)
   enableExa: z.boolean().default(true).description(DEFAULT_LABELS.exa),
   enableParallel: z.boolean().default(true).description(DEFAULT_LABELS.parallel),
@@ -230,7 +242,8 @@ export function resolveOptions(ctx, config) {
       apiKeyEnv: keyEnvName,
       baseURL: config?.[`${id}BaseURL`] ?? DEFAULT_BASE_URLS[id],
       model: config?.[`${id}Model`] ?? DEFAULT_MODELS[id],
-      searchDepth: config?.tavilySearchDepth,
+      ...(id === "tavily" ? { searchDepth: config?.tavilySearchDepth ?? "basic" } : {}),
+      safeSearch: config?.safeSearch !== false,
       label: DEFAULT_LABELS[id] ?? id,
       requiresKey: DEFAULT_KEY_ENV[id] !== undefined,
       resolveApiKey: async () => {
@@ -242,11 +255,20 @@ export function resolveOptions(ctx, config) {
     };
   }
 
+  const language = typeof config?.language === "string" ? config.language.trim() : "";
+
   return {
     enabledBackends,
     numResults: config?.numResults ?? 8,
     concurrency: config?.concurrency ?? 6,
     backendTimeoutMs: config?.backendTimeoutMs ?? 30000,
+    // v2.1 shaping options consumed by the provider merge pipeline.
+    dedupStrategy: config?.dedupStrategy === "url+title" ? "url+title" : "url",
+    rerank: config?.rerank === true,
+    filters: {
+      recency: typeof config?.recency === "string" && config.recency !== "any" ? config.recency : undefined,
+      lang: language.length > 0 ? language : undefined,
+    },
     backends,
     ctx,
   };

package/lib/provider.js

@@ -20,6 +20,8 @@ import {
   recordBackendRequest,
   recordBackendOutcome,
 } from "./util/log.js";
+import { dedupeSources, rerankSources } from "./util/rank.js";
+import { normalizeFilters } from "./util/filters.js";
 
 const PROVIDER_ID = "unified";
 
@@ -107,50 +109,62 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
         );
       }
 
+      // Settings-level default filters, overridden by per-request filters
+      // (normalizeFilters drops unknown/invalid values so typos degrade to
+      // "no filter" instead of breaking the search).
+      const filters = normalizeFilters({
+        ...(opts.filters ?? {}),
+        ...(request?.filters ?? {}),
+      });
+      const effectiveRequest =
+        Object.keys(filters).length > 0 ? { ...request, filters } : request;
+
       // Concurrency limiter
       const limit = createLimiter(opts.concurrency ?? 6);
 
-      // Wrap each backend call with timeout
-      const runWithTimeout = (backend, req, sig, beOpts, hooks) => {
-        const timeoutMs = backendTimeoutMs;
+      // Run one backend under the per-backend timeout. The internal timer's
+      // controller is merged into the caller's signal; when the INTERNAL timer
+      // fired (caller did not abort), a backend abort-shaped failure is
+      // reclassified as WEB_PROVIDER_ERROR so timeouts surface as real
+      // failures instead of being silently demoted like user cancellations.
+      const runBackend = ({ id, backend, beOpts, hooks }) => {
         const controller = new AbortController();
-        const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
-        
-        // Merge signals
-        const mergedSignal = sig 
-          ? AbortSignal.any([sig, controller.signal])
+        const mergedSignal = signal
+          ? AbortSignal.any([signal, controller.signal])
           : controller.signal;
-
-        return Promise.race([
-          backend.search(req, mergedSignal, beOpts, hooks),
-          new Promise((_, reject) => {
-            controller.signal.addEventListener('abort', () => {
-              if (controller.signal.reason === 'timeout' || !sig?.aborted) {
-                reject(new WebError(`Backend timed out after ${timeoutMs}ms`, "WEB_PROVIDER_ERROR"));
-              }
-            });
+        const timeoutId = setTimeout(() => controller.abort(), backendTimeoutMs);
+        return backend
+          .search(effectiveRequest, mergedSignal, beOpts, hooks)
+          .catch((err) => {
+            if (controller.signal.aborted && !(signal && signal.aborted)) {
+              throw new WebError(
+                `backend "${id}" timed out after ${backendTimeoutMs}ms`,
+                "WEB_PROVIDER_ERROR",
+                { cause: err instanceof Error ? err : undefined },
+              );
+            }
+            throw err;
           })
-        ]).finally(() => clearTimeout(timeoutId));
+          .finally(() => clearTimeout(timeoutId));
       };
 
       const results = await Promise.allSettled(
         eligible.map(({ id, backend, beOpts }) => {
-          const recordRequest = (req) => recordBackendRequest(ctx, id, req);
-          const recordOutcome = (out) => recordBackendOutcome(ctx, id, out);
-          const hooks = { recordRequest, recordOutcome };
-          
-          return limit(async () => {
-            return runWithTimeout(backend, request, signal, beOpts, hooks);
-          });
+          const hooks = {
+            recordRequest: (req) => recordBackendRequest(ctx, id, req),
+            recordOutcome: (out) => recordBackendOutcome(ctx, id, out),
+          };
+          return limit(() => runBackend({ id, backend, beOpts, hooks }));
         }),
       );
 
       throwIfSearchAborted(signal);
 
-      // Merge: dedup by url (case-insensitive normalized), keep first-seen
-      // non-empty title/snippet/publishedAt. Tolerate soft-null settled values.
-      const seen = new Set();
-      const merged = [];
+      // Collect every fulfilled backend's sources in fan-out priority order,
+      // then dedupe/rank ONCE over the full set. (The previous loop stopped at
+      // maxResults mid-merge, which silently dropped later backends' entries
+      // before cross-backend fill-in and ranking could see them.)
+      const collected = [];
       let mergedContent;
       let hardAborted = false;
       let hardError = null;
@@ -161,25 +175,7 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
           if (mergedContent === undefined && typeof settled.value?.content === "string" && settled.value.content.length > 0) {
             mergedContent = settled.value.content;
           }
-          const srcs = settled.value?.sources ?? [];
-          for (const s of srcs) {
-            if (!s || typeof s.url !== "string" || s.url.length === 0) continue;
-            const key = s.url.toLowerCase();
-            if (seen.has(key)) {
-              // Merge fields into the existing entry (fill-in missing ones).
-              const existing = merged.find((m) => m.url.toLowerCase() === key);
-              if (existing) {
-                if (!existing.title && s.title) existing.title = s.title;
-                if (!existing.snippet && s.snippet) existing.snippet = s.snippet;
-                if (!existing.publishedAt && s.publishedAt) existing.publishedAt = s.publishedAt;
-              }
-              continue;
-            }
-            seen.add(key);
-            merged.push({ ...s });
-            if (merged.length >= maxResults) break;
-          }
-          if (merged.length >= maxResults) break;
+          for (const s of settled.value?.sources ?? []) collected.push(s);
         } else {
           // settled.status === "rejected"
           const err = settled.reason;
@@ -199,13 +195,24 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
         throw searchAborted(signal);
       }
 
+      // Dedup: "url" preserves the historical URL-key behavior; "url+title"
+      // additionally collapses same-story syndicated copies across backends.
+      const strategy = opts.dedupStrategy === "url+title" ? "url+title" : "url";
+      const deduped = dedupeSources(collected, strategy);
+      // Optional deterministic relevance rerank (stable ties keep fan-out order).
+      const ordered =
+        opts.rerank === true
+          ? rerankSources(deduped, typeof request?.query === "string" ? request.query : "")
+          : deduped;
+
       // If we got ANY sources, return them even if some backends failed.
-      if (merged.length > 0) {
-        const sources = merged.slice(0, maxResults);
-        const wasCapped =
-          merged.length > maxResults ||
-          results.some((r) => r.status === "fulfilled" && (r.value?.sources?.length ?? 0) > maxResults);
-        return { sources, ...(mergedContent !== undefined ? { content: mergedContent } : {}), truncated: sources.length >= maxResults && wasCapped };
+      if (ordered.length > 0) {
+        const sources = ordered.slice(0, maxResults);
+        return {
+          sources,
+          ...(mergedContent !== undefined ? { content: mergedContent } : {}),
+          truncated: ordered.length > maxResults,
+        };
       }
 
       // All enabled backends failed — synthesize a meaningful WEB_PROVIDER_ERROR.

package/lib/util/filters.js

@@ -0,0 +1,104 @@
+/**
+ * @module dsh-websearch/util/filters
+ *
+ * Normalized cross-backend search filters.
+ *
+ * The provider resolves ONE canonical filter shape from settings + request:
+ *   { recency?: "day"|"week"|"month"|"year", lang?: string, region?: string }
+ *
+ * Each REST backend maps only the dimensions it supports (via the apply*
+ * helpers below); unsupported dimensions are silently ignored so a single
+ * settings panel can drive every backend without lying about reach.
+ * All functions are pure and unit-tested in tests/filters.test.js.
+ */
+
+/** Canonical recency values accepted from settings/request. */
+export const RECENCY_VALUES = ["day", "week", "month", "year"];
+
+/**
+ * Backend-specific recency vocabularies:
+ *   brave: freshness=pd|pw|pm|py ; tavily: time_range=day|week|month|year;
+ *   serper: tbs=qdr:d|w|m|y (Google date syntax); searxng: time_range=*;
+ *   ddg: df=d|w|m|y (html endpoint date filter).
+ */
+const RECENCY_MAP = {
+  brave: { day: "pd", week: "pw", month: "pm", year: "py" },
+  tavily: { day: "day", week: "week", month: "month", year: "year" },
+  serper: { day: "qdr:d", week: "qdr:w", month: "qdr:m", year: "qdr:y" },
+  searxng: { day: "day", week: "week", month: "month", year: "year" },
+  ddg: { day: "d", week: "w", month: "m", year: "y" },
+};
+
+/**
+ * Validate + normalize a raw filter bag. Unknown keys and out-of-vocabulary
+ * values are dropped: a typo must degrade to "no filter", never break search.
+ */
+export function normalizeFilters(raw) {
+  if (!raw || typeof raw !== "object") return {};
+  const out = {};
+  if (typeof raw.recency === "string" && RECENCY_VALUES.includes(raw.recency)) {
+    out.recency = raw.recency;
+  }
+  if (typeof raw.lang === "string" && raw.lang.trim().length > 0) out.lang = raw.lang.trim();
+  if (typeof raw.region === "string" && raw.region.trim().length > 0) out.region = raw.region.trim();
+  return out;
+}
+
+/** Backend-specific recency token, or undefined when unsupported/unset. */
+export function recencyToken(backendId, filters) {
+  const v = filters && filters.recency;
+  if (!v) return undefined;
+  const row = RECENCY_MAP[backendId];
+  return row ? row[v] : undefined;
+}
+
+/** Brave: freshness + optional country/search_lang (ISO-ish codes passed through). */
+export function applyBraveFilters(url, filters) {
+  const fresh = recencyToken("brave", filters);
+  if (fresh) url.searchParams.set("freshness", fresh);
+  if (filters && filters.region) url.searchParams.set("country", filters.region);
+  if (filters && filters.lang) url.searchParams.set("search_lang", filters.lang);
+  return url;
+}
+
+/** Tavily: time_range body field. */
+export function applyTavilyFilters(body, filters) {
+  const tr = recencyToken("tavily", filters);
+  if (tr) body.time_range = tr;
+  return body;
+}
+
+/** Serper: Google tbs date syntax + hl (interface lang) / gl (country). */
+export function applySerperFilters(body, filters) {
+  const tbs = recencyToken("serper", filters);
+  if (tbs) body.tbs = tbs;
+  if (filters && filters.lang) body.hl = filters.lang;
+  if (filters && filters.region) body.gl = filters.region;
+  return body;
+}
+
+/** SearXNG: native time_range + language query params. */
+export function applySearxngFilters(url, filters) {
+  const tr = recencyToken("searxng", filters);
+  if (tr) url.searchParams.set("time_range", tr);
+  if (filters && filters.lang) url.searchParams.set("language", filters.lang);
+  return url;
+}
+
+/** DuckDuckGo html endpoint: df (date filter) form field, best-effort. */
+export function applyDdgFilters(formParams, filters) {
+  const df = recencyToken("ddg", filters);
+  if (df) formParams.set("df", df);
+  return formParams;
+}
+
+/**
+ * Safe-search parameter per backend (null = omit / unsupported):
+ *   searxng -> "1"|"0" ; brave -> "strict"|"off".
+ */
+export function safeSearchParams(backendId, enabled) {
+  if (enabled === undefined || enabled === null) return null;
+  if (backendId === "searxng") return enabled ? "1" : "0";
+  if (backendId === "brave") return enabled ? "strict" : "off";
+  return null;
+}

package/lib/util/mcp-client.js

@@ -7,7 +7,7 @@
 
 import { WebError } from "@deepseek-ai/dsh-web";
 
-const UA = "dsh-websearch/2.0";
+const UA = "dsh-websearch/2.1";
 const PROTOCOL_VERSION = "2025-06-18";
 const SESSION_TTL_MS = 10 * 60 * 1000; // 10 minutes
 

package/lib/util/rank.js

@@ -0,0 +1,121 @@
+/**
+ * @module dsh-websearch/util/rank
+ *
+ * Pure result-set post-processing for the unified provider:
+ *   - dedupeSources(sources, strategy) : cross-backend dedup with field fill-in
+ *   - rerankSources(sources, query)    : deterministic query-relevance ordering
+ *
+ * Both are pure, dependency-free and unit-tested in tests/rank.test.js.
+ * Determinism matters: identical inputs must always produce identical output
+ * so cached/replayed searches stay stable.
+ */
+
+/** Lowercase + tokenize into a Set. ASCII words (len>=2) plus CJK chars singly,
+ * so Chinese/Japanese titles still produce meaningful overlap signals. */
+export function tokenize(text) {
+  const tokens = new Set();
+  if (typeof text !== "string" || text.length === 0) return tokens;
+  const lower = text.toLowerCase();
+  // ASCII word runs of length >= 2 ...
+  for (const w of lower.match(/[a-z0-9]{2,}/g) ?? []) tokens.add(w);
+  // ... plus individual CJK chars (U+3400-U+9FFF common range covers CJK).
+  for (const ch of lower.match(/[\u3400-\u9fff]/g) ?? []) tokens.add(ch);
+  return tokens;
+}
+
+/** Jaccard similarity of two token sets: |A∩B| / |A∪B| ∈ [0,1]. */
+export function tokenSimilarity(a, b) {
+  if (a.size === 0 && b.size === 0) return 0;
+  let inter = 0;
+  for (const t of a) if (b.has(t)) inter++;
+  const union = a.size + b.size - inter;
+  return union === 0 ? 0 : inter / union;
+}
+
+/** Titles this similar (Jaccard >= threshold) count as the same story when
+ * strategy is "url+title" (aggregator mirrors / syndicated copies). */
+const TITLE_DUP_THRESHOLD = 0.9;
+
+/**
+ * Dedupe an ordered source list.
+ *
+ * strategy "url"       — key on lowercased URL; later duplicates fill in any
+ *                        missing title/snippet/publishedAt fields (existing
+ *                        behavior, preserved verbatim).
+ * strategy "url+title" — additionally drop later entries whose TITLE overlaps
+ *                        an already-kept entry's title at >=0.9 Jaccard (both
+ *                        titles non-empty). Different URLs reporting the same
+ *                        story collapse to the higher-priority backend's hit.
+ *
+ * The input array is not mutated; returns a fresh array.
+ */
+export function dedupeSources(sources, strategy) {
+  const out = [];
+  const byUrl = new Map(); // urlKey -> kept source
+  const keptTitles = [];   // [{ tokens }] only when strategy is url+title
+
+  for (const s of sources ?? []) {
+    if (!s || typeof s.url !== "string" || s.url.length === 0) continue;
+    const key = s.url.toLowerCase();
+    const existing = byUrl.get(key);
+    if (existing) {
+      if (!existing.title && s.title) existing.title = s.title;
+      if (!existing.snippet && s.snippet) existing.snippet = s.snippet;
+      if (!existing.publishedAt && s.publishedAt) existing.publishedAt = s.publishedAt;
+      continue;
+    }
+
+    if (strategy === "url+title" && typeof s.title === "string" && s.title.length > 0) {
+      const tokens = tokenize(s.title);
+      const dupOf = keptTitles.find((k) => tokenSimilarity(k.tokens, tokens) >= TITLE_DUP_THRESHOLD);
+      if (dupOf) {
+        // Same story from another outlet: enrich the kept entry with any
+        // fields the mirror carries and the original lacks.
+        if (!dupOf.source.title && s.title) dupOf.source.title = s.title;
+        if (!dupOf.source.snippet && s.snippet) dupOf.source.snippet = s.snippet;
+        if (!dupOf.source.publishedAt && s.publishedAt) dupOf.source.publishedAt = s.publishedAt;
+        continue;
+      }
+    }
+
+    const copy = { ...s };
+    byUrl.set(key, copy);
+    out.push(copy);
+    if (strategy === "url+title" && typeof s.title === "string" && s.title.length > 0) {
+      // backfill this kept entry into its title bucket for later comparisons
+      keptTitles.push({ tokens: tokenize(s.title), source: copy });
+    }
+  }
+  return out;
+}
+
+/**
+ * Score one source against the query tokens:
+ *   title match ×3, snippet match ×2, URL match ×1.
+ */
+function scoreSource(source, queryTokens) {
+  let score = 0;
+  const titleTokens = tokenize(source.title ?? "");
+  const snippetTokens = tokenize(source.snippet ?? "");
+  const urlTokens = tokenize(String(source.url ?? "").replace(/[^a-z0-9]+/g, " "));
+  for (const q of queryTokens) {
+    if (titleTokens.has(q)) score += 3;
+    if (snippetTokens.has(q)) score += 2;
+    if (urlTokens.has(q)) score += 1;
+  }
+  return score;
+}
+
+/**
+ * Deterministic relevance rerank. Stable sort by descending score; equal
+ * scores keep their original relative order (fan-out priority wins ties).
+ * Returns a fresh array; input untouched.
+ */
+export function rerankSources(sources, query) {
+  const queryTokens = tokenize(query);
+  if (queryTokens.size === 0 || !Array.isArray(sources)) return [...(sources ?? [])];
+  return sources
+    .map((s, index) => ({ s, index, score: scoreSource(s, queryTokens) }))
+    .sort((a, b) => (b.score - a.score) || (a.index - b.index))
+    .map((e) => e.s);
+}

package/lib/util/rpc.js

@@ -24,7 +24,7 @@ import { WebError } from "@deepseek-ai/dsh-web";
 import { throwIfSearchAborted, searchAborted, isAbortError } from "./abort.js";
 
 /** User-Agent header; bumped with the package version. */
-export const UA = "dsh-websearch/2.0";
+export const UA = "dsh-websearch/2.1";
 
 /** MCP protocol version we negotiate. */
 const PROTOCOL_VERSION = "2025-06-18";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@240xu/dsh-websearch",
-  "version": "2.0.6",
+  "version": "2.1.0",
   "description": "Unified web search provider for DeepSeek Harness (ctx.web): 11 backends — Exa + Parallel + DDG + SearXNG (keyless), DeepSeek + Anthropic + OpenAI + Brave + Tavily + Serper + Mojeek (key-gated via Settings credentials). Registers provider id \"unified\" for zero-config web_search tool.",
   "type": "module",
   "main": "lib/index.js",