@240xu/dsh-websearch 2.3.0 → 2.7.2

package/lib/prompting.js

@@ -0,0 +1,105 @@
+/**
+ * dsh-websearch — 系统性搜索提示词（纯函数，零依赖）。
+ *
+ * 依据权威资料（2024-2026 官方文档与一手论文）实现三项必做：
+ *   1. 查询整形（query shaping）：后端扇出前对口语化长查询做确定性归一，
+ *      并导出给宿主/代理层使用的查询构造指引。
+ *      依据：Tavily Best Practices（agent 风格 ≤1500 字符查询、子查询分解）
+ *      https://docs.tavily.com/documentation/best-practices/best-practices-search
+ *      Exa Search Best Practices（语义丰富、描述性查询）
+ *      https://exa.ai/docs/search-best-practices
+ *   2. 结果呈现头（result-presentation header）：模型看到的结果块以「来源
+ *      甄别 + 引用要求」开头，要求每条事实注明来源链接、优先官方/一手来源、
+ *      时效敏感主题优先较新结果。
+ *      依据：Anthropic Web Search Tool（结果块置于问题之前、引用来源）
+ *      https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool
+ *      OpenAI Prompt Engineering（分解 + 持久性指令）
+ *      https://platform.openai.com/docs/guides/prompt-engineering
+ *   3. 多面查询分解指引（导出常量，供宿主/代理层按需使用；不对本插件内
+ *      的单次请求盲目多查询——盲目多查询可能劣化，见 arXiv:2404.01037）。
+ *      依据：RAG-Fusion / 多查询变体 https://github.com/Raudaschl/rag-fusion
+ *      Step-Back Prompting https://arxiv.org/abs/2310.06117
+ *      HyDE https://arxiv.org/abs/2212.10496
+ * @module dsh-websearch/prompting
+ */
+
+/** 查询硬上限（Tavily：agent 查询应简短聚焦；>1500 字符视为跑题长文）。 */
+export const QUERY_MAX_CHARS = 1500;
+
+/** 去口语化前缀（中英常见寒暄/请求词，确定性剥离，不改写语义）。 */
+const FILLER_PREFIX_RE =
+  /^(?:please|pls|kindly|could you(?: please)?|can you(?: please)?|help me(?: to)?|search for|search|look up|google|find me|find|i(?:'d| would)? like to know|tell me|what is|who is|when is|where is|"|「|『)\s*/i;
+const FILLER_PREFIX_RE_ZH =
+  /^(?:请|麻烦|帮我?|帮忙|帮我查一?下|查一?下|查询|搜索一下?|搜一下?|查一下?|帮我找|找一下?|我想知道|请问)/;
+
+/**
+ * 确定性查询整形：剥离口语前缀（循环多次以处理叠加寒暄）、压缩空白、
+ * 截断到 QUERY_MAX_CHARS。绝不改写实体/版本号/日期。
+ * @param {string} query
+ * @returns {string}
+ */
+export function shapeQuery(query) {
+  if (typeof query !== "string") return "";
+  let q = query.replace(/\s+/g, " ").trim();
+  // 中文寒暄与英文寒暄交替剥离，直到稳定。
+  for (let i = 0; i < 6; i++) {
+    const before = q;
+    q = q.replace(FILLER_PREFIX_RE, "").trim();
+    q = q.replace(FILLER_PREFIX_RE_ZH, "").trim();
+    if (q === before) break;
+  }
+  if (q.length > QUERY_MAX_CHARS) q = q.slice(0, QUERY_MAX_CHARS).trim();
+  return q;
+}
+
+/**
+ * 结果呈现头（英文）。置于模型可见结果块最前（Anthropic：结果先于问题
+ * 显著提升引用召回）。要求：逐条标注 [n]、优先官方/一手来源、时效敏感
+ * 主题优先较新结果、每条事实给出来源链接、与问题无关的条目忽略。
+ */
+export const RESULT_HEADER_EN =
+  "Web search results (merged from multiple engines, deduplicated by URL). " +
+  "Each entry: [n] title — URL. Prefer official / primary sources; prefer newer " +
+  "results for fast-moving topics; ignore entries whose snippet does not actually " +
+  "answer the question. Cite the URL for every fact you take from a result.";
+
+/** 结果呈现头（中文）。 */
+export const RESULT_HEADER_ZH =
+  "网页搜索结果（多引擎合并、已按 URL 去重）。每条格式：[n] 标题 — 链接。" +
+  "优先采用官方/一手来源；时效敏感主题优先较新结果；摘要与问题无关的条目直接忽略。" +
+  "引用结果中的任何事实时必须注明来源链接。";
+
+/**
+ * 查询构造指引（导出给宿主/代理层注入 agent 系统提示用；本插件不发送）。
+ * EN 面向 agent，ZH 为同义说明。要点：一条聚焦查询、保留专名/版本/日期、
+ * 多面请求按维度拆成多条子查询、首轮结果不佳时先写 2-3 句假想答案再以其
+ * 关键短语重搜（HyDE）、持续检索直到每个维度都有答案。
+ */
+export const QUERY_GUIDANCE_EN =
+  "When searching the web: issue ONE concise, agent-style query (keep named " +
+  "entities, version numbers and dates; drop conversational filler). If the " +
+  "request has 2+ independent facets, issue separate focused queries per facet " +
+  "and merge findings. If the first round returns poor results, write a 2-3 " +
+  "sentence hypothetical ideal answer and re-query with its key phrases. " +
+  "Keep searching until every facet is answered or demonstrably unavailable.";
+
+/** 查询构造指引（中文版）。 */
+export const QUERY_GUIDANCE_ZH =
+  "联网检索时：一次发出一条简洁的 agent 风格查询（保留专名/版本号/日期，去掉口语寒暄）。" +
+  "若请求含 2 个以上独立维度，按维度分别发起聚焦查询并汇总结论。" +
+  "首轮结果不佳时，先写 2-3 句假想的理想答案，再用其中的关键短语重新检索。" +
+  "持续检索直到每个维度都有答案或确认不可得。";
+
+/**
+ * 按语言取结果呈现头。
+ * @param {"zh"|"en"|string} [lang]
+ */
+export function resultHeader(lang) {
+  return String(lang || "").toLowerCase().startsWith("zh") ? RESULT_HEADER_ZH : RESULT_HEADER_EN;
+}
+
+/**
+ * 遥测行自标注（零成本修复）：模型可能把后端失败遥测误读为结果来源；
+ * 显式声明「仅诊断，不是来源」。随结果块一并呈现。
+ */
+export const TELEMETRY_TAG = " (diagnostic only, not a source)";

package/lib/provider.js

@@ -22,9 +22,44 @@ import {
 } from "./util/log.js";
 import { dedupeSources, rerankSources } from "./util/rank.js";
 import { normalizeFilters } from "./util/filters.js";
+import { shapeQuery, resultHeader, TELEMETRY_TAG } from "./prompting.js";
+import { cacheKeyFor } from "./cache.js";
+import { foldBreaker, isCooled, cooledAnnotation } from "./breaker.js";
 
 const PROVIDER_ID = "unified";
 
+// Module-level per-backend health telemetry (id -> entry), updated on every
+// backend completion during a real search. Purely passive: nothing writes here
+// outside search(), so the read-only /api/unified-search/health route can
+// report the LAST known reachability without firing probes or burning quota.
+// Entry shape: { ok: boolean, at: ms, ms: duration, count?: number,
+//                error?: string (already truncated to 80), lastOkAt: ms|null }
+const latestBackendHealth = new Map();
+
+/** Read-only access for the health route (never mutate the returned map). */
+export function getLatestBackendHealth() {
+  return latestBackendHealth;
+}
+
+function recordBackendHealth(id, entry, breakerCfg) {
+  const prev = latestBackendHealth.get(id);
+  const breaker = foldBreaker(prev, entry.ok === true, entry.at, breakerCfg ?? undefined);
+  latestBackendHealth.set(id, {
+    ...entry,
+    lastOkAt: entry.ok === true ? entry.at : (prev && prev.lastOkAt) || null,
+    failCount: breaker.failCount,
+    cooledUntil: breaker.cooledUntil,
+  });
+}
+
+/**
+ * Read-only access to one backend's breaker state (failCount / cooledUntil).
+ */
+export function getBackendBreaker(id) {
+  const e = latestBackendHealth.get(id);
+  return e ? { failCount: e.failCount ?? 0, cooledUntil: e.cooledUntil ?? 0 } : { failCount: 0, cooledUntil: 0 };
+}
+
 /**
  * Simple concurrency limiter using a semaphore pattern.
  * No external dependency needed.
@@ -52,6 +87,47 @@ function createLimiter(concurrency) {
   };
 }
 
+/**
+ * PHASE 1 (collect): fold settled fan-out results into the raw merge inputs.
+ * Sources keep fan-out priority order; dedupe/rank over the FULL set happens
+ * later (a mid-merge maxResults cap would silently drop later backends'
+ * entries before cross-backend fill-in and ranking could see them).
+ * URL scheme whitelist is enforced here — javascript:/data: payloads from
+ * compromised or poisoned backends never reach model/GUI consumers.
+ */
+function collectFanoutResults(results, eligible, signal) {
+  const collected = [];
+  let mergedContent;
+  let hardAborted = false;
+  let hardError = null;
+  const failures = [];
+
+  let backendIndex = 0;
+  for (const settled of results) {
+    const failedId = eligible[backendIndex] ? eligible[backendIndex].id : "unknown";
+    backendIndex++;
+    if (settled.status === "fulfilled") {
+      if (mergedContent === undefined && typeof settled.value?.content === "string" && settled.value.content.length > 0) {
+        mergedContent = settled.value.content;
+      }
+      for (const src of settled.value?.sources ?? []) {
+        if (src && typeof src.url === "string" && /^https?:\/\//i.test(src.url)) collected.push(src);
+      }
+    } else {
+      const err = settled.reason;
+      if (err instanceof WebError && err.code === "WEB_ABORTED") {
+        // Per-backend abort -> soft unless the caller signal fired.
+        if (signal?.aborted === true) hardAborted = true;
+        else failures.push({ id: failedId, err });
+      } else {
+        failures.push({ id: failedId, err });
+        if (hardError === null) hardError = err;
+      }
+    }
+  }
+  return { collected, mergedContent, hardAborted, hardError, failures };
+}
+
 /**
  * Build the UnifiedSearchProvider.
  *
@@ -60,7 +136,7 @@ function createLimiter(concurrency) {
  * @param {Function} deps.resolveOptions  () => resolved options snapshot
  * @param {object} deps.backends     keyed backend registry
  */
-export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
+export function createUnifiedSearchProvider({ ctx, resolveOptions, backends, cache, history }) {
   return {
     id: PROVIDER_ID,
 
@@ -87,19 +163,89 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
     async search(request, signal) {
       throwIfSearchAborted(signal);
       const opts = resolveOptions();
-      const maxResults = request.maxResults ?? opts.numResults ?? 8;
+      const filters = normalizeFilters({
+      ...(opts.filters ?? {}),
+      ...(request?.filters ?? {}),
+      });
+        let effReq = { ...request, maxResults: Math.max(1, Number(request?.maxResults) || opts.numResults || 8) };
+      // 系统性查询整形（Tavily/Exa best practices）：扇出前对口语化查询做
+      // 确定性归一 —— 剥离寒暄前缀、压缩空白、截断到 1500 字符；
+      // 专名/版本号/日期原样保留，不改写语义。
+      effReq.query = shapeQuery(effReq.query);
+      if (Object.keys(filters).length > 0) effReq = { ...effReq, filters };
+      const maxResults = effReq.maxResults;
       const concurrency = opts.concurrency ?? 6;
       const backendTimeoutMs = opts.backendTimeoutMs ?? 30000;
 
-      // Pick enabled & available backends.
-      const eligible = [];
+      // v2.7.0 disk cache: keyed on (shaped query + filters + maxResults).
+      // A hit short-circuits the fan-out entirely; sources are byte-identical
+      // to the first search and content carries a "cache hit, age Ns" note.
+      // Cache failures never break the search (cache API never throws).
+      const cacheEnabled = opts.cacheEnabled === true && cache;
+      let cacheKey = null;
+      if (cacheEnabled) {
+        cacheKey = cacheKeyFor({ query: effReq.query, filters, maxResults, backends: opts.enabledBackends });
+        const hit = cache.get(cacheKey);
+        if (hit && Array.isArray(hit.value?.sources) && hit.value.sources.length > 0) {
+          // Defense in depth: the fresh path drops non-http(s) schemes before
+          // merge; a tampered cache file must not bypass that whitelist.
+          const safeSources = hit.value.sources.filter(
+            (src) => src && typeof src.url === "string" && /^https?:\/\//i.test(src.url),
+          );
+          if (safeSources.length === 0) {
+            // whole entry poisoned or filtered empty -> treat as a miss
+            cacheKey = null;
+          } else {
+          let hitContent = typeof hit.value.content === "string" ? hit.value.content : undefined;
+          const cacheNote = `[websearch cache] cache hit, age ${hit.age}s`;
+          hitContent = hitContent === undefined ? cacheNote : hitContent + "\n\n" + cacheNote;
+          history?.record({
+            query: request?.query,
+            time: Date.now(),
+            resultCount: hit.value.sources.length,
+            backendsOk: 0,
+            backendsTotal: 0,
+          });
+          return {
+            sources: safeSources,
+            ...(hitContent !== undefined ? { content: hitContent } : {}),
+            truncated: hit.value.truncated === true,
+          };
+          }
+        }
+      }
+
+      // Pick enabled & available backends. With the breaker enabled, a
+      // backend inside its 60s cooldown window is skipped (annotated in
+      // telemetry as "id ⏸cooled"). Fail-open: if EVERY candidate is cooled,
+      // run them all anyway — the breaker must never make search worse than
+      // no breaker.
+      const breakerCfg = opts.breakerEnabled === false ? null : {
+        threshold: opts.breakerThreshold,
+        cooldownMs: opts.breakerCooldownMs,
+      };
+      const nowMs = Date.now();
+      const candidates = [];
       for (const id of opts.enabledBackends) {
         const be = backends[id];
         if (!be) continue;
         const beOpts = opts.backends?.[id] ?? {};
         const avail = be.available(beOpts);
         if (!avail) continue;
-        eligible.push({ id, backend: be, beOpts });
+        candidates.push({ id, backend: be, beOpts });
+      }
+      const cooled = new Map();
+      let eligible = candidates;
+      if (breakerCfg) {
+        const active = candidates.filter((c) => {
+          if (isCooled(latestBackendHealth.get(c.id), nowMs)) {
+            cooled.set(c.id, latestBackendHealth.get(c.id));
+            return false;
+          }
+          return true;
+        });
+        if (active.length > 0) eligible = active;
+        else cooled.clear(); // all cooled -> fail-open
       }
 
       if (eligible.length === 0) {
@@ -112,16 +258,10 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
       // Settings-level default filters, overridden by per-request filters
       // (normalizeFilters drops unknown/invalid values so typos degrade to
       // "no filter" instead of breaking the search).
-      const filters = normalizeFilters({
-        ...(opts.filters ?? {}),
-        ...(request?.filters ?? {}),
-      });
-      const effectiveRequest =
-        Object.keys(filters).length > 0 ? { ...request, filters } : request;
 
       const telemetry = [];
       // Concurrency limiter
-      const limit = createLimiter(opts.concurrency ?? 6);
+      const limit = createLimiter(concurrency);
 
       // Run one backend under the per-backend timeout. The internal timer's
       // controller is merged into the caller's signal; when the INTERNAL timer
@@ -130,20 +270,35 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
       // failures instead of being silently demoted like user cancellations.
       const runBackend = ({ id, backend, beOpts, hooks }) => {
         const startedAtMs = Date.now();
+        // Trust boundary: baseURL is operator-configurable and API keys are sent to it.
+        // Enforce http(s); plain http only for private hosts (self-hosted SearXNG).
+        // Reject embedded credentials. Failure is a per-backend soft error.
+        if (beOpts.baseURL !== undefined) {
+          let bu = null;
+          try { bu = new URL(beOpts.baseURL); } catch { bu = null; }
+          const privHost = bu ? /^(localhost|127(\.\d{1,3}){3}|10(\.\d{1,3}){3}|192\.168(\.\d{1,3}){3}|172\.(1[6-9]|2\d|3[01])(\.\d{1,3}){3})$/.test(bu.hostname) : false;
+          const schemeOk = bu && (bu.protocol === "https:" || (bu.protocol === "http:" && privHost));
+          if (!schemeOk || bu.username || bu.password) {
+            throw new WebError(`backend "${id}" has an invalid baseURL: https required (plain http only for private hosts), credentials-in-URL rejected`, "WEB_PROVIDER_ERROR");
+          }
+        }
         const controller = new AbortController();
         const mergedSignal = signal
           ? AbortSignal.any([signal, controller.signal])
           : controller.signal;
         const timeoutId = setTimeout(() => controller.abort(), backendTimeoutMs);
         return backend
-          .search(effectiveRequest, mergedSignal, beOpts, hooks)
+          .search(effReq, mergedSignal, beOpts, hooks)
           .then(
             (value) => {
               telemetry.push({ id, ok: true, ms: Date.now() - startedAtMs, count: value?.sources?.length ?? 0 });
+              recordBackendHealth(id, { ok: true, at: Date.now(), ms: Date.now() - startedAtMs, count: value?.sources?.length ?? 0 }, breakerCfg);
               return value;
             },
             (err) => {
-              telemetry.push({ id, ok: false, ms: Date.now() - startedAtMs, error: String(err instanceof Error ? err.message : err).slice(0, 80) });
+              const msg = String(err instanceof Error ? err.message : err).slice(0, 80);
+              telemetry.push({ id, ok: false, ms: Date.now() - startedAtMs, error: msg });
+              recordBackendHealth(id, { ok: false, at: Date.now(), ms: Date.now() - startedAtMs, error: msg }, breakerCfg);
               throw err;
             },
           )
@@ -172,45 +327,18 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
 
       throwIfSearchAborted(signal);
 
-      // Collect every fulfilled backend's sources in fan-out priority order,
-      // then dedupe/rank ONCE over the full set. (The previous loop stopped at
-      // maxResults mid-merge, which silently dropped later backends' entries
-      // before cross-backend fill-in and ranking could see them.)
-      const collected = [];
-      let mergedContent;
-      let hardAborted = false;
-      let hardError = null;
-      const failures = [];
-
-      let backendIndex = 0;
-      for (const settled of results) {
-        const failedId = eligible[backendIndex] ? eligible[backendIndex].id : "unknown";
-        backendIndex++;
-        if (settled.status === "fulfilled") {
-          if (mergedContent === undefined && typeof settled.value?.content === "string" && settled.value.content.length > 0) {
-            mergedContent = settled.value.content;
-          }
-          for (const s of settled.value?.sources ?? []) collected.push(s);
-        } else {
-          // settled.status === "rejected"
-          const err = settled.reason;
-          if (err instanceof WebError && err.code === "WEB_ABORTED") {
-            // Per-backend abort → soft unless caller signal fired.
-            if (signal?.aborted === true) hardAborted = true;
-            else failures.push({ id: failedId, err });
-          } else {
-            failures.push({ id: failedId, err });
-            if (hardError === null) hardError = err;
-          }
-        }
-      }
+      // PHASE 1 collect: fold the settled fan-out into raw sources (fan-out
+      // priority order), first backend content, and per-backend failures.
+      const { collected, mergedContent, hardAborted, hardError, failures } =
+        collectFanoutResults(results, eligible, signal);
 
       // Caller-cancellation check (hard abort fires before we yield results).
       if (signal?.aborted === true || hardAborted) {
         throw searchAborted(signal);
       }
 
-      // Dedup: "url" preserves the historical URL-key behavior; "url+title"
+      // PHASE 2 dedupe / PHASE 3 rerank: "url" preserves the historical
+      // URL-key behavior; "url+title"
       // additionally collapses same-story syndicated copies across backends.
       const strategy = opts.dedupStrategy === "url+title" ? "url+title" : "url";
       const deduped = dedupeSources(collected, strategy);
@@ -223,11 +351,37 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
       // If we got ANY sources, return them even if some backends failed.
       if (ordered.length > 0) {
         const sources = ordered.slice(0, maxResults);
+        // 结果呈现头（Anthropic web-search / OpenAI grounding 官方指引）：
+        // 置于结果块最前，要求逐条引用来源 URL、优先官方/一手与较新来源；
+        // 语言按查询文本自动选择（含 CJK 用中文头）。可用
+        // opts.resultHeader === false 关闭。
         let outContent = mergedContent;
-        if (opts.resultTelemetry !== false && telemetry.length > 0) {
-          const line = "[websearch backends] " + telemetry
-            .map((t) => t.id + (t.ok ? " ✓" : " ✗") + t.ms + "ms" + (t.ok ? "/" + t.count : ""))
-            .join(" · ");
+        if (opts.resultHeader !== false && (sources.length > 0 || outContent !== undefined)) {
+          const header = resultHeader(typeof request?.query === "string" && /[\u4e00-\u9fff]/.test(request.query) ? "zh" : "en");
+          outContent = outContent === undefined ? header : header + "\n\n" + outContent;
+        }
+        // Store the pre-telemetry core (sources + headered content) so a cache
+        // hit reproduces the exact sources and the model-facing header.
+        if (cacheKey && cache) {
+          cache.set(cacheKey, { sources, content: outContent, truncated: ordered.length > maxResults });
+        }
+        history?.record({
+          query: request?.query,
+          time: Date.now(),
+          resultCount: sources.length,
+          backendsOk: telemetry.filter((t) => t.ok).length,
+          backendsTotal: telemetry.length,
+        });
+        if (opts.resultTelemetry !== false && (telemetry.length > 0 || cooled.size > 0)) {
+          const cooledParts = [];
+          for (const [cid, centry] of cooled) {
+            const ann = cooledAnnotation(centry, Date.now()) || "⏸cooled";
+            cooledParts.push(cid + " " + ann);
+          }
+          const parts = telemetry
+            .map((t) => t.id + (t.ok ? " ✓" : " ✗") + t.ms + "ms" + (t.ok ? "/" + t.count : (t.error ? " (" + String(t.error).slice(0, 40) + ")" : "")))
+            .concat(cooledParts);
+          const line = "[websearch backends]" + TELEMETRY_TAG + " " + parts.join(" · ");
           outContent = outContent === undefined ? line : outContent + "\n\n" + line;
         }
         return {
@@ -244,7 +398,7 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
         .join("; ");
       const allKeyMissing =
         failures.length > 0 &&
-        failures.every((f) => f.err instanceof WebError && f.err.code === "WEB_PROVIDER_UNAVAILABLE");
+        failures.every((f) => f.err instanceof WebError && (f.err.code === "WEB_PROVIDER_UNAVAILABLE" || f.err.code === "WEB_PROVIDER_CREDENTIAL_MISSING"));
       const hint = allKeyMissing
         ? " (every enabled backend is missing its API key - add keys in Settings > Web Search, or enable a keyless backend)"
         : "";

package/lib/util/log.js

@@ -56,7 +56,7 @@ export function recordBackendOutcome(ctx, backendId, outcome) {
     const logger = ctx && ctx.logger;
     if (!logger) return;
     const status = outcome && typeof outcome.status === "string" ? outcome.status : "";
-    const line = "[websearch:" + backendId + "] outcome " + JSON.stringify(outcome ?? {});
+    const line = "[websearch:" + backendId + "] outcome " + JSON.stringify(outcome ?? {}).slice(0, 500);
     if (status === "error" && typeof logger.warn === "function") logger.warn(line);
     else if (typeof logger.debug === "function") logger.debug(line);
   } catch {

package/lib/util/mcp-client.js

@@ -7,7 +7,7 @@
 
 import { WebError } from "@deepseek-ai/dsh-web";
 
-const UA = "dsh-websearch/2.1";
+const UA = "dsh-websearch/2.3";
 const PROTOCOL_VERSION = "2025-06-18";
 const SESSION_TTL_MS = 10 * 60 * 1000; // 10 minutes
 
@@ -58,7 +58,7 @@ async function ensureSession(url, signal) {
   }
 
   if (!initResponse.ok) {
-    const text = await initResponse.text().catch(() => "");
+    const text = await initResponse.text().catch(() => "").then((tx) => tx.slice(0, 300));
     throw new WebError(`MCP initialize HTTP ${initResponse.status}: ${text}`, "WEB_PROVIDER_ERROR");
   }
 
@@ -75,7 +75,7 @@ async function ensureSession(url, signal) {
   };
   
   try {
-    await fetch(url, {
+    const notifyResponse = await fetch(url, {
       method: "POST",
       headers: {
         "Content-Type": "application/json",
@@ -86,6 +86,8 @@ async function ensureSession(url, signal) {
       signal,
       redirect: "error",
     });
+    // Drain the discarded body so undici returns the socket to the pool.
+    notifyResponse.body?.cancel?.().catch(() => {});
   } catch (error) {
     // Best-effort: Exa requires this, Parallel tolerates absence
     if (signal?.aborted) throw error;
@@ -104,17 +106,16 @@ async function ensureSession(url, signal) {
 function parseEnvelope(text, expectedId) {
   // Try SSE format first: "event: message\ndata: {...}\n\n"
   if (text.includes("data:")) {
-    const lines = text.trim().split("\n");
-    let dataLine = null;
-    for (const line of lines) {
-      if (line.startsWith("data:")) {
-        dataLine = line.slice(5).trim();
-        break;
-      }
-    }
-    if (dataLine) {
+    // SSE spec: one JSON payload may span multiple data: lines - join them.
+    const joined = text
+      .trim()
+      .split("\n")
+      .filter((l) => l.startsWith("data:"))
+      .map((l) => l.slice(5).trim())
+      .join("");
+    if (joined) {
       try {
-        return JSON.parse(dataLine);
+        return JSON.parse(joined);
       } catch {}
     }
   }
@@ -205,7 +206,7 @@ export async function mcpCall(url, toolName, args, extraHeaders = {}, signal) {
   }
 
   if (!response.ok) {
-    const text = await response.text().catch(() => "");
+    const text = await response.text().catch(() => "").then((tx) => tx.slice(0, 300));
     throw new WebError(`MCP tools/call HTTP ${response.status}: ${text}`, "WEB_PROVIDER_ERROR");
   }
 

package/lib/util/rank.js

@@ -97,7 +97,7 @@ function scoreSource(source, queryTokens) {
   let score = 0;
   const titleTokens = tokenize(source.title ?? "");
   const snippetTokens = tokenize(source.snippet ?? "");
-  const urlTokens = tokenize(String(source.url ?? "").replace(/[^a-z0-9]+/g, " "));
+  const urlTokens = tokenize(String(source.url ?? ""));
   for (const q of queryTokens) {
     if (titleTokens.has(q)) score += 3;
     if (snippetTokens.has(q)) score += 2;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@240xu/dsh-websearch",
-  "version": "2.3.0",
-  "description": "Unified web search provider for DeepSeek Harness (ctx.web): 11 backends \u2014 Exa + Parallel + DDG + SearXNG (keyless), DeepSeek + Anthropic + OpenAI + Brave + Tavily + Serper + Mojeek (key-gated via Settings credentials). Registers provider id \"unified\" for zero-config web_search tool.",
+  "version": "2.7.2",
+  "description": "Unified web search provider for DeepSeek Harness (ctx.web): 11 backends — Exa + Parallel + DDG + SearXNG (keyless), DeepSeek + Anthropic + OpenAI + Brave + Tavily + Serper + Mojeek (key-gated via Settings credentials). Registers provider id \"unified\" for zero-config web_search tool. v2.7.0: disk result cache, per-backend circuit breaker, append-only search history API. v2.7.1 review fixes: cache key includes enabledBackends, scheme whitelist on cache-hit path, serialized history writes, trust fence on /api/websearch/* routes, cooldownMs=0 disables breaker window, user-language settings labels.",
   "type": "module",
   "main": "lib/index.js",
   "exports": {