dsh-free-search 0.7.5 → 0.8.0

package/lib/cache.js

@@ -0,0 +1,49 @@
+// research 文档持久缓存（node:sqlite，零依赖）。Node <22.5 无 node:sqlite 时整体降级为 no-op，插件照常工作。
+// ponytail: cache_search 用 LIKE 而非 FTS5 MATCH——个人规模（百级行）LIKE 即时返回，且无 FTS5 查询语法转义坑；行数过万再升 FTS5 虚表。
+
+import { createRequire } from "node:module";
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+
+let db = null;
+try {
+  const req = createRequire(import.meta.url);
+  const { DatabaseSync } = req("node:sqlite");
+  // 缓存位置可配置：RESEARCH_CACHE_DIR 优先（放哪、是否随备份走由用户决定），
+  // 默认 ~/.cache/dsh-free-search/research.db；删掉该目录即清空缓存，无后台任务残留。
+  const dir = process.env.RESEARCH_CACHE_DIR || join(homedir(), ".cache", "dsh-free-search");
+  mkdirSync(dir, { recursive: true });
+  db = new DatabaseSync(join(dir, "research.db"));
+  db.exec("CREATE TABLE IF NOT EXISTS pages(url TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT '', content TEXT NOT NULL, fetched_at INTEGER NOT NULL)");
+} catch { /* Node <22.5：缓存禁用 */ }
+
+/** 新鲜则返回 {title, content, fetched_at}，否则 null。 */
+export function getFresh(url, maxAgeMs) {
+  if (!db) return null;
+  const r = db.prepare("SELECT title, content, fetched_at FROM pages WHERE url = ?").get(url);
+  if (!r || Date.now() - r.fetched_at > maxAgeMs) return null;
+  return r;
+}
+
+export function put(url, title, content) {
+  if (!db) return;
+  db.prepare("DELETE FROM pages WHERE url = ?").run(url);
+  db.prepare("INSERT INTO pages(url, title, content, fetched_at) VALUES (?, ?, ?, ?)").run(url, title || "", content, Date.now());
+}
+
+/** 全文 LIKE 检索已缓存页面（7 天内），返回 [{url, title, excerpt, fetched_at}]。 */
+const STALE_MS = 7 * 24 * 3600 * 1000; // 与 research.js 的 DOC_CACHE_MAX_AGE_MS 对齐
+export function search(query, limit = 8) {
+  if (!db) return [];
+  db.prepare("DELETE FROM pages WHERE fetched_at < ?").run(Date.now() - STALE_MS); // ponytail: 查询时惰性 GC，无后台任务
+  const like = `%${String(query).replace(/[%_]/g, " ")}%`; // ponytail: 通配符转空格而非 ESCAPE 子句，查询词含 %/_ 属罕见
+  const rows = db
+    .prepare("SELECT url, title, content, fetched_at FROM pages WHERE (title LIKE ? OR content LIKE ?) AND fetched_at > ? ORDER BY fetched_at DESC LIMIT ?")
+    .all(like, like, Date.now() - STALE_MS, limit);
+  return rows.map((r) => {
+    const i = r.content.toLowerCase().indexOf(String(query).toLowerCase());
+    const start = Math.max(0, i - 40);
+    return { url: r.url, title: r.title, excerpt: (start > 0 ? "…" : "") + r.content.slice(start, start + 160).replace(/\s+/g, " "), fetched_at: new Date(r.fetched_at).toISOString() };
+  });
+}

package/lib/client.js

@@ -133,7 +133,7 @@ window.__ModuleLoader__.load({
         searchEngine: "搜索引擎",
         visit: "访问官网 →",
         getKey: "获取 API Key →",
-        engineHint: "Multi Search 会并发搜索路由/优先级前 3 个已启用引擎并合并去重；Auto 根据查询语言及时间过滤自动优选单个引擎。Bing 是稳定的免费引擎。API KEY 引擎需在下方填写凭据。",
+        engineHint: "Multi Search 会按查询语言并发对应语言池的全部已启用引擎并合并去重（中文/英文池可在下方编辑，每池最低 2 个引擎）；Auto 根据查询语言及时间过滤自动优选单个引擎。Bing 是稳定的免费引擎。API KEY 引擎需在下方填写凭据。",
         apiKeys: "API 密钥（可选）",
         anysearchPh: (c) => c ? "AnySearch API 密钥（已配置）" : "AnySearch API 密钥（可选，不填免费匿名，填了提额）",
         exaPh: (c) => c ? "Exa API 密钥（已配置）" : "Exa API 密钥（可选，不填也可免费使用）",
@@ -164,6 +164,12 @@ window.__ModuleLoader__.load({
         fallbackPriority: "全局回退优先级",
         fallbackPriorityHint: "从上到下为回退顺序。首选引擎仍先尝试；Auto 保留语言/时间路由，但自定义后同一分组内及后续回退按此顺序。禁用引擎不会执行，但会保留当前位置。",
         fallbackPriorityReset: "恢复默认顺序",
+        zhPoolLabel: "中文查询池（zhPool）",
+        enPoolLabel: "英文查询池（enPool）",
+        languagePoolHint: "Auto 路由的起始组与 Multi 模式的并发池：按查询语言二选一，Multi 并发池内全部引擎（不再是前 3 个）。每池最低 2 个引擎、无上限；禁用/冷却/缺 key 的引擎会自动跳过。",
+        poolAddFirst: "添加引擎…",
+        poolRemove: "移除",
+        poolReset: "恢复默认池",
         fallbackOnLabel: "回退触发条件（fallbackOn）",
         fallbackOnHint: "勾选哪些失败类别允许触发回退到下一个引擎。全部勾选＝保持现状（搜索永不直接失败）；取消某类＝遇到该类失败就停下并把该引擎的错误报出，不再换引擎。缺 key、被禁用、不支持时间过滤不算失败，始终会换引擎。",
         classQuota: "额度/预算耗尽 (quota)",
@@ -225,7 +231,7 @@ window.__ModuleLoader__.load({
         searchEngine: "Search engine",
         visit: "Visit website →",
         getKey: "Get API Key →",
-        engineHint: "Multi Search queries the top 3 enabled routed/prioritized engines concurrently and merges/deduplicates results. Auto smartly routes a single engine by query language and time filter. API KEY engines need credentials below.",
+        engineHint: "Multi Search queries every enabled engine of the query's language pool concurrently and merges/deduplicates results (zh/en pools editable below, minimum 2 engines each). Auto smartly routes a single engine by query language and time filter. API KEY engines need credentials below.",
         apiKeys: "API keys (optional)",
         anysearchPh: (c) => c ? "AnySearch API key (configured)" : "AnySearch API key (optional, free anonymous without; key raises quota)",
         exaPh: (c) => c ? "Exa API key (configured)" : "Exa API key (optional, free without)",
@@ -256,6 +262,12 @@ window.__ModuleLoader__.load({
         fallbackPriority: "Global fallback priority",
         fallbackPriorityHint: "Top to bottom is the fallback order. The preferred engine is still tried first. Auto keeps language/time routing, while customized priority orders engines within a route group and the remaining fallback chain. Disabled engines keep their position but are skipped.",
         fallbackPriorityReset: "Reset default order",
+        zhPoolLabel: "Chinese query pool (zhPool)",
+        enPoolLabel: "English query pool (enPool)",
+        languagePoolHint: "The route head for Auto and the concurrent pool for Multi: picked by query language. Multi runs every engine in the pool (no longer top 3). Each pool keeps a minimum of 2 engines, no upper limit; disabled/cooling/keyless engines are skipped automatically.",
+        poolAddFirst: "Add engine…",
+        poolRemove: "Remove",
+        poolReset: "Reset default pool",
         fallbackOnLabel: "Fallback triggers (fallbackOn)",
         fallbackOnHint: "Choose which failure classes may trigger a fallback to the next engine. All checked = current behaviour (search never fails outright); unchecking one means such a failure stops the search and surfaces that engine's error instead of trying another engine. Missing keys, disabled engines and unsupported time filters are not failures and always advance.",
         classQuota: "Quota/budget exhausted (quota)",
@@ -313,8 +325,8 @@ window.__ModuleLoader__.load({
       },
     };
     const tt = (lang) => I18N[lang === "en" ? "en" : "zh"];
-    // 当前插件版本（与 lib/index.js 的 PLUGIN_VERSION 保持一致）
-    const PLUGIN_VERSION = "0.7.5";
+    // 当前插件版本（与 lib/index.js 的 PLUGIN_VERSION 保持一致；跟随上游 v0.7.5）
+    const PLUGIN_VERSION = "0.8.0";
     const ENGINES = [
       { id: "auto", label: "Auto · 智能路由", badge: "AUTO", link: null },
       { id: "multi", label: "Multi Search · 多引擎", badge: "MULTI", link: null },
@@ -342,11 +354,20 @@ window.__ModuleLoader__.load({
       { id: "claude", label: "Claude · 内置搜索", badge: "API KEY", link: "https://console.anthropic.com/settings/keys" },
     ];
     const ENGINE_IDS = ENGINES.filter((engine) => engine.id !== "auto" && engine.id !== "multi").map((engine) => engine.id);
+    // 与 lib/index.js 的 EXPLICIT_ONLY_ENGINES 同源：按次计费、只在显式选中时运行，因此不提供入池候选。
+    const EXPLICIT_ONLY_ENGINES = ["openai", "gemini", "claude"];
     const DEFAULT_FALLBACK_ORDER = ["exa", "tavily", "keenable", "firecrawl", "parallel", "perplexity", "serpbase", "serply", "deepseek-official", "you", "baidu", "kimi", "aliyun", "doubao", "bing", "anysearch", "ddg", "ddg-lite", "searxng"];
     const normalizeFallbackOrder = (value) => {
       const configured = Array.isArray(value) ? value.filter((id) => ENGINE_IDS.includes(id)) : [];
       return [...new Set([...configured, ...DEFAULT_FALLBACK_ORDER, ...ENGINE_IDS])];
     };
+    // 语言分池（评审意见 B）：zh/en 两池默认值与 lib/index.js 保持一致
+    const DEFAULT_ZH_POOL = ["bing", "baidu", "aliyun", "anysearch"];
+    const DEFAULT_EN_POOL = ["bing", "exa", "tavily"];
+    const normalizePoolIds = (value, fallback) => {
+      const configured = [...new Set((Array.isArray(value) ? value : []).filter((id) => ENGINE_IDS.includes(id)))];
+      return configured.length > 0 ? configured : fallback.slice();
+    };
     const FAILURE_CLASSES = [
       { id: "quota", labelKey: "classQuota" },
       { id: "auth", labelKey: "classAuth" },
@@ -460,6 +481,8 @@ window.__ModuleLoader__.load({
       const [provider, setProvider] = react.useState("bing");
       const [disabledEngines, setDisabledEngines] = react.useState([]);
       const [fallbackOrder, setFallbackOrder] = react.useState(DEFAULT_FALLBACK_ORDER);
+      const [zhPool, setZhPool] = react.useState(DEFAULT_ZH_POOL);
+      const [enPool, setEnPool] = react.useState(DEFAULT_EN_POOL);
       const [fallbackOn, setFallbackOn] = react.useState(FAILURE_CLASSES.map((cls) => cls.id));
       const [safeSearch, setSafeSearch] = react.useState("off");
       const [bingMarket, setBingMarket] = react.useState("zh-CN");
@@ -514,6 +537,8 @@ window.__ModuleLoader__.load({
                   : []
               );
               setFallbackOrder(normalizeFallbackOrder(v.fallbackOrder));
+              setZhPool(normalizePoolIds(v.zhPool, DEFAULT_ZH_POOL));
+              setEnPool(normalizePoolIds(v.enPool, DEFAULT_EN_POOL));
               setFallbackOn(normalizeFallbackOn(v.fallbackOn));
               setSafeSearch(v.safeSearch === "strict" || v.safeSearch === "moderate" ? v.safeSearch : "off");
               setBingMarket(v.bingMarket === undefined ? "zh-CN" : v.bingMarket);
@@ -642,6 +667,8 @@ window.__ModuleLoader__.load({
           const ops = [{ op: "set", path: ["provider"], value: provider }];
           ops.push({ op: "set", path: ["disabledEngines"], value: disabledEngines });
           ops.push({ op: "set", path: ["fallbackOrder"], value: normalizeFallbackOrder(fallbackOrder) });
+          ops.push({ op: "set", path: ["zhPool"], value: normalizePoolIds(zhPool, DEFAULT_ZH_POOL) });
+          ops.push({ op: "set", path: ["enPool"], value: normalizePoolIds(enPool, DEFAULT_EN_POOL) });
           ops.push({ op: "set", path: ["fallbackOn"], value: normalizeFallbackOn(fallbackOn) });
           ops.push({ op: "set", path: ["lang"], value: lang });
           ops.push({ op: "set", path: ["keyStorage"], value: keyStorage });
@@ -783,6 +810,134 @@ window.__ModuleLoader__.load({
         setFailed(false);
         setSaveError("");
       };
+      // 语言分池（评审意见 B）：两块池列表，行结构复用回退优先级（↑↓ 排序），外加增删；每池最低 2 个引擎、不设上限
+      const renderPoolField = (label, pool, setPool, defaultPool, withHint) => {
+        const removable = pool.length > 2;
+        const candidates = ENGINE_IDS.filter((id) => !pool.includes(id) && !EXPLICIT_ONLY_ENGINES.includes(id));
+        return react_jsx_runtime.jsx("div", {
+          className: "dshfs-field",
+          children: [
+            react_jsx_runtime.jsx("div", {
+              className: "dshfs-label",
+              children: label,
+            }),
+            react_jsx_runtime.jsx("div", {
+              className: "dshfs-priorityList",
+              children: pool.map((engineId, index) => {
+                const engine = ENGINES.find((item) => item.id === engineId);
+                if (!engine) return null;
+                const isDisabled = disabledEngines.includes(engineId);
+                return react_jsx_runtime.jsx("div", {
+                  className: "dshfs-priorityRow" + (isDisabled ? " dshfs-priorityRowDisabled" : ""),
+                  children: [
+                    react_jsx_runtime.jsx("span", { className: "dshfs-priorityIndex", children: String(index + 1) }),
+                    react_jsx_runtime.jsx("span", {
+                      className: "dshfs-priorityName",
+                      children: `${engine.label} (${engine.badge})${isDisabled ? " · " + t.priorityDisabled : ""}`,
+                    }),
+                    react_jsx_runtime.jsx("span", {
+                      className: "dshfs-priorityActions",
+                      children: [
+                        react_jsx_runtime.jsx("button", {
+                          type: "button",
+                          className: "dshfs-btn dshfs-priorityBtn",
+                          disabled: !ready || saving || index === 0,
+                          onClick: () => {
+                            setPool((prev) => {
+                              const next = [...prev];
+                              [next[index - 1], next[index]] = [next[index], next[index - 1]];
+                              return next;
+                            });
+                            setDirty(true);
+                            setFailed(false);
+                            setSaveError("");
+                          },
+                          "aria-label": `${engine.label} ↑`,
+                          title: "↑",
+                          children: "↑",
+                        }, "up"),
+                        react_jsx_runtime.jsx("button", {
+                          type: "button",
+                          className: "dshfs-btn dshfs-priorityBtn",
+                          disabled: !ready || saving || index === pool.length - 1,
+                          onClick: () => {
+                            setPool((prev) => {
+                              const next = [...prev];
+                              [next[index + 1], next[index]] = [next[index], next[index + 1]];
+                              return next;
+                            });
+                            setDirty(true);
+                            setFailed(false);
+                            setSaveError("");
+                          },
+                          "aria-label": `${engine.label} ↓`,
+                          title: "↓",
+                          children: "↓",
+                        }, "down"),
+                        react_jsx_runtime.jsx("button", {
+                          type: "button",
+                          className: "dshfs-btn dshfs-priorityBtn",
+                          disabled: !ready || saving || !removable,
+                          onClick: () => {
+                            setPool((prev) => prev.filter((id) => id !== engineId));
+                            setDirty(true);
+                            setFailed(false);
+                            setSaveError("");
+                          },
+                          "aria-label": `${engine.label} ${t.poolRemove}`,
+                          title: t.poolRemove,
+                          children: "−",
+                        }, "remove"),
+                      ],
+                    }),
+                  ],
+                }, engineId);
+              }),
+            }),
+            react_jsx_runtime.jsx("div", {
+              children: [
+                react_jsx_runtime.jsx("select", {
+                  className: "dshfs-select",
+                  value: "",
+                  style: { colorScheme: selectColorScheme },
+                  disabled: !ready || saving || candidates.length === 0,
+                  onChange: (e) => {
+                    const id = e.target.value;
+                    if (!id) return;
+                    setPool((prev) => [...prev, id]);
+                    setDirty(true);
+                    setFailed(false);
+                    setSaveError("");
+                  },
+                  children: [
+                    react_jsx_runtime.jsx("option", { value: "", children: t.poolAddFirst }, "placeholder"),
+                    ...candidates.map((id) => {
+                      const engine = ENGINES.find((item) => item.id === id);
+                      return react_jsx_runtime.jsx("option", { value: id, children: `${engine.label} (${engine.badge})` }, id);
+                    }),
+                  ],
+                }),
+                react_jsx_runtime.jsx("button", {
+                  type: "button",
+                  className: "dshfs-btn",
+                  disabled: !ready || saving || pool.join(",") === defaultPool.join(","),
+                  onClick: () => {
+                    setPool(defaultPool.slice());
+                    setDirty(true);
+                    setFailed(false);
+                    setSaveError("");
+                  },
+                  children: t.poolReset,
+                }, "reset"),
+              ],
+            }),
+            ...(withHint ? [react_jsx_runtime.jsx("p", {
+              className: "dshfs-hint",
+              children: t.languagePoolHint,
+            })] : []),
+          ],
+        });
+      };
       const badgeClass =
         currentEngine.badge === "AUTO"
           ? "dshfs-badge dshfs-badgeAuto"
@@ -987,6 +1142,8 @@ react_jsx_runtime.jsx("div", {
                       }),
                     ],
                   }),
+                  renderPoolField(t.zhPoolLabel, zhPool, setZhPool, DEFAULT_ZH_POOL, false),
+                  renderPoolField(t.enPoolLabel, enPool, setEnPool, DEFAULT_EN_POOL, true),
                   react_jsx_runtime.jsx("div", {
                     className: "dshfs-field",
                     children: [

package/lib/fetch_provider.js

@@ -0,0 +1,94 @@
+// 质量链 fetch provider：接管宿主 web_fetch（替代基础 http provider）。
+// T1 直连（跟随重定向，仅文本类内容）→ T2 exa contents（$1/1000 页，清洗正文 markdown，文本质量好且便宜）
+// → T3 firecrawl（Markdown，反爬/JS/PDF）→ T4 crawl4ai（Markdown 备份）。
+// firecrawl 额度保护：仅在前层（直连/exa）失败时才走；exa 失败静默降级不抛。
+// 契约：WebFetchResult { url, statusCode, body: {kind:'html'|'text', content}, truncated }（CLOSED 联合类型）。
+// ponytail: 无服务级令牌桶——web_fetch 为单发用户/agent 驱动调用，非扇出；若未来批量调用再引入共享调度。
+
+import https from "node:https";
+import { callExa, callFirecrawl, callCrawl4ai } from "./fetch_upstream.js";
+
+const T1_MIN_CHARS = 300;
+const TEXTUAL = /text\/|json|xml|javascript|application\/json/i;
+
+/** GET 跟随重定向（≤5 跳），返回最终 URL / 状态 / 正文 / 内容类型。 */
+function httpsGetFollow(url, { headers = {}, timeoutMs = 20000, signal } = {}) {
+  return new Promise((resolve, reject) => {
+    const hop = (current, left) => {
+      const u = new URL(current);
+      const req = https.request(
+        { hostname: u.hostname, port: u.port || 443, path: u.pathname + u.search, method: "GET", headers, timeout: timeoutMs, signal },
+        (res) => {
+          if ([301, 302, 303, 307, 308].includes(res.statusCode) && res.headers.location && left > 0) {
+            res.resume();
+            const next = new URL(res.headers.location, current).toString();
+            return hop(next, left - 1);
+          }
+          const chunks = [];
+          res.on("data", (c) => chunks.push(c));
+          res.on("end", () => resolve({ url: current, status: res.statusCode, text: Buffer.concat(chunks).toString("utf8"), contentType: res.headers["content-type"] || "" }));
+        }
+      );
+      req.on("timeout", () => req.destroy(new Error(`timeout after ${timeoutMs}ms`)));
+      req.on("error", reject);
+      req.end();
+    };
+    hop(url, 5);
+  });
+}
+
+const BROWSERISH = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml,*/*", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8" };
+
+export function createFetchProvider({ resolveApiKey, logger }) {
+  return {
+    id: "fs-quality-fetch",
+    /** 廉价本地可用性检查（seam 契约要求，禁止网络调用）。 */
+    available() { return true; },
+    async fetch(request, signal) {
+      const url = String(request?.url || "");
+      if (!/^https:\/\//i.test(url)) throw new Error(`fs-quality-fetch: only https URLs are supported (got "${url.slice(0, 60)}")`);
+      const errors = [];
+
+      // T1：直连（文本类内容；二进制交给后层——exa 取正文，PDF 等仍由 firecrawl 处理）
+      try {
+        const r = await httpsGetFollow(url, { headers: BROWSERISH, signal });
+        if (r.status < 400 && TEXTUAL.test(r.contentType) && r.text.length >= T1_MIN_CHARS) {
+          return { url: r.url, statusCode: r.status, body: { kind: "html", content: r.text }, truncated: false };
+        }
+        if (r.status >= 400) errors.push(`direct HTTP ${r.status}`);
+        else errors.push(`direct: ${r.contentType || "unknown type"} ${r.text.length}B`);
+      } catch (e) { errors.push(`direct: ${e.message}`); }
+
+      // T2：exa contents（$1/1000 页）——直连失败（反爬/JS/过短）时承接；firecrawl 额度保护：仅在本层失败后才走 T3
+      const exaKey = await resolveApiKey("EXA_API_KEY", "exaApiKey").catch(() => undefined);
+      if (exaKey) {
+        try {
+          const text = await callExa(url, exaKey, signal);
+          if (text.trim().length >= T1_MIN_CHARS) return { url, statusCode: 200, body: { kind: "text", content: text }, truncated: false };
+          errors.push("exa: short content");
+        } catch (e) { errors.push(`exa: ${e.message}`); }
+      }
+
+      const firecrawlKey = await resolveApiKey("FIRECRAWL_API_KEY", "firecrawlApiKey").catch(() => undefined);
+      if (firecrawlKey) {
+        try {
+          const md = await callFirecrawl(url, firecrawlKey);
+          if (md.trim().length >= T1_MIN_CHARS) return { url, statusCode: 200, body: { kind: "text", content: md }, truncated: false };
+          errors.push("firecrawl: short content");
+        } catch (e) { errors.push(`firecrawl: ${e.message}`); }
+      }
+
+      const crawl4aiKey = await resolveApiKey("CRAWL4AI_API_KEY", undefined).catch(() => undefined);
+      if (crawl4aiKey) {
+        try {
+          const text = await callCrawl4ai(url, crawl4aiKey);
+          if (text.trim().length >= T1_MIN_CHARS) return { url, statusCode: 200, body: { kind: "text", content: text }, truncated: false };
+          errors.push("crawl4ai: short content");
+        } catch (e) { errors.push(`crawl4ai: ${e.message}`); }
+      }
+
+      logger?.warn?.(`fs-quality-fetch: all tiers failed for ${url}: ${errors.join("; ")}`);
+      throw new Error(`fs-quality-fetch: could not fetch ${url} (${errors.join("; ")})`);
+    },
+  };
+}

package/lib/fetch_upstream.js

@@ -0,0 +1,106 @@
+// fetch_upstream：web_fetch 质量链的云端抓取层（自 research.js 摘取，原生通道版专用）。
+// 依赖面：仅 node:https。fetch_provider.js 从这里拿 callExa / callFirecrawl / callCrawl4ai。
+// 注意：不用全局 fetch(undici) 访问 api.crawl4ai.com——其 Pingora 网关会空 401 拦截 undici 特征。
+
+import https from "node:https";
+
+// ---------- 受控替身注入点（测试基建；默认全 null = 生产直连真实服务，行为不变） ----------
+export const CONTROLLED_STUBS = { exa: null, firecrawl: null, crawl4ai: null };
+
+export function htmlToText(html) {
+  return String(html)
+    .replace(/<script[\s\S]*?<\/script>/gi, " ")
+    .replace(/<style[\s\S]*?<\/style>/gi, " ")
+    .replace(/<noscript[\s\S]*?<\/noscript>/gi, " ")
+    .replace(/<!--[\s\S]*?-->/g, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&nbsp;/gi, " ")
+    .replace(/&amp;/gi, "&")
+    .replace(/&lt;/gi, "<")
+    .replace(/&gt;/gi, ">")
+    .replace(/&quot;/gi, '"')
+    .replace(/&#39;/gi, "'")
+    .replace(/[ \t]+/g, " ")
+    .replace(/\n\s*\n+/g, "\n\n")
+    .trim();
+}
+
+// ---------- 云端抓取（node:https，保留头大小写；undici 会被 crawl4ai 网关拦） ----------
+
+export function httpsJson(method, url, { headers = {}, body, timeoutMs = 30000, signal } = {}) {
+  return new Promise((resolve, reject) => {
+    const u = new URL(url);
+    const req = https.request(
+      { hostname: u.hostname, port: u.port || 443, path: u.pathname + u.search, method, headers, timeout: timeoutMs },
+      (res) => {
+        const chunks = [];
+        res.on("data", (c) => chunks.push(c));
+        res.on("end", () => resolve({ status: res.statusCode, text: Buffer.concat(chunks).toString("utf8"), location: res.headers.location || "" }));
+      }
+    );
+    req.on("timeout", () => req.destroy(new Error(`timeout after ${timeoutMs}ms`)));
+    req.on("error", reject);
+    if (signal) {
+      const onAbort = () => req.destroy(new Error("aborted"));
+      if (signal.aborted) onAbort();
+      else signal.addEventListener("abort", onAbort, { once: true });
+    }
+    if (body) req.write(body);
+    req.end();
+  });
+}
+
+/** firecrawl /v2/scrape：返回 markdown。undici 特征对其无碍，但统一走 httpsJson。 */
+export async function fetchViaFirecrawl(url, apiKey, signal) {
+  const r = await httpsJson("POST", "https://api.firecrawl.dev/v2/scrape", {
+    headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", Accept: "application/json" },
+    body: JSON.stringify({ url, formats: ["markdown"] }),
+    signal,
+  });
+  if (r.status !== 200) throw new Error(`firecrawl HTTP ${r.status}`);
+  const data = JSON.parse(r.text);
+  const md = data?.data?.markdown;
+  if (!md) throw new Error("firecrawl: no markdown");
+  return md;
+}
+
+/** crawl4ai POST /scrape（注意：此部署无 /v1 前缀；认证头 X-API-Key）。 */
+export async function fetchViaCrawl4ai(url, apiKey, signal) {
+  const r = await httpsJson("POST", "https://api.crawl4ai.com/scrape", {
+    headers: { "X-API-Key": apiKey, "Content-Type": "application/json", "User-Agent": "dsh-free-search/0.8" },
+    body: JSON.stringify({ url }),
+    signal,
+  });
+  if (r.status === 429) throw new Error("crawl4ai rate-limited (community plan)");
+  if (r.status !== 200) throw new Error(`crawl4ai HTTP ${r.status}`);
+  const data = JSON.parse(r.text);
+  const content = data?.markdown || (data?.html ? htmlToText(data.html) : "");
+  if (!content) throw new Error("crawl4ai: empty content");
+  return content;
+}
+
+/** exa POST /contents（$1/1000 页）：{ids:[url], text:true} → results[].text（清洗正文 markdown）。
+ * 端点形状（https://exa.ai/docs/reference/get-contents，2026-10-05 核实）：
+ *  - 认证 Authorization: Bearer（x-api-key 亦可）；ids 与 urls 二选一等价（ids 向后兼容）。
+ *  - text 布尔/对象均可（对象形如 {maxCharacters}；此处取全文不加限）。
+ *  - 响应 results[]（含 text）+ statuses[]（每 URL 成败，失败带 error.tag/httpStatusCode）。
+ * 与 fetch_provider 质量链共用：T2 层。 */
+export async function fetchViaExa(url, apiKey, signal) {
+  const r = await httpsJson("POST", "https://api.exa.ai/contents", {
+    headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", Accept: "application/json" },
+    body: JSON.stringify({ ids: [url], text: true }),
+    signal,
+  });
+  if (r.status !== 200) throw new Error(`exa HTTP ${r.status}`);
+  const data = JSON.parse(r.text);
+  const st = (data?.statuses ?? []).find((s) => s.id === url) ?? data?.statuses?.[0];
+  if (st && st.status !== "success") throw new Error(`exa: status ${st.status}${st.error?.httpStatusCode ? ` (HTTP ${st.error.httpStatusCode})` : ""}`);
+  const hit = (data?.results ?? []).find((x) => x.id === url || x.url === url) ?? data?.results?.[0];
+  const text = hit?.text;
+  if (!text) throw new Error("exa: no text");
+  return text;
+}
+
+export const callFirecrawl = (u, k, s) => (CONTROLLED_STUBS.firecrawl ?? fetchViaFirecrawl)(u, k, s);
+export const callCrawl4ai = (u, k, s) => (CONTROLLED_STUBS.crawl4ai ?? fetchViaCrawl4ai)(u, k, s);
+export const callExa = (u, k, s) => (CONTROLLED_STUBS.exa ?? fetchViaExa)(u, k, s);