dsh-free-search 0.8.4 → 0.8.5

package/lib/client.js

@@ -355,7 +355,7 @@ window.__ModuleLoader__.load({
     };
     const tt = (lang) => I18N[lang === "en" ? "en" : "zh"];
     // 当前插件版本（与 lib/index.js 的 PLUGIN_VERSION 保持一致；跟随上游 v0.7.5）
-    const PLUGIN_VERSION = "0.8.4";
+    const PLUGIN_VERSION = "0.8.5";
     const ENGINES = [
       { id: "auto", label: "Auto · 智能路由", badge: "AUTO", link: null },
       { id: "multi", label: "Multi Search · 多引擎", badge: "MULTI", link: null },

package/lib/fetch_provider.js

@@ -6,12 +6,78 @@
 // ponytail: 无服务级令牌桶——web_fetch 为单发用户/agent 驱动调用，非扇出；若未来批量调用再引入共享调度。
 
 import https from "node:https";
+import zlib from "node:zlib";
+import { spawn } from "node:child_process";
+import { existsSync } from "node:fs";
 import { callExa, callFirecrawl, callCrawl4ai } from "./fetch_upstream.js";
 
 const T1_MIN_CHARS = 300;
 const TEXTUAL = /text\/|json|xml|javascript|application\/json/i;
 
-/** GET 跟随重定向（≤5 跳），返回最终 URL / 状态 / 正文 / 内容类型。 */
+// ---------- T1.5：系统浏览器无头渲染（执行 JS 过挑战；无浏览器/失败/超时则静默跳过） ----------
+const BROWSER_CANDIDATES = [
+  "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
+  "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
+  "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
+  "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
+  "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+  "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
+];
+let cachedBrowserPath; // undefined = 未检测；null = 无浏览器
+function findSystemBrowser() {
+  if (cachedBrowserPath !== undefined) return cachedBrowserPath;
+  cachedBrowserPath = null;
+  for (const p of BROWSER_CANDIDATES) {
+    try {
+      if (existsSync(p)) {
+        cachedBrowserPath = p;
+        break;
+      }
+    } catch {
+      // 忽略检测错误
+    }
+  }
+  return cachedBrowserPath;
+}
+
+function browserDumpDom(browserPath, url, { timeoutMs = 20000, signal } = {}) {
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn(browserPath, ["--headless=new", "--disable-gpu", "--no-first-run", "--dump-dom", "--virtual-time-budget=10000", url], {
+        stdio: ["ignore", "pipe", "ignore"],
+        windowsHide: true,
+      });
+    } catch {
+      resolve(null);
+      return;
+    }
+    let out = "";
+    let done = false;
+    const finish = (ok) => {
+      if (done) return;
+      done = true;
+      try {
+        child.kill();
+      } catch {
+        // 忽略
+      }
+      resolve(ok && out.length > 0 ? out : null);
+    };
+    child.stdout.on("data", (c) => {
+      if (out.length < 5 * 1024 * 1024) out += c.toString("utf8");
+    });
+    child.on("error", () => finish(false));
+    child.on("exit", () => finish(true));
+    setTimeout(() => finish(false), timeoutMs);
+    if (signal) {
+      if (signal.aborted) finish(false);
+      else signal.addEventListener("abort", () => finish(false), { once: true });
+    }
+  });
+}
+
+/** GET 跟随重定向（≤5 跳），返回最终 URL / 状态 / 正文 / 内容类型；支持 gzip/br/deflate 解压。 */
 function httpsGetFollow(url, { headers = {}, timeoutMs = 20000, signal } = {}) {
   return new Promise((resolve, reject) => {
     const hop = (current, left) => {
@@ -24,9 +90,19 @@ function httpsGetFollow(url, { headers = {}, timeoutMs = 20000, signal } = {}) {
             const next = new URL(res.headers.location, current).toString();
             return hop(next, left - 1);
           }
+          const enc = String(res.headers["content-encoding"] || "").toLowerCase();
+          let stream = res;
+          try {
+            if (enc.includes("br")) stream = res.pipe(zlib.createBrotliDecompress());
+            else if (enc.includes("gzip")) stream = res.pipe(zlib.createGunzip());
+            else if (enc.includes("deflate")) stream = res.pipe(zlib.createInflate());
+          } catch {
+            stream = res;
+          }
           const chunks = [];
-          res.on("data", (c) => chunks.push(c));
-          res.on("end", () => resolve({ url: current, status: res.statusCode, text: Buffer.concat(chunks).toString("utf8"), contentType: res.headers["content-type"] || "" }));
+          stream.on("data", (c) => chunks.push(c));
+          stream.on("end", () => resolve({ url: current, status: res.statusCode, text: Buffer.concat(chunks).toString("utf8"), contentType: res.headers["content-type"] || "" }));
+          stream.on("error", reject);
         }
       );
       req.on("timeout", () => req.destroy(new Error(`timeout after ${timeoutMs}ms`)));
@@ -37,7 +113,22 @@ function httpsGetFollow(url, { headers = {}, timeoutMs = 20000, signal } = {}) {
   });
 }
 
-const BROWSERISH = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml,*/*", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8" };
+// 完整浏览器请求头（Chrome 128 形态）：部分反爬站（如百度系）对精简头直接 403，完整头可通过。
+// 注意：声明了 Accept-Encoding 就必须解压（httpsGetFollow 已支持 gzip/br/deflate）。
+const BROWSERISH = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
+  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+  "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+  "Accept-Encoding": "gzip, deflate, br",
+  "sec-ch-ua": '"Chromium";v="128", "Not;A=Brand";v="24", "Google Chrome";v="128"',
+  "sec-ch-ua-mobile": "?0",
+  "sec-ch-ua-platform": '"Windows"',
+  "Sec-Fetch-Dest": "document",
+  "Sec-Fetch-Mode": "navigate",
+  "Sec-Fetch-Site": "none",
+  "Sec-Fetch-User": "?1",
+  "Upgrade-Insecure-Requests": "1",
+};
 
 export function createFetchProvider({ resolveApiKey, logger }) {
   return {
@@ -48,10 +139,17 @@ export function createFetchProvider({ resolveApiKey, logger }) {
       const url = String(request?.url || "");
       if (!/^https:\/\//i.test(url)) throw new Error(`fs-quality-fetch: only https URLs are supported (got "${url.slice(0, 60)}")`);
       const errors = [];
+      // Referer：伪装为"站内点击"（部分反爬站对无 Referer 的匿名请求更严格）
+      const headers = { ...BROWSERISH };
+      try {
+        headers.Referer = new URL(url).origin + "/";
+      } catch {
+        // URL 已在上面校验过，理论到不了这里
+      }
 
       // T1：直连（文本类内容；二进制交给后层——exa 取正文，PDF 等仍由 firecrawl 处理）
       try {
-        const r = await httpsGetFollow(url, { headers: BROWSERISH, signal });
+        const r = await httpsGetFollow(url, { headers, signal });
         if (r.status < 400 && TEXTUAL.test(r.contentType) && r.text.length >= T1_MIN_CHARS) {
           return { url: r.url, statusCode: r.status, body: { kind: "html", content: r.text }, truncated: false };
         }
@@ -59,6 +157,20 @@ export function createFetchProvider({ resolveApiKey, logger }) {
         else errors.push(`direct: ${r.contentType || "unknown type"} ${r.text.length}B`);
       } catch (e) { errors.push(`direct: ${e.message}`); }
 
+      // T1.5：系统浏览器无头渲染（执行 JS 过挑战，抓直连拿不到的页面；无浏览器自动跳过）
+      const browserPath = findSystemBrowser();
+      if (browserPath) {
+        try {
+          const html = await browserDumpDom(browserPath, url, { signal });
+          if (html && html.length >= T1_MIN_CHARS && !/403 Forbidden|\bcaptcha\b/i.test(html.slice(0, 3000))) {
+            return { url, statusCode: 200, body: { kind: "html", content: html }, truncated: false };
+          }
+          errors.push(html ? `browser: short or challenge page (${html.length}B)` : "browser: no output (timeout or launch failure)");
+        } catch (e) {
+          errors.push(`browser: ${e.message}`);
+        }
+      }
+
       // T2：exa contents（$1/1000 页）——直连失败（反爬/JS/过短）时承接；firecrawl 额度保护：仅在本层失败后才走 T3
       const exaKey = await resolveApiKey("EXA_API_KEY", "exaApiKey").catch(() => undefined);
       if (exaKey) {

package/lib/index.js

@@ -240,7 +240,7 @@ function routeEngines(query, { timeRange, disabledEngines = [], fallbackOrder =
 }
 
 // 当前插件版本（发布时与 package.json 同步；跟随上游 v0.7.5，不在 PR 里 bump）
-const PLUGIN_VERSION = "0.8.4";
+const PLUGIN_VERSION = "0.8.5";
 // 检查更新的 npm registry 元数据地址（dsh-free-search 是 npmjs 上的公开包）
 const NPM_REGISTRY_URL = "https://registry.npmjs.org/dsh-free-search/latest";
 const PLUGIN_NPM_URL = "https://www.npmjs.com/package/dsh-free-search";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-free-search",
-  "version": "0.8.4",
+  "version": "0.8.5",
   "description": "Free web search for DeepSeek Harness: 24 engines (Bing/DuckDuckGo/AnySearch/SearXNG/Exa/Tavily/Keenable/Firecrawl/Parallel keyless; Zhihu global/site via MCP and Perplexity/SerpBase/DeepSeek/You.com/Baidu/Kimi/Aliyun/Doubao with key; OpenAI/Gemini/Claude model built-in search, explicit-only) + auto smart routing + global engine enable/disable & fallback priority + editable zh/en language pools + multi mode (parallel full language-pool merge) + multi_search cross-source tool + failure-aware fallback (quota/auth cooldown, transient retry, fallbackOn) + time filtering + platform search + web_fetch, with web settings UI.",
   "type": "module",
   "main": "lib/index.js",