@240xu/dsh-websearch 2.8.4 → 2.8.5

package/lib/backends/ddg.js

@@ -30,20 +30,21 @@ export const ddgBackend = {
     throwIfSearchAborted(signal);
     const params = new URLSearchParams({ q: request.query, b: Math.random().toString().slice(2) });
     applyDdgFilters(params, normalizeFilters(request.filters));
-    const body = params.toString();
+    const target = `${DDG_ENDPOINT}?${params.toString()}`;
     hooks.recordRequest?.({ endpoint: DDG_ENDPOINT, toolName: "html", body: { q: request.query }, headers: {} });
 
     let text;
     try {
-      const res = await egressFetch(DDG_ENDPOINT, {
-        method: "POST",
+      // GET form: DDG's html endpoint still serves the same results page, and
+      // unlike the POST form it is not rejected by the endpoint's bot wall
+      // for non-browser clients (POST is JA3 + method double-gated).
+      const res = await egressFetch(target, {
+        method: "GET",
         redirect: "error",
         headers: {
-          "Content-Type": "application/x-www-form-urlencoded",
           "Accept": "text/html",
           "User-Agent": "dsh-websearch/2.3",
         },
-        body,
         ...(signal !== undefined ? { signal } : {}),
       });
       if (!res.ok) {

package/lib/util/egress-fetch.js

@@ -10,6 +10,14 @@
 //
 // When no matching proxy env is set, or the target host sits in NO_PROXY, it
 // defers to the native fetch, so behavior everywhere else is unchanged.
+//
+// Optional TLS-egress forwarder: when DSH_EGRESS_FORWARDER is set (a local
+// sidecar exposing a POST /fwd API backed by a Chromium TLS stack), requests
+// are relayed through it with the upstream proxy carried in the sidecar spec
+// (DSH_EGRESS_FORWARDER_PROXY, defaulting to HTTPS_PROXY). This exists because
+// some upstreams (Exa, DuckDuckGo) gate on the client TLS fingerprint (JA3)
+// rather than on the exit IP; the Node TLS fingerprint is rejected where a
+// real browser's is accepted. Unset the env and behavior is byte-identical.
 
 import http from "node:http";
 import https from "node:https";
@@ -17,13 +25,33 @@ import https from "node:https";
 const REDIRECT_STATUS = new Set([301, 302, 303, 307, 308]);
 const MAX_REDIRECTS = 5;
 
-/** fetch-compatible entry: tunnels via HTTPS_PROXY/HTTP_PROXY when configured. */
+/** fetch-compatible entry: forwarder first, then HTTPS_PROXY/HTTP_PROXY tunnel. */
 export function egressFetch(url, init = {}) {
+  const forwarder = forwarderFor(url);
+  if (forwarder !== undefined) return forwarderFetch(String(url), init, forwarder);
   const proxy = proxyFor(url);
   if (proxy === undefined) return fetch(url, init);
   return tunneledFetch(String(url), init, proxy, 0);
 }
 
+/** Resolve the TLS-egress forwarder for a request target (NO_PROXY aware). */
+function forwarderFor(url) {
+  let target;
+  try {
+    target = new URL(String(url));
+  } catch {
+    return undefined;
+  }
+  if (target.protocol !== "https:" && target.protocol !== "http:") return undefined;
+  const env = process.env;
+  const endpoint = env.DSH_EGRESS_FORWARDER;
+  if (endpoint === undefined || endpoint === "") return undefined;
+  if (inNoProxy(target, env.NO_PROXY || env.no_proxy)) return undefined;
+  const proxy = env.DSH_EGRESS_FORWARDER_PROXY
+    || env.HTTPS_PROXY || env.https_proxy || env.HTTP_PROXY || env.http_proxy || "";
+  return { endpoint, proxy: proxy === "" ? undefined : proxy };
+}
+
 /** Resolve the proxy URL for a request target (NO_PROXY aware). */
 function proxyFor(url) {
   let target;
@@ -110,6 +138,43 @@ async function tunneledFetch(url, init, proxy, depth) {
   return response;
 }
 
+/**
+ * Relay through the TLS-egress sidecar: POST /fwd with an
+ * x-sidecar-spec base64 JSON header; the sidecar performs the upstream
+ * request with its own (Chromium) TLS stack and streams the upstream
+ * status/headers/body back verbatim. Upstream dial failures surface as
+ * 502 + x-sidecar-error, which we rethrow as fetch-style TypeError.
+ */
+async function forwarderFetch(url, init, forwarder) {
+  const method = String(init.method ?? "GET").toUpperCase();
+  const headers = normalizeHeaders(init.headers);
+  const body = normalizeBody(init.body);
+  const spec = { url, method, headers, timeout_ms: 90000 };
+  if (forwarder.proxy !== undefined) spec.proxy = forwarder.proxy;
+  if (init.redirect !== undefined) spec.redirect = init.redirect;
+  let response;
+  try {
+    response = await fetch(forwarder.endpoint, {
+      method: "POST",
+      headers: {
+        "content-type": "application/octet-stream",
+        "x-sidecar-spec": Buffer.from(JSON.stringify(spec)).toString("base64"),
+      },
+      ...(body !== undefined ? { body } : {}),
+      ...(init.signal !== undefined ? { signal: init.signal } : {}),
+    });
+  } catch (error) {
+    throw new TypeError(`fetch failed: forwarder unreachable (${sanitize(error)})`, { cause: error });
+  }
+  const marker = response.headers.get("x-sidecar-error");
+  if (response.status === 502 && marker !== null) {
+    let detail = "";
+    try { detail = (await response.text()).slice(0, 200); } catch {}
+    throw new TypeError(`fetch failed: forwarder ${marker}${detail !== "" ? ` (${detail})` : ""}`);
+  }
+  return response;
+}
+
 /* ------------------------------------------------------------------ */
 /* transports                                                          */
 /* ------------------------------------------------------------------ */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@240xu/dsh-websearch",
-  "version": "2.8.4",
+  "version": "2.8.5",
   "description": "Unified web search provider for DeepSeek Harness (ctx.web): 11 backends — Exa + Parallel + DDG + SearXNG (keyless), DeepSeek + Anthropic + OpenAI + Brave + Tavily + Serper + Mojeek (key-gated via Settings credentials). Registers provider id \"unified\" for zero-config web_search. 统一网页搜索聚合：11 后端并发扇出、去重合并，零依赖。See CHANGELOG.md for version history.",
   "type": "module",
   "main": "lib/index.js",