@240xu/dsh-websearch 2.0.0

package/lib/backends/exa.js

@@ -0,0 +1,97 @@
+// exa.js — Exa search backend (streamable-http MCP, keyless).
+// Exa's MCP server is at https://mcp.exa.ai/mcp. It exposes:
+//   - web_search_exa   { query, numResults }  -> searching
+//   - web_fetch_exa    { urls, maxCharacters } -> page content (not used here)
+// Responses come back as text/event-stream where the JSON envelope holds
+// result.content[0].text as human-readable blocks of "Title:/URL:/Published:
+// .../Highlights: .../  ..." separated by "\n\n---\n\n"; we split into source objects.
+// Exa is keyless — its public MCP endpoint requires no API key.
+
+import { mcpCall, invalidateSession } from "../util/mcp-client.js";
+import { WebError } from "@deepseek-ai/dsh-web";
+
+const EXA_ENDPOINT = "https://mcp.exa.ai/mcp";
+const EXA_TOOL = "web_search_exa";
+
+export const exaBackend = {
+  id: "exa",
+  requiresCredential: false,
+
+  available() {
+    return true; // keyless endpoint
+  },
+
+  async search(request, signal, options, hooks = {}) {
+    const args = {
+      query: request.query,
+      numResults: request.maxResults ?? 8,
+    };
+    hooks.recordRequest?.({
+      endpoint: EXA_ENDPOINT,
+      toolName: EXA_TOOL,
+      body: args,
+      headers: {},
+    });
+
+    let result;
+    try {
+      result = await mcpCall(EXA_ENDPOINT, EXA_TOOL, args, {}, signal);
+    } catch (err) {
+      if (err instanceof WebError) throw err;
+      throw new WebError(
+        `exa search failed: ${String(err?.message ?? err)}`,
+        "WEB_PROVIDER_ERROR",
+        { cause: err },
+      );
+    }
+
+    hooks.recordOutcome?.({ status: "ok" });
+    const sources = parseExaResult(result, request.maxResults ?? 8);
+
+    if (result?.isError === true) {
+      throw new WebError(
+        `exa returned an MCP error: ${String(result.content?.[0]?.text ?? "unknown")}`,
+        "WEB_PROVIDER_ERROR",
+      );
+    }
+
+    return { sources, content: undefined, truncated: false };
+  },
+};
+
+export function parseExaResult(result, maxResults) {
+  const text = result?.content?.[0]?.text;
+  if (typeof text !== "string" || text.length === 0) return [];
+  const blocks = text.split("\n\n---\n\n");
+  const sources = [];
+  for (const block of blocks) {
+    const url = matchField(block, /^URL:\s*(.+)$/m);
+    if (!url) continue;
+    const source = { url: url.trim() };
+    const title = matchField(block, /^Title:\s*(.+)$/m);
+    if (title) source.title = title.trim();
+    const published = matchField(block, /^Published:\s*(.+)$/m);
+    if (published) source.publishedAt = published.trim();
+    const snippet = joinHighlights(block);
+    if (snippet) source.snippet = snippet;
+    sources.push(source);
+    if (sources.length >= maxResults) break;
+  }
+  return sources;
+}
+
+function matchField(block, re) {
+  const m = re.exec(block);
+  return m ? m[1].trim() : undefined;
+}
+
+function joinHighlights(block) {
+  const m = /^Highlights:\s*$/m.exec(block);
+  if (!m) return undefined;
+  const tail = block.slice(m.index + m[0].length);
+  const bullets = [...tail.matchAll(/^\s+-\s+(.+)$/gm)].map((b) => b[1].trim());
+  if (bullets.length === 0) return undefined;
+  return bullets.join(" … ");
+}
+
+export default exaBackend;

package/lib/backends/index.js

@@ -0,0 +1,121 @@
+/**
+ * Unified backend registry for dsh-websearch.
+ * Auto-discovers and exports all backend implementations.
+ */
+
+import { exaBackend } from "./exa.js";
+import { parallelBackend } from "./parallel.js";
+import { ddgBackend } from "./ddg.js";
+import { deepseekBackend, anthropicBackend } from "./anthropic-like.js";
+import { openaiBackend } from "./openai.js";
+import { braveBackend } from "./brave.js";
+import { tavilyBackend } from "./tavily.js";
+import { serperBackend } from "./serper.js";
+import { searxngBackend } from "./searxng.js";
+import { mojeekBackend } from "./mojeek.js";
+
+// Individual backend exports for direct import
+export { exaBackend } from "./exa.js";
+export { parallelBackend } from "./parallel.js";
+export { ddgBackend } from "./ddg.js";
+export { deepseekBackend, anthropicBackend } from "./anthropic-like.js";
+export { openaiBackend } from "./openai.js";
+export { braveBackend } from "./brave.js";
+export { tavilyBackend } from "./tavily.js";
+export { serperBackend } from "./serper.js";
+export { searxngBackend } from "./searxng.js";
+export { mojeekBackend } from "./mojeek.js";
+
+
+/** All registered backends. Order = default fan-out priority. */
+export const BACKENDS = {
+  exa: exaBackend,
+  parallel: parallelBackend,
+  ddg: ddgBackend,
+  searxng: searxngBackend,      // keyless, new
+  brave: braveBackend,          // key-gated, new
+  tavily: tavilyBackend,        // key-gated, new
+  serper: serperBackend,        // key-gated, new
+  mojeek: mojeekBackend,        // key-gated, new
+  deepseek: deepseekBackend,
+  anthropic: anthropicBackend,
+  openai: openaiBackend,
+};
+
+/** All backend IDs in registry order. */
+export const ALL_BACKEND_IDS = Object.keys(BACKENDS);
+
+/** Default enabled backends (keyless only). */
+export const DEFAULT_ENABLED = ["exa", "parallel", "ddg", "searxng"];
+
+/** Environment variable names for key-gated backends. */
+export const DEFAULT_KEY_ENV = {
+  deepseek: "DEEPSEEK_API_KEY",
+  anthropic: "ANTHROPIC_API_KEY",
+  openai: "OPENAI_API_KEY",
+  brave: "BRAVE_API_KEY",
+  tavily: "TAVILY_API_KEY",
+  serper: "SERPER_API_KEY",
+  mojeek: "MOJEEK_API_KEY",
+};
+
+/** Default base URLs for each backend. */
+export const DEFAULT_BASE_URLS = {
+  exa: "https://mcp.exa.ai/mcp",
+  parallel: "https://search.parallel.ai/mcp",
+  ddg: "https://html.duckduckgo.com/html/",
+  searxng: "https://searx.be",
+  brave: "https://api.search.brave.com/res/v1/web/search",
+  tavily: "https://api.tavily.com/search",
+  serper: "https://google.serper.dev/search",
+  mojeek: "https://api.mojeek.com/v1/search",
+  deepseek: "https://api.deepseek.com/anthropic/v1",
+  anthropic: "https://api.anthropic.com/v1",
+  openai: "https://api.openai.com/v1",
+};
+
+/** Backend type classification for GUI. */
+export const BACKEND_TYPES = {
+  exa: "mcp",
+  parallel: "mcp",
+  ddg: "keyless",
+  searxng: "keyless",
+  brave: "credential",
+  tavily: "credential",
+  serper: "credential",
+  mojeek: "credential",
+  deepseek: "credential",
+  anthropic: "credential",
+  openai: "credential",
+};
+
+/** Human-readable labels for GUI. */
+export const BACKEND_LABELS = {
+  exa: "Exa (MCP, 免费无 Key)",
+  parallel: "Parallel (MCP, 免费无 Key)",
+  ddg: "DuckDuckGo (HTML 抓取, 兜底)",
+  searxng: "SearXNG (元搜索, 免费无 Key)",
+  brave: "Brave Search (独立索引, 2000/月免费)",
+  tavily: "Tavily (AI 专用, 含答案摘要)",
+  serper: "Serper.dev (Google 抓取, 2500/月免费)",
+  mojeek: "Mojeek (独立索引, 1000/天免费)",
+  deepseek: "DeepSeek (原生 web_search, 需 Key)",
+  anthropic: "Anthropic (Claude 原生 web_search, 需 Key)",
+  openai: "OpenAI (Responses API web_search, 需 Key)",
+};
+
+/** Backend descriptions for GUI tooltips. */
+export const BACKEND_DESCRIPTIONS = {
+  exa: "流式 HTTP MCP 协议，原生 web_search_exa 工具，结果含标题/URL/高亮摘要",
+  parallel: "流式 HTTP MCP 协议，原生 web_search 工具，返回结构化 JSON 含摘要数组",
+  ddg: "HTML 页面抓取解析，通过 uddg= 参数还原真实 URL，零依赖兜底",
+  searxng: "开源元搜索引擎聚合，可指向公共实例或自托管，隐私友好",
+  brave: "Brave 独立搜索索引，REST API，免费 2000 次/月，结果质量高",
+  tavily: "AI 专用搜索，可返回 AI 生成答案 + 结构化结果，支持深度搜索模式",
+  serper: "抓取 Google SERP，结构化 organic[] 结果，免费 2500 次/月，速度极快",
+  mojeek: "英国独立搜索引擎，无追踪，免费 1000 次/天，英文为主",
+  deepseek: "DeepSeek Anthropic 兼容 API + 原生 web_search_20250305 工具",
+  anthropic: "Claude 官方 Messages API + web_search_20250305 原生工具",
+  openai: "OpenAI Responses API + 原生 web_search 工具，解析 url_citation 注释",
+};
+

package/lib/backends/mojeek.js

@@ -0,0 +1,103 @@
+/**
+ * Mojeek backend for dsh-websearch.
+ * REST API: https://api.mojeek.com/v1/search
+ * Auth: Authorization: Bearer {apiKey} header or api_key query param
+ * Free tier: 1,000 requests/day
+ */
+
+import { WebError } from "@deepseek-ai/dsh-web";
+import { throwIfSearchAborted, searchAborted, isAbortError } from "../util/abort.js";
+
+const MOJEEK_ENDPOINT = "https://api.mojeek.com/v1/search";
+
+/** @type {import("../provider.js").Backend} */
+export const mojeekBackend = {
+  id: "mojeek",
+  requiresCredential: true,
+
+  available(options) {
+    return typeof options?.resolveApiKey === "function";
+  },
+
+  async search(request, signal, options, hooks) {
+    throwIfSearchAborted(signal);
+    const backendOpts = options ?? {};
+
+    const apiKey = await backendOpts.resolveApiKey?.();
+    if (!apiKey) {
+      throw new WebError("Mojeek: API key not configured", "WEB_PROVIDER_UNAVAILABLE");
+    }
+
+    const baseURL = backendOpts.baseURL ?? MOJEEK_ENDPOINT;
+    const count = Math.min(request.maxResults ?? 8, 20);
+    
+    const url = new URL(baseURL);
+    url.searchParams.set("q", request.query);
+    url.searchParams.set("fmt", "json");
+    url.searchParams.set("count", String(count));
+    url.searchParams.set("api_key", apiKey);
+
+    const headers = {
+      "Accept": "application/json",
+      "User-Agent": "dsh-websearch/2.0",
+    };
+
+    hooks?.recordRequest?.({
+      endpoint: baseURL,
+      toolName: "mojeek_search",
+      body: { q: request.query, count },
+    });
+
+    let response;
+    try {
+      response = await fetch(url, {
+        method: "GET",
+        headers,
+        signal,
+      });
+    } catch (error) {
+      if (signal?.aborted === true || isAbortError(error)) throw searchAborted(signal, error);
+      throw new WebError("Mojeek request failed: " + error, "WEB_PROVIDER_ERROR", { cause: error });
+    }
+
+    if (!response.ok) {
+      const text = await response.text().catch(() => "");
+      hooks?.recordOutcome?.({ status: "error", message: "HTTP " + response.status + ": " + text });
+      throw new WebError("Mojeek API error (HTTP " + response.status + "): " + text, "WEB_PROVIDER_ERROR");
+    }
+
+    let data;
+    try {
+      data = await response.json();
+    } catch (error) {
+      throw new WebError("Mojeek: invalid JSON response", "WEB_PROVIDER_ERROR", { cause: error });
+    }
+
+    const sources = parseMojeekResult(data, request.maxResults ?? 8);
+    hooks?.recordOutcome?.({ status: "success", count: sources.length });
+    return { sources };
+  },
+};
+
+/**
+ * Parse Mojeek API response.
+ * @param {object} data
+ * @param {number} maxResults
+ */
+export function parseMojeekResult(data, maxResults) {
+  const sources = [];
+  const results = data?.response?.results ?? [];
+  
+  for (const item of results) {
+    if (sources.length >= maxResults) break;
+    if (!item?.url) continue;
+    sources.push({
+      url: item.url,
+      ...(item.title ? { title: item.title } : {}),
+      ...(item.desc ? { snippet: item.desc.slice(0, 500) } : {}),
+      ...(item.last_updated ? { publishedAt: item.last_updated } : {}),
+    });
+  }
+  
+  return sources;
+}

package/lib/backends/openai.js

@@ -0,0 +1,171 @@
+// openai.js — OpenAI Responses API backend (key-gated)
+// Uses POST /responses with the built-in "web_search" tool (Responses API
+// native web search). The Responses API returns:
+//   output: [
+//     { type:"web_search_call", id, status:"completed" },
+//     { type:"message", content:[ { type:"output_text", text, annotations:
+//       [ { type:"url_citation", url, title, start_index, end_index } ] } ] }
+//   ]
+// Sources = url_citation annotations (deduped by url); the surrounding output_text
+// segment ~[start,end] can serve as a rough snippet when the API doesn't
+// return a separate excerpt.
+
+import { throwIfSearchAborted, searchAborted, isAbortError } from "../util/abort.js";
+import { WebError } from "@deepseek-ai/dsh-web";
+
+const UA = "dsh-websearch/2.0";
+
+export const openaiBackend = {
+  id: "openai",
+  requiresCredential: true,
+
+  /**
+   * options: { apiKey, baseURL?, model? }
+   */
+  available(options) {
+    const okKey = (typeof options?.apiKey === "string" && options.apiKey.length > 0)
+      || typeof options?.resolveApiKey === "function";
+    const urlOk = typeof options?.baseURL === "string" && URL.canParse(options.baseURL);
+    return okKey && urlOk;
+  },
+
+  /**
+   * @param {WebSearchRequest} request
+   * @param {AbortSignal} signal
+   * @param {object} options   { apiKey, baseURL?, model? }
+   * @param {{ recordRequest?: Function, recordOutcome?: Function }} [hooks]
+   * @returns {Promise<WebSearchResult>}
+   */
+  async search(request, signal, options, hooks = {}) {
+    throwIfSearchAborted(signal);
+    const apiKey = options?.apiKey ?? await options?.resolveApiKey?.();
+    if (!apiKey || apiKey.length === 0) {
+      hooks.recordOutcome?.({ status: "error", message: "credential missing" });
+      throw new WebError(
+        "openai search has no API key",
+        "WEB_PROVIDER_CREDENTIAL_MISSING",
+      );
+    }
+    const baseURL = (options.baseURL || "https://api.openai.com/v1").replace(/\/$/, "");
+    const model = options.model || "gpt-5";
+    const endpoint = `${baseURL}/responses`;
+
+    const body = {
+      model,
+      tools: [{ type: "web_search" }],
+      input: request.query,
+    };
+
+    hooks.recordRequest?.({
+      endpoint,
+      toolName: "openai:web_search",
+      body: { model, query: request.query },
+      headers: {},
+    });
+
+    throwIfSearchAborted(signal);
+
+    let response;
+    try {
+      response = await fetch(endpoint, {
+        method: "POST",
+        redirect: "error",
+        headers: {
+          "Authorization": `Bearer ${apiKey}`,
+          "Content-Type": "application/json",
+          "Accept": "application/json",
+          "User-Agent": UA,
+        },
+        body: JSON.stringify(body),
+        ...(signal !== undefined ? { signal } : {}),
+      });
+    } catch (err) {
+      if (signal?.aborted === true || isAbortError(err)) throw searchAborted(signal, err);
+      hooks.recordOutcome?.({ status: "error", message: String(err?.message ?? err) });
+      throw new WebError(
+        `openai search request failed: ${String(err?.message ?? err)}`,
+        "WEB_PROVIDER_ERROR",
+        { cause: err },
+      );
+    }
+
+    if (!response.ok) {
+      let message = `openai API error (HTTP ${response.status})`;
+      try {
+        const parsed = await response.json();
+        const detail =
+          typeof parsed.error === "string"
+            ? parsed.error
+            : parsed.error?.message ?? parsed.message;
+        if (detail !== undefined && detail.length > 0) message = detail;
+      } catch (err) {
+        if (signal?.aborted === true || isAbortError(err)) throw searchAborted(signal, err);
+      }
+      hooks.recordOutcome?.({ status: "error", message });
+      throw new WebError(message, "WEB_PROVIDER_ERROR");
+    }
+
+    try {
+      const json = await response.json();
+      const sources = parseOpenAIResponses(json, request.maxResults ?? 8);
+      hooks.recordOutcome?.({ status: "ok", sources: sources.length });
+      return { sources, content: undefined, truncated: false };
+    } catch (err) {
+      if (signal?.aborted === true || isAbortError(err)) throw searchAborted(signal, err);
+      if (err instanceof WebError) {
+        hooks.recordOutcome?.({ status: "error", message: err.message });
+        throw err;
+      }
+      hooks.recordOutcome?.({ status: "error", message: String(err?.message ?? err) });
+      throw new WebError(
+        `openai returned an unprocessable response body: ${String(err?.message ?? err)}`,
+        "WEB_PROVIDER_ERROR",
+        { cause: err },
+      );
+    }
+  },
+};
+
+/**
+ * Parse a /responses body into WebSearchSource[]. Sources come from
+ * output[].content[].annotations[] with type "url_citation". Each carries
+ * { url, title, start_index, end_index } — we optionally slice the surrounding
+ * output_text to produce a rough snippet.
+ *
+ * @param {object} json  the /responses body
+ * @param {number} maxResults
+ * @returns {Array<{url:string,title?:string,snippet?:string}>}
+ */
+export function parseOpenAIResponses(json, maxResults) {
+  if (!json || !Array.isArray(json.output)) return [];
+  const seen = new Set();
+  const sources = [];
+
+  for (const out of json.output) {
+    if (out.type !== "message" || !Array.isArray(out.content)) continue;
+    for (const block of out.content) {
+      if (block.type !== "output_text" || typeof block.text !== "string") continue;
+      for (const ann of block.annotations ?? []) {
+        if (ann.type !== "url_citation" || !ann.url || seen.has(ann.url)) continue;
+        seen.add(ann.url);
+        const source = { url: ann.url };
+        if (ann.title && ann.title.length > 0) source.title = ann.title;
+        // Rough snippet: the text slice around the citation. May be verbose;
+        // cap to 240 chars as a courtesy. Only emit if non-empty & informative.
+        if (
+          typeof ann.start_index === "number" &&
+          typeof ann.end_index === "number" &&
+          ann.end_index > ann.start_index
+        ) {
+          const slice = block.text.slice(ann.start_index, ann.end_index).trim();
+          if (slice.length > 0) source.snippet = slice.slice(0, 240);
+        }
+        sources.push(source);
+        if (sources.length >= maxResults) return sources;
+      }
+    }
+  }
+  return sources;
+}
+
+export default openaiBackend;

package/lib/backends/parallel.js

@@ -0,0 +1,105 @@
+// parallel.js — Parallel search backend (streamable-http MCP, keyless).
+// Parallel's MCP server is at https://search.parallel.ai/mcp. It exposes:
+//   - web_search  { objective, search_queries[], session_id?, model_name? }
+//   - web_fetch   { urls, objective?, full_content? }
+// The search_queries array should be 3-6 keyword-level variants; we map the
+// raw query into a single-element query array plus that same query as the
+// objective — that's their most atomic usage. session_id is a conversation
+// stable identifier; we generate one per process so call correlation is
+// consistent across searches. model_name is advisory metadata for their rate
+// limiter; we use a stable sentinel.
+//
+// Response comes back as a JSON envelope whose result.content[0].text is a
+// JSON-string with { search_id, results: [{ url, title, publish_date, excerpts:[] }] }.
+// Parallel is keyless — its public MCP endpoint requires no API key.
+
+import { mcpCall, invalidateSession } from "../util/mcp-client.js";
+import { WebError } from "@deepseek-ai/dsh-web";
+
+const PARALLEL_ENDPOINT = "https://search.parallel.ai/mcp";
+const PARALLEL_TOOL = "web_search";
+
+/** Per-process session id — stable across searches so Parallel's free-tier
+ * rate limiter correlates calls consistently. */
+const SESSION_ID =
+  "dsh-websearch-" +
+  process.pid?.toString(36) + "-" +
+  Math.random().toString(36).slice(2, 10);
+
+export const parallelBackend = {
+  id: "parallel",
+  requiresCredential: false,
+
+  available() {
+    return true; // keyless endpoint
+  },
+
+  async search(request, signal, options, hooks = {}) {
+    const args = {
+      objective: request.query,
+      search_queries: [request.query],
+      session_id: SESSION_ID,
+      model_name: "dsh-websearch",
+    };
+    hooks.recordRequest?.({
+      endpoint: PARALLEL_ENDPOINT,
+      toolName: PARALLEL_TOOL,
+      body: args,
+      headers: {},
+    });
+
+    let result;
+    try {
+      result = await mcpCall(PARALLEL_ENDPOINT, PARALLEL_TOOL, args, {}, signal);
+    } catch (err) {
+      if (err instanceof WebError) throw err;
+      throw new WebError(
+        `parallel search failed: ${String(err?.message ?? err)}`,
+        "WEB_PROVIDER_ERROR",
+        { cause: err },
+      );
+    }
+
+    hooks.recordOutcome?.({ status: "ok" });
+    const sources = parseParallelResult(result, request.maxResults ?? 8);
+
+    if (result?.isError === true) {
+      throw new WebError(
+        `parallel returned an MCP error: ${String(result.content?.[0]?.text ?? "unknown")}`,
+        "WEB_PROVIDER_ERROR",
+      );
+    }
+
+    return { sources, content: undefined, truncated: false };
+  },
+};
+
+export function parseParallelResult(result, maxResults) {
+  const text = result?.content?.[0]?.text;
+  if (typeof text !== "string" || text.length === 0) return [];
+  let parsed;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return [];
+  }
+  const results = Array.isArray(parsed?.results) ? parsed.results : [];
+  const sources = [];
+  for (const r of results) {
+    if (typeof r?.url !== "string" || r.url.length === 0) continue;
+    const source = { url: r.url };
+    if (r.title && r.title.length > 0) source.title = r.title;
+    const excerpt = Array.isArray(r.excerpts) && r.excerpts.length > 0 ? r.excerpts[0] : undefined;
+    if (typeof excerpt === "string" && excerpt.length > 0) {
+      source.snippet = excerpt.length > 240 ? excerpt.slice(0, 240) + "…" : excerpt;
+    }
+    if (r.publish_date && r.publish_date !== "null" && r.publish_date.length > 0) {
+      source.publishedAt = r.publish_date;
+    }
+    sources.push(source);
+    if (sources.length >= maxResults) break;
+  }
+  return sources;
+}
+
+export default parallelBackend;