@240xu/dsh-websearch 2.3.0 → 2.7.2

package/lib/backends/index.js

@@ -1,121 +0,0 @@
-/**
- * Unified backend registry for dsh-websearch.
- * Auto-discovers and exports all backend implementations.
- */
-
-import { exaBackend } from "./exa.js";
-import { parallelBackend } from "./parallel.js";
-import { ddgBackend } from "./ddg.js";
-import { deepseekBackend, anthropicBackend } from "./anthropic-like.js";
-import { openaiBackend } from "./openai.js";
-import { braveBackend } from "./brave.js";
-import { tavilyBackend } from "./tavily.js";
-import { serperBackend } from "./serper.js";
-import { searxngBackend } from "./searxng.js";
-import { mojeekBackend } from "./mojeek.js";
-
-// Individual backend exports for direct import
-export { exaBackend } from "./exa.js";
-export { parallelBackend } from "./parallel.js";
-export { ddgBackend } from "./ddg.js";
-export { deepseekBackend, anthropicBackend } from "./anthropic-like.js";
-export { openaiBackend } from "./openai.js";
-export { braveBackend } from "./brave.js";
-export { tavilyBackend } from "./tavily.js";
-export { serperBackend } from "./serper.js";
-export { searxngBackend } from "./searxng.js";
-export { mojeekBackend } from "./mojeek.js";
-
-
-/** All registered backends. Order = default fan-out priority. */
-export const BACKENDS = {
-  exa: exaBackend,
-  parallel: parallelBackend,
-  ddg: ddgBackend,
-  searxng: searxngBackend,      // keyless, new
-  brave: braveBackend,          // key-gated, new
-  tavily: tavilyBackend,        // key-gated, new
-  serper: serperBackend,        // key-gated, new
-  mojeek: mojeekBackend,        // key-gated, new
-  deepseek: deepseekBackend,
-  anthropic: anthropicBackend,
-  openai: openaiBackend,
-};
-
-/** All backend IDs in registry order. */
-export const ALL_BACKEND_IDS = Object.keys(BACKENDS);
-
-/** Default enabled backends (keyless only). */
-export const DEFAULT_ENABLED = ["exa", "parallel", "ddg", "searxng"];
-
-/** Environment variable names for key-gated backends. */
-export const DEFAULT_KEY_ENV = {
-  deepseek: "DEEPSEEK_API_KEY",
-  anthropic: "ANTHROPIC_API_KEY",
-  openai: "OPENAI_API_KEY",
-  brave: "BRAVE_API_KEY",
-  tavily: "TAVILY_API_KEY",
-  serper: "SERPER_API_KEY",
-  mojeek: "MOJEEK_API_KEY",
-};
-
-/** Default base URLs for each backend. */
-export const DEFAULT_BASE_URLS = {
-  exa: "https://mcp.exa.ai/mcp",
-  parallel: "https://search.parallel.ai/mcp",
-  ddg: "https://html.duckduckgo.com/html/",
-  searxng: "https://searx.be",
-  brave: "https://api.search.brave.com/res/v1/web/search",
-  tavily: "https://api.tavily.com/search",
-  serper: "https://google.serper.dev/search",
-  mojeek: "https://api.mojeek.com/v1/search",
-  deepseek: "https://api.deepseek.com/anthropic/v1",
-  anthropic: "https://api.anthropic.com/v1",
-  openai: "https://api.openai.com/v1",
-};
-
-/** Backend type classification for GUI. */
-export const BACKEND_TYPES = {
-  exa: "mcp",
-  parallel: "mcp",
-  ddg: "keyless",
-  searxng: "keyless",
-  brave: "credential",
-  tavily: "credential",
-  serper: "credential",
-  mojeek: "credential",
-  deepseek: "credential",
-  anthropic: "credential",
-  openai: "credential",
-};
-
-/** Human-readable labels for GUI. */
-export const BACKEND_LABELS = {
-  exa: "Exa (MCP, 免费无 Key)",
-  parallel: "Parallel (MCP, 免费无 Key)",
-  ddg: "DuckDuckGo (HTML 抓取, 兜底)",
-  searxng: "SearXNG (元搜索, 免费无 Key)",
-  brave: "Brave Search (独立索引, 2000/月免费)",
-  tavily: "Tavily (AI 专用, 含答案摘要)",
-  serper: "Serper.dev (Google 抓取, 2500/月免费)",
-  mojeek: "Mojeek (独立索引, 1000/天免费)",
-  deepseek: "DeepSeek (原生 web_search, 需 Key)",
-  anthropic: "Anthropic (Claude 原生 web_search, 需 Key)",
-  openai: "OpenAI (Responses API web_search, 需 Key)",
-};
-
-/** Backend descriptions for GUI tooltips. */
-export const BACKEND_DESCRIPTIONS = {
-  exa: "流式 HTTP MCP 协议，原生 web_search_exa 工具，结果含标题/URL/高亮摘要",
-  parallel: "流式 HTTP MCP 协议，原生 web_search 工具，返回结构化 JSON 含摘要数组",
-  ddg: "HTML 页面抓取解析，通过 uddg= 参数还原真实 URL，零依赖兜底",
-  searxng: "开源元搜索引擎聚合，可指向公共实例或自托管，隐私友好",
-  brave: "Brave 独立搜索索引，REST API，免费 2000 次/月，结果质量高",
-  tavily: "AI 专用搜索，可返回 AI 生成答案 + 结构化结果，支持深度搜索模式",
-  serper: "抓取 Google SERP，结构化 organic[] 结果，免费 2500 次/月，速度极快",
-  mojeek: "英国独立搜索引擎，无追踪，免费 1000 次/天，英文为主",
-  deepseek: "DeepSeek Anthropic 兼容 API + 原生 web_search_20250305 工具",
-  anthropic: "Claude 官方 Messages API + web_search_20250305 原生工具",
-  openai: "OpenAI Responses API + 原生 web_search 工具，解析 url_citation 注释",
-};
-

package/lib/util/rpc.js

@@ -1,262 +0,0 @@
-/**
- * @module dsh-websearch/util/rpc
- *
- * Minimal MCP JSON-RPC 2.0 client for the streamable-http transport used by
- * Exa (https://mcp.exa.ai/mcp) and Parallel (https://search.parallel.ai/mcp).
- *
- * Streamable-HTTP MCP servers require a 2-step handshake per session:
- *   1. POST initialize → server replies with its capabilities and (for Exa)
- *      a Mcp-Session-Id header. Parallel also returns this header.
- *   2. POST notifications/initialized → notify the server the client is ready.
- *      (Exa rejects tools/call without this; Parallel tolerates its absence
- *      but sending it is harmless.)
- *   3. POST tools/call with the Mcp-Session-Id header → returns either a plain
- *      JSON body OR a text/event-stream of "data: {…}\n\n" frames.
- *
- * We cache the Mcp-Session-Id per server URL inside the process so subsequent
- * searches skip the handshake (and one cache entry survives across calls).
- * Sessions are NOT shared across processes (each dsh-web worker owns its own).
- *
- * Failures are zh-folded into WebError so the provider fan-out can demote them
- * to soft nulls when only one backend fails.
- */
-import { WebError } from "@deepseek-ai/dsh-web";
-import { throwIfSearchAborted, searchAborted, isAbortError } from "./abort.js";
-
-/** User-Agent header; bumped with the package version. */
-export const UA = "dsh-websearch/2.1";
-
-/** MCP protocol version we negotiate. */
-const PROTOCOL_VERSION = "2025-06-18";
-
-/** Per-URL session cache: url → { sessionId, initializedAt }. */
-const sessionCache = new Map();
-const SESSION_TTL_MS = 10 * 60 * 1000; // Exa sessions live indefinitely; this is a courtesy cap.
-
-/** Monotonic request id so concurrent inits don't collide. */
-let nextReqId = 1;
-
-/**
- * Initialize (or reuse) a streamable-http MCP session.
- * Sends initialize + notifications/initialized. Returns the Mcp-Session-Id,
- * caching it so callers don't pay the handshake more than once per TTL.
- *
- * @param {string} url
- * @param {AbortSignal} [signal]
- * @returns {Promise<string|null>} Mcp-Session-Id (or null if the server omits it).
- * @throws {WebError} on transport error or abort.
- */
-async function ensureSession(url, signal) {
-  throwIfSearchAborted(signal);
-  const cached = sessionCache.get(url);
-  if (cached && Date.now() - cached.initializedAt < SESSION_TTL_MS && cached.sessionId) {
-    return cached.sessionId;
-  }
-
-  const initId = nextReqId++;
-  const initBody = JSON.stringify({
-    jsonrpc: "2.0",
-    id: initId,
-    method: "initialize",
-    params: {
-      protocolVersion: PROTOCOL_VERSION,
-      capabilities: {},
-      clientInfo: { name: "dsh-websearch", version: "1.0" },
-    },
-  });
-
-  let res;
-  try {
-    res = await fetch(url, {
-      method: "POST",
-      redirect: "error",
-      headers: {
-        "Content-Type": "application/json",
-        "Accept": "application/json, text/event-stream",
-        "User-Agent": UA,
-      },
-      body: initBody,
-      ...(signal !== undefined ? { signal } : {}),
-    });
-  } catch (error) {
-    if (signal?.aborted === true || isAbortError(error)) throw searchAborted(signal, error);
-    throw new WebError(
-      `mcp initialize fetch failed for ${url}: ${String(error?.message ?? error)}`,
-      "WEB_PROVIDER_ERROR",
-      { cause: error },
-    );
-  }
-  throwIfSearchAborted(signal);
-  if (!res.ok) {
-    let detail = "";
-    try { detail = (await res.text()).slice(0, 300); } catch {}
-    throw new WebError(
-      `mcp initialize HTTP ${res.status} for ${url}${detail ? ": " + detail : ""}`,
-      "WEB_PROVIDER_ERROR",
-    );
-  }
-
-  // Drain the initialize response body (we don't need its content).
-  await res.text();
-
-  const sessionId = res.headers.get("Mcp-Session-Id") || null;
-
-  // Send notifications/initialized (some servers reject tools/call without it).
-  if (sessionId) {
-    try {
-      await fetch(url, {
-        method: "POST",
-        redirect: "error",
-        headers: {
-          "Content-Type": "application/json",
-          "Accept": "application/json, text/event-stream",
-          "User-Agent": UA,
-          "Mcp-Session-Id": sessionId,
-        },
-        body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }),
-        ...(signal !== undefined ? { signal } : {}),
-      });
-    } catch (error) {
-      // notifications/initialized is best-effort; ignore transport errors here.
-      // (If the session is gone we'll discover that via tools/call.) Re-raise aborts.
-      if (signal?.aborted === true || isAbortError(error)) throw searchAborted(signal, error);
-    }
-  }
-
-  sessionCache.set(url, { sessionId, initializedAt: Date.now() });
-  return sessionId;
-}
-
-/**
- * Parse a streamable-HTTP MCP response body into the first JSON-RPC envelope.
- *
- * Servers respond with EITHER:
- *   - a single JSON object (Parallel: {"jsonrpc":"2.0","id":N,"result":{…}} or {...,"error":{…}}), OR
- *   - a text/event-stream of "event: <type>\n data: {…}\n\n" frames (Exa).
- *
- * We scan line-by-line; SSE data: frames take precedence, otherwise we fall back
- * to scanning whole-line JSON.
- *
- * @param {string} text
- * @param {number} reqId  the request id we expect to be echoed back
- * @returns {{result?:object, error?:object}}
- * @throws if no parseable envelope is found.
- */
-function parseEnvelope(text, reqId) {
-  const lines = String(text).split("\n");
-  let lastJson = null;
-  for (const raw of lines) {
-    const line = raw.trim();
-    if (!line) continue;
-    let json;
-    if (line.startsWith("data: ")) {
-      json = line.slice(6).trim();
-    } else if (line.startsWith("{")) {
-      json = line;
-    } else {
-      continue;
-    }
-    if (!json.startsWith("{")) continue;
-    let obj;
-    try { obj = JSON.parse(json); } catch { continue; }
-    // Prefer the envelope matching our request id; otherwise keep the last seen.
-    if (typeof obj.id === "number" && obj.id === reqId) return obj;
-    lastJson = obj;
-  }
-  return lastJson;
-}
-
-/**
- * JSON-RPC 2.0 tools/call against a streamable-http MCP server (after the
- * initialize + notifications/initialized handshake is established or reused).
- *
- * @param {string} url       the MCP server endpoint
- * @param {string} toolName  the tool to invoke (e.g. "web_search_exa", "web_search")
- * @param {object} args      the tool's arguments (per its input schema)
- * @param {object} [extraHeaders]
- * @param {AbortSignal} [signal]
- * @returns {Promise<object>} the RPC `result` payload (the `content` array etc.)
- * @throws {WebError} on HTTP failure, RPC error, abort, or unparseable response.
- */
-export async function rpcCall(url, toolName, args, extraHeaders, signal) {
-  const sessionId = await ensureSession(url, signal);
-  const reqId = nextReqId++;
-
-  const headers = {
-    "Content-Type": "application/json",
-    "Accept": "application/json, text/event-stream",
-    "User-Agent": UA,
-    ...(sessionId ? { "Mcp-Session-Id": sessionId } : {}),
-    ...(extraHeaders || {}),
-  };
-  const body = JSON.stringify({
-    jsonrpc: "2.0",
-    id: reqId,
-    method: "tools/call",
-    params: { name: toolName, arguments: args },
-  });
-
-  let res;
-  try {
-    res = await fetch(url, {
-      method: "POST",
-      redirect: "error",
-      headers,
-      body,
-      ...(signal !== undefined ? { signal } : {}),
-    });
-  } catch (error) {
-    if (signal?.aborted === true || isAbortError(error)) throw searchAborted(signal, error);
-    throw new WebError(
-      `mcp rpc ${toolName} fetch failed: ${String(error?.message ?? error)}`,
-      "WEB_PROVIDER_ERROR",
-      { cause: error },
-    );
-  }
-  throwIfSearchAborted(signal);
-
-  if (!res.ok) {
-    let detail = "";
-    try { detail = (await res.text()).slice(0, 300); } catch {}
-    // If the server rejected the request id/session, nuke the cached session
-    // so the next call re-initializes. 404/401/403 typically mean session death.
-    if (res.status === 401 || res.status === 403 || res.status === 404) {
-      sessionCache.delete(url);
-    }
-    throw new WebError(
-      `mcp rpc ${toolName} HTTP ${res.status}${detail ? ": " + detail : ""}`,
-      "WEB_PROVIDER_ERROR",
-    );
-  }
-
-  const text = await res.text();
-  throwIfSearchAborted(signal);
-
-  const env = parseEnvelope(text, reqId);
-  if (!env) {
-    throw new WebError(
-      `mcp rpc ${toolName} returned an unparseable response`,
-      "WEB_PROVIDER_ERROR",
-    );
-  }
-  if (env.error) {
-    throw new WebError(
-      `mcp rpc ${toolName} error: ${env.error.message || JSON.stringify(env.error)}`,
-      "WEB_PROVIDER_ERROR",
-    );
-  }
-  if (!env.result) {
-    throw new WebError(
-      `mcp rpc ${toolName} returned a response with no result`,
-      "WEB_PROVIDER_ERROR",
-    );
-  }
-  return env.result;
-}
-
-/** Drop the cached session for a url (used by backends after a fatal error). */
-export function invalidateSession(url) {
-  sessionCache.delete(url);
-}
-
-export { throwIfSearchAborted, searchAborted, isAbortError };