@endday/search-mcp 1.0.0 → 1.0.2

package/src/search/engines/brave.impl.js

@@ -1,223 +0,0 @@
-import { ApiError } from "../../core/errors.js";
-import {
-  fetchSearchText,
-  isChallengeResponse,
-  throwBlockedUpstreamError,
-} from "../engineRequest.js";
-import {
-  ensureAbsoluteUrl,
-  mapLanguage,
-  mapTimeRange,
-} from "../engineUtils.js";
-import { cleanText, parseHtml } from "../../core/html.js";
-import { normalizeResults } from "../ranking.js";
-
-const BRAVE_TIME_RANGE = {
-  day: "pd",
-  week: "pw",
-  month: "pm",
-  year: "py",
-};
-
-const BRAVE_LANGUAGE = {
-  en: "en-us",
-  "en-us": "en-us",
-  "en-gb": "en-gb",
-  zh: "zh-hans",
-  "zh-cn": "zh-hans",
-  "zh-tw": "zh-hant",
-};
-
-const BRAVE_COUNTRY = {
-  en: "us",
-  "en-us": "us",
-  "en-gb": "gb",
-  zh: "cn",
-  "zh-cn": "cn",
-  "zh-tw": "tw",
-};
-
-const BRAVE_CHALLENGE_PATTERNS = [
-  /name=["']captcha["']/i,
-  /id=["'][^"']*captcha[^"']*["']/i,
-];
-
-function isBraveChallengeResponse(source) {
-  const text = String(source || "");
-
-  return (
-    isChallengeResponse(text, BRAVE_CHALLENGE_PATTERNS) ||
-    ((/verify you are human/i.test(text) || /unusual traffic/i.test(text)) &&
-      /<form\b/i.test(text))
-  );
-}
-
-function throwBraveChallengeError() {
-  throwBlockedUpstreamError({
-    engine: "Brave",
-    surface: "html",
-  });
-}
-
-export function parseBraveResults(html) {
-  if (isBraveChallengeResponse(html)) {
-    throwBraveChallengeError();
-  }
-
-  const root = parseHtml(html);
-  const resultNodes = root
-    .querySelectorAll(".snippet")
-    .filter((node) => node.getAttribute("data-type") === "web");
-  const results = [];
-
-  for (const node of resultNodes) {
-    const linkNode =
-      node.querySelector("a.l1[href]") || node.querySelector("a[href]");
-    const titleNode =
-      node.querySelector(".title") || node.querySelector(".search-snippet-title");
-    const descriptionNode =
-      node.querySelector(".generic-snippet .content") ||
-      node.querySelector(".content");
-
-    if (!linkNode || !titleNode) {
-      continue;
-    }
-
-    results.push({
-      title: cleanText(titleNode.innerHTML || titleNode.text),
-      url: ensureAbsoluteUrl(
-        linkNode.getAttribute("href"),
-        "https://search.brave.com"
-      ),
-      description: cleanText(
-        descriptionNode?.innerHTML || descriptionNode?.text || ""
-      ),
-    });
-  }
-
-  if (results.length === 0) {
-    throw new ApiError({
-      status: 502,
-      code: "UPSTREAM_PARSE_ERROR",
-      category: "upstream",
-      message: "Brave parser could not find organic results",
-    });
-  }
-
-  return normalizeResults(results);
-}
-
-export function parseBraveNewsResults(html) {
-  if (isBraveChallengeResponse(html)) {
-    throwBraveChallengeError();
-  }
-
-  const root = parseHtml(html);
-  const resultNodes = root
-    .querySelectorAll('.snippet')
-    .filter((node) => node.getAttribute("data-type") === "news");
-  const results = [];
-
-  for (const node of resultNodes) {
-    const linkNode = node.querySelector("a.l1[href]") || node.querySelector("a[href]");
-    const titleNode = node.querySelector(".title");
-    const descriptionNode = node.querySelector(".generic-snippet .description") || node.querySelector(".content");
-    const sourceNode = node.querySelector(".site-name-content .desktop-small-semibold");
-    const publishedNode =
-      node.querySelector(".site-name-content .desktop-small-regular.t-tertiary") ||
-      node.querySelector(".age-snippet");
-
-    if (!linkNode || !titleNode) {
-      continue;
-    }
-
-    results.push({
-      title: cleanText(titleNode.innerHTML || titleNode.text),
-      url: ensureAbsoluteUrl(
-        linkNode.getAttribute("href"),
-        "https://search.brave.com"
-      ),
-      description: cleanText(
-        descriptionNode?.innerHTML || descriptionNode?.text || ""
-      ),
-      source_name: cleanText(sourceNode?.innerHTML || sourceNode?.text || ""),
-      published_text: cleanText(
-        publishedNode?.innerHTML || publishedNode?.text || ""
-      ),
-    });
-  }
-
-  if (results.length === 0) {
-    throw new ApiError({
-      status: 502,
-      code: "UPSTREAM_PARSE_ERROR",
-      category: "upstream",
-      message: "Brave News parser could not find organic results",
-    });
-  }
-
-  return normalizeResults(results);
-}
-
-async function searchBrave(params) {
-  const { vertical = "web", query, language, time_range, signal, runtimeContext } = params;
-  const searchUrl = new URL(
-    vertical === "news"
-      ? "https://search.brave.com/news"
-      : "https://search.brave.com/search"
-  );
-  searchUrl.searchParams.set("q", query);
-  searchUrl.searchParams.set("spellcheck", "0");
-  searchUrl.searchParams.set("source", "web");
-  searchUrl.searchParams.set("summary", "0");
-
-  const timeFilter = mapTimeRange(time_range, BRAVE_TIME_RANGE);
-  if (timeFilter) {
-    searchUrl.searchParams.set("tf", timeFilter);
-  }
-
-  const html = await fetchSearchText(searchUrl.toString(), {
-    engine: "brave",
-    engineLabel: "Brave",
-    signal,
-    language,
-    cookies: {
-      country: mapLanguage(language, BRAVE_COUNTRY, "us"),
-      ui_lang: mapLanguage(language, BRAVE_LANGUAGE, "en-us"),
-      useLocation: "0",
-      summarizer: "0",
-      safesearch: "off",
-    },
-    referrer: "https://search.brave.com/",
-    runtimeContext,
-    blockedStatuses: [403, 429],
-    isBlocked: isBraveChallengeResponse,
-    blockedSurface: "html",
-  });
-
-  return vertical === "news" ? parseBraveNewsResults(html) : parseBraveResults(html);
-}
-
-export const braveAdapter = {
-  name: "brave",
-  label: "Brave",
-  priority: 90,
-  tier: "secondary",
-  requestPolicy: {
-    retryAttempts: 0,
-    minRequestIntervalMs: 250,
-  },
-  supports: {
-    verticals: ["web", "news"],
-    language: true,
-    time_range: true,
-    pageno: false,
-    news: {
-      pageno: false,
-    },
-  },
-  isAvailable: () => true,
-  search: searchBrave,
-};
-
-export default searchBrave;

package/src/search/engines/brave.js

@@ -1,2 +0,0 @@
-export * from "./brave.impl.js";
-export { default } from "./brave.impl.js";

package/src/search/engines/duckduckgo.impl.js

@@ -1,164 +0,0 @@
-import { ApiError } from "../../core/errors.js";
-import {
-  fetchSearchText,
-  isChallengeResponse,
-  throwBlockedUpstreamError,
-} from "../engineRequest.js";
-import {
-  ensureAbsoluteUrl,
-  mapLanguage,
-  mapTimeRange,
-  resolvePageNumber,
-} from "../engineUtils.js";
-import { cleanText, parseHtml } from "../../core/html.js";
-import { normalizeResults } from "../ranking.js";
-
-const DUCKDUCKGO_LANGUAGE = {
-  en: "us-en",
-  "en-us": "us-en",
-  "en-gb": "uk-en",
-  zh: "cn-zh",
-  "zh-cn": "cn-zh",
-  "zh-tw": "tw-zh",
-};
-
-const DUCKDUCKGO_TIME_RANGE = {
-  day: "d",
-  week: "w",
-  month: "m",
-  year: "y",
-};
-
-const DUCKDUCKGO_CHALLENGE_PATTERNS = [
-  /anomaly\.js/i,
-  /bots use DuckDuckGo too/i,
-  /automated requests/i,
-];
-
-function isDuckDuckGoChallengeResponse(source) {
-  const text = String(source || "");
-
-  return (
-    isChallengeResponse(text, DUCKDUCKGO_CHALLENGE_PATTERNS) ||
-    (/verify you are human/i.test(text) && /<form\b/i.test(text))
-  );
-}
-
-function throwDuckDuckGoChallengeError() {
-  throwBlockedUpstreamError({
-    engine: "DuckDuckGo",
-    surface: "html",
-  });
-}
-
-function extractDuckDuckGoUrl(rawUrl) {
-  const absoluteUrl = ensureAbsoluteUrl(rawUrl, "https://duckduckgo.com");
-
-  try {
-    const parsed = new URL(absoluteUrl);
-    const uddg = parsed.searchParams.get("uddg");
-    return uddg ? decodeURIComponent(uddg) : absoluteUrl;
-  } catch (_) {
-    return absoluteUrl;
-  }
-}
-
-export function parseDuckDuckGoResults(html) {
-  if (isDuckDuckGoChallengeResponse(html)) {
-    throwDuckDuckGoChallengeError();
-  }
-
-  const root = parseHtml(html);
-  const resultNodes = root.querySelectorAll(".result");
-  const results = [];
-
-  for (const node of resultNodes) {
-    const linkNode =
-      node.querySelector("a.result__a[href]") || node.querySelector("a[href]");
-    const snippetNode =
-      node.querySelector(".result__snippet") ||
-      node.querySelector(".result__body");
-
-    if (!linkNode) {
-      continue;
-    }
-
-    results.push({
-      title: cleanText(linkNode.innerHTML || linkNode.text),
-      url: extractDuckDuckGoUrl(linkNode.getAttribute("href")),
-      description: cleanText(snippetNode?.innerHTML || snippetNode?.text || ""),
-    });
-  }
-
-  if (results.length === 0) {
-    throw new ApiError({
-      status: 502,
-      code: "UPSTREAM_PARSE_ERROR",
-      category: "upstream",
-      message: "DuckDuckGo parser could not find organic results",
-    });
-  }
-
-  return normalizeResults(results);
-}
-
-async function searchDuckDuckGo(params) {
-  const { query, language, time_range, pageno, signal, runtimeContext } = params;
-  const page = resolvePageNumber(pageno);
-  const locale = mapLanguage(language, DUCKDUCKGO_LANGUAGE, "wt-wt");
-
-  if (page > 0) {
-    throw new ApiError({
-      status: 400,
-      code: "UNSUPPORTED_PARAMETER",
-      category: "validation",
-      message: "DuckDuckGo HTML pagination is not supported",
-    });
-  }
-
-  const timeFilter = mapTimeRange(time_range, DUCKDUCKGO_TIME_RANGE);
-  const html = await fetchSearchText("https://html.duckduckgo.com/html/", {
-    engine: "duckduckgo",
-    engineLabel: "DuckDuckGo",
-    signal,
-    language,
-    method: "POST",
-    form: {
-      q: query,
-      kl: locale,
-      ...(timeFilter ? { df: timeFilter } : {}),
-    },
-    cookies: {
-      kl: locale,
-      ...(timeFilter ? { df: timeFilter } : {}),
-    },
-    referrer: "https://html.duckduckgo.com/",
-    origin: "https://html.duckduckgo.com",
-    runtimeContext,
-    blockedStatuses: [403, 429],
-    isBlocked: isDuckDuckGoChallengeResponse,
-    blockedSurface: "html",
-  });
-
-  return parseDuckDuckGoResults(html);
-}
-
-export const duckDuckGoAdapter = {
-  name: "duckduckgo",
-  label: "DuckDuckGo",
-  priority: 95,
-  tier: "primary",
-  requestPolicy: {
-    retryAttempts: 0,
-    minRequestIntervalMs: 150,
-  },
-  supports: {
-    language: true,
-    time_range: true,
-    pageno: false,
-  },
-  isAvailable: () => true,
-  search: searchDuckDuckGo,
-};
-
-export default searchDuckDuckGo;

package/src/search/engines/duckduckgo.js

@@ -1,2 +0,0 @@
-export * from "./duckduckgo.impl.js";
-export { default } from "./duckduckgo.impl.js";

package/src/search/engines/mojeek.impl.js

@@ -1,115 +0,0 @@
-import { ApiError } from "../../core/errors.js";
-import {
-  fetchSearchText,
-  isChallengeResponse,
-  throwBlockedUpstreamError,
-} from "../engineRequest.js";
-import { resolvePageNumber } from "../engineUtils.js";
-import { cleanText, parseHtml } from "../../core/html.js";
-import { normalizeResults } from "../ranking.js";
-
-const MOJEEK_CHALLENGE_PATTERNS = [
-  /name=["']captcha["']/i,
-  /id=["'][^"']*captcha[^"']*["']/i,
-];
-
-function isMojeekChallengeResponse(source) {
-  const text = String(source || "");
-
-  return (
-    isChallengeResponse(text, MOJEEK_CHALLENGE_PATTERNS) ||
-    ((/verify you are human/i.test(text) || /unusual traffic/i.test(text)) &&
-      /<form\b/i.test(text))
-  );
-}
-
-function throwMojeekChallengeError() {
-  throwBlockedUpstreamError({
-    engine: "Mojeek",
-    surface: "html",
-  });
-}
-
-export function parseMojeekResults(html) {
-  if (isMojeekChallengeResponse(html)) {
-    throwMojeekChallengeError();
-  }
-
-  const root = parseHtml(html);
-  const resultNodes = root.querySelectorAll("ul.results-standard li");
-  const results = [];
-
-  for (const node of resultNodes) {
-    const linkNode =
-      node.querySelector("h2 a.title[href]") || node.querySelector("h2 a[href]");
-    const descriptionNode = node.querySelector("p.s");
-
-    if (!linkNode) {
-      continue;
-    }
-
-    results.push({
-      title: cleanText(linkNode.innerHTML || linkNode.text),
-      url: linkNode.getAttribute("href"),
-      description: cleanText(
-        descriptionNode?.innerHTML || descriptionNode?.text || ""
-      ),
-    });
-  }
-
-  if (results.length === 0) {
-    throw new ApiError({
-      status: 502,
-      code: "UPSTREAM_PARSE_ERROR",
-      category: "upstream",
-      message: "Mojeek parser could not find organic results",
-    });
-  }
-
-  return normalizeResults(results);
-}
-
-async function searchMojeek(params) {
-  const { query, language, pageno, signal, runtimeContext } = params;
-  const searchUrl = new URL("https://www.mojeek.com/search");
-  searchUrl.searchParams.set("q", query);
-
-  const page = resolvePageNumber(pageno);
-  if (page > 0) {
-    searchUrl.searchParams.set("s", String(page * 10 + 1));
-  }
-
-  const html = await fetchSearchText(searchUrl.toString(), {
-    engine: "mojeek",
-    engineLabel: "Mojeek",
-    signal,
-    language,
-    referrer: "https://www.mojeek.com/",
-    runtimeContext,
-    blockedStatuses: [403, 429],
-    isBlocked: isMojeekChallengeResponse,
-    blockedSurface: "html",
-  });
-
-  return parseMojeekResults(html);
-}
-
-export const mojeekAdapter = {
-  name: "mojeek",
-  label: "Mojeek",
-  priority: 80,
-  tier: "secondary",
-  requestPolicy: {
-    retryAttempts: 0,
-    minRequestIntervalMs: 250,
-  },
-  supports: {
-    language: true,
-    time_range: false,
-    pageno: true,
-  },
-  isAvailable: () => true,
-  search: searchMojeek,
-};
-
-export default searchMojeek;

package/src/search/engines/mojeek.js

@@ -1,2 +0,0 @@
-export * from "./mojeek.impl.js";
-export { default } from "./mojeek.impl.js";

package/src/search/engines/qwant.impl.js

@@ -1,188 +0,0 @@
-import { ApiError } from "../../core/errors.js";
-import {
-  fetchSearchText,
-  isChallengeResponse,
-  throwBlockedUpstreamError,
-} from "../engineRequest.js";
-import {
-  ensureAbsoluteUrl,
-  mapLanguage,
-  resolvePageNumber,
-} from "../engineUtils.js";
-import { cleanText, parseHtml } from "../../core/html.js";
-import { normalizeResults } from "../ranking.js";
-
-const QWANT_LANGUAGE = {
-  en: "en_US",
-  "en-us": "en_US",
-  "en-gb": "en_GB",
-  zh: "zh_CN",
-  "zh-cn": "zh_CN",
-  "zh-tw": "zh_TW",
-  fr: "fr_FR",
-  de: "de_DE",
-  es: "es_ES",
-  it: "it_IT",
-};
-
-const QWANT_CHALLENGE_PATTERNS = [
-  /<title>\s*Service unavailable\s*<\/title>/i,
-  /name=["']captcha["']/i,
-  /id=["'][^"']*captcha[^"']*["']/i,
-];
-
-function isQwantChallengeResponse(source) {
-  const text = String(source || "");
-  const hasResultArticles = /<section\b[\s\S]*<article\b/i.test(text);
-
-  return (
-    (isChallengeResponse(text, QWANT_CHALLENGE_PATTERNS) && !hasResultArticles) ||
-    ((/verify you are human/i.test(text) || /unusual traffic/i.test(text)) &&
-      /<form\b/i.test(text))
-  );
-}
-
-function throwQwantChallengeError() {
-  throwBlockedUpstreamError({
-    engine: "Qwant",
-    surface: "html",
-  });
-}
-
-function normalizeQwantUrl(rawUrl) {
-  const value = String(rawUrl || "").trim();
-
-  if (!value) {
-    return "";
-  }
-
-  if (/^https?:\/\//i.test(value) || value.startsWith("//")) {
-    return ensureAbsoluteUrl(value, "https://www.qwant.com");
-  }
-
-  if (/^[\w.-]+\.[a-z]{2,}(?:[/:?#]|$)/i.test(value)) {
-    return `https://${value}`;
-  }
-
-  return ensureAbsoluteUrl(value, "https://www.qwant.com");
-}
-
-function extractQwantResultUrl(node, linkNode) {
-  const visibleUrlNode =
-    node.querySelector("span.url.partner") ||
-    node.querySelector(".url.partner") ||
-    node.querySelector(".url");
-  const visibleUrl = cleanText(
-    visibleUrlNode?.innerHTML || visibleUrlNode?.text || ""
-  );
-
-  return normalizeQwantUrl(visibleUrl || linkNode.getAttribute("href"));
-}
-
-export function parseQwantResults(html) {
-  if (isQwantChallengeResponse(html)) {
-    throwQwantChallengeError();
-  }
-
-  const root = parseHtml(html);
-  const resultNodes = root.querySelectorAll("section article");
-  const results = [];
-
-  for (const node of resultNodes) {
-    if (node.querySelector("span.tooltip")) {
-      continue;
-    }
-
-    const linkNode = node.querySelector("h2 a[href]") || node.querySelector("a[href]");
-    if (!linkNode) {
-      continue;
-    }
-
-    const descriptionNode = node.querySelector("p");
-
-    results.push({
-      title: cleanText(linkNode.innerHTML || linkNode.text),
-      url: extractQwantResultUrl(node, linkNode),
-      description: cleanText(
-        descriptionNode?.innerHTML || descriptionNode?.text || ""
-      ),
-    });
-  }
-
-  const normalized = normalizeResults(results);
-  if (normalized.length === 0) {
-    throw new ApiError({
-      status: 502,
-      code: "UPSTREAM_PARSE_ERROR",
-      category: "upstream",
-      message: "Qwant parser could not find organic results",
-    });
-  }
-
-  return normalized;
-}
-
-async function searchQwant(params) {
-  const { query, language, time_range, pageno, signal, runtimeContext } = params;
-
-  if (time_range) {
-    throw new ApiError({
-      status: 400,
-      code: "UNSUPPORTED_PARAMETER",
-      category: "validation",
-      message: "Qwant Lite time_range filtering is not supported",
-    });
-  }
-
-  const page = resolvePageNumber(pageno);
-  if (page > 4) {
-    throw new ApiError({
-      status: 400,
-      code: "UNSUPPORTED_PARAMETER",
-      category: "validation",
-      message: "Qwant Lite supports at most five result pages",
-    });
-  }
-
-  const locale = mapLanguage(language, QWANT_LANGUAGE, "en_US");
-  const searchUrl = new URL("https://lite.qwant.com/");
-  searchUrl.searchParams.set("q", query);
-  searchUrl.searchParams.set("locale", locale.toLowerCase());
-  searchUrl.searchParams.set("l", locale.split("_")[0]);
-  searchUrl.searchParams.set("s", "1");
-  searchUrl.searchParams.set("p", String(page + 1));
-
-  const html = await fetchSearchText(searchUrl.toString(), {
-    engine: "qwant",
-    engineLabel: "Qwant",
-    signal,
-    language,
-    referrer: "https://www.qwant.com/",
-    runtimeContext,
-    blockedStatuses: [403, 429, 503],
-    isBlocked: isQwantChallengeResponse,
-    blockedSurface: "html",
-  });
-
-  return parseQwantResults(html);
-}
-
-export const qwantAdapter = {
-  name: "qwant",
-  label: "Qwant",
-  priority: 85,
-  tier: "secondary",
-  requestPolicy: {
-    retryAttempts: 0,
-    minRequestIntervalMs: 250,
-  },
-  supports: {
-    language: true,
-    time_range: false,
-    pageno: true,
-  },
-  isAvailable: () => true,
-  search: searchQwant,
-};
-
-export default searchQwant;

package/src/search/engines/qwant.js

@@ -1,2 +0,0 @@
-export * from "./qwant.impl.js";
-export { default } from "./qwant.impl.js";