@endday/search-mcp 1.0.0 → 1.0.1

package/src/search/engines/startpage.impl.js

@@ -1,237 +0,0 @@
-import { ApiError } from "../../core/errors.js";
-import {
-  fetchSearchText,
-  isChallengeResponse,
-  throwBlockedUpstreamError,
-} from "../engineRequest.js";
-import {
-  mapLanguage,
-  resolvePageNumber,
-} from "../engineUtils.js";
-import { cleanText, extractBalancedSegment } from "../../core/html.js";
-import { normalizeResults } from "../ranking.js";
-
-const STARTPAGE_LANGUAGE = {
-  en: "english",
-  zh: "chinese_simplified",
-  "zh-cn": "chinese_simplified",
-  "zh-tw": "chinese_traditional",
-};
-
-const STARTPAGE_CHALLENGE_PATTERNS = [
-  /\/sp\/captcha\b/i,
-  /name=["']captcha["']/i,
-];
-const STARTPAGE_SC_TTL_MS = 15 * 60 * 1000;
-let cachedStartpageSc = {
-  value: "",
-  expiresAt: 0,
-};
-
-function isStartpageChallengeResponse(source) {
-  const text = String(source || "");
-
-  return (
-    isChallengeResponse(text, STARTPAGE_CHALLENGE_PATTERNS) ||
-    ((/verify you are human/i.test(text) || /unusual traffic/i.test(text)) &&
-      /<form\b/i.test(text))
-  );
-}
-
-function throwStartpageChallengeError(surface) {
-  throwBlockedUpstreamError({
-    engine: "Startpage",
-    surface,
-  });
-}
-
-function extractStartpageScToken(html) {
-  const match =
-    html.match(/<input\b[^>]*name=["']sc["'][^>]*value=["']([^"']+)["'][^>]*>/i) ||
-    html.match(/<input\b[^>]*value=["']([^"']+)["'][^>]*name=["']sc["'][^>]*>/i);
-
-  return match?.[1]?.trim() || "";
-}
-
-function buildStartpagePreferences(languageValue) {
-  const preferences = [
-    ["disable_family_filter", "1"],
-    ["enable_post_method", "1"],
-    ["instant_answers", "0"],
-    ["num_of_results", "10"],
-  ];
-
-  if (languageValue) {
-    preferences.push(
-      ["lang_homepage", languageValue],
-      ["language", languageValue],
-      ["lui", languageValue]
-    );
-  }
-
-  return encodeURIComponent(
-    preferences.map(([key, value]) => `${key}EEE${value}`).join("N1N")
-  );
-}
-
-async function fetchStartpageScToken({ signal, language, runtimeContext }) {
-  if (cachedStartpageSc.value && cachedStartpageSc.expiresAt > Date.now()) {
-    return cachedStartpageSc.value;
-  }
-
-  try {
-    const html = await fetchSearchText("https://www.startpage.com/", {
-      engine: "startpage",
-      engineLabel: "Startpage",
-      signal,
-      language,
-      referrer: "https://www.startpage.com/",
-      runtimeContext,
-      blockedStatuses: [403, 429],
-      isBlocked: isStartpageChallengeResponse,
-      blockedSurface: "home",
-    });
-    const token = extractStartpageScToken(html);
-
-    if (token) {
-      cachedStartpageSc = {
-        value: token,
-        expiresAt: Date.now() + STARTPAGE_SC_TTL_MS,
-      };
-    }
-
-    return token;
-  } catch (_) {
-    return "";
-  }
-}
-
-export function resetStartpageRequestState() {
-  cachedStartpageSc = {
-    value: "",
-    expiresAt: 0,
-  };
-}
-
-function extractStartpageResultArray(html) {
-  const markerIndex = [
-    '"display_type":"web-google"',
-    '"display_type":"web-results"',
-    '"display_type":"web"',
-  ]
-    .map((marker) => html.indexOf(marker))
-    .filter((index) => index >= 0)
-    .sort((left, right) => left - right)[0];
-
-  if (typeof markerIndex !== "number") {
-    return null;
-  }
-
-  const resultsMarker = '"results":';
-  const resultsIndex = html.indexOf(resultsMarker, markerIndex);
-  if (resultsIndex === -1) {
-    return null;
-  }
-
-  const arrayStart = html.indexOf("[", resultsIndex);
-  if (arrayStart === -1) {
-    return null;
-  }
-
-  return JSON.parse(extractBalancedSegment(html, arrayStart));
-}
-
-export function parseStartpageResults(html) {
-  if (isStartpageChallengeResponse(html)) {
-    throwStartpageChallengeError("html");
-  }
-
-  const items = extractStartpageResultArray(html);
-
-  if (!Array.isArray(items)) {
-    throw new ApiError({
-      status: 502,
-      code: "UPSTREAM_PARSE_ERROR",
-      category: "upstream",
-      message: "Startpage parser could not find result payload",
-    });
-  }
-
-  return normalizeResults(
-    items
-      .filter((item) => item?.clickUrl && item?.title)
-      .map((item) => ({
-        title: cleanText(item.title),
-        url: item.clickUrl,
-        description: cleanText(item.description || ""),
-      }))
-  );
-}
-
-async function searchStartpage(params) {
-  const { query, language, time_range, pageno, signal, runtimeContext } = params;
-
-  if (time_range) {
-    throw new ApiError({
-      status: 400,
-      code: "UNSUPPORTED_PARAMETER",
-      category: "validation",
-      message: "Startpage time_range filtering is not supported",
-    });
-  }
-
-  const page = resolvePageNumber(pageno);
-  const languageValue = mapLanguage(language, STARTPAGE_LANGUAGE, "");
-  const sc = await fetchStartpageScToken({ signal, language, runtimeContext });
-  const html = await fetchSearchText("https://www.startpage.com/sp/search", {
-    engine: "startpage",
-    engineLabel: "Startpage",
-    signal,
-    language,
-    method: "POST",
-    form: {
-      query,
-      cat: "web",
-      segment: "startpage.udog",
-      ...(page > 0 ? { page: String(page + 1) } : {}),
-      ...(languageValue
-        ? {
-            language: languageValue,
-            lui: languageValue,
-          }
-        : {}),
-      ...(sc ? { sc } : {}),
-    },
-    cookies: {
-      preferences: buildStartpagePreferences(languageValue),
-    },
-    referrer: "https://www.startpage.com/",
-    origin: "https://www.startpage.com",
-    runtimeContext,
-    blockedStatuses: [403, 429],
-    isBlocked: isStartpageChallengeResponse,
-    blockedSurface: "html",
-  });
-
-  return parseStartpageResults(html);
-}
-
-export const startpageAdapter = {
-  name: "startpage",
-  label: "Startpage",
-  priority: 100,
-  tier: "primary",
-  requestPolicy: {
-    retryAttempts: 0,
-    minRequestIntervalMs: 200,
-  },
-  supports: {
-    language: true,
-    time_range: false,
-    pageno: true,
-  },
-  isAvailable: () => true,
-  search: searchStartpage,
-};
-
-export default searchStartpage;

package/src/search/engines/startpage.js

@@ -1,2 +0,0 @@
-export * from "./startpage.impl.js";
-export { default } from "./startpage.impl.js";

package/src/search/engines/toutiao.impl.js

@@ -1,265 +0,0 @@
-import { ApiError } from "../../core/errors.js";
-import {
-  fetchSearchText,
-  isChallengeResponse,
-  throwBlockedUpstreamError,
-} from "../engineRequest.js";
-import { cleanText, parseHtml } from "../../core/html.js";
-import { normalizeResults } from "../ranking.js";
-
-const TOUTIAO_CHALLENGE_PATTERNS = [
-  /安全验证/i,
-  /captcha.*验证/i,
-];
-
-function isToutiaoChallengeResponse(source) {
-  const text = String(source || "");
-  return (
-    isChallengeResponse(text, TOUTIAO_CHALLENGE_PATTERNS) ||
-    (text.length < 1000 && /<form\b/i.test(text)) ||
-    /"challenge_code"\s*:\s*1366/.test(text) ||
-    /"template_key"\s*:\s*"71-undefined"/.test(text)
-  );
-}
-
-function throwToutiaoChallengeError() {
-  throwBlockedUpstreamError({
-    engine: "Toutiao",
-    surface: "html",
-  });
-}
-
-/**
- * Decode a Toutiao search-jump redirect URL.
- * Links in the SSR HTML look like:
- *   https://sou.toutiao.com/search/jump?url=https%3A%2F%2Fexample.com%2Farticle
- * We extract and decode the embedded target URL.
- */
-function decodeToutiaoJumpUrl(href) {
-  try {
-    const url = new URL(href);
-    const target = url.searchParams.get("url");
-    if (!target) return href;
-    const decoded = decodeURIComponent(target);
-    if (decoded.includes("search/jump?url=")) {
-      return decodeToutiaoJumpUrl(decoded);
-    }
-    return decoded;
-  } catch (_) {
-    return href;
-  }
-}
-
-/**
- * Extract description text from a result-content card.
- *
- * Toutiao SSR DOM structure per search result:
- *   div.result-content
- *     script/style (hydrate/render code, ignored)
- *     div
- *       div.cs-view.cs-view-block.cs-card
- *         div.cs-view.cs-view-block.cs-card-header
- *           a[href] (title link)
- *         div.cs-view.cs-view-block.cs-card-content
- *           (description text, may be inside nested divs)
- */
-function extractCardDescription(card) {
-  // Look for cs-card-content, which holds the description
-  const contentNode = card.querySelector(".cs-card-content");
-  if (contentNode) {
-    // The description text is usually in a direct child div without a link
-    const descDivs = contentNode.querySelectorAll("div");
-    for (const d of descDivs) {
-      const txt = cleanText(d.textContent || "").trim();
-      // Skip text that's just the title repeated, or too short, or contains JS
-      if (
-        txt.length > 15 &&
-        txt.length < 500 &&
-        !txt.includes("druid") &&
-        !txt.includes("PerfTag") &&
-        !txt.includes("script")
-      ) {
-        // Prefer longer, more descriptive text
-        if (txt.length > 40) {
-          return txt.slice(0, 300);
-        }
-      }
-    }
-    // Fallback: the content node's own text (excluding nested links)
-    const fullText = cleanText(contentNode.textContent || "").trim();
-    // Remove the title portion from the description
-    const titleLink = card.querySelector("a[href]");
-    const titleText = titleLink ? (titleLink.textContent || "").trim() : "";
-    const descOnly = fullText.replace(titleText, "").trim();
-    if (descOnly.length > 15 && descOnly.length < 500 && !descOnly.includes("druid")) {
-      return descOnly.slice(0, 300);
-    }
-  }
-
-  // Fallback: search all divs in the card for description-like text
-  const allDivs = card.querySelectorAll("div");
-  for (const d of allDivs) {
-    const txt = cleanText(d.textContent || "").trim();
-    if (
-      txt.length > 30 &&
-      txt.length < 500 &&
-      !txt.includes("druid") &&
-      !txt.includes("PerfTag") &&
-      !txt.includes("script") &&
-      !txt.includes("换一换")
-    ) {
-      return txt.slice(0, 300);
-    }
-  }
-
-  return "";
-}
-
-export function parseToutiaoResults(html) {
-  if (isToutiaoChallengeResponse(html)) {
-    throwToutiaoChallengeError();
-  }
-
-  const root = parseHtml(html);
-  const seen = new Set();
-  const results = [];
-
-  // Find the search result list container
-  const resultList = root.querySelector(".s-result-list");
-
-  if (!resultList) {
-    throw new ApiError({
-      status: 502,
-      code: "UPSTREAM_PARSE_ERROR",
-      category: "upstream",
-      message: "Toutiao parser could not find s-result-list container",
-    });
-  }
-
-  // Each div.result-content inside s-result-list is one search result card.
-  // Skip cards inside s-side-list (hot trending sidebar).
-  const cards = resultList.querySelectorAll(".result-content");
-
-  for (const card of cards) {
-    // Exclude the hot trending sidebar entirely
-    if (card.closest(".s-side-list")) continue;
-
-    // Exclude ad/promotion cards: data-test-card-id="67-toutiao_web"
-    // Keep organic results: "67-homepage" (official site), "26-aft_ciyu_detail" (word definition)
-    const adMarker = card.querySelector("[data-test-card-id='67-toutiao_web']");
-    if (adMarker) continue;
-
-    // Exclude related-search suggestions: data-test-card-id="20-undefined"
-    const relatedMarker = card.querySelector("[data-test-card-id^='20-']");
-    if (relatedMarker) continue;
-
-    // Find the primary title link
-    const links = card.querySelectorAll("a[href]");
-    const titleLink = links.find((a) => {
-      const href = a.getAttribute("href") || "";
-      const text = (a.textContent || "").trim();
-      return (
-        href.includes("search/jump?url=") ||
-        href.includes("/article/") ||
-        href.includes("m.douyinhanyu.com") ||
-        href.includes("baike.com") ||
-        href.includes("m.toutiaoimg.cn") ||
-        href.includes("cloud.tencent.com")
-      ) && text.length > 3 && !/^\d{1,2}:\d{2}/.test(text);
-    });
-
-    if (!titleLink) continue;
-
-    const title = cleanText(titleLink.textContent || titleLink.innerHTML || "").trim();
-    const href = titleLink.getAttribute("href") || "";
-
-    if (!title || title.length < 3 || title.length > 150) continue;
-
-    // Skip UI/navigation titles
-    if (
-      title.includes("换一换") ||
-      title.includes("首页") ||
-      title.includes("登录") ||
-      title.includes("去西瓜搜") ||
-      title.includes("去抖音搜") ||
-      title.includes("查看详情") ||
-      title.includes("播放") ||
-      title.startsWith("无障碍") ||
-      title.startsWith("相关搜索")
-    ) continue;
-
-    const targetUrl = decodeToutiaoJumpUrl(href);
-    if (!targetUrl || targetUrl.startsWith("#") || targetUrl.startsWith("/")) continue;
-
-    // Skip trending items and internal search navigation
-    if (targetUrl.includes("/trending")) continue;
-    if (targetUrl.includes("so.toutiao.com/search") && !targetUrl.includes("toutiao.com/a")) continue;
-
-    // Deduplicate by target URL
-    try {
-      const canonical = new URL(targetUrl).toString().toLowerCase();
-      if (seen.has(canonical)) continue;
-      seen.add(canonical);
-    } catch (_) {
-      if (seen.has(targetUrl)) continue;
-      seen.add(targetUrl);
-    }
-
-    // Extract description from cs-card-content
-    const description = extractCardDescription(card);
-
-    results.push({ title, url: targetUrl, description });
-  }
-
-  if (results.length === 0) {
-    throw new ApiError({
-      status: 502,
-      code: "UPSTREAM_PARSE_ERROR",
-      category: "upstream",
-      message: "Toutiao parser could not find organic results",
-    });
-  }
-
-  return normalizeResults(results);
-}
-
-async function searchToutiao(params) {
-  const { query, signal, runtimeContext } = params;
-  const searchUrl = new URL("https://so.toutiao.com/search");
-  searchUrl.searchParams.set("keyword", query);
-  searchUrl.searchParams.set("dvpf", "pc");
-  searchUrl.searchParams.set("source", "input");
-
-  const html = await fetchSearchText(searchUrl.toString(), {
-    engine: "toutiao",
-    engineLabel: "Toutiao",
-    signal,
-    referrer: "https://so.toutiao.com/",
-    runtimeContext,
-    blockedStatuses: [403, 429],
-    isBlocked: isToutiaoChallengeResponse,
-    blockedSurface: "html",
-  });
-
-  return parseToutiaoResults(html);
-}
-
-export const toutiaoAdapter = {
-  name: "toutiao",
-  label: "Toutiao",
-  priority: 65,
-  tier: "experimental",
-  requestPolicy: {
-    retryAttempts: 0,
-    minRequestIntervalMs: 500,
-  },
-  supports: {
-    language: true,
-    time_range: false,
-    pageno: false,
-  },
-  isAvailable: () => true,
-  search: searchToutiao,
-};
-
-export default searchToutiao;

package/src/search/engines/toutiao.js

@@ -1,2 +0,0 @@
-export * from "./toutiao.impl.js";
-export { default } from "./toutiao.impl.js";