@endday/search-mcp 1.0.0

package/src/search/engines/startpage.impl.js

@@ -0,0 +1,237 @@
+import { ApiError } from "../../core/errors.js";
+import {
+  fetchSearchText,
+  isChallengeResponse,
+  throwBlockedUpstreamError,
+} from "../engineRequest.js";
+import {
+  mapLanguage,
+  resolvePageNumber,
+} from "../engineUtils.js";
+import { cleanText, extractBalancedSegment } from "../../core/html.js";
+import { normalizeResults } from "../ranking.js";
+
+const STARTPAGE_LANGUAGE = {
+  en: "english",
+  zh: "chinese_simplified",
+  "zh-cn": "chinese_simplified",
+  "zh-tw": "chinese_traditional",
+};
+
+const STARTPAGE_CHALLENGE_PATTERNS = [
+  /\/sp\/captcha\b/i,
+  /name=["']captcha["']/i,
+];
+const STARTPAGE_SC_TTL_MS = 15 * 60 * 1000;
+let cachedStartpageSc = {
+  value: "",
+  expiresAt: 0,
+};
+
+function isStartpageChallengeResponse(source) {
+  const text = String(source || "");
+
+  return (
+    isChallengeResponse(text, STARTPAGE_CHALLENGE_PATTERNS) ||
+    ((/verify you are human/i.test(text) || /unusual traffic/i.test(text)) &&
+      /<form\b/i.test(text))
+  );
+}
+
+function throwStartpageChallengeError(surface) {
+  throwBlockedUpstreamError({
+    engine: "Startpage",
+    surface,
+  });
+}
+
+function extractStartpageScToken(html) {
+  const match =
+    html.match(/<input\b[^>]*name=["']sc["'][^>]*value=["']([^"']+)["'][^>]*>/i) ||
+    html.match(/<input\b[^>]*value=["']([^"']+)["'][^>]*name=["']sc["'][^>]*>/i);
+
+  return match?.[1]?.trim() || "";
+}
+
+function buildStartpagePreferences(languageValue) {
+  const preferences = [
+    ["disable_family_filter", "1"],
+    ["enable_post_method", "1"],
+    ["instant_answers", "0"],
+    ["num_of_results", "10"],
+  ];
+
+  if (languageValue) {
+    preferences.push(
+      ["lang_homepage", languageValue],
+      ["language", languageValue],
+      ["lui", languageValue]
+    );
+  }
+
+  return encodeURIComponent(
+    preferences.map(([key, value]) => `${key}EEE${value}`).join("N1N")
+  );
+}
+
+async function fetchStartpageScToken({ signal, language, runtimeContext }) {
+  if (cachedStartpageSc.value && cachedStartpageSc.expiresAt > Date.now()) {
+    return cachedStartpageSc.value;
+  }
+
+  try {
+    const html = await fetchSearchText("https://www.startpage.com/", {
+      engine: "startpage",
+      engineLabel: "Startpage",
+      signal,
+      language,
+      referrer: "https://www.startpage.com/",
+      runtimeContext,
+      blockedStatuses: [403, 429],
+      isBlocked: isStartpageChallengeResponse,
+      blockedSurface: "home",
+    });
+    const token = extractStartpageScToken(html);
+
+    if (token) {
+      cachedStartpageSc = {
+        value: token,
+        expiresAt: Date.now() + STARTPAGE_SC_TTL_MS,
+      };
+    }
+
+    return token;
+  } catch (_) {
+    return "";
+  }
+}
+
+export function resetStartpageRequestState() {
+  cachedStartpageSc = {
+    value: "",
+    expiresAt: 0,
+  };
+}
+
+function extractStartpageResultArray(html) {
+  const markerIndex = [
+    '"display_type":"web-google"',
+    '"display_type":"web-results"',
+    '"display_type":"web"',
+  ]
+    .map((marker) => html.indexOf(marker))
+    .filter((index) => index >= 0)
+    .sort((left, right) => left - right)[0];
+
+  if (typeof markerIndex !== "number") {
+    return null;
+  }
+
+  const resultsMarker = '"results":';
+  const resultsIndex = html.indexOf(resultsMarker, markerIndex);
+  if (resultsIndex === -1) {
+    return null;
+  }
+
+  const arrayStart = html.indexOf("[", resultsIndex);
+  if (arrayStart === -1) {
+    return null;
+  }
+
+  return JSON.parse(extractBalancedSegment(html, arrayStart));
+}
+
+export function parseStartpageResults(html) {
+  if (isStartpageChallengeResponse(html)) {
+    throwStartpageChallengeError("html");
+  }
+
+  const items = extractStartpageResultArray(html);
+
+  if (!Array.isArray(items)) {
+    throw new ApiError({
+      status: 502,
+      code: "UPSTREAM_PARSE_ERROR",
+      category: "upstream",
+      message: "Startpage parser could not find result payload",
+    });
+  }
+
+  return normalizeResults(
+    items
+      .filter((item) => item?.clickUrl && item?.title)
+      .map((item) => ({
+        title: cleanText(item.title),
+        url: item.clickUrl,
+        description: cleanText(item.description || ""),
+      }))
+  );
+}
+
+async function searchStartpage(params) {
+  const { query, language, time_range, pageno, signal, runtimeContext } = params;
+
+  if (time_range) {
+    throw new ApiError({
+      status: 400,
+      code: "UNSUPPORTED_PARAMETER",
+      category: "validation",
+      message: "Startpage time_range filtering is not supported",
+    });
+  }
+
+  const page = resolvePageNumber(pageno);
+  const languageValue = mapLanguage(language, STARTPAGE_LANGUAGE, "");
+  const sc = await fetchStartpageScToken({ signal, language, runtimeContext });
+  const html = await fetchSearchText("https://www.startpage.com/sp/search", {
+    engine: "startpage",
+    engineLabel: "Startpage",
+    signal,
+    language,
+    method: "POST",
+    form: {
+      query,
+      cat: "web",
+      segment: "startpage.udog",
+      ...(page > 0 ? { page: String(page + 1) } : {}),
+      ...(languageValue
+        ? {
+            language: languageValue,
+            lui: languageValue,
+          }
+        : {}),
+      ...(sc ? { sc } : {}),
+    },
+    cookies: {
+      preferences: buildStartpagePreferences(languageValue),
+    },
+    referrer: "https://www.startpage.com/",
+    origin: "https://www.startpage.com",
+    runtimeContext,
+    blockedStatuses: [403, 429],
+    isBlocked: isStartpageChallengeResponse,
+    blockedSurface: "html",
+  });
+
+  return parseStartpageResults(html);
+}
+
+export const startpageAdapter = {
+  name: "startpage",
+  label: "Startpage",
+  priority: 100,
+  tier: "primary",
+  requestPolicy: {
+    retryAttempts: 0,
+    minRequestIntervalMs: 200,
+  },
+  supports: {
+    language: true,
+    time_range: false,
+    pageno: true,
+  },
+  isAvailable: () => true,
+  search: searchStartpage,
+};
+
+export default searchStartpage;

package/src/search/engines/startpage.js

@@ -0,0 +1,2 @@
+export * from "./startpage.impl.js";
+export { default } from "./startpage.impl.js";

package/src/search/engines/toutiao.impl.js

@@ -0,0 +1,265 @@
+import { ApiError } from "../../core/errors.js";
+import {
+  fetchSearchText,
+  isChallengeResponse,
+  throwBlockedUpstreamError,
+} from "../engineRequest.js";
+import { cleanText, parseHtml } from "../../core/html.js";
+import { normalizeResults } from "../ranking.js";
+
+const TOUTIAO_CHALLENGE_PATTERNS = [
+  /安全验证/i,
+  /captcha.*验证/i,
+];
+
+function isToutiaoChallengeResponse(source) {
+  const text = String(source || "");
+  return (
+    isChallengeResponse(text, TOUTIAO_CHALLENGE_PATTERNS) ||
+    (text.length < 1000 && /<form\b/i.test(text)) ||
+    /"challenge_code"\s*:\s*1366/.test(text) ||
+    /"template_key"\s*:\s*"71-undefined"/.test(text)
+  );
+}
+
+function throwToutiaoChallengeError() {
+  throwBlockedUpstreamError({
+    engine: "Toutiao",
+    surface: "html",
+  });
+}
+
+/**
+ * Decode a Toutiao search-jump redirect URL.
+ * Links in the SSR HTML look like:
+ *   https://sou.toutiao.com/search/jump?url=https%3A%2F%2Fexample.com%2Farticle
+ * We extract and decode the embedded target URL.
+ */
+function decodeToutiaoJumpUrl(href) {
+  try {
+    const url = new URL(href);
+    const target = url.searchParams.get("url");
+    if (!target) return href;
+    const decoded = decodeURIComponent(target);
+    if (decoded.includes("search/jump?url=")) {
+      return decodeToutiaoJumpUrl(decoded);
+    }
+    return decoded;
+  } catch (_) {
+    return href;
+  }
+}
+
+/**
+ * Extract description text from a result-content card.
+ *
+ * Toutiao SSR DOM structure per search result:
+ *   div.result-content
+ *     script/style (hydrate/render code, ignored)
+ *     div
+ *       div.cs-view.cs-view-block.cs-card
+ *         div.cs-view.cs-view-block.cs-card-header
+ *           a[href] (title link)
+ *         div.cs-view.cs-view-block.cs-card-content
+ *           (description text, may be inside nested divs)
+ */
+function extractCardDescription(card) {
+  // Look for cs-card-content, which holds the description
+  const contentNode = card.querySelector(".cs-card-content");
+  if (contentNode) {
+    // The description text is usually in a direct child div without a link
+    const descDivs = contentNode.querySelectorAll("div");
+    for (const d of descDivs) {
+      const txt = cleanText(d.textContent || "").trim();
+      // Skip text that's just the title repeated, or too short, or contains JS
+      if (
+        txt.length > 15 &&
+        txt.length < 500 &&
+        !txt.includes("druid") &&
+        !txt.includes("PerfTag") &&
+        !txt.includes("script")
+      ) {
+        // Prefer longer, more descriptive text
+        if (txt.length > 40) {
+          return txt.slice(0, 300);
+        }
+      }
+    }
+    // Fallback: the content node's own text (excluding nested links)
+    const fullText = cleanText(contentNode.textContent || "").trim();
+    // Remove the title portion from the description
+    const titleLink = card.querySelector("a[href]");
+    const titleText = titleLink ? (titleLink.textContent || "").trim() : "";
+    const descOnly = fullText.replace(titleText, "").trim();
+    if (descOnly.length > 15 && descOnly.length < 500 && !descOnly.includes("druid")) {
+      return descOnly.slice(0, 300);
+    }
+  }
+
+  // Fallback: search all divs in the card for description-like text
+  const allDivs = card.querySelectorAll("div");
+  for (const d of allDivs) {
+    const txt = cleanText(d.textContent || "").trim();
+    if (
+      txt.length > 30 &&
+      txt.length < 500 &&
+      !txt.includes("druid") &&
+      !txt.includes("PerfTag") &&
+      !txt.includes("script") &&
+      !txt.includes("换一换")
+    ) {
+      return txt.slice(0, 300);
+    }
+  }
+
+  return "";
+}
+
+export function parseToutiaoResults(html) {
+  if (isToutiaoChallengeResponse(html)) {
+    throwToutiaoChallengeError();
+  }
+
+  const root = parseHtml(html);
+  const seen = new Set();
+  const results = [];
+
+  // Find the search result list container
+  const resultList = root.querySelector(".s-result-list");
+
+  if (!resultList) {
+    throw new ApiError({
+      status: 502,
+      code: "UPSTREAM_PARSE_ERROR",
+      category: "upstream",
+      message: "Toutiao parser could not find s-result-list container",
+    });
+  }
+
+  // Each div.result-content inside s-result-list is one search result card.
+  // Skip cards inside s-side-list (hot trending sidebar).
+  const cards = resultList.querySelectorAll(".result-content");
+
+  for (const card of cards) {
+    // Exclude the hot trending sidebar entirely
+    if (card.closest(".s-side-list")) continue;
+
+    // Exclude ad/promotion cards: data-test-card-id="67-toutiao_web"
+    // Keep organic results: "67-homepage" (official site), "26-aft_ciyu_detail" (word definition)
+    const adMarker = card.querySelector("[data-test-card-id='67-toutiao_web']");
+    if (adMarker) continue;
+
+    // Exclude related-search suggestions: data-test-card-id="20-undefined"
+    const relatedMarker = card.querySelector("[data-test-card-id^='20-']");
+    if (relatedMarker) continue;
+
+    // Find the primary title link
+    const links = card.querySelectorAll("a[href]");
+    const titleLink = links.find((a) => {
+      const href = a.getAttribute("href") || "";
+      const text = (a.textContent || "").trim();
+      return (
+        href.includes("search/jump?url=") ||
+        href.includes("/article/") ||
+        href.includes("m.douyinhanyu.com") ||
+        href.includes("baike.com") ||
+        href.includes("m.toutiaoimg.cn") ||
+        href.includes("cloud.tencent.com")
+      ) && text.length > 3 && !/^\d{1,2}:\d{2}/.test(text);
+    });
+
+    if (!titleLink) continue;
+
+    const title = cleanText(titleLink.textContent || titleLink.innerHTML || "").trim();
+    const href = titleLink.getAttribute("href") || "";
+
+    if (!title || title.length < 3 || title.length > 150) continue;
+
+    // Skip UI/navigation titles
+    if (
+      title.includes("换一换") ||
+      title.includes("首页") ||
+      title.includes("登录") ||
+      title.includes("去西瓜搜") ||
+      title.includes("去抖音搜") ||
+      title.includes("查看详情") ||
+      title.includes("播放") ||
+      title.startsWith("无障碍") ||
+      title.startsWith("相关搜索")
+    ) continue;
+
+    const targetUrl = decodeToutiaoJumpUrl(href);
+    if (!targetUrl || targetUrl.startsWith("#") || targetUrl.startsWith("/")) continue;
+
+    // Skip trending items and internal search navigation
+    if (targetUrl.includes("/trending")) continue;
+    if (targetUrl.includes("so.toutiao.com/search") && !targetUrl.includes("toutiao.com/a")) continue;
+
+    // Deduplicate by target URL
+    try {
+      const canonical = new URL(targetUrl).toString().toLowerCase();
+      if (seen.has(canonical)) continue;
+      seen.add(canonical);
+    } catch (_) {
+      if (seen.has(targetUrl)) continue;
+      seen.add(targetUrl);
+    }
+
+    // Extract description from cs-card-content
+    const description = extractCardDescription(card);
+
+    results.push({ title, url: targetUrl, description });
+  }
+
+  if (results.length === 0) {
+    throw new ApiError({
+      status: 502,
+      code: "UPSTREAM_PARSE_ERROR",
+      category: "upstream",
+      message: "Toutiao parser could not find organic results",
+    });
+  }
+
+  return normalizeResults(results);
+}
+
+async function searchToutiao(params) {
+  const { query, signal, runtimeContext } = params;
+  const searchUrl = new URL("https://so.toutiao.com/search");
+  searchUrl.searchParams.set("keyword", query);
+  searchUrl.searchParams.set("dvpf", "pc");
+  searchUrl.searchParams.set("source", "input");
+
+  const html = await fetchSearchText(searchUrl.toString(), {
+    engine: "toutiao",
+    engineLabel: "Toutiao",
+    signal,
+    referrer: "https://so.toutiao.com/",
+    runtimeContext,
+    blockedStatuses: [403, 429],
+    isBlocked: isToutiaoChallengeResponse,
+    blockedSurface: "html",
+  });
+
+  return parseToutiaoResults(html);
+}
+
+export const toutiaoAdapter = {
+  name: "toutiao",
+  label: "Toutiao",
+  priority: 65,
+  tier: "experimental",
+  requestPolicy: {
+    retryAttempts: 0,
+    minRequestIntervalMs: 500,
+  },
+  supports: {
+    language: true,
+    time_range: false,
+    pageno: false,
+  },
+  isAvailable: () => true,
+  search: searchToutiao,
+};
+
+export default searchToutiao;

package/src/search/engines/toutiao.js

@@ -0,0 +1,2 @@
+export * from "./toutiao.impl.js";
+export { default } from "./toutiao.impl.js";