@endday/search-mcp 1.0.0 → 1.0.2

package/src/search/ranking.impl.js

@@ -1,381 +0,0 @@
-import { GENERATED_BLOCKLIST } from "../../data/blocklist.generated.js";
-
-export const normalizeResults = (results) =>
-  results
-    .map((result) => {
-      const {
-        title,
-        name,
-        url,
-        link,
-        href,
-        description,
-        content,
-        snippet,
-        ...rest
-      } = result || {};
-
-      return {
-        ...rest,
-        title: String(title || name || "").trim(),
-        url: String(url || link || href || "").trim(),
-        description: String(description || content || snippet || "").trim(),
-      };
-    })
-    .filter((result) => result.url && result.title);
-
-const TRACKING_QUERY_PARAMS = new Set([
-  "fbclid",
-  "gclid",
-  "msclkid",
-  "mc_cid",
-  "mc_eid",
-  "ref_src",
-  "srsltid",
-]);
-const BLOCKED_HOSTS = new Set(GENERATED_BLOCKLIST.domains || []);
-const SOURCE_AUTHORITY_RULES = [
-  {
-    domains: ["cloudflare.com", "openai.com"],
-    source_type: "official",
-    authority_score: 85,
-  },
-  {
-    domains: ["developers.cloudflare.com", "platform.openai.com"],
-    source_type: "official",
-    authority_score: 90,
-  },
-  {
-    domains: ["deepseek.com"],
-    source_type: "official",
-    authority_score: 90,
-    ai_model_boost: 20,
-  },
-  {
-    domains: ["huggingface.co"],
-    source_type: "model_repo",
-    authority_score: 70,
-    ai_model_boost: 15,
-  },
-  {
-    domains: ["github.com"],
-    source_type: "code_repo",
-    authority_score: 55,
-    ai_model_boost: 10,
-  },
-  {
-    domains: ["arxiv.org", "openreview.net"],
-    source_type: "paper",
-    authority_score: 60,
-    ai_model_boost: 15,
-  },
-  {
-    domains: [
-      "artificialanalysis.ai",
-      "lmarena.ai",
-      "livebench.ai",
-      "paperswithcode.com",
-      "scale.com",
-      "swebench.com",
-      "vals.ai",
-    ],
-    source_type: "benchmark",
-    authority_score: 55,
-    ai_model_boost: 15,
-  },
-  {
-    domains: ["semianalysis.com"],
-    source_type: "analysis",
-    authority_score: 50,
-    ai_model_boost: 10,
-  },
-  {
-    domains: [
-      "apnews.com",
-      "bloomberg.com",
-      "ft.com",
-      "reuters.com",
-      "theverge.com",
-      "wsj.com",
-    ],
-    source_type: "media",
-    authority_score: 35,
-  },
-  {
-    domains: [
-      "caixin.com",
-      "eet-china.com",
-      "infoq.cn",
-      "news.cn",
-      "stcn.com",
-      "yicai.com",
-    ],
-    source_type: "media",
-    authority_score: 28,
-  },
-];
-const AI_MODEL_QUERY_RE =
-  /ai|agent|benchmark|deepseek|gpt|llm|model|性能|模型|推理|评测|测评|基准|代码|上下文|开源/i;
-const PDF_PATH_RE = /\.pdf(?:$|[?#])/i;
-const HAN_SEGMENT_RE = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]{2,}/gu;
-const MOSTLY_LATIN_QUERY_RE = /^[\p{Script=Latin}\p{Number}\s'".,!?():/_+-]+$/u;
-
-function normalizeUrlPath(pathname) {
-  const normalized = pathname.replace(/\/+$/, "");
-  return normalized || "/";
-}
-
-function hostnameMatches(hostname, domain) {
-  return hostname === domain || hostname.endsWith(`.${domain}`);
-}
-
-function getMatchedAuthorityRule(hostname) {
-  return SOURCE_AUTHORITY_RULES.find((rule) =>
-    rule.domains.some((domain) => hostnameMatches(hostname, domain))
-  );
-}
-
-function getSourceAuthority(rawUrl, query) {
-  try {
-    const url = new URL(rawUrl);
-    const hostname = url.hostname.toLowerCase();
-    const rule = getMatchedAuthorityRule(hostname);
-    const isAiModelQuery = AI_MODEL_QUERY_RE.test(String(query || ""));
-    const mobilePenalty = hostname.startsWith("m.") ? -5 : 0;
-    const pdfPenalty = PDF_PATH_RE.test(`${url.pathname}${url.search}`) ? -15 : 0;
-
-    if (!rule) {
-      return {
-        source_type: PDF_PATH_RE.test(`${url.pathname}${url.search}`)
-          ? "document"
-          : "unknown",
-        authority_score: mobilePenalty + pdfPenalty,
-      };
-    }
-
-    return {
-      source_type: rule.source_type,
-      authority_score:
-        rule.authority_score +
-        mobilePenalty +
-        pdfPenalty +
-        (isAiModelQuery ? rule.ai_model_boost || 0 : 0),
-    };
-  } catch (_) {
-    return {
-      source_type: "unknown",
-      authority_score: 0,
-    };
-  }
-}
-
-function isBlockedHostname(hostname) {
-  const parts = String(hostname || "").toLowerCase().split(".");
-
-  for (let index = 0; index <= parts.length - 2; index += 1) {
-    const candidate = parts.slice(index).join(".");
-    if (BLOCKED_HOSTS.has(candidate)) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
-function isBlockedUrl(rawUrl) {
-  try {
-    const url = new URL(rawUrl);
-    return isBlockedHostname(url.hostname);
-  } catch (_) {
-    return false;
-  }
-}
-
-export function canonicalizeUrl(rawUrl) {
-  try {
-    const url = new URL(rawUrl);
-    url.hash = "";
-    url.hostname = url.hostname.toLowerCase();
-    url.pathname = normalizeUrlPath(url.pathname);
-
-    [...url.searchParams.keys()].forEach((key) => {
-      if (key.startsWith("utm_") || TRACKING_QUERY_PARAMS.has(key)) {
-        url.searchParams.delete(key);
-      }
-    });
-
-    return url.toString();
-  } catch (_) {
-    return String(rawUrl || "").trim();
-  }
-}
-
-function tokenizeQuery(query) {
-  const normalizedQuery = String(query || "");
-  const tokens = normalizedQuery
-    .toLowerCase()
-    .split(/\s+/)
-    .map((item) => item.trim())
-    .filter((item) => item.length >= 2);
-
-  const hanSegments = normalizedQuery.match(HAN_SEGMENT_RE) || [];
-
-  return [...new Set([...tokens, ...hanSegments])];
-}
-
-function normalizeComparableText(value) {
-  return String(value || "")
-    .toLowerCase()
-    .replace(/[^\p{Letter}\p{Number}\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+/gu, "");
-}
-
-function isMostlyLatinQuery(query) {
-  const normalized = String(query || "").trim();
-  return MOSTLY_LATIN_QUERY_RE.test(normalized) && /[a-z]/i.test(normalized);
-}
-
-function getHanPenalty(text) {
-  const matches = String(text || "").match(HAN_SEGMENT_RE) || [];
-  return matches.join("").length >= 6 ? -30 : 0;
-}
-
-function calculateResultScore({
-  queryTokens,
-  queryPhrase,
-  query,
-  title,
-  description,
-  url,
-  engine,
-  sourceType,
-  enginePriority,
-  position,
-}) {
-  const normalizedTitle = title.toLowerCase();
-  const normalizedDescription = description.toLowerCase();
-  const comparableTitle = normalizeComparableText(title);
-  const comparableDescription = normalizeComparableText(description);
-  const comparableUrl = normalizeComparableText(url);
-  const titleMatches = queryTokens.filter((token) =>
-    normalizedTitle.includes(token)
-  ).length;
-  const descriptionMatches = queryTokens.filter((token) =>
-    normalizedDescription.includes(token)
-  ).length;
-  const titlePhraseMatch =
-    queryPhrase.length >= 4 && comparableTitle.includes(queryPhrase);
-  const descriptionPhraseMatch =
-    queryPhrase.length >= 4 && comparableDescription.includes(queryPhrase);
-  const urlPhraseMatch =
-    queryPhrase.length >= 4 && comparableUrl.includes(queryPhrase);
-  const hasAnyTokenMatch = titleMatches > 0 || descriptionMatches > 0;
-  const officialPhraseBoost =
-    sourceType === "official" && (titlePhraseMatch || urlPhraseMatch) ? 20 : 0;
-  const englishEngineBoost = isMostlyLatinQuery(query)
-    ? {
-        brave: 18,
-        bing: 15,
-        yahoo: 2,
-        mojeek: -4,
-        baidu: -20,
-      }[engine] || 0
-    : 0;
-  const latinQueryHanPenalty =
-    isMostlyLatinQuery(query) && sourceType !== "official"
-      ? getHanPenalty(`${title} ${description}`)
-      : 0;
-  const hanQueryNoMatchPenalty =
-    !isMostlyLatinQuery(query) &&
-    queryTokens.length > 0 &&
-    !hasAnyTokenMatch &&
-    !titlePhraseMatch &&
-    !descriptionPhraseMatch &&
-    !urlPhraseMatch
-      ? -60
-      : 0;
-
-  return (
-    enginePriority +
-    Math.max(0, 30 - position * 2) +
-    titleMatches * 6 +
-    descriptionMatches * 2 +
-    (titlePhraseMatch ? 35 : 0) +
-    (descriptionPhraseMatch ? 10 : 0) +
-    (urlPhraseMatch ? 20 : 0) +
-    officialPhraseBoost +
-    englishEngineBoost +
-    latinQueryHanPenalty +
-    hanQueryNoMatchPenalty
-  );
-}
-
-const LOW_QUALITY_ENGINE_THRESHOLD = 80;
-
-export function dedupeAndRankResults({ engineResults, query, registry }) {
-  const queryTokens = tokenizeQuery(query);
-  const queryPhrase = normalizeComparableText(query);
-  const deduped = new Map();
-  // Track which engines contributed to each URL for cross-engine penalty
-  const urlEngineCount = new Map();
-
-  for (const { engine, results } of engineResults) {
-    const enginePriority = registry[engine]?.priority || 0;
-
-    normalizeResults(results).forEach((result, index) => {
-      const canonicalUrl = canonicalizeUrl(result.url);
-      if (isBlockedUrl(canonicalUrl)) {
-        return;
-      }
-
-      const sourceAuthority = getSourceAuthority(canonicalUrl, query);
-      const candidate = {
-        ...result,
-        url: canonicalUrl,
-        engine,
-        ...sourceAuthority,
-        score:
-          calculateResultScore({
-            queryTokens,
-            queryPhrase,
-            query,
-            title: result.title,
-            description: result.description,
-            url: canonicalUrl,
-            engine,
-            sourceType: sourceAuthority.source_type,
-            enginePriority,
-            position: index,
-          }) + sourceAuthority.authority_score,
-      };
-
-      // Track engine contribution count for this URL
-      const prevCount = urlEngineCount.get(canonicalUrl) || 0;
-      urlEngineCount.set(canonicalUrl, prevCount + 1);
-
-      const existing = deduped.get(canonicalUrl);
-      if (!existing || candidate.score > existing.score) {
-        deduped.set(canonicalUrl, candidate);
-      } else if (!existing.description && candidate.description) {
-        existing.description = candidate.description;
-      }
-    });
-  }
-
-  // Penalize results that only appear in a single low-quality engine
-  // (likely irrelevant / low-quality results that no other engine confirmed)
-  const LOW_PRIORITY_SOLE_PENALTY = -40;
-
-  return [...deduped.values()]
-    .map((result) => {
-      const enginePriority = registry[result.engine]?.priority || 0;
-      const engineCount = urlEngineCount.get(result.url) || 1;
-
-      if (engineCount === 1 && enginePriority < LOW_QUALITY_ENGINE_THRESHOLD) {
-        return { ...result, score: result.score + LOW_PRIORITY_SOLE_PENALTY };
-      }
-
-      return result;
-    })
-    .sort((left, right) => right.score - left.score)
-    .map(({ score, ...result }) => result);
-}

package/src/search/ranking.js

@@ -1 +0,0 @@
-export * from "./ranking.impl.js";

package/src/search/requestPolicy.impl.js

@@ -1,137 +0,0 @@
-import { env } from "../../envs.js";
-
-const upstreamThrottleState = new Map();
-
-function parseNonNegativeInt(value, fallback) {
-  const parsed = Number.parseInt(value ?? String(fallback), 10);
-  if (Number.isNaN(parsed) || parsed < 0) {
-    return fallback;
-  }
-
-  return parsed;
-}
-
-export function resetUpstreamRequestPolicyState() {
-  upstreamThrottleState.clear();
-}
-
-export function getEngineTier(adapter) {
-  return String(adapter?.tier || "secondary").toLowerCase();
-}
-
-export function buildEnginePolicy(adapter = {}) {
-  const tier = getEngineTier(adapter);
-  const policy = adapter.requestPolicy || {};
-  const retryAttempts =
-    policy.retryAttempts ??
-    parseNonNegativeInt(
-      env[`UPSTREAM_${tier.toUpperCase()}_RETRY_ATTEMPTS`],
-      parseNonNegativeInt(env.UPSTREAM_RETRY_ATTEMPTS, 1)
-    );
-  const retryDelayMs =
-    policy.retryDelayMs ??
-    parseNonNegativeInt(env.UPSTREAM_RETRY_DELAY_MS, 200);
-  const minRequestIntervalMs =
-    policy.minRequestIntervalMs ??
-    parseNonNegativeInt(
-      env[`UPSTREAM_${tier.toUpperCase()}_MIN_REQUEST_INTERVAL_MS`],
-      parseNonNegativeInt(env.UPSTREAM_MIN_REQUEST_INTERVAL_MS, 150)
-    );
-
-  return {
-    tier,
-    retryAttempts,
-    retryDelayMs,
-    minRequestIntervalMs,
-  };
-}
-
-export function groupEnginesByTier(engineNames, registry) {
-  const groups = new Map();
-
-  for (const engineName of engineNames) {
-    const adapter = registry[engineName];
-    const tier = getEngineTier(adapter);
-    if (!groups.has(tier)) {
-      groups.set(tier, []);
-    }
-
-    groups.get(tier).push(engineName);
-  }
-
-  return groups;
-}
-
-export function getTierExecutionOrder(groups) {
-  const primaryTiers = new Set(
-    (env.SEARCH_PRIMARY_TIERS || []).map((item) => String(item).toLowerCase())
-  );
-  const secondaryTiers = new Set(
-    (env.SEARCH_SECONDARY_TIERS || []).map((item) => String(item).toLowerCase())
-  );
-  const experimentalTiers = new Set(
-    (env.SEARCH_EXPERIMENTAL_TIERS || []).map((item) => String(item).toLowerCase())
-  );
-  const known = [];
-
-  for (const tier of [...groups.keys()]) {
-    if (primaryTiers.has(tier) || secondaryTiers.has(tier) || experimentalTiers.has(tier)) {
-      continue;
-    }
-
-    known.push(tier);
-  }
-
-  return [
-    ...[...primaryTiers].filter((tier) => groups.has(tier)),
-    ...[...secondaryTiers].filter((tier) => groups.has(tier)),
-    ...[...experimentalTiers].filter((tier) => groups.has(tier)),
-    ...known.sort(),
-  ];
-}
-
-export async function enforceEngineThrottle(engineName, policy) {
-  const minInterval = parseNonNegativeInt(policy?.minRequestIntervalMs, 0);
-  if (minInterval <= 0) {
-    return;
-  }
-
-  const now = Date.now();
-  const key = String(engineName);
-  const nextAllowedAt = upstreamThrottleState.get(key) || 0;
-  const delay = nextAllowedAt - now;
-
-  if (delay > 0) {
-    await new Promise((resolve) => setTimeout(resolve, delay));
-  }
-
-  upstreamThrottleState.set(key, Date.now() + minInterval);
-}
-
-export function shouldRetryUpstream(error, attempt, policy) {
-  if (attempt >= parseNonNegativeInt(policy?.retryAttempts, 0)) {
-    return false;
-  }
-
-  const status = error?.status || error?.details?.upstream_status || 0;
-  const code = String(error?.code || "");
-
-  if (code === "UPSTREAM_BLOCKED") {
-    return false;
-  }
-
-  if (code === "ABORT_ERR" || code === "TIMEOUT") {
-    return true;
-  }
-
-  return status === 408 || status === 429 || status >= 500 || status === 0;
-}
-
-export async function sleepBeforeRetry(policy) {
-  const delay = parseNonNegativeInt(policy?.retryDelayMs, 0);
-  if (delay <= 0) {
-    return;
-  }
-
-  await new Promise((resolve) => setTimeout(resolve, delay));
-}

package/src/search/requestPolicy.js

@@ -1 +0,0 @@
-export * from "./requestPolicy.impl.js";

package/src/search/upstreamSession.impl.js

@@ -1,148 +0,0 @@
-import { env } from "../../envs.js";
-import { sha256Hex } from "../core/crypto.js";
-import { getStateKv, normalizeExpirationTtl } from "../platform/stateKv.js";
-
-const UPSTREAM_SESSION_PREFIX = "upstream-session:v1";
-const memoryState = new Map();
-
-function nowSeconds() {
-  return Math.floor(Date.now() / 1000);
-}
-
-function buildMemoryKey(clientId, engine) {
-  return `${clientId}:${engine}`;
-}
-
-async function buildKvKey(clientId, engine) {
-  return `${UPSTREAM_SESSION_PREFIX}:${await sha256Hex(`${clientId}:${engine}`)}`;
-}
-
-function pickStableIndex(seed, size) {
-  if (!size || size < 1) {
-    return 0;
-  }
-
-  let hash = 0;
-  const value = String(seed || "default");
-  for (let index = 0; index < value.length; index += 1) {
-    hash = (hash * 31 + value.charCodeAt(index)) >>> 0;
-  }
-
-  return hash % size;
-}
-
-export function resetUpstreamSessionState() {
-  memoryState.clear();
-}
-
-async function loadFromKv(clientId, engine) {
-  const kv = getStateKv();
-  if (!kv) {
-    return null;
-  }
-
-  try {
-    return await kv.get(await buildKvKey(clientId, engine), "json");
-  } catch (_) {
-    return null;
-  }
-}
-
-async function saveToKv(clientId, engine, value) {
-  const kv = getStateKv();
-  if (!kv) {
-    return false;
-  }
-
-  try {
-    await kv.put(await buildKvKey(clientId, engine), JSON.stringify(value), {
-      expirationTtl: normalizeExpirationTtl(env.UPSTREAM_SESSION_TTL_SECONDS, 3600),
-    });
-    return true;
-  } catch (_) {
-    return false;
-  }
-}
-
-function loadFromMemory(clientId, engine) {
-  const record = memoryState.get(buildMemoryKey(clientId, engine));
-  if (!record) {
-    return null;
-  }
-
-  if (record.expiresAt <= nowSeconds()) {
-    memoryState.delete(buildMemoryKey(clientId, engine));
-    return null;
-  }
-
-  return record.value;
-}
-
-function saveToMemory(clientId, engine, value) {
-  memoryState.set(buildMemoryKey(clientId, engine), {
-    value,
-    expiresAt: nowSeconds() + normalizeExpirationTtl(env.UPSTREAM_SESSION_TTL_SECONDS, 3600),
-  });
-}
-
-export async function getUpstreamSession(clientId, engine, profiles = []) {
-  if (!clientId || !engine || profiles.length === 0) {
-    return null;
-  }
-
-  const existing = (await loadFromKv(clientId, engine)) || loadFromMemory(clientId, engine);
-  if (existing?.profileId) {
-    return existing;
-  }
-
-  const selectedIndex = pickStableIndex(`${clientId}:${engine}`, profiles.length);
-  const selectedProfile = profiles[selectedIndex];
-  const created = {
-    profileId: selectedProfile.id,
-    createdAt: nowSeconds(),
-    lastUsedAt: nowSeconds(),
-    cookies: {},
-  };
-
-  if (!(await saveToKv(clientId, engine, created))) {
-    saveToMemory(clientId, engine, created);
-  }
-
-  return created;
-}
-
-export async function updateUpstreamSession(clientId, engine, patch = {}) {
-  if (!clientId || !engine) {
-    return null;
-  }
-
-  const current =
-    (await loadFromKv(clientId, engine)) ||
-    loadFromMemory(clientId, engine) || {
-      profileId: patch.profileId || "",
-      createdAt: nowSeconds(),
-      cookies: {},
-    };
-
-  const next = {
-    ...current,
-    ...patch,
-    cookies: {
-      ...(current.cookies || {}),
-      ...(patch.cookies || {}),
-    },
-    lastUsedAt: nowSeconds(),
-  };
-
-  if (!(await saveToKv(clientId, engine, next))) {
-    saveToMemory(clientId, engine, next);
-  }
-
-  return next;
-}
-
-export function createDeferredUpstreamSessionWriter(clientId, engine, patch = {}) {
-  return async function writeUpstreamSession() {
-    await updateUpstreamSession(clientId, engine, patch);
-  };
-}

package/src/search/upstreamSession.js

@@ -1 +0,0 @@
-export * from "./upstreamSession.impl.js";