@endday/search-mcp 1.0.0

package/envs.js

@@ -0,0 +1,129 @@
+const createDefaultEnv = () => ({
+  DEFAULT_TIMEOUT: "4000",
+  SUPPORTED_ENGINES: [
+    "baidu",
+    "startpage",
+    "duckduckgo",
+    "brave",
+    "qwant",
+    "yahoo",
+    "mojeek",
+    "bing",
+    "toutiao",
+  ],
+  DEFAULT_ENGINES: ["bing", "brave", "yahoo", "mojeek"],
+  DEFAULT_ENGINES_ZH: ["baidu", "bing"],
+  DEFAULT_ENGINES_NON_ZH: ["bing", "brave", "yahoo", "mojeek"],
+  DEFAULT_LANGUAGE: "en",
+  FALLBACK_MIN_RESULTS: "6",
+  FALLBACK_MIN_CONTRIBUTING_ENGINES: "2",
+  SEARCH_PRIMARY_TIERS: ["primary"],
+  SEARCH_SECONDARY_TIERS: ["secondary"],
+  SEARCH_EXPERIMENTAL_TIERS: ["experimental"],
+  SEARCH_TIER_HEDGE_DELAY_MS: "250",
+  EDGE_CACHE_TTL_SECONDS: "30",
+  CACHE_TTL_SECONDS: "300",
+  STALE_CACHE_TTL_SECONDS: "1800",
+  RATE_LIMIT_WINDOW_SECONDS: "60",
+  RATE_LIMIT_MAX_REQUESTS: "60",
+  UPSTREAM_RETRY_ATTEMPTS: "1",
+  UPSTREAM_RETRY_DELAY_MS: "200",
+  UPSTREAM_PRIMARY_RETRY_ATTEMPTS: "1",
+  UPSTREAM_SECONDARY_RETRY_ATTEMPTS: "0",
+  UPSTREAM_EXPERIMENTAL_RETRY_ATTEMPTS: "0",
+  UPSTREAM_SESSION_TTL_SECONDS: "3600",
+  UPSTREAM_MIN_REQUEST_INTERVAL_MS: "150",
+  UPSTREAM_PRIMARY_MIN_REQUEST_INTERVAL_MS: "100",
+  UPSTREAM_SECONDARY_MIN_REQUEST_INTERVAL_MS: "250",
+  UPSTREAM_EXPERIMENTAL_MIN_REQUEST_INTERVAL_MS: "500",
+  HEALTH_FAILURE_THRESHOLD: "2",
+  HEALTH_COOLDOWN_SECONDS: "180",
+  HEALTH_STATE_TTL_SECONDS: "3600",
+  CORS_ALLOWED_ORIGINS: ["*"],
+  CORS_ALLOWED_HEADERS: ["Authorization", "Content-Type", "x-api-key"],
+  AUTH_REQUIRED: "false",
+  TOKEN: null,
+  CF_BROWSER_RENDERING_ACCOUNT_ID: null,
+  CF_BROWSER_RENDERING_API_TOKEN: null,
+  SEARCH_KV: null,
+  SEARCH_STATE_KV: null,
+});
+
+function normalizeStringArray(value, fallback) {
+  if (Array.isArray(value)) {
+    return value.map((item) => String(item).trim()).filter(Boolean);
+  }
+
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    if (!trimmed) {
+      return [...fallback];
+    }
+
+    if (trimmed.startsWith("[")) {
+      try {
+        return normalizeStringArray(JSON.parse(trimmed), fallback);
+      } catch (_) {
+        return [...fallback];
+      }
+    }
+
+    return trimmed
+      .split(",")
+      .map((item) => item.trim())
+      .filter(Boolean);
+  }
+
+  return [...fallback];
+}
+
+function resetEnv(target) {
+  const defaults = createDefaultEnv();
+  Object.keys(target).forEach((key) => {
+    delete target[key];
+  });
+  Object.assign(target, defaults);
+}
+
+export const env = createDefaultEnv();
+
+export const setEnv = (newEnv = {}) => {
+  resetEnv(env);
+  Object.assign(env, newEnv);
+  env.SUPPORTED_ENGINES = normalizeStringArray(
+    env.SUPPORTED_ENGINES,
+    createDefaultEnv().SUPPORTED_ENGINES
+  );
+  env.DEFAULT_ENGINES = normalizeStringArray(
+    env.DEFAULT_ENGINES,
+    createDefaultEnv().DEFAULT_ENGINES
+  );
+  env.DEFAULT_ENGINES_ZH = normalizeStringArray(
+    env.DEFAULT_ENGINES_ZH,
+    createDefaultEnv().DEFAULT_ENGINES_ZH
+  );
+  env.DEFAULT_ENGINES_NON_ZH = normalizeStringArray(
+    env.DEFAULT_ENGINES_NON_ZH,
+    createDefaultEnv().DEFAULT_ENGINES_NON_ZH
+  );
+  env.CORS_ALLOWED_ORIGINS = normalizeStringArray(
+    env.CORS_ALLOWED_ORIGINS,
+    createDefaultEnv().CORS_ALLOWED_ORIGINS
+  );
+  env.CORS_ALLOWED_HEADERS = normalizeStringArray(
+    env.CORS_ALLOWED_HEADERS,
+    createDefaultEnv().CORS_ALLOWED_HEADERS
+  );
+  env.SEARCH_PRIMARY_TIERS = normalizeStringArray(
+    env.SEARCH_PRIMARY_TIERS,
+    createDefaultEnv().SEARCH_PRIMARY_TIERS
+  );
+  env.SEARCH_SECONDARY_TIERS = normalizeStringArray(
+    env.SEARCH_SECONDARY_TIERS,
+    createDefaultEnv().SEARCH_SECONDARY_TIERS
+  );
+  env.SEARCH_EXPERIMENTAL_TIERS = normalizeStringArray(
+    env.SEARCH_EXPERIMENTAL_TIERS,
+    createDefaultEnv().SEARCH_EXPERIMENTAL_TIERS
+  );
+};

package/index.d.ts

@@ -0,0 +1,191 @@
+import type { Server } from "@modelcontextprotocol/sdk/server/index.js";
+
+export type SourceType =
+  | "official"
+  | "model_repo"
+  | "code_repo"
+  | "paper"
+  | "benchmark"
+  | "analysis"
+  | "media"
+  | "blog"
+  | "community"
+  | "low_credibility"
+  | "disinformation"
+  | "document"
+  | "unknown";
+
+export type TimeRange = "day" | "week" | "month" | "year";
+
+export type ResultItem = {
+  title: string;
+  description: string;
+  url: string;
+  source_name?: string;
+  published_text?: string;
+  source_type?: SourceType;
+  authority_score?: number;
+};
+
+export type SkippedEngineReason =
+  | "unsupported_engine"
+  | "unsupported_vertical"
+  | "unavailable_engine"
+  | "unsupported_time_range"
+  | "unsupported_pageno";
+
+export type SkippedEngine = {
+  engine: string;
+  reason: SkippedEngineReason;
+};
+
+export type LocationSource =
+  | "auto"
+  | "explicit"
+  | "disabled"
+  | "unavailable";
+
+export type SourceFilters = {
+  include_source_types: string[];
+  exclude_source_types: string[];
+  min_authority_score: number | null;
+  active: boolean;
+};
+
+export type SearchLocalOptions = {
+  vertical?: "web" | "news";
+  language?: string;
+  search_lang?: string;
+  ui_lang?: string;
+  location?: string;
+  time_range?: TimeRange;
+  pageno?: number;
+  count?: number;
+  offset?: number;
+  min_authority_score?: number | string | null;
+  include_source_types?: string[];
+  exclude_source_types?: string[];
+  clientId?: string;
+};
+
+export type SearchAllParams = {
+  vertical?: "web" | "news";
+  query: string;
+  engines: string[];
+  language?: string;
+  time_range?: TimeRange;
+  pageno?: number;
+  clientId?: string;
+  runtimeContext?: unknown;
+};
+
+export type SearchResponse = {
+  vertical?: "web" | "news";
+  query: string;
+  effective_query?: string;
+  location?: string | null;
+  location_source?: LocationSource;
+  location_context?: {
+    value: string;
+    source: LocationSource;
+    mode: string;
+    client: {
+      city: string;
+      region: string;
+      country: string;
+      timezone: string;
+    };
+  };
+  number_of_results: number;
+  count?: number | null;
+  offset?: number;
+  enabled_engines: string[];
+  skipped_engines: SkippedEngine[];
+  unresponsive_engines: string[];
+  source_filters?: SourceFilters;
+  results: Array<ResultItem & { engine: string }>;
+};
+
+export type SearchMetaResponse = {
+  response: SearchResponse;
+  meta: {
+    cache_status: "hit" | "miss" | "revalidated" | "stale-if-error";
+    cache_layer?: string;
+    fallback_order: string[];
+    fallback_path: string[];
+    strategy?: string;
+    engine_timings: Array<{
+      engine: string;
+      duration_ms: number;
+      status: string;
+      result_count: number;
+      tier?: string;
+    }>;
+  };
+};
+
+export interface Env {
+  DEFAULT_TIMEOUT?: string;
+  SUPPORTED_ENGINES?: string[];
+  DEFAULT_ENGINES?: string[];
+  DEFAULT_ENGINES_ZH?: string[];
+  DEFAULT_ENGINES_NON_ZH?: string[];
+  DEFAULT_LANGUAGE?: string;
+  FALLBACK_MIN_RESULTS?: string;
+  FALLBACK_MIN_CONTRIBUTING_ENGINES?: string;
+  SEARCH_PRIMARY_TIERS?: string[];
+  SEARCH_SECONDARY_TIERS?: string[];
+  SEARCH_EXPERIMENTAL_TIERS?: string[];
+  SEARCH_TIER_HEDGE_DELAY_MS?: string;
+  EDGE_CACHE_TTL_SECONDS?: string;
+  CACHE_TTL_SECONDS?: string;
+  STALE_CACHE_TTL_SECONDS?: string;
+  RATE_LIMIT_WINDOW_SECONDS?: string;
+  RATE_LIMIT_MAX_REQUESTS?: string;
+  UPSTREAM_RETRY_ATTEMPTS?: string;
+  UPSTREAM_RETRY_DELAY_MS?: string;
+  UPSTREAM_PRIMARY_RETRY_ATTEMPTS?: string;
+  UPSTREAM_SECONDARY_RETRY_ATTEMPTS?: string;
+  UPSTREAM_EXPERIMENTAL_RETRY_ATTEMPTS?: string;
+  UPSTREAM_SESSION_TTL_SECONDS?: string;
+  UPSTREAM_MIN_REQUEST_INTERVAL_MS?: string;
+  UPSTREAM_PRIMARY_MIN_REQUEST_INTERVAL_MS?: string;
+  UPSTREAM_SECONDARY_MIN_REQUEST_INTERVAL_MS?: string;
+  UPSTREAM_EXPERIMENTAL_MIN_REQUEST_INTERVAL_MS?: string;
+  HEALTH_FAILURE_THRESHOLD?: string;
+  HEALTH_COOLDOWN_SECONDS?: string;
+  HEALTH_STATE_TTL_SECONDS?: string;
+  CORS_ALLOWED_ORIGINS?: string[] | string;
+  CORS_ALLOWED_HEADERS?: string[] | string;
+  AUTH_REQUIRED?: string;
+  TOKEN?: string | null;
+  CF_BROWSER_RENDERING_ACCOUNT_ID?: string | null;
+  CF_BROWSER_RENDERING_API_TOKEN?: string | null;
+  SEARCH_KV?: unknown;
+  SEARCH_STATE_KV?: unknown;
+}
+
+export type McpConfig = {
+  mode: "local";
+  jinaApiKey: string;
+  jinaBaseUrl: string;
+  upstreamClient: string;
+  localClientId: string;
+  allEngines: string[];
+};
+
+export const env: Env;
+export function setEnv(newEnv?: Partial<Env>): void;
+export function loadMcpConfig(): McpConfig;
+export function main(): Promise<void>;
+export function createServer(config: McpConfig): Server;
+export function startServer(server: Server): Promise<void>;
+export function searchLocal(
+  query: string,
+  engines?: string[] | null,
+  options?: SearchLocalOptions
+): Promise<SearchResponse>;
+export function searchAll(params: SearchAllParams): Promise<SearchResponse>;
+export function searchAllWithMeta(
+  params: SearchAllParams
+): Promise<SearchMetaResponse>;

package/index.js

@@ -0,0 +1,6 @@
+export { env, setEnv } from "./envs.js";
+export { loadMcpConfig } from "./src/mcp/config.js";
+export { main } from "./src/mcp/index.js";
+export { searchLocal } from "./src/mcp/local/search.js";
+export { createServer, startServer } from "./src/mcp/server.js";
+export { searchAll, searchAllWithMeta } from "./src/search/gateway.js";

package/mcp/search-mcp.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+import { main } from "../src/mcp/index.js";
+
+main().catch((error) => {
+  console.error("Fatal error:", error);
+  process.exit(1);
+});

package/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@endday/search-mcp",
+  "version": "1.0.0",
+  "description": "Local MCP server for aggregated web search and content extraction",
+  "type": "module",
+  "main": "./index.js",
+  "types": "./index.d.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "bin": {
+    "search-mcp": "mcp/search-mcp.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "import": "./index.js"
+    },
+    "./cli": "./mcp/search-mcp.js"
+  },
+  "files": [
+    "data/blocklist.generated.js",
+    "index.js",
+    "mcp/search-mcp.js",
+    "src/**/*.js",
+    "envs.js",
+    "index.d.ts"
+  ],
+  "scripts": {
+    "start": "node mcp/search-mcp.js",
+    "test": "node --test test",
+    "smoke": "node scripts/mcp-local-smoke.js",
+    "update:blocklist": "node scripts/update-blocklist.js",
+    "docs:dev": "npm --prefix docs-site run docs:dev",
+    "docs:build": "npm --prefix docs-site run docs:build",
+    "docs:preview": "npm --prefix docs-site run docs:preview"
+  },
+  "keywords": [
+    "mcp",
+    "search",
+    "web-search",
+    "self-hosted",
+    "ai-agent",
+    "openclaw",
+    "privacy",
+    "aggregated-search",
+    "startpage",
+    "mojeek",
+    "brave",
+    "duckduckgo",
+    "bing",
+    "qwant",
+    "yahoo"
+  ],
+  "author": "endday",
+  "license": "GPL-3.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/endday/search-mcp.git"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.4",
+    "@mozilla/readability": "^0.6.0",
+    "impit": "^0.14.2",
+    "linkedom": "^0.18.12",
+    "node-html-parser": "^7.1.0"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  }
+}

package/src/content/extract.impl.js

@@ -0,0 +1,228 @@
+import { Readability } from "@mozilla/readability";
+import { DOMParser } from "linkedom/worker";
+
+import { cleanText, parseHtml } from "../core/html.js";
+
+const NOISE_SELECTOR =
+  "script, style, noscript, nav, footer, header, aside, form, iframe, svg, canvas, button, input, select, textarea";
+const CANDIDATE_SELECTOR =
+  "article, main, section, div, [role=main], .article, .post, .content, .entry-content, #content";
+const POSITIVE_RE =
+  /article|body|content|entry|hentry|main|page|post|story|text|正文|内容|文章/i;
+const NEGATIVE_RE =
+  /ad|banner|comment|combx|contact|footer|header|menu|meta|nav|promo|related|remark|rss|share|sidebar|social|tag|tool|widget|广告|评论|导航|分享|推荐|相关阅读/i;
+
+function getNodeText(node) {
+  return cleanText(node?.text || "");
+}
+
+function getMeta(root, selector) {
+  return cleanText(root.querySelector(selector)?.getAttribute("content") || "");
+}
+
+function getNodeSignal(node) {
+  return `${node.getAttribute?.("id") || ""} ${node.getAttribute?.("class") || ""}`;
+}
+
+function scoreCandidate(node) {
+  const text = getNodeText(node);
+  const textLength = text.length;
+
+  if (textLength < 80) {
+    return {
+      node,
+      score: 0,
+      textLength,
+      linkDensity: 1,
+      paragraphCount: 0,
+    };
+  }
+
+  const linkTextLength = node
+    .querySelectorAll("a")
+    .reduce((total, link) => total + getNodeText(link).length, 0);
+  const paragraphCount = node.querySelectorAll("p").filter((p) => {
+    return getNodeText(p).length >= 20;
+  }).length;
+  const commaCount = (text.match(/[，,。.!?！？；;]/g) || []).length;
+  const signal = getNodeSignal(node);
+  const linkDensity = textLength ? linkTextLength / textLength : 1;
+  let score = textLength + paragraphCount * 120 + commaCount * 12;
+
+  if (POSITIVE_RE.test(signal)) {
+    score += 350;
+  }
+
+  if (NEGATIVE_RE.test(signal)) {
+    score -= 500;
+  }
+
+  score *= Math.max(0.05, 1 - linkDensity);
+
+  return {
+    node,
+    score,
+    textLength,
+    linkDensity,
+    paragraphCount,
+  };
+}
+
+function cleanTree(root) {
+  root.querySelectorAll(NOISE_SELECTOR).forEach((node) => node.remove());
+  root.querySelectorAll("a").forEach((link) => {
+    const href = String(link.getAttribute("href") || "").trim();
+    if (/^\s*javascript:/i.test(href)) {
+      link.removeAttribute("href");
+    }
+  });
+}
+
+function stripUnsafeHtml(html) {
+  return String(html || "")
+    .replace(/\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi, "")
+    .replace(/\s+style\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi, "")
+    .replace(/\s+href\s*=\s*(['"])\s*javascript:[\s\S]*?\1/gi, "");
+}
+
+function pickBestCandidate(root) {
+  const candidates = root
+    .querySelectorAll(CANDIDATE_SELECTOR)
+    .map(scoreCandidate)
+    .sort((a, b) => b.score - a.score);
+
+  const best = candidates[0];
+  if (best?.score > 0) {
+    return best;
+  }
+
+  const body = root.querySelector("body") || root;
+  return scoreCandidate(body);
+}
+
+function getPageMetadata(html) {
+  const root = parseHtml(html);
+
+  return {
+    title:
+      getMeta(root, 'meta[property="og:title"]') ||
+      getMeta(root, 'meta[name="twitter:title"]') ||
+      cleanText(root.querySelector("title")?.text || ""),
+    description:
+      getMeta(root, 'meta[property="og:description"]') ||
+      getMeta(root, 'meta[name="twitter:description"]') ||
+      getMeta(root, 'meta[name="description"]'),
+    site_name: getMeta(root, 'meta[property="og:site_name"]'),
+    author: getMeta(root, 'meta[name="author"]'),
+    published_time:
+      getMeta(root, 'meta[property="article:published_time"]') ||
+      getMeta(root, 'meta[name="date"]') ||
+      getMeta(root, 'meta[name="pubdate"]'),
+    image:
+      getMeta(root, 'meta[property="og:image"]') ||
+      getMeta(root, 'meta[name="twitter:image"]'),
+    lang: root.querySelector("html")?.getAttribute("lang") || "",
+  };
+}
+
+function extractWithReadability(html, url) {
+  const document = new DOMParser().parseFromString(html, "text/html");
+
+  if (url && document.head) {
+    const base = document.createElement("base");
+    base.setAttribute("href", url);
+    document.head.appendChild(base);
+  }
+
+  const article = new Readability(document, {
+    keepClasses: true,
+  }).parse();
+
+  if (!article?.content) {
+    return null;
+  }
+
+  return article;
+}
+
+function normalizeReadabilityArticle(article, metadata, url) {
+  const text = cleanText(article.textContent || article.content);
+  const contentHtml = stripUnsafeHtml(article.content);
+
+  if (text.length < 80) {
+    return null;
+  }
+
+  return {
+    url: article.url || url,
+    source: "direct-fetch",
+    extractor: "readability",
+    title: cleanText(article.title || metadata.title || ""),
+    description: cleanText(article.excerpt || metadata.description || ""),
+    metadata: {
+      ...metadata,
+      title: cleanText(article.title || metadata.title || ""),
+      description: cleanText(article.excerpt || metadata.description || ""),
+      site_name: cleanText(article.siteName || metadata.site_name || ""),
+      author: cleanText(article.byline || metadata.author || ""),
+      lang: article.lang || metadata.lang || "",
+    },
+    html: contentHtml,
+    text,
+    excerpt: text.slice(0, 500),
+    stats: {
+      text_length: text.length,
+      html_length: contentHtml.length,
+      score: null,
+      link_density: null,
+      paragraph_count: (contentHtml.match(/<p\b/gi) || []).length,
+    },
+  };
+}
+
+function extractPageContentWithHeuristics(html, url) {
+  const root = parseHtml(html);
+  const metadata = getPageMetadata(html);
+
+  cleanTree(root);
+
+  const candidate = pickBestCandidate(root);
+  const text = getNodeText(candidate.node);
+  const contentHtml = stripUnsafeHtml(candidate.node?.toString() || "");
+
+  return {
+    url,
+    source: "direct-fetch",
+    extractor: "heuristic",
+    title: metadata.title,
+    description: metadata.description,
+    metadata,
+    html: contentHtml,
+    text,
+    excerpt: text.slice(0, 500),
+    stats: {
+      text_length: text.length,
+      html_length: contentHtml.length,
+      score: Math.round(candidate.score),
+      link_density: Number(candidate.linkDensity.toFixed(3)),
+      paragraph_count: candidate.paragraphCount,
+    },
+  };
+}
+
+export async function extractPageContent(html, url) {
+  try {
+    const metadata = getPageMetadata(html);
+    const article = extractWithReadability(html, url);
+    const extracted = normalizeReadabilityArticle(article, metadata, url);
+
+    if (extracted) {
+      return extracted;
+    }
+  } catch (_) {
+    // Fall back to the local heuristic extractor when the library cannot parse
+    // a page or when Worker bundling/runtime behavior differs by site.
+  }
+
+  return extractPageContentWithHeuristics(html, url);
+}

package/src/content/extract.js

@@ -0,0 +1 @@
+export * from "./extract.impl.js";