@endday/search-mcp 1.0.0 → 1.0.1

package/src/content/fetch.impl.js

@@ -1,400 +0,0 @@
-import { ApiError } from "../core/errors.js";
-import { extractPageContent } from "./extract.js";
-import { getRandomBrowserProfile } from "../search/engineUtils.js";
-import { normalizePositiveInteger } from "../routes/requestParams.js";
-import { fetchWithOptionalCurlImpersonate } from "../platform/nodeHttpClient.js";
-
-const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
-const MAX_SAFE_REDIRECTS = 5;
-
-function parseIpv4Address(value) {
-  const match = String(value || "").match(
-    /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/
-  );
-
-  if (!match) {
-    return null;
-  }
-
-  const octets = match.slice(1).map((part) => Number.parseInt(part, 10));
-  return {
-    octets,
-    valid: octets.every((part) => part >= 0 && part <= 255),
-  };
-}
-
-function isBlockedIpv4Address(octets) {
-  const [first, second] = octets;
-
-  return (
-    first === 0 ||
-    first === 10 ||
-    first === 127 ||
-    (first === 100 && second >= 64 && second <= 127) ||
-    (first === 169 && second === 254) ||
-    (first === 172 && second >= 16 && second <= 31) ||
-    (first === 192 && second === 0) ||
-    (first === 192 && second === 168) ||
-    (first === 198 && (second === 18 || second === 19)) ||
-    first >= 224
-  );
-}
-
-function parseIpv6Hextets(value) {
-  let normalized = String(value || "")
-    .toLowerCase()
-    .replace(/^\[|\]$/g, "")
-    .split("%")[0];
-
-  if (!normalized.includes(":")) {
-    return null;
-  }
-
-  const embeddedIpv4Match = normalized.match(/(\d{1,3}(?:\.\d{1,3}){3})$/);
-  if (embeddedIpv4Match) {
-    const parsedIpv4 = parseIpv4Address(embeddedIpv4Match[1]);
-    if (!parsedIpv4?.valid) {
-      return null;
-    }
-
-    const [a, b, c, d] = parsedIpv4.octets;
-    const replacement = `${((a << 8) | b).toString(16)}:${(
-      (c << 8) |
-      d
-    ).toString(16)}`;
-    normalized =
-      normalized.slice(0, normalized.length - embeddedIpv4Match[1].length) +
-      replacement;
-  }
-
-  const compressionParts = normalized.split("::");
-  if (compressionParts.length > 2) {
-    return null;
-  }
-
-  const hasCompression = compressionParts.length === 2;
-  const leftParts = compressionParts[0] ? compressionParts[0].split(":") : [];
-  const rightParts =
-    hasCompression && compressionParts[1] ? compressionParts[1].split(":") : [];
-  const parts = [...leftParts, ...rightParts];
-
-  if (parts.some((part) => !/^[0-9a-f]{1,4}$/.test(part))) {
-    return null;
-  }
-
-  if (!hasCompression && parts.length !== 8) {
-    return null;
-  }
-
-  const fillCount = hasCompression ? 8 - parts.length : 0;
-  if (fillCount < 1 && hasCompression) {
-    return null;
-  }
-
-  return [
-    ...leftParts.map((part) => Number.parseInt(part, 16)),
-    ...Array(fillCount).fill(0),
-    ...rightParts.map((part) => Number.parseInt(part, 16)),
-  ];
-}
-
-function getEmbeddedIpv4FromIpv6(hextets) {
-  if (!hextets || hextets.length !== 8) {
-    return null;
-  }
-
-  const lastIpv4 = [
-    hextets[6] >> 8,
-    hextets[6] & 255,
-    hextets[7] >> 8,
-    hextets[7] & 255,
-  ];
-
-  const isIpv4Mapped =
-    hextets.slice(0, 5).every((part) => part === 0) && hextets[5] === 0xffff;
-  const isIpv4Compatible = hextets.slice(0, 6).every((part) => part === 0);
-  const isNat64WellKnown =
-    hextets[0] === 0x0064 &&
-    hextets[1] === 0xff9b &&
-    hextets.slice(2, 6).every((part) => part === 0);
-
-  return isIpv4Mapped || isIpv4Compatible || isNat64WellKnown ? lastIpv4 : null;
-}
-
-function isBlockedIpv6Address(hextets) {
-  if (!hextets || hextets.length !== 8) {
-    return false;
-  }
-
-  const first = hextets[0];
-  const embeddedIpv4 = getEmbeddedIpv4FromIpv6(hextets);
-
-  return (
-    hextets.every((part) => part === 0) ||
-    hextets.slice(0, 7).every((part) => part === 0) ||
-    (first & 0xfe00) === 0xfc00 ||
-    (first & 0xffc0) === 0xfe80 ||
-    (first & 0xff00) === 0xff00 ||
-    (embeddedIpv4 ? isBlockedIpv4Address(embeddedIpv4) : false)
-  );
-}
-
-function isBlockedTargetHostname(hostname) {
-  const normalized = hostname.toLowerCase().replace(/^\[|\]$/g, "");
-
-  if (
-    normalized === "localhost" ||
-    normalized.endsWith(".localhost") ||
-    normalized.endsWith(".local")
-  ) {
-    return true;
-  }
-
-  const parsedIpv4 = parseIpv4Address(normalized);
-  if (parsedIpv4) {
-    return !parsedIpv4.valid || isBlockedIpv4Address(parsedIpv4.octets);
-  }
-
-  return isBlockedIpv6Address(parseIpv6Hextets(normalized));
-}
-
-function buildNavigationHeaders(profile) {
-  const headers = {
-    Accept:
-      "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-    "Accept-Language": "en-US,en;q=0.9",
-    "User-Agent": profile.ua,
-    "Sec-Fetch-Dest": "document",
-    "Sec-Fetch-Mode": "navigate",
-    "Sec-Fetch-Site": "none",
-    "Sec-Fetch-User": "?1",
-  };
-
-  if (profile.headers?.["sec-ch-ua"]) {
-    headers["Sec-Ch-Ua"] = profile.headers["sec-ch-ua"];
-    headers["Sec-Ch-Ua-Platform"] = profile.headers["sec-ch-ua-platform"];
-    headers["Sec-Ch-Ua-Mobile"] = profile.headers["sec-ch-ua-mobile"];
-  }
-
-  return headers;
-}
-
-export function normalizeTargetUrl(value) {
-  const rawUrl = String(value || "").trim();
-
-  if (!rawUrl) {
-    throw new ApiError({
-      status: 400,
-      code: "MISSING_URL",
-      category: "validation",
-      message: "Please provide 'url' parameter",
-    });
-  }
-
-  let parsedUrl;
-  try {
-    parsedUrl = new URL(rawUrl);
-  } catch (_) {
-    throw new ApiError({
-      status: 400,
-      code: "INVALID_URL",
-      category: "validation",
-      message: "The 'url' parameter must be a valid URL",
-    });
-  }
-
-  if (!["http:", "https:"].includes(parsedUrl.protocol)) {
-    throw new ApiError({
-      status: 400,
-      code: "INVALID_URL",
-      category: "validation",
-      message: "Only http and https URLs are supported",
-    });
-  }
-
-  if (isBlockedTargetHostname(parsedUrl.hostname)) {
-    throw new ApiError({
-      status: 400,
-      code: "INVALID_URL",
-      category: "validation",
-      message: "Localhost and private network URLs are not supported",
-    });
-  }
-
-  return parsedUrl.toString();
-}
-
-function getSafeRedirectUrl(response, currentUrl) {
-  if (!REDIRECT_STATUSES.has(response.status)) {
-    return null;
-  }
-
-  const location = response.headers.get("location");
-  if (!location) {
-    return null;
-  }
-
-  return normalizeTargetUrl(new URL(location, currentUrl).toString());
-}
-
-async function cancelResponseBody(response) {
-  try {
-    await response.body?.cancel();
-  } catch (_) {
-    // best effort
-  }
-}
-
-async function fetchWithSafeRedirects(targetUrl, init = {}) {
-  let currentUrl = normalizeTargetUrl(targetUrl);
-
-  for (let redirectCount = 0; redirectCount <= MAX_SAFE_REDIRECTS; redirectCount += 1) {
-    const response = await fetchWithOptionalCurlImpersonate(currentUrl, {
-      ...init,
-      redirect: "manual",
-    });
-    const redirectUrl = getSafeRedirectUrl(response, currentUrl);
-
-    if (!redirectUrl) {
-      return response;
-    }
-
-    await cancelResponseBody(response);
-    currentUrl = redirectUrl;
-  }
-
-  throw new ApiError({
-    status: 508,
-    code: "TOO_MANY_REDIRECTS",
-    category: "upstream",
-    message: "Target URL redirected too many times",
-  });
-}
-
-export async function verifySafeRedirectChain(targetUrl) {
-  const profile = getRandomBrowserProfile("default");
-  const response = await fetchWithSafeRedirects(targetUrl, {
-    method: "GET",
-    headers: {
-      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-      Range: "bytes=0-0",
-      "User-Agent": profile.ua,
-    },
-  });
-
-  await cancelResponseBody(response);
-}
-
-async function readResponseTextWithLimit(response, maxBytes) {
-  const contentLength = Number.parseInt(
-    response.headers.get("content-length") || "0",
-    10
-  );
-
-  if (contentLength > maxBytes) {
-    throw new ApiError({
-      status: 413,
-      code: "CONTENT_TOO_LARGE",
-      category: "upstream",
-      message: `Upstream response is larger than ${maxBytes} bytes`,
-    });
-  }
-
-  if (!response.body) {
-    const text = await response.text();
-    if (new TextEncoder().encode(text).length > maxBytes) {
-      throw new ApiError({
-        status: 413,
-        code: "CONTENT_TOO_LARGE",
-        category: "upstream",
-        message: `Upstream response is larger than ${maxBytes} bytes`,
-      });
-    }
-    return text;
-  }
-
-  const reader = response.body.getReader();
-  const decoder = new TextDecoder();
-  const chunks = [];
-  let receivedBytes = 0;
-
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) {
-      break;
-    }
-
-    const chunk = value instanceof Uint8Array ? value : new TextEncoder().encode(value);
-    receivedBytes += chunk.byteLength;
-
-    if (receivedBytes > maxBytes) {
-      await reader.cancel();
-      throw new ApiError({
-        status: 413,
-        code: "CONTENT_TOO_LARGE",
-        category: "upstream",
-        message: `Upstream response is larger than ${maxBytes} bytes`,
-      });
-    }
-
-    chunks.push(decoder.decode(chunk, { stream: true }));
-  }
-
-  chunks.push(decoder.decode());
-  return chunks.join("");
-}
-
-export function normalizeContentMaxBytes(params) {
-  return normalizePositiveInteger(params.max_bytes || params.maxBytes, 1_500_000, {
-    min: 50_000,
-    max: 5_000_000,
-  });
-}
-
-export async function fetchReadableContent(targetUrl, maxBytes) {
-  const normalizedTargetUrl = normalizeTargetUrl(targetUrl);
-  const profile = getRandomBrowserProfile("default");
-  const contentHeaders = buildNavigationHeaders(profile);
-  const upstreamResponse = await fetchWithSafeRedirects(normalizedTargetUrl, {
-    headers: contentHeaders,
-  });
-  const contentType = upstreamResponse.headers.get("content-type") || "";
-
-  if (!upstreamResponse.ok) {
-    throw new ApiError({
-      status: upstreamResponse.status >= 500 ? 502 : upstreamResponse.status,
-      code: "UPSTREAM_HTTP_ERROR",
-      category: "upstream",
-      message: `Upstream returned HTTP ${upstreamResponse.status}`,
-      details: {
-        upstream_status: upstreamResponse.status,
-      },
-    });
-  }
-
-  if (
-    contentType &&
-    !/text\/html|application\/xhtml\+xml|application\/xml|text\/xml/i.test(contentType)
-  ) {
-    throw new ApiError({
-      status: 415,
-      code: "UNSUPPORTED_CONTENT_TYPE",
-      category: "upstream",
-      message: `Unsupported content type: ${contentType}`,
-    });
-  }
-
-  const html = await readResponseTextWithLimit(upstreamResponse, maxBytes);
-
-  const payload = await extractPageContent(
-    html,
-    upstreamResponse.url || normalizedTargetUrl
-  );
-
-  return {
-    ...payload,
-    requested_url: normalizedTargetUrl,
-    content_type: contentType || null,
-    max_bytes: maxBytes,
-  };
-}

package/src/content/fetch.js

@@ -1 +0,0 @@
-export * from "./fetch.impl.js";

package/src/core/crypto.js

@@ -1,7 +0,0 @@
-export async function sha256Hex(value) {
-  const encoded = new TextEncoder().encode(value);
-  const digest = await crypto.subtle.digest("SHA-256", encoded);
-  return [...new Uint8Array(digest)]
-    .map((item) => item.toString(16).padStart(2, "0"))
-    .join("");
-}

package/src/core/errors.impl.js

@@ -1,52 +0,0 @@
-export class ApiError extends Error {
-  constructor({
-    status = 500,
-    code = "INTERNAL_ERROR",
-    message = "Unexpected error",
-    category = "internal",
-    details,
-  } = {}) {
-    super(message);
-    this.name = "ApiError";
-    this.status = status;
-    this.code = code;
-    this.category = category;
-    this.details = details;
-  }
-}
-
-export function normalizeError(error, { engine } = {}) {
-  if (error instanceof ApiError) {
-    return error;
-  }
-
-  if (error?.name === "AbortError") {
-    return new ApiError({
-      status: 504,
-      code: "UPSTREAM_TIMEOUT",
-      category: "upstream",
-      message: engine
-        ? `${engine} request timed out`
-        : "Upstream request timed out",
-      details: engine ? { engine } : undefined,
-    });
-  }
-
-  return new ApiError({
-    status: 502,
-    code: "UPSTREAM_ERROR",
-    category: "upstream",
-    message: error?.message || "Upstream request failed",
-    details: engine ? { engine } : undefined,
-  });
-}
-
-export function toErrorPayload(error) {
-  const normalized = normalizeError(error);
-  return {
-    error: normalized.category,
-    code: normalized.code,
-    message: normalized.message,
-    ...(normalized.details ? { details: normalized.details } : {}),
-  };
-}

package/src/core/errors.js

@@ -1 +0,0 @@
-export * from "./errors.impl.js";

package/src/core/html.impl.js

@@ -1,69 +0,0 @@
-import { parse } from "node-html-parser";
-
-export function parseHtml(html) {
-  return parse(html, {
-    comment: false,
-    lowerCaseTagName: false,
-    blockTextElements: {
-      script: true,
-      noscript: false,
-      style: false,
-      pre: true,
-    },
-  });
-}
-
-export function cleanText(input) {
-  if (!input) {
-    return "";
-  }
-
-  const parsed = parse(`<span>${String(input)}</span>`);
-  return parsed.text
-    .replace(/\u00a0/g, " ")
-    .replace(/\s+/g, " ")
-    .trim();
-}
-
-export function extractBalancedSegment(source, startIndex) {
-  const opening = source[startIndex];
-  const closing = opening === "{" ? "}" : "]";
-
-  let depth = 0;
-  let inString = false;
-  let escaped = false;
-
-  for (let index = startIndex; index < source.length; index += 1) {
-    const char = source[index];
-
-    if (escaped) {
-      escaped = false;
-      continue;
-    }
-
-    if (char === "\\") {
-      escaped = true;
-      continue;
-    }
-
-    if (char === '"') {
-      inString = !inString;
-      continue;
-    }
-
-    if (inString) {
-      continue;
-    }
-
-    if (char === opening) {
-      depth += 1;
-    } else if (char === closing) {
-      depth -= 1;
-      if (depth === 0) {
-        return source.slice(startIndex, index + 1);
-      }
-    }
-  }
-
-  throw new Error("Unable to extract balanced JSON segment");
-}

package/src/core/html.js

@@ -1 +0,0 @@
-export * from "./html.impl.js";

package/src/mcp/config.js

@@ -1,75 +0,0 @@
-import { env, setEnv } from "../../envs.js";
-import { createHash } from "node:crypto";
-
-function buildDefaultLocalClientId() {
-  const seed = [
-    process.env.SEARCH_MCP_SESSION_ID || "",
-    process.pid,
-    process.cwd(),
-  ].join(":");
-
-  const digest = createHash("sha256").update(seed).digest("hex").slice(0, 12);
-  return `mcp-local:${digest}`;
-}
-
-function normalizeMode(value) {
-  return "local";
-}
-
-function applyLocalEnvFromProcess() {
-  const overrides = {};
-  const keys = [
-    "SUPPORTED_ENGINES",
-    "DEFAULT_ENGINES",
-    "DEFAULT_ENGINES_ZH",
-    "DEFAULT_ENGINES_NON_ZH",
-    "DEFAULT_LANGUAGE",
-    "DEFAULT_TIMEOUT",
-    "FALLBACK_MIN_RESULTS",
-    "FALLBACK_MIN_CONTRIBUTING_ENGINES",
-    "SEARCH_PRIMARY_TIERS",
-    "SEARCH_SECONDARY_TIERS",
-    "SEARCH_EXPERIMENTAL_TIERS",
-    "EDGE_CACHE_TTL_SECONDS",
-    "CACHE_TTL_SECONDS",
-    "STALE_CACHE_TTL_SECONDS",
-    "UPSTREAM_RETRY_ATTEMPTS",
-    "UPSTREAM_RETRY_DELAY_MS",
-    "UPSTREAM_PRIMARY_RETRY_ATTEMPTS",
-    "UPSTREAM_SECONDARY_RETRY_ATTEMPTS",
-    "UPSTREAM_EXPERIMENTAL_RETRY_ATTEMPTS",
-    "UPSTREAM_MIN_REQUEST_INTERVAL_MS",
-    "UPSTREAM_PRIMARY_MIN_REQUEST_INTERVAL_MS",
-    "UPSTREAM_SECONDARY_MIN_REQUEST_INTERVAL_MS",
-    "UPSTREAM_EXPERIMENTAL_MIN_REQUEST_INTERVAL_MS",
-    "HEALTH_FAILURE_THRESHOLD",
-    "HEALTH_COOLDOWN_SECONDS",
-  ];
-
-  for (const key of keys) {
-    if (process.env[key] !== undefined) {
-      overrides[key] = process.env[key];
-    }
-  }
-
-  overrides.SUPPORTED_ENGINES ||= env.SUPPORTED_ENGINES.join(",");
-  setEnv(overrides);
-}
-
-export function loadMcpConfig() {
-  const mode = normalizeMode(process.env.SEARCH_MCP_MODE);
-
-  applyLocalEnvFromProcess();
-
-  return {
-    mode,
-    jinaApiKey: process.env.JINA_API_KEY || "",
-    jinaBaseUrl: process.env.JINA_BASE_URL || "https://r.jina.ai/",
-    upstreamClient: process.env.SEARCH_MCP_UPSTREAM_CLIENT || "auto",
-    localClientId:
-      process.env.SEARCH_MCP_CLIENT_ID ||
-      process.env.SEARCH_MCP_SESSION_ID ||
-      buildDefaultLocalClientId(),
-    allEngines: [...env.SUPPORTED_ENGINES],
-  };
-}

package/src/mcp/format.js

@@ -1,44 +0,0 @@
-export const MAX_CHARS_MIN = 500;
-export const MAX_CHARS_MAX = 20000;
-export const MAX_CHARS_DEFAULT = 4000;
-
-export function normalizeMaxChars(value) {
-  return Number.isInteger(value)
-    ? Math.min(Math.max(value, MAX_CHARS_MIN), MAX_CHARS_MAX)
-    : MAX_CHARS_DEFAULT;
-}
-
-export function truncateText(text, maxChars) {
-  return text.length > maxChars
-    ? text.slice(0, maxChars) + `\n[...${text.length - maxChars} chars truncated]`
-    : text;
-}
-
-export function formatSearchResponse(result) {
-  const lines = result.results.map((item, index) => {
-    const score = item.authority_score ?? 0;
-    const type = item.source_type || "unknown";
-    const tag = score > 0 || type !== "unknown" ? ` ${type}(${score})` : "";
-    const desc = String(item.description || "").slice(0, 120);
-    const descPart = desc ? ` | ${desc}` : "";
-    const sourcePart = item.source_name ? ` | source: ${item.source_name}` : "";
-    const publishedPart = item.published_text ? ` | published: ${item.published_text}` : "";
-    return `${index + 1}. ${item.title}${sourcePart}${publishedPart}${descPart} | ${item.url}${tag}`;
-  });
-
-  return lines.join("\n");
-}
-
-export function formatContentResponse(result, maxChars) {
-  const text = String(result.text || result.excerpt || "");
-  const header = [
-    result.title ? `# ${result.title}` : null,
-    result.url ? result.url : null,
-    result.extractor ? `(${result.extractor})` : null,
-  ]
-    .filter(Boolean)
-    .join(" ");
-
-  const excerpt = truncateText(text, maxChars);
-  return header ? `${header}\n${excerpt}` : excerpt;
-}

package/src/mcp/index.js

@@ -1,10 +0,0 @@
-import { loadMcpConfig } from "./config.js";
-import { createServer, startServer } from "./server.js";
-
-export async function main() {
-  const config = loadMcpConfig();
-  const server = createServer(config);
-  await startServer(server);
-  console.error("Search MCP Server running on stdio");
-  console.error(`Mode: ${config.mode}`);
-}

package/src/mcp/local/content.js

@@ -1,26 +0,0 @@
-import { extractPageContent } from "../../content/extract.js";
-import {
-  fetchReadableContent,
-  normalizeContentMaxBytes,
-  normalizeTargetUrl,
-} from "../../content/fetch.js";
-
-export function requireUrl(args) {
-  const url = args.url;
-  if (!url || typeof url !== "string") {
-    throw new Error("url required");
-  }
-  return url;
-}
-
-export async function contentLocal(targetUrl, options = {}) {
-  const maxBytes = normalizeContentMaxBytes({
-    max_bytes: options.max_bytes,
-  });
-  return fetchReadableContent(targetUrl, maxBytes);
-}
-
-export async function extractHtmlLocal(targetUrl, html) {
-  const normalizedUrl = normalizeTargetUrl(targetUrl);
-  return extractPageContent(html, normalizedUrl);
-}