data-aggregator-mcp 1.0.0

package/src/tools/scraper.ts

@@ -0,0 +1,217 @@
+/**
+ * fetch_webpage - Structured web scraping.
+ *
+ * Fetches a URL and returns structured JSON with:
+ *   - Page title
+ *   - Meta description
+ *   - Headings hierarchy
+ *   - Main text content (HTML stripped)
+ *   - Selected content via CSS-selector-like path (basic support)
+ *   - Links found on the page
+ *
+ * Uses rotating User-Agent strings for basic anti-scraping.
+ */
+
+import { cache, TTL } from "../utils/cache.js";
+import { rateLimiter } from "../utils/rate-limiter.js";
+import { httpGetText, formatError } from "../utils/http.js";
+import type { ScrapedPage, ToolResult } from "../types.js";
+
+// ─── Basic HTML Parsing Helpers ────────────────────────────────────────────
+// We do simple regex-based extraction to avoid heavy deps like cheerio.
+
+function extractTag(html: string, tag: string): string {
+  const re = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}>`, "i");
+  const m = html.match(re);
+  return m ? stripHtml(m[1]!).trim() : "";
+}
+
+function extractMetaContent(html: string, name: string): string {
+  // Match both name="..." and property="..."
+  const re = new RegExp(
+    `<meta\\s+(?:[^>]*?(?:name|property)\\s*=\\s*["']${name}["'][^>]*?content\\s*=\\s*["']([^"']*)["']|[^>]*?content\\s*=\\s*["']([^"']*)["'][^>]*?(?:name|property)\\s*=\\s*["']${name}["'])`,
+    "i"
+  );
+  const m = html.match(re);
+  return (m?.[1] ?? m?.[2] ?? "").trim();
+}
+
+function extractHeadings(html: string): { level: number; text: string }[] {
+  const headings: { level: number; text: string }[] = [];
+  const re = /<(h[1-6])[^>]*>([\s\S]*?)<\/\1>/gi;
+  let match;
+  while ((match = re.exec(html)) !== null) {
+    const level = parseInt(match[1]!.charAt(1), 10);
+    const text = stripHtml(match[2]!).trim();
+    if (text) headings.push({ level, text });
+  }
+  return headings;
+}
+
+function extractLinks(html: string, baseUrl: string): { text: string; href: string }[] {
+  const links: { text: string; href: string }[] = [];
+  const re = /<a\s+[^>]*href\s*=\s*["']([^"'#]+)["'][^>]*>([\s\S]*?)<\/a>/gi;
+  let match;
+  const seen = new Set<string>();
+
+  while ((match = re.exec(html)) !== null) {
+    let href = match[1]!.trim();
+    const text = stripHtml(match[2]!).trim();
+    if (!text || !href) continue;
+
+    // Resolve relative URLs
+    try {
+      href = new URL(href, baseUrl).href;
+    } catch {
+      continue;
+    }
+
+    if (!seen.has(href)) {
+      seen.add(href);
+      links.push({ text: text.slice(0, 200), href });
+    }
+
+    if (links.length >= 50) break;
+  }
+
+  return links;
+}
+
+function stripHtml(html: string): string {
+  return html
+    .replace(/<script[\s\S]*?<\/script>/gi, " ")
+    .replace(/<style[\s\S]*?<\/style>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&nbsp;/gi, " ")
+    .replace(/&amp;/gi, "&")
+    .replace(/&lt;/gi, "<")
+    .replace(/&gt;/gi, ">")
+    .replace(/&quot;/gi, '"')
+    .replace(/&#39;/gi, "'")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function extractMainContent(html: string): string {
+  // Try to find <main>, <article>, or role="main" first
+  const mainPatterns = [
+    /<main[^>]*>([\s\S]*?)<\/main>/i,
+    /<article[^>]*>([\s\S]*?)<\/article>/i,
+    /<div[^>]*role\s*=\s*["']main["'][^>]*>([\s\S]*?)<\/div>/i,
+  ];
+
+  for (const pattern of mainPatterns) {
+    const match = html.match(pattern);
+    if (match) {
+      const text = stripHtml(match[1]!);
+      if (text.length > 100) return text.slice(0, 10_000);
+    }
+  }
+
+  // Fall back to stripping the body
+  const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+  const body = bodyMatch ? bodyMatch[1]! : html;
+  return stripHtml(body).slice(0, 10_000);
+}
+
+/**
+ * Very basic CSS-selector-like content extraction.
+ * Supports: tag, .class, #id (single selector only).
+ */
+function extractBySelector(html: string, selector: string): string {
+  let pattern: RegExp;
+
+  if (selector.startsWith("#")) {
+    const id = selector.slice(1);
+    pattern = new RegExp(
+      `<([a-z][a-z0-9]*)\\s+[^>]*id\\s*=\\s*["']${id}["'][^>]*>([\\s\\S]*?)<\\/\\1>`,
+      "i"
+    );
+  } else if (selector.startsWith(".")) {
+    const cls = selector.slice(1);
+    pattern = new RegExp(
+      `<([a-z][a-z0-9]*)\\s+[^>]*class\\s*=\\s*["'][^"']*\\b${cls}\\b[^"']*["'][^>]*>([\\s\\S]*?)<\\/\\1>`,
+      "i"
+    );
+  } else {
+    // Plain tag name
+    pattern = new RegExp(`<${selector}[^>]*>([\\s\\S]*?)<\\/${selector}>`, "i");
+  }
+
+  const match = html.match(pattern);
+  if (!match) return "";
+
+  // The captured content is in group 2 for id/class patterns, group 1 for tag
+  const content = match[2] ?? match[1] ?? "";
+  return stripHtml(content).slice(0, 10_000);
+}
+
+// ─── Public handler ────────────────────────────────────────────────────────
+
+export async function fetchWebpage(args: {
+  url: string;
+  selector?: string;
+}): Promise<ToolResult> {
+  try {
+    const url = args.url.trim();
+
+    // Validate URL
+    try {
+      new URL(url);
+    } catch {
+      return {
+        content: [{ type: "text", text: `Invalid URL: "${url}"` }],
+        isError: true,
+      };
+    }
+
+    const cacheKey = `scrape:${url}:${args.selector ?? ""}`;
+    const cached = cache.get<ScrapedPage>(cacheKey);
+    if (cached) {
+      return {
+        content: [
+          { type: "text", text: JSON.stringify({ ...cached, cached: true }, null, 2) },
+        ],
+      };
+    }
+
+    await rateLimiter.acquire("generic");
+
+    const html = await httpGetText(url, {
+      rotateUserAgent: true,
+      headers: {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+      },
+    });
+
+    const result: ScrapedPage = {
+      url,
+      title: extractTag(html, "title"),
+      metaDescription:
+        extractMetaContent(html, "description") ||
+        extractMetaContent(html, "og:description"),
+      headings: extractHeadings(html),
+      mainContent: extractMainContent(html),
+      links: extractLinks(html, url),
+      timestamp: new Date().toISOString(),
+    };
+
+    if (args.selector) {
+      result.selectedContent = extractBySelector(html, args.selector);
+    }
+
+    cache.set(cacheKey, result, TTL.SCRAPE);
+
+    return {
+      content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+    };
+  } catch (err) {
+    return {
+      content: [
+        { type: "text", text: `Error scraping webpage: ${formatError(err)}` },
+      ],
+      isError: true,
+    };
+  }
+}

package/src/tools/stocks.ts

@@ -0,0 +1,205 @@
+/**
+ * query_stocks - Fetch stock and crypto market data.
+ *
+ * Free data sources:
+ *   - CoinGecko API (crypto, no key required)
+ *   - Yahoo Finance v8 unofficial API (stocks, no key required)
+ *   - Alpha Vantage (stocks, optional key via ALPHA_VANTAGE_KEY env)
+ */
+
+import { cache, TTL } from "../utils/cache.js";
+import { rateLimiter } from "../utils/rate-limiter.js";
+import { httpGetJson, formatError } from "../utils/http.js";
+import type { StockQuote, CryptoQuote, ToolResult } from "../types.js";
+
+// ─── CoinGecko ID mapping for common crypto symbols ────────────────────────
+
+const CRYPTO_MAP: Record<string, string> = {
+  btc: "bitcoin",
+  eth: "ethereum",
+  sol: "solana",
+  ada: "cardano",
+  dot: "polkadot",
+  matic: "matic-network",
+  avax: "avalanche-2",
+  link: "chainlink",
+  doge: "dogecoin",
+  shib: "shiba-inu",
+  xrp: "ripple",
+  bnb: "binancecoin",
+  ltc: "litecoin",
+  uni: "uniswap",
+  atom: "cosmos",
+  near: "near",
+  apt: "aptos",
+  arb: "arbitrum",
+  op: "optimism",
+  sui: "sui",
+};
+
+function isCrypto(symbol: string): boolean {
+  return symbol.toLowerCase() in CRYPTO_MAP;
+}
+
+function coingeckoId(symbol: string): string {
+  return CRYPTO_MAP[symbol.toLowerCase()] ?? symbol.toLowerCase();
+}
+
+// ─── Crypto via CoinGecko ──────────────────────────────────────────────────
+
+async function fetchCryptoQuote(symbol: string, vsCurrency = "usd"): Promise<CryptoQuote> {
+  const id = coingeckoId(symbol);
+  const url =
+    `https://api.coingecko.com/api/v3/coins/${id}` +
+    `?localization=false&tickers=false&community_data=false&developer_data=false`;
+
+  await rateLimiter.acquire("coingecko");
+  const data: any = await httpGetJson(url);
+
+  const md = data.market_data;
+  return {
+    symbol: (data.symbol as string).toUpperCase(),
+    name: data.name,
+    price: md.current_price[vsCurrency] ?? md.current_price.usd,
+    currency: vsCurrency.toUpperCase(),
+    change24h: md.price_change_24h ?? 0,
+    changePercent24h: md.price_change_percentage_24h ?? 0,
+    volume24h: md.total_volume[vsCurrency] ?? md.total_volume.usd ?? 0,
+    marketCap: md.market_cap[vsCurrency] ?? md.market_cap.usd ?? 0,
+    high24h: md.high_24h?.[vsCurrency],
+    low24h: md.low_24h?.[vsCurrency],
+    timestamp: new Date().toISOString(),
+    source: "CoinGecko",
+  };
+}
+
+// ─── Stocks via Yahoo Finance (unofficial v8 endpoint) ─────────────────────
+
+async function fetchStockQuoteYahoo(symbol: string): Promise<StockQuote> {
+  const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}?range=1d&interval=1d`;
+
+  await rateLimiter.acquire("generic");
+
+  const data: any = await httpGetJson(url, { rotateUserAgent: true });
+  const result = data?.chart?.result?.[0];
+  if (!result) throw new Error(`No data found for symbol "${symbol}"`);
+
+  const meta = result.meta;
+  const price = meta.regularMarketPrice ?? meta.previousClose;
+  const prevClose = meta.chartPreviousClose ?? meta.previousClose ?? price;
+
+  return {
+    symbol: meta.symbol ?? symbol.toUpperCase(),
+    price,
+    currency: (meta.currency ?? "USD").toUpperCase(),
+    change: +(price - prevClose).toFixed(4),
+    changePercent: prevClose ? +(((price - prevClose) / prevClose) * 100).toFixed(4) : 0,
+    volume: meta.regularMarketVolume,
+    high: meta.regularMarketDayHigh,
+    low: meta.regularMarketDayLow,
+    open: meta.regularMarketOpen ?? meta.openPrice,
+    previousClose: prevClose,
+    timestamp: new Date().toISOString(),
+    source: "Yahoo Finance",
+  };
+}
+
+// ─── Stocks via Alpha Vantage (if key is available) ────────────────────────
+
+async function fetchStockQuoteAlphaVantage(
+  symbol: string,
+  apiKey: string
+): Promise<StockQuote> {
+  const url =
+    `https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol=${encodeURIComponent(symbol)}&apikey=${apiKey}`;
+
+  await rateLimiter.acquire("generic");
+  const data: any = await httpGetJson(url);
+  const q = data["Global Quote"];
+
+  if (!q || !q["05. price"]) {
+    throw new Error(`Alpha Vantage returned no data for "${symbol}"`);
+  }
+
+  const price = parseFloat(q["05. price"]);
+  const prevClose = parseFloat(q["08. previous close"]);
+
+  return {
+    symbol: q["01. symbol"] ?? symbol.toUpperCase(),
+    price,
+    currency: "USD",
+    change: parseFloat(q["09. change"]) || 0,
+    changePercent: parseFloat(q["10. change percent"]?.replace("%", "")) || 0,
+    volume: parseInt(q["06. volume"], 10) || undefined,
+    high: parseFloat(q["03. high"]) || undefined,
+    low: parseFloat(q["04. low"]) || undefined,
+    open: parseFloat(q["02. open"]) || undefined,
+    previousClose: prevClose || undefined,
+    timestamp: new Date().toISOString(),
+    source: "Alpha Vantage",
+  };
+}
+
+// ─── Public handler ────────────────────────────────────────────────────────
+
+export async function queryStocks(args: {
+  symbol: string;
+  type?: "stock" | "crypto" | "auto";
+}): Promise<ToolResult> {
+  try {
+    const symbol = args.symbol.trim();
+    const assetType =
+      args.type === "crypto" || (args.type !== "stock" && isCrypto(symbol))
+        ? "crypto"
+        : "stock";
+
+    const cacheKey = `stocks:${assetType}:${symbol.toLowerCase()}`;
+
+    // Check cache
+    const cached = cache.get<StockQuote | CryptoQuote>(cacheKey);
+    if (cached) {
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify({ ...cached, cached: true }, null, 2),
+          },
+        ],
+      };
+    }
+
+    let result: StockQuote | CryptoQuote;
+
+    if (assetType === "crypto") {
+      result = await fetchCryptoQuote(symbol);
+    } else {
+      // Try Yahoo first; fall back to Alpha Vantage if a key is available
+      try {
+        result = await fetchStockQuoteYahoo(symbol);
+      } catch (yahooErr) {
+        const avKey = process.env.ALPHA_VANTAGE_KEY;
+        if (avKey) {
+          result = await fetchStockQuoteAlphaVantage(symbol, avKey);
+        } else {
+          throw yahooErr;
+        }
+      }
+    }
+
+    cache.set(cacheKey, result, TTL.MARKET);
+
+    return {
+      content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
+    };
+  } catch (err) {
+    return {
+      content: [
+        {
+          type: "text",
+          text: `Error fetching market data: ${formatError(err)}`,
+        },
+      ],
+      isError: true,
+    };
+  }
+}

package/src/tools/weather.ts

@@ -0,0 +1,216 @@
+/**
+ * query_weather - Weather data via Open-Meteo API.
+ *
+ * Open-Meteo is 100% free, no API key required.
+ * Provides current conditions + 7-day forecast.
+ * Accepts city name (geocoded) or direct lat/lon.
+ */
+
+import { cache, TTL } from "../utils/cache.js";
+import { rateLimiter } from "../utils/rate-limiter.js";
+import { httpGetJson, formatError } from "../utils/http.js";
+import type { WeatherData, CurrentWeather, DailyForecast, ToolResult } from "../types.js";
+
+// ─── WMO Weather Code Descriptions ────────────────────────────────────────
+
+const WMO_CODES: Record<number, string> = {
+  0: "Clear sky",
+  1: "Mainly clear",
+  2: "Partly cloudy",
+  3: "Overcast",
+  45: "Fog",
+  48: "Depositing rime fog",
+  51: "Light drizzle",
+  53: "Moderate drizzle",
+  55: "Dense drizzle",
+  56: "Light freezing drizzle",
+  57: "Dense freezing drizzle",
+  61: "Slight rain",
+  63: "Moderate rain",
+  65: "Heavy rain",
+  66: "Light freezing rain",
+  67: "Heavy freezing rain",
+  71: "Slight snowfall",
+  73: "Moderate snowfall",
+  75: "Heavy snowfall",
+  77: "Snow grains",
+  80: "Slight rain showers",
+  81: "Moderate rain showers",
+  82: "Violent rain showers",
+  85: "Slight snow showers",
+  86: "Heavy snow showers",
+  95: "Thunderstorm",
+  96: "Thunderstorm with slight hail",
+  99: "Thunderstorm with heavy hail",
+};
+
+function weatherDescription(code: number): string {
+  return WMO_CODES[code] ?? `Unknown (code ${code})`;
+}
+
+// ─── Geocoding via Open-Meteo ──────────────────────────────────────────────
+
+interface GeoResult {
+  name: string;
+  latitude: number;
+  longitude: number;
+  timezone: string;
+  country?: string;
+  admin1?: string;
+}
+
+async function geocodeCity(city: string): Promise<GeoResult> {
+  const url = `https://geocoding-api.open-meteo.com/v1/search?name=${encodeURIComponent(city)}&count=1&language=en&format=json`;
+
+  await rateLimiter.acquire("open-meteo");
+  const data: any = await httpGetJson(url);
+
+  if (!data.results || data.results.length === 0) {
+    throw new Error(`Could not find location "${city}". Try a different spelling or use latitude/longitude.`);
+  }
+
+  const r = data.results[0];
+  return {
+    name: [r.name, r.admin1, r.country].filter(Boolean).join(", "),
+    latitude: r.latitude,
+    longitude: r.longitude,
+    timezone: r.timezone ?? "UTC",
+  };
+}
+
+// ─── Weather fetch ─────────────────────────────────────────────────────────
+
+async function fetchWeather(
+  lat: number,
+  lon: number,
+  locationName: string,
+  timezone: string,
+  units: "celsius" | "fahrenheit"
+): Promise<WeatherData> {
+  const tempUnit = units === "fahrenheit" ? "fahrenheit" : "celsius";
+  const windUnit = units === "fahrenheit" ? "mph" : "kmh";
+
+  const params = new URLSearchParams({
+    latitude: String(lat),
+    longitude: String(lon),
+    timezone,
+    temperature_unit: tempUnit,
+    wind_speed_unit: windUnit,
+    current: "temperature_2m,relative_humidity_2m,apparent_temperature,precipitation,weather_code,wind_speed_10m,wind_direction_10m,is_day",
+    daily: "weather_code,temperature_2m_max,temperature_2m_min,precipitation_sum,precipitation_probability_max,wind_speed_10m_max,sunrise,sunset",
+    forecast_days: "7",
+  });
+
+  const url = `https://api.open-meteo.com/v1/forecast?${params.toString()}`;
+
+  await rateLimiter.acquire("open-meteo");
+  const data: any = await httpGetJson(url);
+
+  const c = data.current;
+  const current: CurrentWeather = {
+    temperature: c.temperature_2m,
+    feelsLike: c.apparent_temperature,
+    humidity: c.relative_humidity_2m,
+    windSpeed: c.wind_speed_10m,
+    windDirection: c.wind_direction_10m,
+    precipitation: c.precipitation,
+    weatherCode: c.weather_code,
+    weatherDescription: weatherDescription(c.weather_code),
+    isDay: c.is_day === 1,
+    timestamp: new Date().toISOString(),
+  };
+
+  const d = data.daily;
+  const forecast: DailyForecast[] = [];
+  for (let i = 0; i < (d.time?.length ?? 0); i++) {
+    forecast.push({
+      date: d.time[i],
+      temperatureMax: d.temperature_2m_max[i],
+      temperatureMin: d.temperature_2m_min[i],
+      precipitationSum: d.precipitation_sum[i],
+      precipitationProbability: d.precipitation_probability_max[i],
+      windSpeedMax: d.wind_speed_10m_max[i],
+      weatherCode: d.weather_code[i],
+      weatherDescription: weatherDescription(d.weather_code[i]),
+      sunrise: d.sunrise[i],
+      sunset: d.sunset[i],
+    });
+  }
+
+  return {
+    location: locationName,
+    latitude: lat,
+    longitude: lon,
+    timezone,
+    current,
+    forecast,
+    source: "Open-Meteo",
+  };
+}
+
+// ─── Public handler ────────────────────────────────────────────────────────
+
+export async function queryWeather(args: {
+  city?: string;
+  latitude?: number;
+  longitude?: number;
+  units?: "celsius" | "fahrenheit";
+}): Promise<ToolResult> {
+  try {
+    const { city, units = "celsius" } = args;
+    let { latitude, longitude } = args;
+
+    if (!city && (latitude === undefined || longitude === undefined)) {
+      return {
+        content: [
+          {
+            type: "text",
+            text: 'Please provide either a city name or both latitude and longitude.',
+          },
+        ],
+        isError: true,
+      };
+    }
+
+    let locationName: string;
+    let timezone: string;
+
+    if (city) {
+      const geo = await geocodeCity(city);
+      latitude = geo.latitude;
+      longitude = geo.longitude;
+      locationName = geo.name;
+      timezone = geo.timezone;
+    } else {
+      locationName = `${latitude!.toFixed(4)}, ${longitude!.toFixed(4)}`;
+      timezone = "UTC";
+    }
+
+    const cacheKey = `weather:${latitude!.toFixed(2)}:${longitude!.toFixed(2)}:${units}`;
+    const cached = cache.get<WeatherData>(cacheKey);
+    if (cached) {
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify({ ...cached, cached: true }, null, 2),
+          },
+        ],
+      };
+    }
+
+    const weather = await fetchWeather(latitude!, longitude!, locationName, timezone, units);
+    cache.set(cacheKey, weather, TTL.WEATHER);
+
+    return {
+      content: [{ type: "text", text: JSON.stringify(weather, null, 2) }],
+    };
+  } catch (err) {
+    return {
+      content: [
+        { type: "text", text: `Error fetching weather: ${formatError(err)}` },
+      ],
+      isError: true,
+    };
+  }
+}