data-aggregator-mcp 1.0.0

package/src/tools/news.ts

@@ -0,0 +1,238 @@
+/**
+ * search_news - News aggregation from multiple sources.
+ *
+ * Data sources:
+ *   - NewsAPI.org (optional key via NEWS_API_KEY env)
+ *   - Google News RSS (free, no key required)
+ *
+ * Falls back to Google News RSS when no NewsAPI key is configured.
+ */
+
+import { cache, TTL } from "../utils/cache.js";
+import { rateLimiter } from "../utils/rate-limiter.js";
+import { httpGetJson, httpGetText, formatError } from "../utils/http.js";
+import type { NewsArticle, ToolResult } from "../types.js";
+
+// ─── Google News RSS parsing ───────────────────────────────────────────────
+
+function parseRssItems(xml: string): NewsArticle[] {
+  const articles: NewsArticle[] = [];
+  const itemRegex = /<item>([\s\S]*?)<\/item>/gi;
+  let itemMatch;
+
+  while ((itemMatch = itemRegex.exec(xml)) !== null) {
+    const item = itemMatch[1]!;
+
+    const title = extractXmlTag(item, "title");
+    const link = extractXmlTag(item, "link");
+    const pubDate = extractXmlTag(item, "pubDate");
+    const description = stripCdata(extractXmlTag(item, "description"));
+    const source = extractXmlTag(item, "source");
+
+    if (title && link) {
+      articles.push({
+        title: decodeHtmlEntities(title),
+        description: decodeHtmlEntities(stripHtmlTags(description)).slice(0, 500),
+        url: link,
+        source: source || "Google News",
+        publishedAt: pubDate ? new Date(pubDate).toISOString() : new Date().toISOString(),
+      });
+    }
+  }
+
+  return articles;
+}
+
+function extractXmlTag(xml: string, tag: string): string {
+  const re = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}>`, "i");
+  const m = xml.match(re);
+  return m ? m[1]!.trim() : "";
+}
+
+function stripCdata(text: string): string {
+  return text.replace(/<!\[CDATA\[/g, "").replace(/\]\]>/g, "");
+}
+
+function stripHtmlTags(text: string): string {
+  return text.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+}
+
+function decodeHtmlEntities(text: string): string {
+  return text
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&apos;/g, "'");
+}
+
+// ─── Google News RSS fetch ─────────────────────────────────────────────────
+
+async function fetchGoogleNews(args: {
+  query?: string;
+  language?: string;
+  maxResults?: number;
+}): Promise<NewsArticle[]> {
+  const { query, language = "en", maxResults = 10 } = args;
+
+  let url: string;
+  if (query) {
+    url = `https://news.google.com/rss/search?q=${encodeURIComponent(query)}&hl=${language}&gl=US&ceid=US:${language}`;
+  } else {
+    url = `https://news.google.com/rss?hl=${language}&gl=US&ceid=US:${language}`;
+  }
+
+  await rateLimiter.acquire("googlenews");
+  const xml = await httpGetText(url, { rotateUserAgent: true });
+  const articles = parseRssItems(xml);
+
+  return articles.slice(0, maxResults);
+}
+
+// ─── NewsAPI.org fetch ─────────────────────────────────────────────────────
+
+async function fetchNewsApi(args: {
+  query?: string;
+  category?: string;
+  source?: string;
+  language?: string;
+  from?: string;
+  to?: string;
+  maxResults?: number;
+  apiKey: string;
+}): Promise<NewsArticle[]> {
+  const {
+    query,
+    category,
+    source,
+    language = "en",
+    from,
+    to,
+    maxResults = 10,
+    apiKey,
+  } = args;
+
+  await rateLimiter.acquire("newsapi");
+
+  // Use /everything for queries, /top-headlines for category/source browsing
+  let url: string;
+  const params = new URLSearchParams();
+  params.set("apiKey", apiKey);
+  params.set("pageSize", String(Math.min(maxResults, 100)));
+  params.set("language", language);
+
+  if (query) {
+    url = "https://newsapi.org/v2/everything";
+    params.set("q", query);
+    params.set("sortBy", "publishedAt");
+    if (from) params.set("from", from);
+    if (to) params.set("to", to);
+    if (source) params.set("sources", source);
+  } else {
+    url = "https://newsapi.org/v2/top-headlines";
+    if (category) params.set("category", category);
+    if (source) params.set("sources", source);
+    // top-headlines needs country if no source
+    if (!source) params.set("country", "us");
+  }
+
+  const fullUrl = `${url}?${params.toString()}`;
+  const data: any = await httpGetJson(fullUrl);
+
+  if (data.status !== "ok") {
+    throw new Error(data.message || "NewsAPI returned an error");
+  }
+
+  return (data.articles ?? []).map((a: any) => ({
+    title: a.title ?? "",
+    description: a.description ?? "",
+    url: a.url ?? "",
+    source: a.source?.name ?? "Unknown",
+    publishedAt: a.publishedAt ?? new Date().toISOString(),
+    author: a.author ?? undefined,
+    imageUrl: a.urlToImage ?? undefined,
+  }));
+}
+
+// ─── Public handler ────────────────────────────────────────────────────────
+
+export async function searchNews(args: {
+  query?: string;
+  category?: string;
+  source?: string;
+  language?: string;
+  from?: string;
+  to?: string;
+  maxResults?: number;
+}): Promise<ToolResult> {
+  try {
+    const {
+      query,
+      category,
+      source,
+      language = "en",
+      from,
+      to,
+      maxResults = 10,
+    } = args;
+
+    const cacheKey = `news:${query ?? ""}:${category ?? ""}:${source ?? ""}:${language}:${from ?? ""}:${to ?? ""}`;
+    const cached = cache.get<NewsArticle[]>(cacheKey);
+    if (cached) {
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify({ articles: cached, count: cached.length, cached: true }, null, 2),
+          },
+        ],
+      };
+    }
+
+    let articles: NewsArticle[];
+    const newsApiKey = process.env.NEWS_API_KEY;
+
+    if (newsApiKey) {
+      articles = await fetchNewsApi({
+        query,
+        category,
+        source,
+        language,
+        from,
+        to,
+        maxResults,
+        apiKey: newsApiKey,
+      });
+    } else {
+      // Fall back to Google News RSS (category/source filtering not supported)
+      articles = await fetchGoogleNews({ query, language, maxResults });
+    }
+
+    cache.set(cacheKey, articles, TTL.NEWS);
+
+    return {
+      content: [
+        {
+          type: "text",
+          text: JSON.stringify(
+            {
+              articles,
+              count: articles.length,
+              source: newsApiKey ? "NewsAPI.org" : "Google News RSS",
+            },
+            null,
+            2
+          ),
+        },
+      ],
+    };
+  } catch (err) {
+    return {
+      content: [
+        { type: "text", text: `Error fetching news: ${formatError(err)}` },
+      ],
+      isError: true,
+    };
+  }
+}

package/src/tools/public-data.ts

@@ -0,0 +1,325 @@
+/**
+ * query_public_data - Miscellaneous public API queries.
+ *
+ * Sub-commands:
+ *   - wikipedia    : Get a Wikipedia article summary
+ *   - ip_geolocation : Look up geographic info for an IP address
+ *   - dns_lookup   : DNS record lookup for a domain
+ *   - expand_url   : Expand a shortened URL to its final destination
+ *
+ * All APIs used are free and require no API keys.
+ */
+
+import { cache, TTL } from "../utils/cache.js";
+import { rateLimiter } from "../utils/rate-limiter.js";
+import { httpGetJson, httpFetch, formatError } from "../utils/http.js";
+import type {
+  WikipediaSummary,
+  IpGeolocation,
+  DnsRecord,
+  UrlInfo,
+  ToolResult,
+} from "../types.js";
+import { resolve } from "node:dns/promises";
+
+// ─── Wikipedia Summary ─────────────────────────────────────────────────────
+
+async function fetchWikipediaSummary(
+  query: string,
+  language = "en"
+): Promise<WikipediaSummary> {
+  // Use the Wikipedia REST API summary endpoint
+  const encoded = encodeURIComponent(query.replace(/\s+/g, "_"));
+  const url = `https://${language}.wikipedia.org/api/rest_v1/page/summary/${encoded}`;
+
+  await rateLimiter.acquire("wikipedia");
+  const data: any = await httpGetJson(url);
+
+  if (data.type === "disambiguation") {
+    return {
+      title: data.title ?? query,
+      extract: `This is a disambiguation page. ${data.extract ?? "Multiple meanings exist for this term."}`,
+      url: data.content_urls?.desktop?.page ?? `https://${language}.wikipedia.org/wiki/${encoded}`,
+      thumbnail: data.thumbnail?.source,
+      timestamp: new Date().toISOString(),
+    };
+  }
+
+  if (!data.extract) {
+    throw new Error(`No Wikipedia article found for "${query}". Try a different search term.`);
+  }
+
+  return {
+    title: data.title ?? query,
+    extract: data.extract,
+    url: data.content_urls?.desktop?.page ?? `https://${language}.wikipedia.org/wiki/${encoded}`,
+    thumbnail: data.thumbnail?.source,
+    timestamp: new Date().toISOString(),
+  };
+}
+
+// ─── IP Geolocation ────────────────────────────────────────────────────────
+
+async function fetchIpGeolocation(ip?: string): Promise<IpGeolocation> {
+  // ip-api.com is free for non-commercial use, no key required
+  const target = ip ?? ""; // empty string = caller's IP
+  const url = `http://ip-api.com/json/${target}?fields=status,message,country,countryCode,regionName,city,lat,lon,timezone,isp,query`;
+
+  await rateLimiter.acquire("ipgeo");
+  const data: any = await httpGetJson(url);
+
+  if (data.status !== "success") {
+    throw new Error(data.message ?? `Geolocation failed for IP "${ip ?? "self"}"`);
+  }
+
+  return {
+    ip: data.query,
+    country: data.country,
+    countryCode: data.countryCode,
+    region: data.regionName,
+    city: data.city,
+    latitude: data.lat,
+    longitude: data.lon,
+    timezone: data.timezone,
+    isp: data.isp,
+  };
+}
+
+// ─── DNS Lookup ────────────────────────────────────────────────────────────
+
+async function fetchDnsRecords(
+  domain: string,
+  recordType: string
+): Promise<DnsRecord> {
+  await rateLimiter.acquire("dns");
+
+  const type = recordType.toUpperCase();
+  let records: string[] = [];
+
+  try {
+    switch (type) {
+      case "A": {
+        const result = await resolve(domain, "A");
+        records = result;
+        break;
+      }
+      case "AAAA": {
+        const result = await resolve(domain, "AAAA");
+        records = result;
+        break;
+      }
+      case "MX": {
+        const result = await resolve(domain, "MX");
+        records = (result as any[]).map(
+          (r: any) => `${r.priority} ${r.exchange}`
+        );
+        break;
+      }
+      case "TXT": {
+        const result = await resolve(domain, "TXT");
+        records = (result as string[][]).map((r) => r.join(""));
+        break;
+      }
+      case "NS": {
+        const result = await resolve(domain, "NS");
+        records = result;
+        break;
+      }
+      case "CNAME": {
+        const result = await resolve(domain, "CNAME");
+        records = result;
+        break;
+      }
+      case "SOA": {
+        const result = await resolve(domain, "SOA");
+        const soa = result as any;
+        records = [
+          `nsname=${soa.nsname} hostmaster=${soa.hostmaster} serial=${soa.serial}`,
+        ];
+        break;
+      }
+      default:
+        throw new Error(
+          `Unsupported record type: ${type}. Supported: A, AAAA, MX, TXT, NS, CNAME, SOA`
+        );
+    }
+  } catch (err: any) {
+    if (err.code === "ENODATA" || err.code === "ENOTFOUND") {
+      records = [];
+    } else {
+      throw err;
+    }
+  }
+
+  return {
+    domain,
+    type,
+    records,
+  };
+}
+
+// ─── URL Expander ──────────────────────────────────────────────────────────
+
+async function expandUrl(shortUrl: string): Promise<UrlInfo> {
+  await rateLimiter.acquire("generic");
+
+  // Follow redirects by making a HEAD request
+  const response = await httpFetch(shortUrl, {
+    method: "GET",
+    maxRetries: 1,
+    timeoutMs: 10_000,
+    rotateUserAgent: true,
+  });
+
+  return {
+    originalUrl: shortUrl,
+    expandedUrl: response.url,
+    statusCode: response.status,
+    contentType: response.headers.get("content-type") ?? undefined,
+  };
+}
+
+// ─── Public handler ────────────────────────────────────────────────────────
+
+type SubCommand = "wikipedia" | "ip_geolocation" | "dns_lookup" | "expand_url";
+
+export async function queryPublicData(args: {
+  command: SubCommand;
+  query?: string;
+  ip?: string;
+  domain?: string;
+  recordType?: string;
+  url?: string;
+  language?: string;
+}): Promise<ToolResult> {
+  try {
+    const { command } = args;
+
+    switch (command) {
+      case "wikipedia": {
+        const query = args.query;
+        if (!query) {
+          return {
+            content: [
+              { type: "text", text: 'The "query" parameter is required for Wikipedia lookups.' },
+            ],
+            isError: true,
+          };
+        }
+
+        const cacheKey = `wiki:${query.toLowerCase()}:${args.language ?? "en"}`;
+        const cached = cache.get<WikipediaSummary>(cacheKey);
+        if (cached) {
+          return {
+            content: [
+              { type: "text", text: JSON.stringify({ ...cached, cached: true }, null, 2) },
+            ],
+          };
+        }
+
+        const summary = await fetchWikipediaSummary(query, args.language);
+        cache.set(cacheKey, summary, TTL.PUBLIC);
+
+        return {
+          content: [{ type: "text", text: JSON.stringify(summary, null, 2) }],
+        };
+      }
+
+      case "ip_geolocation": {
+        const cacheKey = `ipgeo:${args.ip ?? "self"}`;
+        const cached = cache.get<IpGeolocation>(cacheKey);
+        if (cached) {
+          return {
+            content: [
+              { type: "text", text: JSON.stringify({ ...cached, cached: true }, null, 2) },
+            ],
+          };
+        }
+
+        const geo = await fetchIpGeolocation(args.ip);
+        cache.set(cacheKey, geo, TTL.PUBLIC);
+
+        return {
+          content: [{ type: "text", text: JSON.stringify(geo, null, 2) }],
+        };
+      }
+
+      case "dns_lookup": {
+        const domain = args.domain;
+        if (!domain) {
+          return {
+            content: [
+              { type: "text", text: 'The "domain" parameter is required for DNS lookups.' },
+            ],
+            isError: true,
+          };
+        }
+
+        const recordType = args.recordType ?? "A";
+        const cacheKey = `dns:${domain.toLowerCase()}:${recordType}`;
+        const cached = cache.get<DnsRecord>(cacheKey);
+        if (cached) {
+          return {
+            content: [
+              { type: "text", text: JSON.stringify({ ...cached, cached: true }, null, 2) },
+            ],
+          };
+        }
+
+        const records = await fetchDnsRecords(domain, recordType);
+        cache.set(cacheKey, records, TTL.PUBLIC);
+
+        return {
+          content: [{ type: "text", text: JSON.stringify(records, null, 2) }],
+        };
+      }
+
+      case "expand_url": {
+        const url = args.url;
+        if (!url) {
+          return {
+            content: [
+              { type: "text", text: 'The "url" parameter is required for URL expansion.' },
+            ],
+            isError: true,
+          };
+        }
+
+        const cacheKey = `url:${url}`;
+        const cached = cache.get<UrlInfo>(cacheKey);
+        if (cached) {
+          return {
+            content: [
+              { type: "text", text: JSON.stringify({ ...cached, cached: true }, null, 2) },
+            ],
+          };
+        }
+
+        const info = await expandUrl(url);
+        cache.set(cacheKey, info, TTL.PUBLIC);
+
+        return {
+          content: [{ type: "text", text: JSON.stringify(info, null, 2) }],
+        };
+      }
+
+      default:
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Unknown sub-command: "${command}". Available commands: wikipedia, ip_geolocation, dns_lookup, expand_url`,
+            },
+          ],
+          isError: true,
+        };
+    }
+  } catch (err) {
+    return {
+      content: [
+        { type: "text", text: `Error in public data query: ${formatError(err)}` },
+      ],
+      isError: true,
+    };
+  }
+}