atlas-mcp-web 0.1.0

package/README.md

@@ -0,0 +1,125 @@
+# atlas-mcp-web
+
+**Give your AI agent the ability to understand any web page.** A premium Model Context Protocol (MCP) server that adds 6 powerful web extraction tools to Claude, Cursor, Windsurf, and any MCP-compatible AI agent.
+
+No more "sorry, I can't browse the web for you." Your agent gets instant, structured access to:
+
+- Clean article text for RAG and summarization
+- Complete metadata (Open Graph, Twitter Card, JSON-LD)
+- HTML tables as structured data
+- All links, grouped and classified
+- Contact info (emails, phones, socials)
+- Website tech stack detection (70+ technologies)
+
+## Install
+
+### Claude Desktop
+
+Edit `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or `%APPDATA%\Claude\claude_desktop_config.json` (Windows):
+
+```json
+{
+  "mcpServers": {
+    "atlas-web": {
+      "command": "npx",
+      "args": ["-y", "atlas-mcp-web"]
+    }
+  }
+}
+```
+
+Restart Claude Desktop. The 6 tools will appear in the tool menu.
+
+### Cursor / Windsurf
+
+Add to your MCP config:
+
+```json
+{
+  "mcpServers": {
+    "atlas-web": {
+      "command": "npx",
+      "args": ["-y", "atlas-mcp-web"]
+    }
+  }
+}
+```
+
+### Test locally
+
+```bash
+npx -y atlas-mcp-web
+```
+
+## Tools
+
+### `extract_article`
+
+Pull the main body of any news article or blog post. Strips ads, nav, comments, and boilerplate. Returns clean text plus metadata.
+
+**Input**
+```json
+{ "url": "https://www.bbc.com/news/articles/..." }
+```
+
+**Output**
+```json
+{
+  "url": "...",
+  "title": "Major AI breakthrough announced",
+  "description": "Researchers report...",
+  "authors": ["Jane Doe"],
+  "publishedAt": "2026-04-13T08:00:00Z",
+  "image": "https://...",
+  "siteName": "BBC News",
+  "language": "en",
+  "content": "The full cleaned body of the article...",
+  "wordCount": 842,
+  "readingTimeMinutes": 4,
+  "keywords": ["AI", "research", "machine learning"]
+}
+```
+
+### `extract_metadata`
+
+Complete URL metadata for link previews, SEO audits, and bookmarks: Open Graph, Twitter Card, JSON-LD structured data, favicons, and more.
+
+### `extract_tables`
+
+Pull every HTML table on a page as structured arrays with headers and rows. Perfect for financial data, sports stats, product comparisons.
+
+### `extract_links`
+
+All links on a page, grouped by internal / external / social / email / phone. With anchor text.
+
+### `extract_contact`
+
+Scan a page for contact details: emails, phone numbers, and social media handles (Twitter, LinkedIn, Instagram, Facebook, YouTube, GitHub, TikTok). Ideal for lead generation.
+
+### `detect_tech_stack`
+
+Identify the technologies powering a website: CMS, JS frameworks, CDN, analytics, hosting, ecommerce, marketing tools. 70+ signatures supported.
+
+## Why Cortex?
+
+- **Free and open source** (MIT license)
+- **No API key required** — everything runs locally through your MCP client
+- **Fast** — pure HTTP + Cheerio, no headless browser
+- **Private** — your agent hits the target site directly, no middleman logs your queries
+- **Premium options** — upgrade to the hosted API at [atlas-agent.dev](https://atlas-agent.dev) for proxy rotation, JS rendering, and higher rate limits
+
+## Use cases
+
+- **Research agents** — Give Claude the ability to read any article in full, not just the title
+- **Sales tools** — Automate lead generation by extracting contact info from company pages
+- **SEO audits** — Scan hundreds of competitor URLs for tech stack, metadata, and structured data
+- **RAG pipelines** — Feed clean article content directly into your vector database
+- **Link preview generators** — Power Discord/Slack-style unfurls in your own apps
+
+## License
+
+MIT © Cortex
+
+## Support
+
+Issues and feature requests: [github.com/atlas-agent/mcp-web-extractor/issues](https://github.com/atlas-agent/mcp-web-extractor/issues)

package/dist/index.d.ts

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+/**
+ * Atlas MCP — Web Extractor for AI Agents
+ *
+ * A Model Context Protocol server that gives Claude, Cursor, Windsurf and any
+ * MCP-compatible AI agent the ability to extract clean, structured data from
+ * any web page:
+ *
+ *   - extract_article     Clean article body for RAG pipelines
+ *   - extract_metadata    Open Graph, Twitter Card, JSON-LD
+ *   - extract_tables      All HTML tables as structured rows
+ *   - extract_links       All links grouped by category
+ *   - extract_contact     Emails, phones, social handles
+ *   - detect_tech_stack   CMS, framework, CDN, analytics
+ *
+ * Install:
+ *   npx -y atlas-mcp-web
+ *
+ * Or configure in Claude Desktop / Cursor / Windsurf:
+ *   {
+ *     "mcpServers": {
+ *       "atlas-web": {
+ *         "command": "npx",
+ *         "args": ["-y", "atlas-mcp-web"]
+ *       }
+ *     }
+ *   }
+ */
+export {};

package/dist/index.js

@@ -0,0 +1,194 @@
+#!/usr/bin/env node
+/**
+ * Atlas MCP — Web Extractor for AI Agents
+ *
+ * A Model Context Protocol server that gives Claude, Cursor, Windsurf and any
+ * MCP-compatible AI agent the ability to extract clean, structured data from
+ * any web page:
+ *
+ *   - extract_article     Clean article body for RAG pipelines
+ *   - extract_metadata    Open Graph, Twitter Card, JSON-LD
+ *   - extract_tables      All HTML tables as structured rows
+ *   - extract_links       All links grouped by category
+ *   - extract_contact     Emails, phones, social handles
+ *   - detect_tech_stack   CMS, framework, CDN, analytics
+ *
+ * Install:
+ *   npx -y atlas-mcp-web
+ *
+ * Or configure in Claude Desktop / Cursor / Windsurf:
+ *   {
+ *     "mcpServers": {
+ *       "atlas-web": {
+ *         "command": "npx",
+ *         "args": ["-y", "atlas-mcp-web"]
+ *       }
+ *     }
+ *   }
+ */
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, ListToolsRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
+import { z } from 'zod';
+import { extractArticle } from './tools/article.js';
+import { extractMetadata } from './tools/metadata.js';
+import { extractTables } from './tools/tables.js';
+import { extractLinks } from './tools/links.js';
+import { extractContact } from './tools/contact.js';
+import { detectTechStack } from './tools/techstack.js';
+import { fetchHtml } from './lib/fetch.js';
+const UrlInput = z.object({
+    url: z.string().url().describe('The URL to extract data from'),
+});
+const LinksInput = UrlInput.extend({
+    sameDomainOnly: z
+        .boolean()
+        .optional()
+        .default(false)
+        .describe('If true, only return links on the same domain'),
+});
+const TOOLS = [
+    {
+        name: 'extract_article',
+        description: 'Extract the main article body from any news article, blog post, or long-form web page. Returns clean, readable text plus title, authors, publish date, word count, and keywords. Strips ads, navigation, comments, and boilerplate. Perfect for feeding content into RAG pipelines, summarization, or analysis.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                url: { type: 'string', description: 'URL of the article to extract' },
+            },
+            required: ['url'],
+        },
+    },
+    {
+        name: 'extract_metadata',
+        description: 'Extract all metadata from any URL: Open Graph tags (title, description, image, site_name), Twitter Card metadata, JSON-LD structured data, canonical URL, favicons, and more. Ideal for building link previews, SEO audits, or enriching bookmarks.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                url: { type: 'string', description: 'URL to extract metadata from' },
+            },
+            required: ['url'],
+        },
+    },
+    {
+        name: 'extract_tables',
+        description: 'Extract all HTML tables from a web page as structured arrays. Each table is returned with its headers and rows, ready for analysis. Perfect for scraping financial data, sports stats, product comparisons, or any tabular content.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                url: { type: 'string', description: 'URL containing the tables to extract' },
+            },
+            required: ['url'],
+        },
+    },
+    {
+        name: 'extract_links',
+        description: 'Extract all links from a web page grouped by category: internal, external, social media, email, and phone. Returns anchor text alongside each URL. Useful for link audits, sitemap generation, and finding contact information.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                url: { type: 'string', description: 'URL to extract links from' },
+                sameDomainOnly: {
+                    type: 'boolean',
+                    description: 'If true, only return links on the same domain as the source URL',
+                    default: false,
+                },
+            },
+            required: ['url'],
+        },
+    },
+    {
+        name: 'extract_contact',
+        description: "Scan a web page for contact information: email addresses, phone numbers, and social media handles (Twitter, LinkedIn, Instagram, Facebook, YouTube, GitHub). Returns normalized, deduplicated results. Perfect for lead generation and sales prospecting.",
+        inputSchema: {
+            type: 'object',
+            properties: {
+                url: { type: 'string', description: 'URL to scan for contact info' },
+            },
+            required: ['url'],
+        },
+    },
+    {
+        name: 'detect_tech_stack',
+        description: 'Detect the technologies powering a website: CMS (WordPress, Shopify, etc), JS frameworks (React, Next.js, Vue), CDN (Cloudflare, Fastly), analytics tools (Google Analytics, Mixpanel), hosting (Vercel, Netlify), ecommerce platform, and more. 70+ technologies supported.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                url: { type: 'string', description: 'URL of the website to analyze' },
+            },
+            required: ['url'],
+        },
+    },
+];
+const server = new Server({ name: 'atlas-web', version: '0.1.0' }, { capabilities: { tools: {} } });
+server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOLS }));
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    const { name, arguments: args } = request.params;
+    try {
+        if (name === 'extract_article') {
+            const { url } = UrlInput.parse(args);
+            const { $, html, finalUrl } = await fetchHtml(url);
+            const result = extractArticle({ url: finalUrl, $, html });
+            return toolResult(result);
+        }
+        if (name === 'extract_metadata') {
+            const { url } = UrlInput.parse(args);
+            const { $, finalUrl, headers } = await fetchHtml(url);
+            const result = extractMetadata({ url: finalUrl, $, headers });
+            return toolResult(result);
+        }
+        if (name === 'extract_tables') {
+            const { url } = UrlInput.parse(args);
+            const { $, finalUrl } = await fetchHtml(url);
+            const result = extractTables({ url: finalUrl, $ });
+            return toolResult(result);
+        }
+        if (name === 'extract_links') {
+            const { url, sameDomainOnly } = LinksInput.parse(args);
+            const { $, finalUrl } = await fetchHtml(url);
+            const result = extractLinks({ url: finalUrl, $, sameDomainOnly });
+            return toolResult(result);
+        }
+        if (name === 'extract_contact') {
+            const { url } = UrlInput.parse(args);
+            const { $, html, finalUrl } = await fetchHtml(url);
+            const result = extractContact({ url: finalUrl, $, html });
+            return toolResult(result);
+        }
+        if (name === 'detect_tech_stack') {
+            const { url } = UrlInput.parse(args);
+            const { $, html, finalUrl, headers } = await fetchHtml(url);
+            const result = detectTechStack({ url: finalUrl, $, html, headers });
+            return toolResult(result);
+        }
+        throw new Error(`Unknown tool: ${name}`);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+            content: [{ type: 'text', text: `Error: ${message}` }],
+            isError: true,
+        };
+    }
+});
+function toolResult(data) {
+    return {
+        content: [
+            {
+                type: 'text',
+                text: JSON.stringify(data, null, 2),
+            },
+        ],
+    };
+}
+async function main() {
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    // eslint-disable-next-line no-console
+    console.error('Atlas MCP Web Extractor ready');
+}
+main().catch((err) => {
+    // eslint-disable-next-line no-console
+    console.error('Fatal error:', err);
+    process.exit(1);
+});

package/dist/lib/fetch.d.ts

@@ -0,0 +1,12 @@
+/**
+ * HTTP fetch with polite defaults, redirect following, and Cheerio parsing.
+ */
+import { type CheerioAPI } from 'cheerio';
+export interface FetchResult {
+    $: CheerioAPI;
+    html: string;
+    finalUrl: string;
+    headers: Record<string, string>;
+    statusCode: number;
+}
+export declare function fetchHtml(url: string, timeoutMs?: number): Promise<FetchResult>;

package/dist/lib/fetch.js

@@ -0,0 +1,39 @@
+/**
+ * HTTP fetch with polite defaults, redirect following, and Cheerio parsing.
+ */
+import { load } from 'cheerio';
+const USER_AGENT = 'Mozilla/5.0 (compatible; Cortex-MCP/0.1; +https://atlas-agent.dev/bot)';
+export async function fetchHtml(url, timeoutMs = 20000) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const response = await fetch(url, {
+            headers: {
+                'User-Agent': USER_AGENT,
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                'Accept-Language': 'en-US,en;q=0.9',
+            },
+            redirect: 'follow',
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status} ${response.statusText} when fetching ${url}`);
+        }
+        const html = await response.text();
+        const $ = load(html);
+        const headers = {};
+        response.headers.forEach((value, key) => {
+            headers[key.toLowerCase()] = value;
+        });
+        return {
+            $,
+            html,
+            finalUrl: response.url,
+            headers,
+            statusCode: response.status,
+        };
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}

package/dist/tools/article.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Clean article extraction: title, authors, date, content.
+ * JSON-LD first, then OG/meta, then readability heuristics.
+ */
+import type { CheerioAPI } from 'cheerio';
+export declare function extractArticle({ url, $, }: {
+    url: string;
+    $: CheerioAPI;
+    html: string;
+}): {
+    url: string;
+    title: string;
+    description: string;
+    authors: string[];
+    publishedAt: string | null;
+    image: string | null;
+    siteName: string | null;
+    language: string | null;
+    content: string;
+    wordCount: number;
+    readingTimeMinutes: number;
+    keywords: string[];
+};

package/dist/tools/article.js

@@ -0,0 +1,260 @@
+/**
+ * Clean article extraction: title, authors, date, content.
+ * JSON-LD first, then OG/meta, then readability heuristics.
+ */
+const MIN_CHARS = 280;
+export function extractArticle({ url, $, }) {
+    const meta = extractMeta($);
+    const jsonLd = extractJsonLd($);
+    const title = jsonLd.headline ||
+        meta.ogTitle ||
+        meta.twitterTitle ||
+        ($('h1').first().text() || '').trim() ||
+        ($('title').first().text() || '').trim();
+    const description = jsonLd.description ||
+        meta.ogDescription ||
+        meta.twitterDescription ||
+        meta.description ||
+        '';
+    const authors = jsonLd.authors.length
+        ? jsonLd.authors
+        : extractAuthorsFromMeta($, meta);
+    const publishedAt = jsonLd.datePublished ||
+        meta.articlePublishedTime ||
+        $('time[datetime]').first().attr('datetime') ||
+        null;
+    const image = jsonLd.image || meta.ogImage || meta.twitterImage || null;
+    const siteName = meta.ogSiteName || extractDomain(url);
+    const language = ($('html').attr('lang') || meta.ogLocale || '').split('-')[0].toLowerCase() ||
+        null;
+    let content = extractContentFromArticle($);
+    if (!content || content.length < MIN_CHARS) {
+        content = extractContentByReadability($);
+    }
+    content = cleanText(content);
+    const words = content.split(/\s+/).filter(Boolean);
+    const wordCount = words.length;
+    const readingTimeMinutes = Math.max(1, Math.round(wordCount / 220));
+    return {
+        url,
+        title: cleanText(title),
+        description: cleanText(description),
+        authors,
+        publishedAt,
+        image,
+        siteName,
+        language,
+        content,
+        wordCount,
+        readingTimeMinutes,
+        keywords: extractKeywords($, meta, jsonLd),
+    };
+}
+function extractMeta($) {
+    const meta = {};
+    $('meta').each((_, el) => {
+        const name = ($(el).attr('name') || '').toLowerCase();
+        const property = ($(el).attr('property') || '').toLowerCase();
+        const content = $(el).attr('content') || '';
+        if (!content)
+            return;
+        if (name === 'description')
+            meta.description = content;
+        if (name === 'keywords')
+            meta.keywords = content;
+        if (name === 'author')
+            meta.author = content;
+        if (name === 'twitter:title')
+            meta.twitterTitle = content;
+        if (name === 'twitter:description')
+            meta.twitterDescription = content;
+        if (name === 'twitter:image')
+            meta.twitterImage = content;
+        if (property === 'og:title')
+            meta.ogTitle = content;
+        if (property === 'og:description')
+            meta.ogDescription = content;
+        if (property === 'og:image')
+            meta.ogImage = content;
+        if (property === 'og:site_name')
+            meta.ogSiteName = content;
+        if (property === 'og:locale')
+            meta.ogLocale = content;
+        if (property === 'article:published_time')
+            meta.articlePublishedTime = content;
+        if (property === 'article:modified_time')
+            meta.articleModifiedTime = content;
+        if (property === 'article:author')
+            meta.articleAuthor = content;
+        if (property === 'article:tag') {
+            if (!Array.isArray(meta.articleTags))
+                meta.articleTags = [];
+            meta.articleTags.push(content);
+        }
+    });
+    return meta;
+}
+function extractJsonLd($) {
+    const out = {
+        headline: null,
+        description: null,
+        authors: [],
+        datePublished: null,
+        image: null,
+        keywords: [],
+    };
+    $('script[type="application/ld+json"]').each((_, el) => {
+        const raw = $(el).contents().text();
+        if (!raw)
+            return;
+        let data;
+        try {
+            data = JSON.parse(raw);
+        }
+        catch {
+            return;
+        }
+        const items = Array.isArray(data) ? data : [data];
+        for (const item of items) {
+            collectJsonLdArticle(item, out);
+            if (item && typeof item === 'object' && '@graph' in item) {
+                const graph = item['@graph'];
+                if (Array.isArray(graph))
+                    for (const sub of graph)
+                        collectJsonLdArticle(sub, out);
+            }
+        }
+    });
+    return out;
+}
+function collectJsonLdArticle(item, out) {
+    if (!item || typeof item !== 'object')
+        return;
+    const obj = item;
+    const type = obj['@type'];
+    const isArticle = type === 'NewsArticle' ||
+        type === 'Article' ||
+        type === 'BlogPosting' ||
+        type === 'Report' ||
+        (Array.isArray(type) && type.some((t) => typeof t === 'string' && /Article|Posting/.test(t)));
+    if (!isArticle)
+        return;
+    if (!out.headline && typeof obj.headline === 'string')
+        out.headline = obj.headline;
+    if (!out.description && typeof obj.description === 'string')
+        out.description = obj.description;
+    if (!out.datePublished && typeof obj.datePublished === 'string')
+        out.datePublished = obj.datePublished;
+    if (!out.image) {
+        const img = obj.image;
+        if (typeof img === 'string')
+            out.image = img;
+        else if (Array.isArray(img) && img.length) {
+            const first = img[0];
+            out.image = typeof first === 'string' ? first : first?.url || null;
+        }
+        else if (img && typeof img === 'object') {
+            out.image = img.url || null;
+        }
+    }
+    const authors = [].concat(obj.author || []).flat();
+    for (const a of authors) {
+        if (!a)
+            continue;
+        const name = typeof a === 'string' ? a : a?.name || null;
+        if (name && !out.authors.includes(name))
+            out.authors.push(name);
+    }
+    if (obj.keywords) {
+        const kws = Array.isArray(obj.keywords)
+            ? obj.keywords
+            : String(obj.keywords).split(',').map((k) => k.trim());
+        for (const k of kws)
+            if (k && !out.keywords.includes(k))
+                out.keywords.push(k);
+    }
+}
+function extractAuthorsFromMeta($, meta) {
+    const authors = new Set();
+    if (meta.author)
+        authors.add(meta.author);
+    if (meta.articleAuthor)
+        authors.add(meta.articleAuthor);
+    $('[rel="author"], .author, .byline, .post-author').each((_, el) => {
+        const t = $(el).text().trim();
+        if (t && t.length < 100)
+            authors.add(t);
+    });
+    return Array.from(authors).filter(Boolean).slice(0, 10);
+}
+function extractContentFromArticle($) {
+    const noiseSelectors = [
+        'script', 'style', 'nav', 'header', 'footer', 'aside',
+        '.ad', '.ads', '.advertisement', '.social', '.share', '.newsletter',
+        '[class*="subscribe"]', '[class*="paywall"]', '[class*="related"]',
+        '[class*="comment"]', '[class*="sidebar"]',
+    ];
+    const $clone = $.root().clone();
+    noiseSelectors.forEach((sel) => $clone.find(sel).remove());
+    const selectors = [
+        'article',
+        '[itemprop="articleBody"]',
+        '[role="main"] article',
+        'main article',
+        '.article-body',
+        '.post-content',
+        '.entry-content',
+        '.story-body',
+        'main',
+    ];
+    for (const sel of selectors) {
+        const el = $clone.find(sel).first();
+        if (!el.length)
+            continue;
+        const text = el.find('p').map((_, p) => $(p).text()).get().join('\n\n').trim();
+        if (text.length >= MIN_CHARS)
+            return text;
+    }
+    return '';
+}
+function extractContentByReadability($) {
+    const paragraphs = [];
+    $('p').each((_, el) => {
+        const text = $(el).text().trim();
+        if (text.length >= 40)
+            paragraphs.push(text);
+    });
+    return paragraphs.join('\n\n');
+}
+function extractKeywords($, meta, jsonLd) {
+    const set = new Set();
+    if (meta.keywords) {
+        for (const k of String(meta.keywords).split(',')) {
+            const trimmed = k.trim();
+            if (trimmed)
+                set.add(trimmed);
+        }
+    }
+    if (Array.isArray(meta.articleTags))
+        for (const t of meta.articleTags)
+            set.add(t);
+    for (const k of jsonLd.keywords)
+        set.add(k);
+    return Array.from(set).slice(0, 25);
+}
+function cleanText(text) {
+    if (!text)
+        return '';
+    return String(text)
+        .replace(/\s+/g, ' ')
+        .replace(/\s([.,;:!?])/g, '$1')
+        .trim();
+}
+function extractDomain(url) {
+    try {
+        return new URL(url).hostname.replace(/^www\./, '');
+    }
+    catch {
+        return null;
+    }
+}