@just-every/mcp-read-website-fast 0.1.1

package/dist/index.js

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import { CrawlQueue } from './crawler/queue.js';
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const packageJson = JSON.parse(readFileSync(join(__dirname, '../package.json'), 'utf-8'));
+const program = new Command();
+program
+    .name('mcp')
+    .description('Markdown Content Preprocessor - Extract and convert web content to clean Markdown')
+    .version(packageJson.version);
+program
+    .command('fetch <url>')
+    .description('Fetch a URL and convert to Markdown')
+    .option('-d, --depth <number>', 'Crawl depth (0 = single page)', '0')
+    .option('-c, --concurrency <number>', 'Max concurrent requests', '3')
+    .option('--no-robots', 'Ignore robots.txt')
+    .option('--all-origins', 'Allow cross-origin crawling')
+    .option('-u, --user-agent <string>', 'Custom user agent')
+    .option('--cache-dir <path>', 'Cache directory', '.cache')
+    .option('-t, --timeout <ms>', 'Request timeout in milliseconds', '30000')
+    .option('-o, --output <format>', 'Output format: json, markdown, or both', 'markdown')
+    .action(async (url, options) => {
+    try {
+        const crawlOptions = {
+            depth: parseInt(options.depth, 10),
+            maxConcurrency: parseInt(options.concurrency, 10),
+            respectRobots: options.robots,
+            sameOriginOnly: !options.allOrigins,
+            userAgent: options.userAgent,
+            cacheDir: options.cacheDir,
+            timeout: parseInt(options.timeout, 10)
+        };
+        const queue = new CrawlQueue(crawlOptions);
+        await queue.init();
+        console.error(`Fetching ${url}...`);
+        const results = await queue.crawl(url);
+        if (options.output === 'json') {
+            console.log(JSON.stringify(results, null, 2));
+        }
+        else if (options.output === 'markdown') {
+            results.forEach(result => {
+                if (result.error) {
+                    console.error(`Error for ${result.url}: ${result.error}`);
+                }
+                else if (result.markdown) {
+                    console.log(result.markdown);
+                    if (results.length > 1) {
+                        console.log('\n---\n');
+                    }
+                }
+            });
+        }
+        else if (options.output === 'both') {
+            results.forEach(result => {
+                console.log(`\n## URL: ${result.url}\n`);
+                if (result.error) {
+                    console.error(`Error: ${result.error}`);
+                }
+                else {
+                    console.log(result.markdown);
+                }
+            });
+        }
+        const hasErrors = results.some(r => r.error);
+        if (hasErrors) {
+            process.exit(1);
+        }
+    }
+    catch (error) {
+        console.error('Error:', error instanceof Error ? error.message : error);
+        process.exit(1);
+    }
+});
+program
+    .command('clear-cache')
+    .description('Clear the cache directory')
+    .option('--cache-dir <path>', 'Cache directory', '.cache')
+    .action(async (options) => {
+    try {
+        const { rm } = await import('fs/promises');
+        await rm(options.cacheDir, { recursive: true, force: true });
+        console.log(`Cache cleared: ${options.cacheDir}`);
+    }
+    catch (error) {
+        console.error('Error clearing cache:', error);
+        process.exit(1);
+    }
+});
+program
+    .command('serve')
+    .description('Run as an MCP server')
+    .action(async () => {
+    await import('./serve.js');
+});
+program.parse();

package/dist/internal/fetchMarkdown.d.ts

@@ -0,0 +1,16 @@
+export interface FetchMarkdownOptions {
+    depth?: number;
+    maxConcurrency?: number;
+    respectRobots?: boolean;
+    sameOriginOnly?: boolean;
+    userAgent?: string;
+    cacheDir?: string;
+    timeout?: number;
+}
+export interface FetchMarkdownResult {
+    markdown: string;
+    title?: string;
+    links?: string[];
+    error?: string;
+}
+export declare function fetchMarkdown(url: string, options?: FetchMarkdownOptions): Promise<FetchMarkdownResult>;

package/dist/internal/fetchMarkdown.js

@@ -0,0 +1,36 @@
+import { CrawlQueue } from '../crawler/queue.js';
+export async function fetchMarkdown(url, options = {}) {
+    try {
+        const crawlOptions = {
+            depth: options.depth ?? 0,
+            maxConcurrency: options.maxConcurrency ?? 3,
+            respectRobots: options.respectRobots ?? true,
+            sameOriginOnly: options.sameOriginOnly ?? true,
+            userAgent: options.userAgent,
+            cacheDir: options.cacheDir ?? '.cache',
+            timeout: options.timeout ?? 30000
+        };
+        const queue = new CrawlQueue(crawlOptions);
+        await queue.init();
+        const results = await queue.crawl(url);
+        const mainResult = results[0];
+        if (!mainResult) {
+            return {
+                markdown: '',
+                error: 'No results returned'
+            };
+        }
+        return {
+            markdown: mainResult.markdown,
+            title: mainResult.title,
+            links: mainResult.links,
+            error: mainResult.error
+        };
+    }
+    catch (error) {
+        return {
+            markdown: '',
+            error: error instanceof Error ? error.message : 'Unknown error'
+        };
+    }
+}

package/dist/parser/article.d.ts

@@ -0,0 +1,4 @@
+import { JSDOM } from 'jsdom';
+import { Article } from '../types.js';
+export declare function extractArticle(dom: JSDOM): Article | null;
+export declare function hasContent(html: string): boolean;

package/dist/parser/article.js

@@ -0,0 +1,115 @@
+import { Readability } from '@mozilla/readability';
+export function extractArticle(dom) {
+    const document = dom.window.document;
+    const baseUrl = dom.window.location.href;
+    const articleParagraph = document.querySelector('article p');
+    const hasStrongArticleIndicators = (document.querySelector('article') !== null &&
+        articleParagraph?.textContent && articleParagraph.textContent.length > 200) ||
+        document.querySelector('[itemtype*="BlogPosting"]') !== null ||
+        document.querySelector('[itemtype*="NewsArticle"]') !== null ||
+        document.querySelector('meta[property="article:published_time"]') !== null;
+    if (hasStrongArticleIndicators) {
+        const documentClone = document.cloneNode(true);
+        const reader = new Readability(documentClone);
+        const article = reader.parse();
+        if (article && article.content && article.content.trim().length > 500) {
+            return {
+                title: article.title || 'Untitled',
+                content: article.content || '',
+                textContent: article.textContent || '',
+                length: article.length || 0,
+                excerpt: article.excerpt || '',
+                byline: article.byline || null,
+                dir: article.dir || null,
+                lang: article.lang || null,
+                siteName: article.siteName || null,
+                publishedTime: article.publishedTime || null,
+                baseUrl
+            };
+        }
+    }
+    return extractContentManually(dom);
+}
+function extractContentManually(dom) {
+    try {
+        const document = dom.window.document;
+        const baseUrl = dom.window.location.href;
+        const title = document.querySelector('title')?.textContent ||
+            document.querySelector('h1')?.textContent ||
+            document.querySelector('meta[property="og:title"]')?.getAttribute('content') ||
+            document.querySelector('meta[name="title"]')?.getAttribute('content') ||
+            'Untitled Page';
+        const byline = document.querySelector('meta[name="author"]')?.getAttribute('content') ||
+            document.querySelector('[rel="author"]')?.textContent ||
+            document.querySelector('.author')?.textContent ||
+            null;
+        if (!document.body) {
+            const html = document.documentElement?.innerHTML || '';
+            return {
+                title: title.trim(),
+                content: html,
+                byline,
+                excerpt: '',
+                dir: null,
+                lang: document.documentElement?.lang || null,
+                length: html.length,
+                siteName: null,
+                textContent: document.documentElement?.textContent || '',
+                publishedTime: null,
+                baseUrl
+            };
+        }
+        const contentClone = document.body.cloneNode(true);
+        const selectorsToRemove = [
+            'script', 'style', 'noscript', 'template'
+        ];
+        selectorsToRemove.forEach(selector => {
+            try {
+                contentClone.querySelectorAll(selector).forEach(el => el.remove());
+            }
+            catch (e) {
+            }
+        });
+        const mainContent = contentClone;
+        const content = mainContent.innerHTML || mainContent.textContent || '';
+        return {
+            title: title.trim(),
+            content,
+            byline,
+            excerpt: '',
+            dir: null,
+            lang: document.documentElement?.lang || null,
+            length: content.length,
+            siteName: null,
+            textContent: mainContent.textContent || '',
+            publishedTime: null,
+            baseUrl
+        };
+    }
+    catch (error) {
+        console.error('Error in manual extraction:', error);
+        return {
+            title: 'Error extracting content',
+            content: dom.window.document.body?.innerHTML || dom.window.document.documentElement?.innerHTML || '',
+            byline: null,
+            excerpt: '',
+            dir: null,
+            lang: null,
+            length: 0,
+            siteName: null,
+            textContent: dom.window.document.body?.textContent || '',
+            publishedTime: null,
+            baseUrl: dom.window.location.href
+        };
+    }
+}
+export function hasContent(html) {
+    const lowerHtml = html.toLowerCase();
+    if (lowerHtml.includes('<noscript>') &&
+        !lowerHtml.includes('<article') &&
+        !lowerHtml.includes('<main')) {
+        return false;
+    }
+    const textContent = html.replace(/<[^>]*>/g, '').trim();
+    return textContent.length > 100;
+}

package/dist/parser/dom.d.ts

@@ -0,0 +1,3 @@
+import { JSDOM } from 'jsdom';
+export declare function htmlToDom(html: string, url: string): JSDOM;
+export declare function extractLinks(dom: JSDOM): string[];

package/dist/parser/dom.js

@@ -0,0 +1,53 @@
+import { JSDOM } from 'jsdom';
+export function htmlToDom(html, url) {
+    try {
+        return new JSDOM(html, {
+            url,
+            contentType: 'text/html',
+            includeNodeLocations: false,
+            runScripts: 'outside-only',
+            resources: 'usable',
+            pretendToBeVisual: true
+        });
+    }
+    catch (error) {
+        console.error('Error parsing HTML with JSDOM, trying with minimal options:', error);
+        try {
+            return new JSDOM(html, {
+                url,
+                contentType: 'text/html'
+            });
+        }
+        catch (fallbackError) {
+            console.error('Fallback parsing also failed:', fallbackError);
+            return new JSDOM(`<!DOCTYPE html><html><body>${html}</body></html>`, {
+                url,
+                contentType: 'text/html'
+            });
+        }
+    }
+}
+export function extractLinks(dom) {
+    const document = dom.window.document;
+    const links = [];
+    const baseUrl = dom.window.location.href;
+    const anchorElements = document.querySelectorAll('a[href]');
+    anchorElements.forEach((element) => {
+        try {
+            const href = element.getAttribute('href');
+            if (!href)
+                return;
+            if (href.startsWith('mailto:') ||
+                href.startsWith('tel:') ||
+                href.startsWith('javascript:') ||
+                href.startsWith('#')) {
+                return;
+            }
+            const absoluteUrl = new URL(href, baseUrl).href;
+            links.push(absoluteUrl);
+        }
+        catch {
+        }
+    });
+    return [...new Set(links)];
+}

package/dist/parser/markdown.d.ts

@@ -0,0 +1,9 @@
+import TurndownService from 'turndown';
+export declare function createTurndownService(): TurndownService;
+export declare function htmlToMarkdown(html: string): string;
+export declare function formatArticleMarkdown(article: {
+    title: string;
+    content: string;
+    byline?: string | null;
+    baseUrl?: string;
+}): string;

package/dist/parser/markdown.js

@@ -0,0 +1,134 @@
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+import { JSDOM } from 'jsdom';
+function convertRelativeUrls(html, baseUrl) {
+    try {
+        const dom = new JSDOM(html, { url: baseUrl });
+        const document = dom.window.document;
+        document.querySelectorAll('a[href]').forEach(link => {
+            const href = link.getAttribute('href');
+            if (href && !href.startsWith('http://') && !href.startsWith('https://') &&
+                !href.startsWith('//') && !href.startsWith('mailto:') &&
+                !href.startsWith('tel:') && !href.startsWith('javascript:') &&
+                !href.startsWith('#')) {
+                try {
+                    const absoluteUrl = new URL(href, baseUrl).href;
+                    link.setAttribute('href', absoluteUrl);
+                }
+                catch (e) {
+                }
+            }
+        });
+        document.querySelectorAll('img[src]').forEach(img => {
+            const src = img.getAttribute('src');
+            if (src && !src.startsWith('http://') && !src.startsWith('https://') &&
+                !src.startsWith('//') && !src.startsWith('data:')) {
+                try {
+                    const absoluteUrl = new URL(src, baseUrl).href;
+                    img.setAttribute('src', absoluteUrl);
+                }
+                catch (e) {
+                }
+            }
+        });
+        const bodyElement = document.body || document.documentElement;
+        return bodyElement ? bodyElement.innerHTML : html;
+    }
+    catch (e) {
+        return html;
+    }
+}
+export function createTurndownService() {
+    const turndown = new TurndownService({
+        headingStyle: 'atx',
+        codeBlockStyle: 'fenced',
+        linkStyle: 'inlined',
+        emDelimiter: '_',
+        bulletListMarker: '-',
+        strongDelimiter: '**',
+        hr: '---',
+        blankReplacement: (_content, node) => {
+            return node.isBlock ? '\n\n' : '';
+        },
+        keepReplacement: (content, node) => {
+            return node.isBlock ? '\n\n' + content + '\n\n' : content;
+        },
+        defaultReplacement: (content, node) => {
+            return node.isBlock ? '\n\n' + content + '\n\n' : content;
+        }
+    });
+    turndown.use(gfm);
+    turndown.addRule('media', {
+        filter: ['iframe', 'video', 'audio', 'embed'],
+        replacement: (_content, node) => {
+            const element = node;
+            const src = element.getAttribute('src') || element.getAttribute('data-src');
+            const title = element.getAttribute('title') || element.getAttribute('alt') || 'media';
+            if (src) {
+                return `\n\n[${title}](${src})\n\n`;
+            }
+            return '';
+        }
+    });
+    turndown.addRule('figure', {
+        filter: 'figure',
+        replacement: (content, node) => {
+            const figure = node;
+            const caption = figure.querySelector('figcaption');
+            if (caption) {
+                const captionText = caption.textContent || '';
+                return `\n\n${content.trim()}\n*${captionText}*\n\n`;
+            }
+            return `\n\n${content.trim()}\n\n`;
+        }
+    });
+    return turndown;
+}
+export function htmlToMarkdown(html) {
+    const turndown = createTurndownService();
+    let markdown = turndown.turndown(html);
+    markdown = markdown
+        .replace(/\n{3,}/g, '\n\n')
+        .replace(/\s+$/gm, '')
+        .trim();
+    return markdown;
+}
+export function formatArticleMarkdown(article) {
+    try {
+        const turndown = createTurndownService();
+        let markdown = '';
+        if (article.title && article.title.trim()) {
+            markdown = `# ${article.title}\n\n`;
+        }
+        if (article.byline) {
+            markdown += `*By ${article.byline}*\n\n---\n\n`;
+        }
+        try {
+            const processedContent = article.baseUrl
+                ? convertRelativeUrls(article.content, article.baseUrl)
+                : article.content;
+            markdown += turndown.turndown(processedContent);
+        }
+        catch (conversionError) {
+            console.error('Error converting HTML to markdown:', conversionError);
+            const tempDiv = typeof document !== 'undefined'
+                ? document.createElement('div')
+                : null;
+            if (tempDiv) {
+                tempDiv.innerHTML = article.content;
+                markdown += tempDiv.textContent || article.content;
+            }
+            else {
+                markdown += article.content.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ');
+            }
+        }
+        return markdown
+            .replace(/\n{3,}/g, '\n\n')
+            .replace(/\s+$/gm, '')
+            .trim();
+    }
+    catch (error) {
+        console.error('Fatal error in formatArticleMarkdown:', error);
+        return article.title ? `# ${article.title}\n\n[Content extraction failed]` : '[Content extraction failed]';
+    }
+}

package/dist/serve.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/serve.js

@@ -0,0 +1,171 @@
+#!/usr/bin/env node
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { CallToolRequestSchema, ListToolsRequestSchema, ListResourcesRequestSchema, ReadResourceRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
+let fetchMarkdownModule;
+let fsPromises;
+let pathModule;
+const server = new Server({
+    name: "read-website-fast",
+    version: "0.1.0",
+}, {
+    capabilities: {
+        tools: {},
+        resources: {},
+    },
+});
+const READ_WEBSITE_TOOL = {
+    name: "read_website_fast",
+    description: "Quickly reads webpages and converts to markdown for fast, token efficient web scraping",
+    inputSchema: {
+        type: "object",
+        properties: {
+            url: {
+                type: "string",
+                description: "HTTP/HTTPS URL to fetch and convert to markdown",
+            },
+            depth: {
+                type: "number",
+                description: "Crawl depth (0 = single page)",
+                default: 0,
+            },
+            respectRobots: {
+                type: "boolean",
+                description: "Whether to respect robots.txt",
+                default: true,
+            },
+        },
+        required: ["url"],
+    },
+};
+const RESOURCES = [
+    {
+        uri: "read-website-fast://status",
+        name: "Cache Status",
+        mimeType: "application/json",
+        description: "Get cache status information",
+    },
+    {
+        uri: "read-website-fast://clear-cache",
+        name: "Clear Cache",
+        mimeType: "application/json",
+        description: "Clear the cache directory",
+    },
+];
+server.setRequestHandler(ListToolsRequestSchema, async () => ({
+    tools: [READ_WEBSITE_TOOL],
+}));
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    if (request.params.name !== "read_website_fast") {
+        throw new Error(`Unknown tool: ${request.params.name}`);
+    }
+    if (!fetchMarkdownModule) {
+        fetchMarkdownModule = await import("./internal/fetchMarkdown.js");
+    }
+    const args = request.params.arguments;
+    const result = await fetchMarkdownModule.fetchMarkdown(args.url, {
+        depth: args.depth ?? 0,
+        respectRobots: args.respectRobots ?? true,
+    });
+    if (result.error) {
+        throw new Error(result.error);
+    }
+    return {
+        content: [{ type: "text", text: result.markdown }],
+    };
+});
+server.setRequestHandler(ListResourcesRequestSchema, async () => ({
+    resources: RESOURCES,
+}));
+server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
+    const uri = request.params.uri;
+    if (!fsPromises) {
+        fsPromises = await import("fs/promises");
+    }
+    if (!pathModule) {
+        pathModule = await import("path");
+    }
+    if (uri === "read-website-fast://status") {
+        try {
+            const cacheDir = ".cache";
+            const files = await fsPromises.readdir(cacheDir).catch(() => []);
+            let totalSize = 0;
+            for (const file of files) {
+                const stats = await fsPromises
+                    .stat(pathModule.join(cacheDir, file))
+                    .catch(() => null);
+                if (stats) {
+                    totalSize += stats.size;
+                }
+            }
+            return {
+                contents: [
+                    {
+                        uri,
+                        mimeType: "application/json",
+                        text: JSON.stringify({
+                            cacheSize: totalSize,
+                            cacheFiles: files.length,
+                            cacheSizeFormatted: `${(totalSize / 1024 / 1024).toFixed(2)} MB`,
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                contents: [
+                    {
+                        uri,
+                        mimeType: "application/json",
+                        text: JSON.stringify({
+                            error: "Failed to get cache status",
+                            message: error instanceof Error ? error.message : "Unknown error",
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+    }
+    if (uri === "read-website-fast://clear-cache") {
+        try {
+            await fsPromises.rm(".cache", { recursive: true, force: true });
+            return {
+                contents: [
+                    {
+                        uri,
+                        mimeType: "application/json",
+                        text: JSON.stringify({
+                            status: "success",
+                            message: "Cache cleared successfully",
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                contents: [
+                    {
+                        uri,
+                        mimeType: "application/json",
+                        text: JSON.stringify({
+                            status: "error",
+                            message: error instanceof Error ? error.message : "Failed to clear cache",
+                        }, null, 2),
+                    },
+                ],
+            };
+        }
+    }
+    throw new Error(`Unknown resource: ${uri}`);
+});
+async function runServer() {
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    console.error("read-website-fast MCP server running");
+}
+runServer().catch((error) => {
+    console.error("Server error:", error);
+    process.exit(1);
+});

package/dist/utils/chunker.d.ts

@@ -0,0 +1,26 @@
+export interface ChunkOptions {
+    maxTokens?: number;
+    maxChars?: number;
+    splitOn?: 'heading' | 'paragraph' | 'sentence';
+    overlap?: number;
+}
+export interface Chunk {
+    content: string;
+    index: number;
+    tokens?: number;
+    metadata?: {
+        headings?: string[];
+        startLine?: number;
+        endLine?: number;
+    };
+}
+export declare class MarkdownChunker {
+    private options;
+    constructor(options?: ChunkOptions);
+    chunk(markdown: string): Chunk[];
+    private chunkByHeading;
+    private chunkByParagraph;
+    private chunkBySentence;
+    private getOverlapLines;
+    estimateTokens(text: string): number;
+}