@just-every/mcp-read-website-fast 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Context
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,165 @@
+# MCP Read JustEvery Website
+
+Existing MCP web crawlers are slow and consume large quantities of tokens. This pauses the development process and provides incomplete results as LLMs need to parse whole web pages.
+
+This MCP package fetches web pages locally, strips noise, and converts content to clean Markdown while preserving links. Designed for Claude Code, IDEs and LLM pipelines with minimal token footprint. Crawl sites locally with minimal dependencies.
+
+## MCP Server Configuration
+
+This tool can be used as an MCP (Model Context Protocol) server with Claude Desktop, Cursor, VS Code, and other compatible clients.
+
+## Installation
+
+### Claude Code
+
+```bash
+claude mcp add read-website-fast -s user -- npx -y @just-every/mcp-read-website-fast
+```
+
+### VS Code
+
+```bash
+code --add-mcp '{"name":"read-website-fast","command":"npx","args":["-y","@just-every/mcp-read-website-fast"]}'
+```
+
+### Cursor
+
+```bash
+cursor://anysphere.cursor-deeplink/mcp/install?name=read-website-fast&config=eyJyZWFkLXdlYnNpdGUtZmFzdCI6eyJjb21tYW5kIjoibnB4IiwiYXJncyI6WyIteSIsIkBqdXN0LWV2ZXJ5L21jcC1yZWFkLXdlYnNpdGUtZmFzdCJdfX0=
+```
+
+### JetBrains IDEs
+
+Settings → Tools → AI Assistant → Model Context Protocol (MCP) → Add
+
+Choose “As JSON” and paste:
+
+```json
+{"command":"npx","args":["-y","@just-every/mcp-read-website-fast"]}
+```
+
+Or, in the chat window, type /add and fill in the same JSON—both paths land the server in a single step. 
+
+### Raw JSON (works in any MCP client)
+
+```json
+{
+  "mcpServers": {
+    "read-website-fast": {
+      "command": "npx",
+      "args": ["-y", "@just-every/mcp-read-website-fast"]
+    }
+  }
+}
+```
+
+Drop this into your client’s mcp.json (e.g. .vscode/mcp.json, ~/.cursor/mcp.json, or .mcp.json for Claude).
+
+
+
+## Features
+
+- **Fast startup** using official MCP SDK with lazy loading for optimal performance
+- **Content extraction** using Mozilla Readability (same as Firefox Reader View)
+- **HTML to Markdown** conversion with Turndown + GFM support
+- **Smart caching** with SHA-256 hashed URLs
+- **Polite crawling** with robots.txt support and rate limiting
+- **Concurrent fetching** with configurable depth crawling
+- **Stream-first design** for low memory usage
+- **Link preservation** for knowledge graphs
+- **Optional chunking** for downstream processing
+
+### Available Tools
+
+- `read_website_fast` - Fetches a webpage and converts it to clean markdown
+  - Parameters:
+    - `url` (required): The HTTP/HTTPS URL to fetch
+    - `depth` (optional): Crawl depth (0 = single page)
+    - `respectRobots` (optional): Whether to respect robots.txt
+
+### Available Resources
+
+- `read-website-fast://status` - Get cache statistics
+- `read-website-fast://clear-cache` - Clear the cache directory
+
+## Development Usage
+
+### Install
+
+```bash
+npm install
+npm run build
+```
+
+### Single page fetch
+```bash
+npm run dev fetch https://example.com/article
+```
+
+### Crawl with depth
+```bash
+npm run dev fetch https://example.com --depth 2 --concurrency 5
+```
+
+### Output formats
+```bash
+# Markdown only (default)
+npm run dev fetch https://example.com
+
+# JSON output with metadata
+npm run dev fetch https://example.com --output json
+
+# Both URL and markdown
+npm run dev fetch https://example.com --output both
+```
+
+### CLI Options
+
+- `-d, --depth <number>` - Crawl depth (0 = single page, default: 0)
+- `-c, --concurrency <number>` - Max concurrent requests (default: 3)
+- `--no-robots` - Ignore robots.txt
+- `--all-origins` - Allow cross-origin crawling
+- `-u, --user-agent <string>` - Custom user agent
+- `--cache-dir <path>` - Cache directory (default: .cache)
+- `-t, --timeout <ms>` - Request timeout in milliseconds (default: 30000)
+- `-o, --output <format>` - Output format: json, markdown, or both (default: markdown)
+
+### Clear cache
+```bash
+npm run dev clear-cache
+```
+
+## Architecture
+
+```
+mcp/
+├── src/
+│   ├── crawler/        # URL fetching, queue management, robots.txt
+│   ├── parser/         # DOM parsing, Readability, Turndown conversion
+│   ├── cache/          # Disk-based caching with SHA-256 keys
+│   ├── utils/          # Logger, chunker utilities
+│   └── index.ts        # CLI entry point
+```
+
+## Development
+
+```bash
+# Run in development mode
+npm run dev fetch https://example.com
+
+# Build for production
+npm run build
+
+# Run tests
+npm test
+
+# Type checking
+npm run typecheck
+
+# Linting
+npm run lint
+```
+
+## License
+
+MIT

package/bin/mcp-read-website.js

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+import { existsSync } from 'fs';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const args = process.argv.slice(2);
+
+async function main() {
+  // Default to 'serve' if no arguments provided (for MCP usage)
+  const command = args[0] || 'serve';
+  
+  // Check if compiled dist exists
+  const distExists = existsSync(join(__dirname, '..', 'dist'));
+  
+  if (distExists) {
+    // Use compiled JavaScript for production (fast startup)
+    if (command === 'serve') {
+      const servePath = join(__dirname, '..', 'dist', 'serve.js');
+      await import(servePath);
+    } else {
+      const cliPath = join(__dirname, '..', 'dist', 'index.js');
+      await import(cliPath);
+    }
+  } else {
+    // Fall back to TypeScript with tsx for development
+    try {
+      await import('tsx/esm');
+      
+      if (command === 'serve') {
+        const servePath = join(__dirname, '..', 'src', 'serve.ts');
+        await import(servePath);
+      } else {
+        const cliPath = join(__dirname, '..', 'src', 'index.ts');
+        await import(cliPath);
+      }
+    } catch (error) {
+      console.error('Error: Development dependencies not installed. Please run "npm install" first.');
+      process.exit(1);
+    }
+  }
+}
+
+main().catch(err => {
+  console.error('Error:', err);
+  process.exit(1);
+});

package/dist/cache/disk.d.ts

@@ -0,0 +1,12 @@
+import { CacheEntry } from '../types.js';
+export declare class DiskCache {
+    private cacheDir;
+    constructor(cacheDir?: string);
+    init(): Promise<void>;
+    private getCacheKey;
+    private getCachePath;
+    has(url: string): Promise<boolean>;
+    get(url: string): Promise<CacheEntry | null>;
+    put(url: string, markdown: string, title?: string): Promise<void>;
+    getAge(url: string): Promise<number | null>;
+}

package/dist/cache/disk.js

@@ -0,0 +1,54 @@
+import { createHash } from 'crypto';
+import { mkdir, readFile, writeFile, access } from 'fs/promises';
+import { join } from 'path';
+export class DiskCache {
+    cacheDir;
+    constructor(cacheDir = '.cache') {
+        this.cacheDir = cacheDir;
+    }
+    async init() {
+        await mkdir(this.cacheDir, { recursive: true });
+    }
+    getCacheKey(url) {
+        return createHash('sha256').update(url).digest('hex');
+    }
+    getCachePath(url) {
+        const key = this.getCacheKey(url);
+        return join(this.cacheDir, `${key}.json`);
+    }
+    async has(url) {
+        try {
+            await access(this.getCachePath(url));
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async get(url) {
+        try {
+            const path = this.getCachePath(url);
+            const data = await readFile(path, 'utf-8');
+            return JSON.parse(data);
+        }
+        catch {
+            return null;
+        }
+    }
+    async put(url, markdown, title) {
+        const entry = {
+            url,
+            markdown,
+            timestamp: Date.now(),
+            title
+        };
+        const path = this.getCachePath(url);
+        await writeFile(path, JSON.stringify(entry, null, 2));
+    }
+    async getAge(url) {
+        const entry = await this.get(url);
+        if (!entry)
+            return null;
+        return Date.now() - entry.timestamp;
+    }
+}

package/dist/cache/normalize.d.ts

@@ -0,0 +1,2 @@
+export declare function normalizeUrl(url: string): string;
+export declare function isSameOrigin(url1: string, url2: string): boolean;

package/dist/cache/normalize.js

@@ -0,0 +1,31 @@
+export function normalizeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        if (parsed.pathname !== '/' && parsed.pathname.endsWith('/')) {
+            parsed.pathname = parsed.pathname.slice(0, -1);
+        }
+        const params = Array.from(parsed.searchParams.entries());
+        params.sort(([a], [b]) => a.localeCompare(b));
+        parsed.search = '';
+        params.forEach(([key, value]) => parsed.searchParams.append(key, value));
+        if ((parsed.protocol === 'http:' && parsed.port === '80') ||
+            (parsed.protocol === 'https:' && parsed.port === '443')) {
+            parsed.port = '';
+        }
+        parsed.hash = '';
+        return parsed.href;
+    }
+    catch {
+        return url;
+    }
+}
+export function isSameOrigin(url1, url2) {
+    try {
+        const u1 = new URL(url1);
+        const u2 = new URL(url2);
+        return u1.origin === u2.origin;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/crawler/fetch.d.ts

@@ -0,0 +1,8 @@
+interface FetchOptions {
+    userAgent?: string;
+    timeout?: number;
+    maxRedirections?: number;
+}
+export declare function fetchStream(url: string, options?: FetchOptions): Promise<string>;
+export declare function isValidUrl(url: string): boolean;
+export {};

package/dist/crawler/fetch.js

@@ -0,0 +1,42 @@
+import { fetch } from 'undici';
+export async function fetchStream(url, options = {}) {
+    const { userAgent = 'MCP/0.1 (+https://github.com/just-every/mcp-read-website-fast)', timeout = 30000, maxRedirections = 5 } = options;
+    try {
+        const response = await fetch(url, {
+            headers: {
+                'User-Agent': userAgent,
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                'Accept-Language': 'en-US,en;q=0.5',
+                'DNT': '1',
+                'Connection': 'keep-alive',
+                'Upgrade-Insecure-Requests': '1'
+            },
+            redirect: maxRedirections > 0 ? 'follow' : 'manual',
+            signal: AbortSignal.timeout(timeout)
+        });
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status} for ${url}`);
+        }
+        const contentType = response.headers.get('content-type');
+        if (contentType && !contentType.includes('text/html') &&
+            !contentType.includes('application/xhtml+xml')) {
+            throw new Error(`Non-HTML content type: ${contentType} for ${url}`);
+        }
+        return await response.text();
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            throw new Error(`Failed to fetch ${url}: ${error.message}`);
+        }
+        throw error;
+    }
+}
+export function isValidUrl(url) {
+    try {
+        const parsed = new URL(url);
+        return parsed.protocol === 'http:' || parsed.protocol === 'https:';
+    }
+    catch {
+        return false;
+    }
+}

package/dist/crawler/queue.d.ts

@@ -0,0 +1,14 @@
+import { CrawlOptions, CrawlResult } from '../types.js';
+export declare class CrawlQueue {
+    private visited;
+    private queue;
+    private limit;
+    private cache;
+    private options;
+    private results;
+    constructor(options?: CrawlOptions);
+    init(): Promise<void>;
+    crawl(startUrl: string): Promise<CrawlResult[]>;
+    private processQueue;
+    private processUrl;
+}

package/dist/crawler/queue.js

@@ -0,0 +1,142 @@
+import pLimit from 'p-limit';
+import { normalizeUrl, isSameOrigin } from '../cache/normalize.js';
+import { DiskCache } from '../cache/disk.js';
+import { fetchStream, isValidUrl } from './fetch.js';
+import { isAllowedByRobots, getCrawlDelay } from './robots.js';
+import { htmlToDom, extractLinks } from '../parser/dom.js';
+import { extractArticle } from '../parser/article.js';
+import { formatArticleMarkdown } from '../parser/markdown.js';
+export class CrawlQueue {
+    visited = new Set();
+    queue = [];
+    limit;
+    cache;
+    options;
+    results = [];
+    constructor(options = {}) {
+        this.options = {
+            depth: options.depth ?? 0,
+            maxConcurrency: options.maxConcurrency ?? 3,
+            respectRobots: options.respectRobots ?? true,
+            sameOriginOnly: options.sameOriginOnly ?? true,
+            userAgent: options.userAgent ?? 'MCP/0.1',
+            cacheDir: options.cacheDir ?? '.cache',
+            timeout: options.timeout ?? 30000
+        };
+        this.limit = pLimit(this.options.maxConcurrency);
+        this.cache = new DiskCache(this.options.cacheDir);
+    }
+    async init() {
+        await this.cache.init();
+    }
+    async crawl(startUrl) {
+        const normalizedUrl = normalizeUrl(startUrl);
+        if (!isValidUrl(normalizedUrl)) {
+            throw new Error(`Invalid URL: ${startUrl}`);
+        }
+        this.queue.push(normalizedUrl);
+        await this.processQueue(0);
+        return this.results;
+    }
+    async processQueue(currentDepth) {
+        if (currentDepth > this.options.depth)
+            return;
+        const urls = [...this.queue];
+        this.queue = [];
+        const tasks = urls.map(url => this.limit(() => this.processUrl(url, currentDepth)));
+        await Promise.all(tasks);
+        if (this.queue.length > 0) {
+            await this.processQueue(currentDepth + 1);
+        }
+    }
+    async processUrl(url, depth) {
+        const normalizedUrl = normalizeUrl(url);
+        if (this.visited.has(normalizedUrl))
+            return;
+        this.visited.add(normalizedUrl);
+        try {
+            const cached = await this.cache.get(normalizedUrl);
+            if (cached) {
+                this.results.push({
+                    url: normalizedUrl,
+                    markdown: cached.markdown,
+                    title: cached.title
+                });
+                return;
+            }
+            if (this.options.respectRobots) {
+                const allowed = await isAllowedByRobots(normalizedUrl, this.options.userAgent);
+                if (!allowed) {
+                    this.results.push({
+                        url: normalizedUrl,
+                        markdown: '',
+                        error: 'Blocked by robots.txt'
+                    });
+                    return;
+                }
+                const delay = await getCrawlDelay(normalizedUrl, this.options.userAgent);
+                if (delay > 0) {
+                    await new Promise(resolve => setTimeout(resolve, delay * 1000));
+                }
+            }
+            const html = await fetchStream(normalizedUrl, {
+                userAgent: this.options.userAgent,
+                timeout: this.options.timeout
+            });
+            if (!html || html.trim().length === 0) {
+                this.results.push({
+                    url: normalizedUrl,
+                    markdown: '',
+                    error: 'Empty response from server'
+                });
+                return;
+            }
+            const dom = htmlToDom(html, normalizedUrl);
+            const article = extractArticle(dom);
+            if (!article) {
+                this.results.push({
+                    url: normalizedUrl,
+                    markdown: '',
+                    error: 'Failed to extract article content'
+                });
+                return;
+            }
+            if (!article.content || article.content.trim().length < 50) {
+                this.results.push({
+                    url: normalizedUrl,
+                    markdown: '',
+                    error: 'Page contains minimal extractable content'
+                });
+                return;
+            }
+            const markdown = formatArticleMarkdown(article);
+            await this.cache.put(normalizedUrl, markdown, article.title);
+            let links = [];
+            if (depth < this.options.depth) {
+                links = extractLinks(dom);
+                if (this.options.sameOriginOnly) {
+                    links = links.filter(link => isSameOrigin(normalizedUrl, link));
+                }
+                links.forEach(link => {
+                    const normalized = normalizeUrl(link);
+                    if (!this.visited.has(normalized)) {
+                        this.queue.push(normalized);
+                    }
+                });
+            }
+            this.results.push({
+                url: normalizedUrl,
+                markdown,
+                title: article.title,
+                links: links.length > 0 ? links : undefined
+            });
+        }
+        catch (error) {
+            this.results.push({
+                url: normalizedUrl,
+                markdown: '',
+                error: error instanceof Error ? error.message : 'Unknown error'
+            });
+        }
+    }
+}

package/dist/crawler/robots.d.ts

@@ -0,0 +1,8 @@
+interface RobotsChecker {
+    isAllowed(url: string, userAgent?: string): boolean;
+    getCrawlDelay(userAgent?: string): number | undefined;
+}
+export declare function getRobotsChecker(origin: string, userAgent?: string): Promise<RobotsChecker>;
+export declare function isAllowedByRobots(url: string, userAgent?: string): Promise<boolean>;
+export declare function getCrawlDelay(url: string, userAgent?: string): Promise<number>;
+export {};

package/dist/crawler/robots.js

@@ -0,0 +1,47 @@
+import { fetchStream } from './fetch.js';
+const robotsCache = new Map();
+export async function getRobotsChecker(origin, userAgent = '*') {
+    const cached = robotsCache.get(origin);
+    if (cached)
+        return cached;
+    try {
+        const robotsUrl = new URL('/robots.txt', origin).href;
+        const robotsTxt = await fetchStream(robotsUrl, {
+            timeout: 5000,
+            userAgent
+        });
+        const robotsParserModule = await import('robots-parser');
+        const robotsParser = robotsParserModule.default || robotsParserModule;
+        const robots = robotsParser(robotsUrl, robotsTxt);
+        robotsCache.set(origin, robots);
+        return robots;
+    }
+    catch {
+        const permissive = {
+            isAllowed: () => true,
+            getCrawlDelay: () => undefined
+        };
+        robotsCache.set(origin, permissive);
+        return permissive;
+    }
+}
+export async function isAllowedByRobots(url, userAgent = '*') {
+    try {
+        const { origin } = new URL(url);
+        const checker = await getRobotsChecker(origin, userAgent);
+        return checker.isAllowed(url, userAgent);
+    }
+    catch {
+        return true;
+    }
+}
+export async function getCrawlDelay(url, userAgent = '*') {
+    try {
+        const { origin } = new URL(url);
+        const checker = await getRobotsChecker(origin, userAgent);
+        return checker.getCrawlDelay(userAgent) || 0;
+    }
+    catch {
+        return 0;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};