@just-every/mcp-read-website-fast 0.1.10 → 0.1.13

package/README.md

@@ -11,6 +11,8 @@ Existing MCP web crawlers are slow and consume large quantities of tokens. This
 
 This MCP package fetches web pages locally, strips noise, and converts content to clean Markdown while preserving links. Designed for Claude Code, IDEs and LLM pipelines with minimal token footprint. Crawl sites locally with minimal dependencies.
 
+**Note:** This package now uses [@just-every/crawl](https://www.npmjs.com/package/@just-every/crawl) for its core crawling and markdown conversion functionality.
+
 ## Features
 
 - **Fast startup** using official MCP SDK with lazy loading for optimal performance
@@ -86,7 +88,7 @@ Drop this into your client’s mcp.json (e.g. .vscode/mcp.json, ~/.cursor/mcp.js
 
 ### Available Tools
 
-- `read_website_fast` - Fetches a webpage and converts it to clean markdown
+- `read_website` - Fetches a webpage and converts it to clean markdown
   - Parameters:
     - `url` (required): The HTTP/HTTPS URL to fetch
     - `depth` (optional): Crawl depth (0 = single page)
@@ -144,6 +146,22 @@ npm run dev fetch https://example.com --output both
 npm run dev clear-cache
 ```
 
+## Auto-Restart Feature
+
+The MCP server includes automatic restart capability by default for improved reliability:
+
+- Automatically restarts the server if it crashes
+- Handles unhandled exceptions and promise rejections
+- Implements exponential backoff (max 10 attempts in 1 minute)
+- Logs all restart attempts for monitoring
+- Gracefully handles shutdown signals (SIGINT, SIGTERM)
+
+For development/debugging without auto-restart:
+```bash
+# Run directly without restart wrapper
+npm run serve:dev
+```
+
 ## Architecture
 
 ```
@@ -153,7 +171,9 @@ mcp/
 │   ├── parser/         # DOM parsing, Readability, Turndown conversion
 │   ├── cache/          # Disk-based caching with SHA-256 keys
 │   ├── utils/          # Logger, chunker utilities
-│   └── index.ts        # CLI entry point
+│   ├── index.ts        # CLI entry point
+│   ├── serve.ts        # MCP server entry point
+│   └── serve-restart.ts # Auto-restart wrapper
 ```
 
 ## Development

package/bin/mcp-read-website.js

@@ -18,7 +18,7 @@ async function main() {
   if (distExists) {
     // Use compiled JavaScript for production (fast startup)
     if (command === 'serve') {
-      const servePath = join(__dirname, '..', 'dist', 'serve.js');
+      const servePath = join(__dirname, '..', 'dist', 'serve-restart.js');
       await import(servePath);
     } else {
       const cliPath = join(__dirname, '..', 'dist', 'index.js');
@@ -30,7 +30,7 @@ async function main() {
       await import('tsx/esm');
       
       if (command === 'serve') {
-        const servePath = join(__dirname, '..', 'src', 'serve.ts');
+        const servePath = join(__dirname, '..', 'src', 'serve-restart.ts');
         await import(servePath);
       } else {
         const cliPath = join(__dirname, '..', 'src', 'index.ts');

package/dist/index.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { Command } from 'commander';
-import { CrawlQueue } from './crawler/queue.js';
+import { fetch } from '@just-every/crawl';
 import { readFileSync } from 'fs';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
@@ -34,10 +34,8 @@ program
             cacheDir: options.cacheDir,
             timeout: parseInt(options.timeout, 10),
         };
-        const queue = new CrawlQueue(crawlOptions);
-        await queue.init();
         console.error(`Fetching ${url}...`);
-        const results = await queue.crawl(url);
+        const results = await fetch(url, crawlOptions);
         if (options.output === 'json') {
             console.log(JSON.stringify(results, null, 2));
         }

package/dist/internal/fetchMarkdown.js

@@ -1,4 +1,4 @@
-import { CrawlQueue } from '../crawler/queue.js';
+import { fetch } from '@just-every/crawl';
 export async function fetchMarkdown(url, options = {}) {
     try {
         const crawlOptions = {
@@ -10,9 +10,7 @@ export async function fetchMarkdown(url, options = {}) {
             cacheDir: options.cacheDir ?? '.cache',
             timeout: options.timeout ?? 30000,
         };
-        const queue = new CrawlQueue(crawlOptions);
-        await queue.init();
-        const results = await queue.crawl(url);
+        const results = await fetch(url, crawlOptions);
         const mainResult = results[0];
         if (!mainResult) {
             return {

package/dist/serve-restart.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/serve-restart.js

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+import { spawn } from 'child_process';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const MAX_RESTART_ATTEMPTS = 10;
+const RESTART_WINDOW_MS = 60000;
+const INITIAL_BACKOFF_MS = 1000;
+const MAX_BACKOFF_MS = 30000;
+let restartAttempts = [];
+let currentBackoff = INITIAL_BACKOFF_MS;
+const log = (level, message, ...args) => {
+    const timestamp = new Date().toISOString();
+    console.error(`[${timestamp}] [${level}] [restart-wrapper]`, message, ...args);
+};
+const cleanupRestartAttempts = () => {
+    const now = Date.now();
+    restartAttempts = restartAttempts.filter(timestamp => now - timestamp < RESTART_WINDOW_MS);
+};
+const shouldRestart = () => {
+    cleanupRestartAttempts();
+    if (restartAttempts.length >= MAX_RESTART_ATTEMPTS) {
+        log('ERROR', `Reached maximum restart attempts (${MAX_RESTART_ATTEMPTS}) within ${RESTART_WINDOW_MS}ms`);
+        return false;
+    }
+    return true;
+};
+const getBackoffDelay = () => {
+    const delay = Math.min(currentBackoff, MAX_BACKOFF_MS);
+    currentBackoff = Math.min(currentBackoff * 2, MAX_BACKOFF_MS);
+    return delay;
+};
+const resetBackoff = () => {
+    currentBackoff = INITIAL_BACKOFF_MS;
+};
+const startServer = () => {
+    log('INFO', 'Starting MCP server...');
+    const serverPath = join(__dirname, 'serve.js');
+    const child = spawn(process.execPath, [serverPath], {
+        stdio: 'inherit',
+        env: process.env,
+    });
+    let shuttingDown = false;
+    let restartTimer = null;
+    let startupTimer = setTimeout(() => {
+        log('INFO', 'Server started successfully');
+        resetBackoff();
+    }, 5000);
+    child.on('exit', (code, signal) => {
+        clearTimeout(startupTimer);
+        if (shuttingDown) {
+            log('INFO', 'Server stopped gracefully');
+            process.exit(0);
+            return;
+        }
+        if (code === 0) {
+            log('INFO', 'Server exited cleanly');
+            process.exit(0);
+            return;
+        }
+        log('WARN', `Server exited with code ${code}, signal ${signal}`);
+        if (!shouldRestart()) {
+            log('ERROR', 'Too many restart attempts, giving up');
+            process.exit(1);
+            return;
+        }
+        const backoffDelay = getBackoffDelay();
+        restartAttempts.push(Date.now());
+        log('INFO', `Restarting server in ${backoffDelay}ms (attempt ${restartAttempts.length}/${MAX_RESTART_ATTEMPTS})...`);
+        restartTimer = setTimeout(() => {
+            startServer();
+        }, backoffDelay);
+    });
+    child.on('error', (error) => {
+        log('ERROR', 'Failed to start server:', error);
+        process.exit(1);
+    });
+    const shutdown = (signal) => {
+        if (shuttingDown)
+            return;
+        shuttingDown = true;
+        log('INFO', `Received ${signal}, shutting down...`);
+        if (restartTimer) {
+            clearTimeout(restartTimer);
+        }
+        child.kill(signal);
+        setTimeout(() => {
+            log('WARN', 'Force killing child process');
+            child.kill('SIGKILL');
+        }, 5000);
+    };
+    process.on('SIGINT', () => shutdown('SIGINT'));
+    process.on('SIGTERM', () => shutdown('SIGTERM'));
+};
+process.on('uncaughtException', (error) => {
+    log('ERROR', 'Uncaught exception in restart wrapper:', error);
+    process.exit(1);
+});
+process.on('unhandledRejection', (reason) => {
+    log('ERROR', 'Unhandled rejection in restart wrapper:', reason);
+    process.exit(1);
+});
+log('INFO', 'MCP server restart wrapper starting...');
+log('INFO', `Configuration: max attempts=${MAX_RESTART_ATTEMPTS}, window=${RESTART_WINDOW_MS}ms`);
+startServer();

package/dist/serve.js

@@ -1,13 +1,23 @@
 #!/usr/bin/env node
+console.error('[serve.ts] Process started, PID:', process.pid);
+console.error('[serve.ts] Node version:', process.version);
+console.error('[serve.ts] Current directory:', process.cwd());
 import { Server } from '@modelcontextprotocol/sdk/server/index.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { CallToolRequestSchema, ListToolsRequestSchema, ListResourcesRequestSchema, ReadResourceRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
+import { logger, LogLevel } from './utils/logger.js';
+logger.setLevel(LogLevel.DEBUG);
+logger.info('MCP Server starting up...');
+logger.debug('Node version:', process.version);
+logger.debug('Working directory:', process.cwd());
+logger.debug('Environment:', { LOG_LEVEL: process.env.LOG_LEVEL });
 process.stdin.on('error', () => { });
 process.stdout.on('error', () => { });
 process.stderr.on('error', () => { });
 let fetchMarkdownModule;
 let fsPromises;
 let pathModule;
+logger.debug('Creating MCP server instance...');
 const server = new Server({
     name: 'read-website-fast',
     version: '0.1.0',
@@ -17,12 +27,13 @@ const server = new Server({
         resources: {},
     },
 });
+logger.info('MCP server instance created successfully');
 server.onerror = error => {
-    console.error('[MCP Server Error]', error);
+    logger.error('MCP Server Error:', error);
 };
 const READ_WEBSITE_TOOL = {
-    name: 'read_website_fast',
-    description: 'Quickly reads webpages and converts to markdown for fast, token efficient web scraping',
+    name: 'read_website',
+    description: 'Fast, token-efficient web content extraction - ideal for reading documentation, analyzing content, and gathering information from websites. Converts to clean Markdown while preserving links and structure.',
     inputSchema: {
         type: 'object',
         properties: {
@@ -43,6 +54,13 @@ const READ_WEBSITE_TOOL = {
         },
         required: ['url'],
     },
+    annotations: {
+        title: 'Read Website',
+        readOnlyHint: true,
+        destructiveHint: false,
+        idempotentHint: true,
+        openWorldHint: true,
+    },
 };
 const RESOURCES = [
     {
@@ -58,25 +76,44 @@ const RESOURCES = [
         description: 'Clear the cache directory',
     },
 ];
-server.setRequestHandler(ListToolsRequestSchema, async () => ({
-    tools: [READ_WEBSITE_TOOL],
-}));
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+    logger.debug('Received ListTools request');
+    const response = {
+        tools: [READ_WEBSITE_TOOL],
+    };
+    logger.debug('Returning tools:', response.tools.map(t => t.name));
+    return response;
+});
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
-    if (request.params.name !== 'read_website_fast') {
-        throw new Error(`Unknown tool: ${request.params.name}`);
+    logger.info('Received CallTool request:', request.params.name);
+    logger.debug('Request params:', JSON.stringify(request.params, null, 2));
+    if (request.params.name !== 'read_website') {
+        const error = `Unknown tool: ${request.params.name}`;
+        logger.error(error);
+        throw new Error(error);
     }
     try {
         if (!fetchMarkdownModule) {
+            logger.debug('Lazy loading fetchMarkdown module...');
             fetchMarkdownModule = await import('./internal/fetchMarkdown.js');
+            logger.info('fetchMarkdown module loaded successfully');
         }
         const args = request.params.arguments;
         if (!args.url || typeof args.url !== 'string') {
             throw new Error('URL parameter is required and must be a string');
         }
+        logger.info(`Processing read request for URL: ${args.url}`);
+        logger.debug('Read parameters:', {
+            url: args.url,
+            depth: args.depth,
+            respectRobots: args.respectRobots,
+        });
+        logger.debug('Calling fetchMarkdown...');
         const result = await fetchMarkdownModule.fetchMarkdown(args.url, {
             depth: args.depth ?? 0,
             respectRobots: args.respectRobots ?? true,
         });
+        logger.info('Content fetched successfully');
         if (result.error && result.markdown) {
             return {
                 content: [
@@ -95,14 +132,24 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         };
     }
     catch (error) {
-        console.error('Tool execution error:', error);
+        logger.error('Error fetching content:', error.message);
+        logger.debug('Error stack:', error.stack);
+        logger.debug('Error details:', {
+            name: error.name,
+            code: error.code,
+            ...error,
+        });
         throw new Error(`Failed to fetch content: ${error instanceof Error ? error.message : 'Unknown error'}`);
     }
 });
-server.setRequestHandler(ListResourcesRequestSchema, async () => ({
-    resources: RESOURCES,
-}));
+server.setRequestHandler(ListResourcesRequestSchema, async () => {
+    logger.debug('Received ListResources request');
+    return {
+        resources: RESOURCES,
+    };
+});
 server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
+    logger.debug('Received ReadResource request:', request.params);
     const uri = request.params.uri;
     if (!fsPromises) {
         fsPromises = await import('fs/promises');
@@ -190,49 +237,84 @@ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
     throw new Error(`Unknown resource: ${uri}`);
 });
 async function runServer() {
-    const transport = new StdioServerTransport();
-    transport.onerror = error => {
-        console.error('[Transport Error]', error);
-    };
-    process.on('SIGINT', async () => {
-        console.error('Received SIGINT, shutting down gracefully...');
-        await server.close();
-        process.exit(0);
-    });
-    process.on('SIGTERM', async () => {
-        console.error('Received SIGTERM, shutting down gracefully...');
-        await server.close();
-        process.exit(0);
-    });
-    process.on('uncaughtException', error => {
-        console.error('Uncaught exception:', error);
-        if (error && error.message && error.message.includes('EPIPE')) {
-            console.error('Pipe error detected, keeping server alive');
-            return;
-        }
-        process.exit(1);
-    });
-    process.on('unhandledRejection', (reason, promise) => {
-        console.error('Unhandled rejection at:', promise, 'reason:', reason);
-    });
-    process.stdin.on('end', () => {
-        console.error('Stdin closed, shutting down...');
-        process.exit(0);
-    });
-    process.stdin.on('error', error => {
-        console.error('Stdin error:', error);
-    });
     try {
+        logger.info('Starting MCP server...');
+        logger.debug('Creating StdioServerTransport...');
+        const transport = new StdioServerTransport();
+        logger.debug('Transport created, connecting to server...');
+        transport.onerror = error => {
+            logger.error('Transport Error:', error);
+            if (error?.message?.includes('Connection closed')) {
+                logger.info('Connection closed by client');
+                process.exit(0);
+            }
+        };
+        const cleanup = async (signal) => {
+            logger.info(`Received ${signal}, shutting down gracefully...`);
+            try {
+                await server.close();
+                logger.info('Server closed successfully');
+                process.exit(0);
+            }
+            catch (error) {
+                logger.error('Error during cleanup:', error);
+                process.exit(1);
+            }
+        };
+        process.on('SIGINT', () => cleanup('SIGINT'));
+        process.on('SIGTERM', () => cleanup('SIGTERM'));
+        process.on('uncaughtException', error => {
+            logger.error('Uncaught exception:', error.message);
+            logger.error('Stack trace:', error.stack);
+            logger.debug('Full error object:', error);
+            if (error && error.message && error.message.includes('EPIPE')) {
+                logger.warn('Pipe error detected, keeping server alive');
+                return;
+            }
+            process.exit(1);
+        });
+        process.on('unhandledRejection', (reason, promise) => {
+            logger.error('Unhandled Rejection at:', promise);
+            logger.error('Rejection reason:', reason);
+            logger.debug('Full rejection details:', { reason, promise });
+        });
+        process.on('exit', code => {
+            logger.info(`Process exiting with code: ${code}`);
+        });
+        process.on('warning', warning => {
+            logger.warn('Process warning:', warning.message);
+            logger.debug('Warning details:', warning);
+        });
+        process.stdin.on('end', () => {
+            logger.info('Stdin closed, shutting down...');
+            setTimeout(() => process.exit(0), 100);
+        });
+        process.stdin.on('error', error => {
+            logger.warn('Stdin error:', error);
+        });
         await server.connect(transport);
-        console.error('read-website-fast MCP server running');
+        logger.info('MCP server connected and running successfully!');
+        logger.info('Ready to receive requests');
+        logger.debug('Server details:', {
+            name: 'read-website-fast',
+            version: '0.1.0',
+            pid: process.pid,
+        });
+        setInterval(() => {
+            logger.debug('Server heartbeat - still running...');
+        }, 30000);
         process.stdin.resume();
     }
     catch (error) {
-        console.error('Failed to start server:', error);
-        process.exit(1);
+        logger.error('Failed to start server:', error.message);
+        logger.debug('Startup error details:', error);
+        throw error;
     }
 }
+logger.info('Initializing MCP server...');
 runServer().catch(error => {
-    console.error('Server initialization error:', error);
+    logger.error('Fatal server error:', error.message);
+    logger.error('Stack trace:', error.stack);
+    logger.debug('Full error:', error);
     process.exit(1);
 });

package/dist/utils/logger.js

@@ -23,10 +23,10 @@ export class Logger {
                 console.error(prefix, message, ...args);
                 break;
             case LogLevel.WARN:
-                console.warn(prefix, message, ...args);
+                console.error(prefix, message, ...args);
                 break;
             default:
-                console.log(prefix, message, ...args);
+                console.error(prefix, message, ...args);
         }
     }
     error(message, ...args) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@just-every/mcp-read-website-fast",
-  "version": "0.1.10",
+  "version": "0.1.13",
   "description": "Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown",
   "main": "dist/index.js",
   "bin": {
@@ -18,7 +18,7 @@
     "build:dev": "tsc",
     "dev": "tsx src/index.ts",
     "start": "node dist/index.js",
-    "serve": "tsx src/serve.ts",
+    "serve": "node dist/serve-restart.js",
     "serve:dev": "tsx src/serve.ts",
     "test": "vitest",
     "test:deploy": "vitest run test/deployment.test.ts",
@@ -50,15 +50,9 @@
   "homepage": "https://github.com/just-every/mcp-read-website-fast#readme",
   "license": "MIT",
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.12.1",
-    "@mozilla/readability": "^0.6.0",
-    "commander": "^14.0.0",
-    "jsdom": "^26.1.0",
-    "p-limit": "^6.2.0",
-    "robots-parser": "^3.0.1",
-    "turndown": "^7.1.3",
-    "turndown-plugin-gfm": "^1.0.2",
-    "undici": "^7.10.0"
+    "@just-every/crawl": "^1.0.2",
+    "@modelcontextprotocol/sdk": "^1.12.3",
+    "commander": "^14.0.0"
   },
   "devDependencies": {
     "@types/jsdom": "^21.1.6",

package/dist/cache/disk.d.ts

@@ -1,12 +0,0 @@
-import { CacheEntry } from '../types.js';
-export declare class DiskCache {
-    private cacheDir;
-    constructor(cacheDir?: string);
-    init(): Promise<void>;
-    private getCacheKey;
-    private getCachePath;
-    has(url: string): Promise<boolean>;
-    get(url: string): Promise<CacheEntry | null>;
-    put(url: string, markdown: string, title?: string): Promise<void>;
-    getAge(url: string): Promise<number | null>;
-}

package/dist/cache/disk.js

@@ -1,54 +0,0 @@
-import { createHash } from 'crypto';
-import { mkdir, readFile, writeFile, access } from 'fs/promises';
-import { join } from 'path';
-export class DiskCache {
-    cacheDir;
-    constructor(cacheDir = '.cache') {
-        this.cacheDir = cacheDir;
-    }
-    async init() {
-        await mkdir(this.cacheDir, { recursive: true });
-    }
-    getCacheKey(url) {
-        return createHash('sha256').update(url).digest('hex');
-    }
-    getCachePath(url) {
-        const key = this.getCacheKey(url);
-        return join(this.cacheDir, `${key}.json`);
-    }
-    async has(url) {
-        try {
-            await access(this.getCachePath(url));
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-    async get(url) {
-        try {
-            const path = this.getCachePath(url);
-            const data = await readFile(path, 'utf-8');
-            return JSON.parse(data);
-        }
-        catch {
-            return null;
-        }
-    }
-    async put(url, markdown, title) {
-        const entry = {
-            url,
-            markdown,
-            timestamp: Date.now(),
-            title,
-        };
-        const path = this.getCachePath(url);
-        await writeFile(path, JSON.stringify(entry, null, 2));
-    }
-    async getAge(url) {
-        const entry = await this.get(url);
-        if (!entry)
-            return null;
-        return Date.now() - entry.timestamp;
-    }
-}

package/dist/cache/normalize.d.ts

@@ -1,2 +0,0 @@
-export declare function normalizeUrl(url: string): string;
-export declare function isSameOrigin(url1: string, url2: string): boolean;

package/dist/cache/normalize.js

@@ -1,31 +0,0 @@
-export function normalizeUrl(url) {
-    try {
-        const parsed = new URL(url);
-        if (parsed.pathname !== '/' && parsed.pathname.endsWith('/')) {
-            parsed.pathname = parsed.pathname.slice(0, -1);
-        }
-        const params = Array.from(parsed.searchParams.entries());
-        params.sort(([a], [b]) => a.localeCompare(b));
-        parsed.search = '';
-        params.forEach(([key, value]) => parsed.searchParams.append(key, value));
-        if ((parsed.protocol === 'http:' && parsed.port === '80') ||
-            (parsed.protocol === 'https:' && parsed.port === '443')) {
-            parsed.port = '';
-        }
-        parsed.hash = '';
-        return parsed.href;
-    }
-    catch {
-        return url;
-    }
-}
-export function isSameOrigin(url1, url2) {
-    try {
-        const u1 = new URL(url1);
-        const u2 = new URL(url2);
-        return u1.origin === u2.origin;
-    }
-    catch {
-        return false;
-    }
-}

package/dist/crawler/fetch.d.ts

@@ -1,8 +0,0 @@
-interface FetchOptions {
-    userAgent?: string;
-    timeout?: number;
-    maxRedirections?: number;
-}
-export declare function fetchStream(url: string, options?: FetchOptions): Promise<string>;
-export declare function isValidUrl(url: string): boolean;
-export {};

package/dist/crawler/fetch.js

@@ -1,43 +0,0 @@
-import { fetch } from 'undici';
-export async function fetchStream(url, options = {}) {
-    const { userAgent = 'MCP/0.1 (+https://github.com/just-every/mcp-read-website-fast)', timeout = 30000, maxRedirections = 5, } = options;
-    try {
-        const response = await fetch(url, {
-            headers: {
-                'User-Agent': userAgent,
-                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-                'Accept-Language': 'en-US,en;q=0.5',
-                DNT: '1',
-                Connection: 'keep-alive',
-                'Upgrade-Insecure-Requests': '1',
-            },
-            redirect: maxRedirections > 0 ? 'follow' : 'manual',
-            signal: AbortSignal.timeout(timeout),
-        });
-        if (!response.ok) {
-            throw new Error(`HTTP ${response.status} for ${url}`);
-        }
-        const contentType = response.headers.get('content-type');
-        if (contentType &&
-            !contentType.includes('text/html') &&
-            !contentType.includes('application/xhtml+xml')) {
-            throw new Error(`Non-HTML content type: ${contentType} for ${url}`);
-        }
-        return await response.text();
-    }
-    catch (error) {
-        if (error instanceof Error) {
-            throw new Error(`Failed to fetch ${url}: ${error.message}`);
-        }
-        throw error;
-    }
-}
-export function isValidUrl(url) {
-    try {
-        const parsed = new URL(url);
-        return parsed.protocol === 'http:' || parsed.protocol === 'https:';
-    }
-    catch {
-        return false;
-    }
-}

package/dist/crawler/queue.d.ts

@@ -1,14 +0,0 @@
-import { CrawlOptions, CrawlResult } from '../types.js';
-export declare class CrawlQueue {
-    private visited;
-    private queue;
-    private limit;
-    private cache;
-    private options;
-    private results;
-    constructor(options?: CrawlOptions);
-    init(): Promise<void>;
-    crawl(startUrl: string): Promise<CrawlResult[]>;
-    private processQueue;
-    private processUrl;
-}

package/dist/crawler/queue.js

@@ -1,148 +0,0 @@
-import pLimit from 'p-limit';
-import { normalizeUrl, isSameOrigin } from '../cache/normalize.js';
-import { DiskCache } from '../cache/disk.js';
-import { fetchStream, isValidUrl } from './fetch.js';
-import { isAllowedByRobots, getCrawlDelay } from './robots.js';
-import { htmlToDom, extractLinks } from '../parser/dom.js';
-import { extractArticle } from '../parser/article.js';
-import { formatArticleMarkdown } from '../parser/markdown.js';
-export class CrawlQueue {
-    visited = new Set();
-    queue = [];
-    limit;
-    cache;
-    options;
-    results = [];
-    constructor(options = {}) {
-        this.options = {
-            depth: options.depth ?? 0,
-            maxConcurrency: options.maxConcurrency ?? 3,
-            respectRobots: options.respectRobots ?? true,
-            sameOriginOnly: options.sameOriginOnly ?? true,
-            userAgent: options.userAgent ?? 'MCP/0.1',
-            cacheDir: options.cacheDir ?? '.cache',
-            timeout: options.timeout ?? 30000,
-        };
-        this.limit = pLimit(this.options.maxConcurrency);
-        this.cache = new DiskCache(this.options.cacheDir);
-    }
-    async init() {
-        await this.cache.init();
-    }
-    async crawl(startUrl) {
-        const normalizedUrl = normalizeUrl(startUrl);
-        if (!isValidUrl(normalizedUrl)) {
-            throw new Error(`Invalid URL: ${startUrl}`);
-        }
-        this.queue.push(normalizedUrl);
-        await this.processQueue(0);
-        return this.results;
-    }
-    async processQueue(currentDepth) {
-        if (currentDepth > this.options.depth)
-            return;
-        const urls = [...this.queue];
-        this.queue = [];
-        const tasks = urls.map(url => this.limit(() => this.processUrl(url, currentDepth)));
-        await Promise.all(tasks);
-        if (this.queue.length > 0) {
-            await this.processQueue(currentDepth + 1);
-        }
-    }
-    async processUrl(url, depth) {
-        const normalizedUrl = normalizeUrl(url);
-        if (this.visited.has(normalizedUrl))
-            return;
-        this.visited.add(normalizedUrl);
-        try {
-            const cached = await this.cache.get(normalizedUrl);
-            if (cached) {
-                this.results.push({
-                    url: normalizedUrl,
-                    markdown: cached.markdown,
-                    title: cached.title,
-                });
-                return;
-            }
-            if (this.options.respectRobots) {
-                const allowed = await isAllowedByRobots(normalizedUrl, this.options.userAgent);
-                if (!allowed) {
-                    this.results.push({
-                        url: normalizedUrl,
-                        markdown: '',
-                        error: 'Blocked by robots.txt',
-                    });
-                    return;
-                }
-                const delay = await getCrawlDelay(normalizedUrl, this.options.userAgent);
-                if (delay > 0) {
-                    await new Promise(resolve => setTimeout(resolve, delay * 1000));
-                }
-            }
-            const html = await fetchStream(normalizedUrl, {
-                userAgent: this.options.userAgent,
-                timeout: this.options.timeout,
-            });
-            if (!html || html.trim().length === 0) {
-                this.results.push({
-                    url: normalizedUrl,
-                    markdown: '',
-                    error: 'Empty response from server',
-                });
-                return;
-            }
-            const dom = htmlToDom(html, normalizedUrl);
-            const article = extractArticle(dom);
-            if (!article) {
-                this.results.push({
-                    url: normalizedUrl,
-                    markdown: '',
-                    error: 'Failed to extract article content',
-                });
-                return;
-            }
-            if (!article.content || article.content.trim().length < 50) {
-                const fallbackMarkdown = `# ${article.title || 'Page Content'}\n\n` +
-                    `*Note: This page appears to be JavaScript-rendered. Limited content extracted.*\n\n` +
-                    (article.textContent
-                        ? article.textContent.substring(0, 1000) + '...'
-                        : 'No text content available');
-                this.results.push({
-                    url: normalizedUrl,
-                    markdown: fallbackMarkdown,
-                    title: article.title || normalizedUrl,
-                    error: 'Limited content extracted (JavaScript-rendered page)',
-                });
-                return;
-            }
-            const markdown = formatArticleMarkdown(article);
-            await this.cache.put(normalizedUrl, markdown, article.title);
-            let links = [];
-            if (depth < this.options.depth) {
-                links = extractLinks(dom);
-                if (this.options.sameOriginOnly) {
-                    links = links.filter(link => isSameOrigin(normalizedUrl, link));
-                }
-                links.forEach(link => {
-                    const normalized = normalizeUrl(link);
-                    if (!this.visited.has(normalized)) {
-                        this.queue.push(normalized);
-                    }
-                });
-            }
-            this.results.push({
-                url: normalizedUrl,
-                markdown,
-                title: article.title,
-                links: links.length > 0 ? links : undefined,
-            });
-        }
-        catch (error) {
-            this.results.push({
-                url: normalizedUrl,
-                markdown: '',
-                error: error instanceof Error ? error.message : 'Unknown error',
-            });
-        }
-    }
-}

package/dist/crawler/robots.d.ts

@@ -1,8 +0,0 @@
-interface RobotsChecker {
-    isAllowed(url: string, userAgent?: string): boolean;
-    getCrawlDelay(userAgent?: string): number | undefined;
-}
-export declare function getRobotsChecker(origin: string, userAgent?: string): Promise<RobotsChecker>;
-export declare function isAllowedByRobots(url: string, userAgent?: string): Promise<boolean>;
-export declare function getCrawlDelay(url: string, userAgent?: string): Promise<number>;
-export {};

package/dist/crawler/robots.js

@@ -1,47 +0,0 @@
-import { fetchStream } from './fetch.js';
-const robotsCache = new Map();
-export async function getRobotsChecker(origin, userAgent = '*') {
-    const cached = robotsCache.get(origin);
-    if (cached)
-        return cached;
-    try {
-        const robotsUrl = new URL('/robots.txt', origin).href;
-        const robotsTxt = await fetchStream(robotsUrl, {
-            timeout: 5000,
-            userAgent,
-        });
-        const robotsParserModule = (await import('robots-parser'));
-        const robotsParser = robotsParserModule.default || robotsParserModule;
-        const robots = robotsParser(robotsUrl, robotsTxt);
-        robotsCache.set(origin, robots);
-        return robots;
-    }
-    catch {
-        const permissive = {
-            isAllowed: () => true,
-            getCrawlDelay: () => undefined,
-        };
-        robotsCache.set(origin, permissive);
-        return permissive;
-    }
-}
-export async function isAllowedByRobots(url, userAgent = '*') {
-    try {
-        const { origin } = new URL(url);
-        const checker = await getRobotsChecker(origin, userAgent);
-        return checker.isAllowed(url, userAgent);
-    }
-    catch {
-        return true;
-    }
-}
-export async function getCrawlDelay(url, userAgent = '*') {
-    try {
-        const { origin } = new URL(url);
-        const checker = await getRobotsChecker(origin, userAgent);
-        return checker.getCrawlDelay(userAgent) || 0;
-    }
-    catch {
-        return 0;
-    }
-}

package/dist/parser/article.d.ts

@@ -1,4 +0,0 @@
-import { JSDOM } from 'jsdom';
-import { Article } from '../types.js';
-export declare function extractArticle(dom: JSDOM): Article | null;
-export declare function hasContent(html: string): boolean;

package/dist/parser/article.js

@@ -1,125 +0,0 @@
-import { Readability } from '@mozilla/readability';
-export function extractArticle(dom) {
-    const document = dom.window.document;
-    const baseUrl = dom.window.location.href;
-    const articleParagraph = document.querySelector('article p');
-    const hasStrongArticleIndicators = (document.querySelector('article') !== null &&
-        articleParagraph?.textContent &&
-        articleParagraph.textContent.length > 200) ||
-        document.querySelector('[itemtype*="BlogPosting"]') !== null ||
-        document.querySelector('[itemtype*="NewsArticle"]') !== null ||
-        document.querySelector('meta[property="article:published_time"]') !==
-            null;
-    if (hasStrongArticleIndicators) {
-        const documentClone = document.cloneNode(true);
-        const reader = new Readability(documentClone);
-        const article = reader.parse();
-        if (article && article.content && article.content.trim().length > 500) {
-            return {
-                title: article.title || 'Untitled',
-                content: article.content || '',
-                textContent: article.textContent || '',
-                length: article.length || 0,
-                excerpt: article.excerpt || '',
-                byline: article.byline || null,
-                dir: article.dir || null,
-                lang: article.lang || null,
-                siteName: article.siteName || null,
-                publishedTime: article.publishedTime || null,
-                baseUrl,
-            };
-        }
-    }
-    return extractContentManually(dom);
-}
-function extractContentManually(dom) {
-    try {
-        const document = dom.window.document;
-        const baseUrl = dom.window.location.href;
-        const title = document.querySelector('title')?.textContent ||
-            document.querySelector('h1')?.textContent ||
-            document
-                .querySelector('meta[property="og:title"]')
-                ?.getAttribute('content') ||
-            document
-                .querySelector('meta[name="title"]')
-                ?.getAttribute('content') ||
-            'Untitled Page';
-        const byline = document
-            .querySelector('meta[name="author"]')
-            ?.getAttribute('content') ||
-            document.querySelector('[rel="author"]')?.textContent ||
-            document.querySelector('.author')?.textContent ||
-            null;
-        if (!document.body) {
-            const html = document.documentElement?.innerHTML || '';
-            return {
-                title: title.trim(),
-                content: html,
-                byline,
-                excerpt: '',
-                dir: null,
-                lang: document.documentElement?.lang || null,
-                length: html.length,
-                siteName: null,
-                textContent: document.documentElement?.textContent || '',
-                publishedTime: null,
-                baseUrl,
-            };
-        }
-        const contentClone = document.body.cloneNode(true);
-        const selectorsToRemove = ['script', 'style', 'noscript', 'template'];
-        selectorsToRemove.forEach(selector => {
-            try {
-                contentClone
-                    .querySelectorAll(selector)
-                    .forEach(el => el.remove());
-            }
-            catch {
-            }
-        });
-        const mainContent = contentClone;
-        const content = mainContent.innerHTML || mainContent.textContent || '';
-        return {
-            title: title.trim(),
-            content,
-            byline,
-            excerpt: '',
-            dir: null,
-            lang: document.documentElement?.lang || null,
-            length: content.length,
-            siteName: null,
-            textContent: mainContent.textContent || '',
-            publishedTime: null,
-            baseUrl,
-        };
-    }
-    catch (error) {
-        console.error('Error in manual extraction:', error);
-        return {
-            title: 'Error extracting content',
-            content: dom.window.document.body?.innerHTML ||
-                dom.window.document.documentElement?.innerHTML ||
-                '',
-            byline: null,
-            excerpt: '',
-            dir: null,
-            lang: null,
-            length: 0,
-            siteName: null,
-            textContent: dom.window.document.body?.textContent || '',
-            publishedTime: null,
-            baseUrl: dom.window.location.href,
-        };
-    }
-}
-export function hasContent(html) {
-    const lowerHtml = html.toLowerCase();
-    if (lowerHtml.includes('<noscript>') &&
-        !lowerHtml.includes('<article') &&
-        !lowerHtml.includes('<main')) {
-        return false;
-    }
-    const textContent = html.replace(/<[^>]*>/g, '').trim();
-    return textContent.length > 100;
-}

package/dist/parser/dom.d.ts

@@ -1,3 +0,0 @@
-import { JSDOM } from 'jsdom';
-export declare function htmlToDom(html: string, url: string): JSDOM;
-export declare function extractLinks(dom: JSDOM): string[];

package/dist/parser/dom.js

@@ -1,60 +0,0 @@
-import { JSDOM, VirtualConsole } from 'jsdom';
-export function htmlToDom(html, url) {
-    try {
-        return new JSDOM(html, {
-            url,
-            contentType: 'text/html',
-            includeNodeLocations: false,
-            runScripts: undefined,
-            resources: undefined,
-            pretendToBeVisual: true,
-            virtualConsole: new VirtualConsole().sendTo(console, {
-                omitJSDOMErrors: true,
-            }),
-        });
-    }
-    catch {
-        try {
-            return new JSDOM(html, {
-                url,
-                contentType: 'text/html',
-                virtualConsole: new VirtualConsole().sendTo(console, {
-                    omitJSDOMErrors: true,
-                }),
-            });
-        }
-        catch {
-            return new JSDOM(`<!DOCTYPE html><html><body>${html}</body></html>`, {
-                url,
-                contentType: 'text/html',
-                virtualConsole: new VirtualConsole().sendTo(console, {
-                    omitJSDOMErrors: true,
-                }),
-            });
-        }
-    }
-}
-export function extractLinks(dom) {
-    const document = dom.window.document;
-    const links = [];
-    const baseUrl = dom.window.location.href;
-    const anchorElements = document.querySelectorAll('a[href]');
-    anchorElements.forEach(element => {
-        try {
-            const href = element.getAttribute('href');
-            if (!href)
-                return;
-            if (href.startsWith('mailto:') ||
-                href.startsWith('tel:') ||
-                href.startsWith('javascript:') ||
-                href.startsWith('#')) {
-                return;
-            }
-            const absoluteUrl = new URL(href, baseUrl).href;
-            links.push(absoluteUrl);
-        }
-        catch {
-        }
-    });
-    return [...new Set(links)];
-}

package/dist/parser/markdown.d.ts

@@ -1,9 +0,0 @@
-import TurndownService from 'turndown';
-export declare function createTurndownService(): TurndownService;
-export declare function htmlToMarkdown(html: string): string;
-export declare function formatArticleMarkdown(article: {
-    title: string;
-    content: string;
-    byline?: string | null;
-    baseUrl?: string;
-}): string;

package/dist/parser/markdown.js

@@ -1,147 +0,0 @@
-import TurndownService from 'turndown';
-import { gfm } from 'turndown-plugin-gfm';
-import { JSDOM } from 'jsdom';
-function convertRelativeUrls(html, baseUrl) {
-    try {
-        const dom = new JSDOM(html, { url: baseUrl });
-        const document = dom.window.document;
-        document.querySelectorAll('a[href]').forEach(link => {
-            const href = link.getAttribute('href');
-            if (href &&
-                !href.startsWith('http://') &&
-                !href.startsWith('https://') &&
-                !href.startsWith('//') &&
-                !href.startsWith('mailto:') &&
-                !href.startsWith('tel:') &&
-                !href.startsWith('javascript:') &&
-                !href.startsWith('#')) {
-                try {
-                    const absoluteUrl = new URL(href, baseUrl).href;
-                    link.setAttribute('href', absoluteUrl);
-                }
-                catch {
-                }
-            }
-        });
-        document.querySelectorAll('img[src]').forEach(img => {
-            const src = img.getAttribute('src');
-            if (src &&
-                !src.startsWith('http://') &&
-                !src.startsWith('https://') &&
-                !src.startsWith('//') &&
-                !src.startsWith('data:')) {
-                try {
-                    const absoluteUrl = new URL(src, baseUrl).href;
-                    img.setAttribute('src', absoluteUrl);
-                }
-                catch {
-                }
-            }
-        });
-        const bodyElement = document.body || document.documentElement;
-        return bodyElement ? bodyElement.innerHTML : html;
-    }
-    catch {
-        return html;
-    }
-}
-export function createTurndownService() {
-    const turndown = new TurndownService({
-        headingStyle: 'atx',
-        codeBlockStyle: 'fenced',
-        linkStyle: 'inlined',
-        emDelimiter: '_',
-        bulletListMarker: '-',
-        strongDelimiter: '**',
-        hr: '---',
-        blankReplacement: (_content, node) => {
-            return node.isBlock ? '\n\n' : '';
-        },
-        keepReplacement: (content, node) => {
-            return node.isBlock ? '\n\n' + content + '\n\n' : content;
-        },
-        defaultReplacement: (content, node) => {
-            return node.isBlock ? '\n\n' + content + '\n\n' : content;
-        },
-    });
-    turndown.use(gfm);
-    turndown.addRule('media', {
-        filter: ['iframe', 'video', 'audio', 'embed'],
-        replacement: (_content, node) => {
-            const element = node;
-            const src = element.getAttribute('src') || element.getAttribute('data-src');
-            const title = element.getAttribute('title') ||
-                element.getAttribute('alt') ||
-                'media';
-            if (src) {
-                return `\n\n[${title}](${src})\n\n`;
-            }
-            return '';
-        },
-    });
-    turndown.addRule('figure', {
-        filter: 'figure',
-        replacement: (content, node) => {
-            const figure = node;
-            const caption = figure.querySelector('figcaption');
-            if (caption) {
-                const captionText = caption.textContent || '';
-                return `\n\n${content.trim()}\n*${captionText}*\n\n`;
-            }
-            return `\n\n${content.trim()}\n\n`;
-        },
-    });
-    return turndown;
-}
-export function htmlToMarkdown(html) {
-    const turndown = createTurndownService();
-    let markdown = turndown.turndown(html);
-    markdown = markdown
-        .replace(/\n{3,}/g, '\n\n')
-        .replace(/\s+$/gm, '')
-        .trim();
-    return markdown;
-}
-export function formatArticleMarkdown(article) {
-    try {
-        const turndown = createTurndownService();
-        let markdown = '';
-        if (article.title && article.title.trim()) {
-            markdown = `# ${article.title}\n\n`;
-        }
-        if (article.byline) {
-            markdown += `*By ${article.byline}*\n\n---\n\n`;
-        }
-        try {
-            const processedContent = article.baseUrl
-                ? convertRelativeUrls(article.content, article.baseUrl)
-                : article.content;
-            markdown += turndown.turndown(processedContent);
-        }
-        catch (conversionError) {
-            console.error('Error converting HTML to markdown:', conversionError);
-            const tempDiv = typeof document !== 'undefined'
-                ? document.createElement('div')
-                : null;
-            if (tempDiv) {
-                tempDiv.innerHTML = article.content;
-                markdown += tempDiv.textContent || article.content;
-            }
-            else {
-                markdown += article.content
-                    .replace(/<[^>]*>/g, ' ')
-                    .replace(/\s+/g, ' ');
-            }
-        }
-        return markdown
-            .replace(/\n{3,}/g, '\n\n')
-            .replace(/\s+$/gm, '')
-            .trim();
-    }
-    catch (error) {
-        console.error('Fatal error in formatArticleMarkdown:', error);
-        return article.title
-            ? `# ${article.title}\n\n[Content extraction failed]`
-            : '[Content extraction failed]';
-    }
-}