@just-every/mcp-read-website-fast 0.1.21 → 0.1.23

package/dist/index.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { Command } from 'commander';
 import { fetchMarkdown } from './internal/fetchMarkdown.js';
-import { loadCrawlModule } from './internal/crawlCompat.js';
+import { secureCrawl } from './internal/secureCrawl.js';
 import { readFileSync } from 'fs';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
@@ -43,8 +43,7 @@ program
         }
         console.error(`Fetching ${url}...`);
         if (options.output === 'json') {
-            const { fetch } = await loadCrawlModule();
-            const results = await fetch(url, crawlOptions);
+            const results = await secureCrawl(url, crawlOptions);
             console.log(JSON.stringify(results, null, 2));
         }
         else if (options.output === 'markdown') {
@@ -57,11 +56,13 @@ program
             }
             if (result.error) {
                 console.error(`Error: ${result.error}`);
+                if (!result.markdown) {
+                    process.exit(1);
+                }
             }
         }
         else if (options.output === 'both') {
-            const { fetch } = await loadCrawlModule();
-            const results = await fetch(url, crawlOptions);
+            const results = await secureCrawl(url, crawlOptions);
             results.forEach((result) => {
                 console.log(`\n## URL: ${result.url}\n`);
                 if (result.markdown) {

package/dist/internal/fetchMarkdown.js

@@ -1,5 +1,6 @@
-import { loadCrawlModule } from './crawlCompat.js';
-import { extractMarkdownLinks, filterSameOriginLinks } from '../utils/extractMarkdownLinks.js';
+import { extractMarkdownLinks, filterSameOriginLinks, } from '../utils/extractMarkdownLinks.js';
+import { assertPublicHttpUrl } from '../utils/urlPolicy.js';
+import { secureCrawl } from './secureCrawl.js';
 export async function fetchMarkdown(url, options = {}) {
     try {
         const maxPages = options.maxPages ?? 1;
@@ -11,6 +12,7 @@ export async function fetchMarkdown(url, options = {}) {
             if (visited.has(currentUrl))
                 continue;
             visited.add(currentUrl);
+            await assertPublicHttpUrl(currentUrl);
             const crawlOptions = {
                 depth: 0,
                 maxConcurrency: options.maxConcurrency ?? 3,
@@ -23,8 +25,7 @@ export async function fetchMarkdown(url, options = {}) {
             if (options.cookiesFile) {
                 crawlOptions.cookiesFile = options.cookiesFile;
             }
-            const { fetch } = await loadCrawlModule();
-            const results = await fetch(currentUrl, crawlOptions);
+            const results = await secureCrawl(currentUrl, crawlOptions);
             if (results && results.length > 0) {
                 const result = results[0];
                 allResults.push(result);
@@ -67,7 +68,10 @@ export async function fetchMarkdown(url, options = {}) {
             title: pagesToReturn[0].title,
             links: pagesToReturn.flatMap(r => r.links || []),
             error: pagesToReturn.some(r => r.error)
-                ? `Some pages had errors: ${pagesToReturn.filter(r => r.error).map(r => r.url).join(', ')}`
+                ? `Some pages had errors: ${pagesToReturn
+                    .filter(r => r.error)
+                    .map(r => r.url)
+                    .join(', ')}`
                 : undefined,
         };
     }

package/dist/internal/pinnedDnsDispatcher.d.ts

@@ -0,0 +1,5 @@
+import type { LookupFunction } from 'node:net';
+import { Agent } from 'undici';
+import type { PublicHttpUrlResolution } from '../utils/urlPolicy.js';
+export declare function createPinnedDnsAgent(resolution: PublicHttpUrlResolution): Agent;
+export declare function createPinnedLookup(resolution: PublicHttpUrlResolution): LookupFunction;

package/dist/internal/pinnedDnsDispatcher.js

@@ -0,0 +1,29 @@
+import { Agent } from 'undici';
+export function createPinnedDnsAgent(resolution) {
+    return new Agent({
+        connect: {
+            lookup: createPinnedLookup(resolution),
+        },
+    });
+}
+export function createPinnedLookup(resolution) {
+    return (hostname, options, callback) => {
+        const normalizedHostname = normalizeLookupHostname(hostname);
+        if (normalizedHostname !== resolution.hostname) {
+            callback(new Error(`Unexpected hostname lookup: ${normalizedHostname}`), '');
+            return;
+        }
+        if (options.all) {
+            callback(null, resolution.addresses);
+            return;
+        }
+        const address = resolution.addresses[0];
+        callback(null, address.address, address.family);
+    };
+}
+function normalizeLookupHostname(hostname) {
+    if (hostname.startsWith('[') && hostname.endsWith(']')) {
+        return hostname.slice(1, -1);
+    }
+    return hostname;
+}

package/dist/internal/secureCrawl.d.ts

@@ -0,0 +1,3 @@
+import type { CrawlResult } from '@just-every/crawl';
+import type { FetchMarkdownOptions } from './fetchMarkdown.js';
+export declare function secureCrawl(startUrl: string, options?: FetchMarkdownOptions): Promise<CrawlResult[]>;

package/dist/internal/secureCrawl.js

@@ -0,0 +1,164 @@
+import pLimit from 'p-limit';
+import { DiskCache } from '@just-every/crawl/dist/cache/disk.js';
+import { normalizeUrl, isSameOrigin, } from '@just-every/crawl/dist/cache/normalize.js';
+import { parseNetscapeCookieFile, buildCookieHeaderForUrl, } from '@just-every/crawl/dist/crawler/cookies.js';
+import { isAllowedByRobots, getCrawlDelay, } from '@just-every/crawl/dist/crawler/robots.js';
+import { htmlToDom, extractLinks } from '@just-every/crawl/dist/parser/dom.js';
+import { extractArticle } from '@just-every/crawl/dist/parser/article.js';
+import { formatArticleMarkdown } from '@just-every/crawl/dist/parser/markdown.js';
+import { assertPublicHttpUrl } from '../utils/urlPolicy.js';
+import { secureFetchHtml } from './secureFetchHtml.js';
+export async function secureCrawl(startUrl, options = {}) {
+    const crawler = new SecureCrawler(options);
+    await crawler.init();
+    return crawler.crawl(startUrl);
+}
+class SecureCrawler {
+    visited = new Set();
+    queue = [];
+    limit;
+    cache;
+    options;
+    results = [];
+    cookieJar;
+    constructor(options = {}) {
+        this.options = {
+            depth: options.depth ?? 0,
+            maxConcurrency: options.maxConcurrency ?? 3,
+            respectRobots: options.respectRobots ?? true,
+            sameOriginOnly: options.sameOriginOnly ?? true,
+            userAgent: options.userAgent,
+            cacheDir: options.cacheDir ?? '.cache',
+            timeout: options.timeout ?? 30000,
+            cookiesFile: options.cookiesFile,
+        };
+        this.limit = pLimit(this.options.maxConcurrency);
+        this.cache = new DiskCache(this.options.cacheDir);
+        if (options.cookiesFile) {
+            this.cookieJar = parseNetscapeCookieFile(options.cookiesFile);
+        }
+    }
+    async init() {
+        await this.cache.init();
+    }
+    async crawl(startUrl) {
+        const normalizedUrl = normalizeUrl(startUrl);
+        await assertPublicHttpUrl(normalizedUrl);
+        this.queue.push(normalizedUrl);
+        await this.processQueue(0);
+        return this.results;
+    }
+    async processQueue(currentDepth) {
+        if (currentDepth > this.options.depth) {
+            return;
+        }
+        const urls = [...this.queue];
+        this.queue = [];
+        const tasks = urls.map(url => this.limit(() => this.processUrl(url, currentDepth)));
+        await Promise.all(tasks);
+        if (this.queue.length > 0) {
+            await this.processQueue(currentDepth + 1);
+        }
+    }
+    async processUrl(url, depth) {
+        const normalizedUrl = normalizeUrl(url);
+        if (this.visited.has(normalizedUrl)) {
+            return;
+        }
+        this.visited.add(normalizedUrl);
+        try {
+            await assertPublicHttpUrl(normalizedUrl);
+            const cached = await this.cache.get(normalizedUrl);
+            if (cached) {
+                this.results.push({
+                    url: normalizedUrl,
+                    markdown: cached.markdown,
+                    title: cached.title,
+                });
+                return;
+            }
+            if (this.options.respectRobots) {
+                const allowed = await isAllowedByRobots(normalizedUrl, this.options.userAgent);
+                if (!allowed) {
+                    this.results.push({
+                        url: normalizedUrl,
+                        markdown: '',
+                        error: 'Blocked by robots.txt',
+                    });
+                    return;
+                }
+                const delay = await getCrawlDelay(normalizedUrl, this.options.userAgent);
+                if (delay > 0) {
+                    await new Promise(resolve => setTimeout(resolve, delay * 1000));
+                }
+            }
+            const { html, finalUrl } = await secureFetchHtml(normalizedUrl, {
+                userAgent: this.options.userAgent,
+                timeout: this.options.timeout,
+                cookieHeaderForUrl: redirectedUrl => this.cookieJar
+                    ? buildCookieHeaderForUrl(redirectedUrl, this.cookieJar)
+                    : undefined,
+            });
+            if (!html.trim()) {
+                this.results.push({
+                    url: finalUrl,
+                    markdown: '',
+                    error: 'Empty response from server',
+                });
+                return;
+            }
+            const dom = htmlToDom(html, finalUrl);
+            const article = extractArticle(dom);
+            if (!article) {
+                this.results.push({
+                    url: finalUrl,
+                    markdown: '',
+                    error: 'Failed to extract article content',
+                });
+                return;
+            }
+            if (!article.content || article.content.trim().length < 50) {
+                this.results.push({
+                    url: finalUrl,
+                    markdown: `# ${article.title || 'Page Content'}\n\n` +
+                        '*Note: This page appears to be JavaScript-rendered. Limited content extracted.*\n\n' +
+                        (article.textContent
+                            ? `${article.textContent.substring(0, 1000)}...`
+                            : 'No text content available'),
+                    title: article.title || finalUrl,
+                    error: 'Limited content extracted (JavaScript-rendered page)',
+                });
+                return;
+            }
+            const markdown = formatArticleMarkdown(article);
+            await this.cache.put(finalUrl, markdown, article.title);
+            let links = [];
+            if (depth < this.options.depth) {
+                links = extractLinks(dom);
+                if (this.options.sameOriginOnly) {
+                    links = links.filter(link => isSameOrigin(finalUrl, link));
+                }
+                for (const link of links) {
+                    const normalized = normalizeUrl(link);
+                    if (!this.visited.has(normalized)) {
+                        await assertPublicHttpUrl(normalized);
+                        this.queue.push(normalized);
+                    }
+                }
+            }
+            this.results.push({
+                url: finalUrl,
+                markdown,
+                title: article.title,
+                links: links.length > 0 ? links : undefined,
+            });
+        }
+        catch (error) {
+            this.results.push({
+                url: normalizedUrl,
+                markdown: '',
+                error: error instanceof Error ? error.message : 'Unknown error',
+            });
+        }
+    }
+}

package/dist/internal/secureFetchHtml.d.ts

@@ -0,0 +1,10 @@
+export interface SecureFetchHtmlOptions {
+    userAgent?: string;
+    timeout?: number;
+    maxRedirections?: number;
+    cookieHeaderForUrl?: (url: string) => string | undefined;
+}
+export declare function secureFetchHtml(url: string, options?: SecureFetchHtmlOptions): Promise<{
+    html: string;
+    finalUrl: string;
+}>;

package/dist/internal/secureFetchHtml.js

@@ -0,0 +1,53 @@
+import { fetch } from 'undici';
+import { resolvePublicHttpUrl } from '../utils/urlPolicy.js';
+import { createPinnedDnsAgent } from './pinnedDnsDispatcher.js';
+const DEFAULT_USER_AGENT = 'MCP/0.1 (+https://github.com/just-every/mcp-read-website-fast)';
+export async function secureFetchHtml(url, options = {}) {
+    const maxRedirections = options.maxRedirections ?? 5;
+    let currentUrl = url;
+    for (let redirectCount = 0; redirectCount <= maxRedirections; redirectCount += 1) {
+        const resolution = await resolvePublicHttpUrl(currentUrl);
+        const dispatcher = createPinnedDnsAgent(resolution);
+        try {
+            const response = await fetch(currentUrl, {
+                dispatcher,
+                headers: {
+                    'User-Agent': options.userAgent ?? DEFAULT_USER_AGENT,
+                    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                    'Accept-Language': 'en-US,en;q=0.5',
+                    DNT: '1',
+                    Connection: 'keep-alive',
+                    'Upgrade-Insecure-Requests': '1',
+                    ...(options.cookieHeaderForUrl?.(currentUrl)
+                        ? { Cookie: options.cookieHeaderForUrl(currentUrl) }
+                        : {}),
+                },
+                redirect: 'manual',
+                signal: AbortSignal.timeout(options.timeout ?? 30000),
+            });
+            if (response.status >= 300 && response.status < 400) {
+                const location = response.headers.get('location');
+                await response.body?.cancel();
+                if (!location) {
+                    throw new Error(`Redirect without Location header for ${currentUrl}`);
+                }
+                currentUrl = new URL(location, currentUrl).href;
+                continue;
+            }
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status} for ${currentUrl}`);
+            }
+            const contentType = response.headers.get('content-type');
+            if (contentType &&
+                !contentType.includes('text/html') &&
+                !contentType.includes('application/xhtml+xml')) {
+                throw new Error(`Non-HTML content type: ${contentType} for ${currentUrl}`);
+            }
+            return { html: await response.text(), finalUrl: currentUrl };
+        }
+        finally {
+            await dispatcher.close();
+        }
+    }
+    throw new Error(`Too many redirects for ${url}`);
+}

package/dist/internal/turndownPluginGfmCompat.js

@@ -1,6 +1,16 @@
 import { createRequire } from 'node:module';
-const require = createRequire(import.meta.url);
-const turndownPluginGfmModule = require('turndown-plugin-gfm');
+const requireFromHere = createRequire(import.meta.url);
+function loadTurndownPluginGfmModule() {
+    try {
+        return requireFromHere('turndown-plugin-gfm');
+    }
+    catch {
+        const crawlPackageJsonPath = requireFromHere.resolve('@just-every/crawl/package.json');
+        const requireFromCrawl = createRequire(crawlPackageJsonPath);
+        return requireFromCrawl('turndown-plugin-gfm');
+    }
+}
+const turndownPluginGfmModule = loadTurndownPluginGfmModule();
 export const gfm = turndownPluginGfmModule.gfm ?? turndownPluginGfmModule.default?.gfm;
 if (typeof gfm !== 'function') {
     throw new Error('turndown-plugin-gfm did not provide a usable gfm export');

package/dist/utils/urlPolicy.d.ts

@@ -0,0 +1,14 @@
+export interface PublicResolvedAddress {
+    address: string;
+    family: 4 | 6;
+}
+export interface PublicHttpUrlResolution {
+    url: URL;
+    hostname: string;
+    addresses: PublicResolvedAddress[];
+}
+export declare class UrlPolicyError extends Error {
+    constructor(message: string);
+}
+export declare function assertPublicHttpUrl(url: string): Promise<URL>;
+export declare function resolvePublicHttpUrl(url: string): Promise<PublicHttpUrlResolution>;

package/dist/utils/urlPolicy.js

@@ -0,0 +1,172 @@
+import { lookup } from 'node:dns/promises';
+import { isIP } from 'node:net';
+const IPV4_BLOCKED_RANGES = [
+    [0x00000000n, 0xff000000n],
+    [0x0a000000n, 0xff000000n],
+    [0x64400000n, 0xffc00000n],
+    [0x7f000000n, 0xff000000n],
+    [0xa9fe0000n, 0xffff0000n],
+    [0xac100000n, 0xfff00000n],
+    [0xc0000000n, 0xffffff00n],
+    [0xc0000200n, 0xffffff00n],
+    [0xc0586300n, 0xffffff00n],
+    [0xc0a80000n, 0xffff0000n],
+    [0xc6120000n, 0xfffe0000n],
+    [0xc6336400n, 0xffffff00n],
+    [0xcb007100n, 0xffffff00n],
+    [0xe0000000n, 0xf0000000n],
+    [0xf0000000n, 0xf0000000n],
+];
+const IPV6_BLOCKED_RANGES = [
+    [0n, (1n << 128n) - 1n],
+    [1n, (1n << 128n) - 1n],
+    [0xffffn << 32n, ((1n << 96n) - 1n) << 32n],
+    [0x0064ff9b000000000000000000000000n, prefixMask(96)],
+    [0x01000000000000000000000000000000n, prefixMask(64)],
+    [0x20010000000000000000000000000000n, prefixMask(23)],
+    [0x20020000000000000000000000000000n, prefixMask(16)],
+    [0xfc000000000000000000000000000000n, prefixMask(7)],
+    [0xfe800000000000000000000000000000n, prefixMask(10)],
+    [0xff000000000000000000000000000000n, prefixMask(8)],
+];
+export class UrlPolicyError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'UrlPolicyError';
+    }
+}
+export async function assertPublicHttpUrl(url) {
+    return (await resolvePublicHttpUrl(url)).url;
+}
+export async function resolvePublicHttpUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        throw new UrlPolicyError(`Invalid URL: ${url}`);
+    }
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+        throw new UrlPolicyError(`URL scheme is not allowed: ${parsed.protocol}`);
+    }
+    if (parsed.username || parsed.password) {
+        throw new UrlPolicyError('URL credentials are not allowed');
+    }
+    const hostname = normalizeHostname(parsed.hostname);
+    const hostIpVersion = isIP(hostname);
+    if (hostIpVersion !== 0) {
+        const family = ipVersionToFamily(hostIpVersion);
+        assertPublicIp(hostname, family);
+        return {
+            url: parsed,
+            hostname,
+            addresses: [{ address: hostname, family }],
+        };
+    }
+    let addresses;
+    try {
+        addresses = await lookup(hostname, { all: true, verbatim: true });
+    }
+    catch (error) {
+        throw new UrlPolicyError(`Unable to resolve hostname "${hostname}": ${error instanceof Error ? error.message : 'Unknown DNS error'}`);
+    }
+    if (addresses.length === 0) {
+        throw new UrlPolicyError(`Hostname "${hostname}" did not resolve`);
+    }
+    const publicAddresses = addresses.map(address => {
+        const family = ipVersionToFamily(address.family);
+        assertPublicIp(address.address, family);
+        return { address: address.address, family };
+    });
+    return { url: parsed, hostname, addresses: publicAddresses };
+}
+function normalizeHostname(hostname) {
+    if (hostname.startsWith('[') && hostname.endsWith(']')) {
+        return hostname.slice(1, -1);
+    }
+    return hostname;
+}
+function assertPublicIp(address, family) {
+    if (family === 4) {
+        if (isBlockedIpv4(address)) {
+            throw new UrlPolicyError(`IP address is not allowed: ${address}`);
+        }
+        return;
+    }
+    if (family === 6) {
+        const ipv4Mapped = address
+            .toLowerCase()
+            .match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
+        if (ipv4Mapped) {
+            assertPublicIp(ipv4Mapped[1], 4);
+            return;
+        }
+        if (isBlockedIpv6(address)) {
+            throw new UrlPolicyError(`IP address is not allowed: ${address}`);
+        }
+        return;
+    }
+    throw new UrlPolicyError(`Unknown IP address family for ${address}`);
+}
+function ipVersionToFamily(family) {
+    if (family === 4 || family === 6) {
+        return family;
+    }
+    throw new UrlPolicyError(`Unknown IP address family: ${family}`);
+}
+function isBlockedIpv4(address) {
+    const numeric = ipv4ToNumber(address);
+    return IPV4_BLOCKED_RANGES.some(([range, mask]) => (numeric & mask) === range);
+}
+function ipv4ToNumber(address) {
+    const parts = address.split('.').map(part => Number(part));
+    if (parts.length !== 4 ||
+        parts.some(part => !Number.isInteger(part) || part < 0 || part > 255)) {
+        throw new UrlPolicyError(`Invalid IPv4 address: ${address}`);
+    }
+    return ((BigInt(parts[0]) << 24n) +
+        (BigInt(parts[1]) << 16n) +
+        (BigInt(parts[2]) << 8n) +
+        BigInt(parts[3]));
+}
+function isBlockedIpv6(address) {
+    const numeric = ipv6ToBigInt(address);
+    return IPV6_BLOCKED_RANGES.some(([range, mask]) => (numeric & mask) === range);
+}
+function ipv6ToBigInt(address) {
+    const lower = address.toLowerCase();
+    const [head = '', tail = ''] = lower.split('::');
+    const headParts = parseIpv6Parts(head);
+    const tailParts = parseIpv6Parts(tail);
+    const missing = 8 - headParts.length - tailParts.length;
+    if (lower.includes('::')) {
+        if (missing < 1) {
+            throw new UrlPolicyError(`Invalid IPv6 address: ${address}`);
+        }
+    }
+    else if (headParts.length !== 8) {
+        throw new UrlPolicyError(`Invalid IPv6 address: ${address}`);
+    }
+    const parts = lower.includes('::')
+        ? [...headParts, ...Array(missing).fill(0), ...tailParts]
+        : headParts;
+    return parts.reduce((acc, part) => (acc << 16n) + BigInt(part), 0n);
+}
+function parseIpv6Parts(value) {
+    if (!value) {
+        return [];
+    }
+    return value.split(':').map(part => {
+        const parsed = Number.parseInt(part, 16);
+        if (!part ||
+            !Number.isInteger(parsed) ||
+            parsed < 0 ||
+            parsed > 0xffff) {
+            throw new UrlPolicyError(`Invalid IPv6 address part: ${part}`);
+        }
+        return parsed;
+    });
+}
+function prefixMask(prefixLength) {
+    return ((1n << BigInt(prefixLength)) - 1n) << BigInt(128 - prefixLength);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@just-every/mcp-read-website-fast",
-  "version": "0.1.21",
+  "version": "0.1.23",
   "description": "Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown",
   "main": "dist/index.js",
   "bin": {
@@ -53,6 +53,7 @@
     "@just-every/crawl": "^1.0.8",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "commander": "^14.0.3",
+    "turndown-plugin-gfm": "^1.0.2",
     "uuid": "^13.0.0"
   },
   "devDependencies": {