@just-every/mcp-read-website-fast 0.1.20 → 0.1.21

package/dist/index.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { Command } from 'commander';
-import { fetch } from '@just-every/crawl';
 import { fetchMarkdown } from './internal/fetchMarkdown.js';
+import { loadCrawlModule } from './internal/crawlCompat.js';
 import { readFileSync } from 'fs';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
@@ -43,6 +43,7 @@ program
         }
         console.error(`Fetching ${url}...`);
         if (options.output === 'json') {
+            const { fetch } = await loadCrawlModule();
             const results = await fetch(url, crawlOptions);
             console.log(JSON.stringify(results, null, 2));
         }
@@ -59,6 +60,7 @@ program
             }
         }
         else if (options.output === 'both') {
+            const { fetch } = await loadCrawlModule();
             const results = await fetch(url, crawlOptions);
             results.forEach((result) => {
                 console.log(`\n## URL: ${result.url}\n`);

package/dist/internal/crawlCompat.d.ts

@@ -0,0 +1,4 @@
+import type { CrawlOptions } from '@just-every/crawl';
+export type { CrawlOptions };
+type CrawlModule = typeof import('@just-every/crawl');
+export declare function loadCrawlModule(): Promise<CrawlModule>;

package/dist/internal/crawlCompat.js

@@ -0,0 +1,11 @@
+import { ensureTurndownPluginCompat } from './turndownPluginCompat.js';
+let crawlModulePromise;
+export async function loadCrawlModule() {
+    if (!crawlModulePromise) {
+        crawlModulePromise = (async () => {
+            await ensureTurndownPluginCompat();
+            return import('@just-every/crawl');
+        })();
+    }
+    return crawlModulePromise;
+}

package/dist/internal/fetchMarkdown.js

@@ -1,4 +1,4 @@
-import { fetch } from '@just-every/crawl';
+import { loadCrawlModule } from './crawlCompat.js';
 import { extractMarkdownLinks, filterSameOriginLinks } from '../utils/extractMarkdownLinks.js';
 export async function fetchMarkdown(url, options = {}) {
     try {
@@ -23,6 +23,7 @@ export async function fetchMarkdown(url, options = {}) {
             if (options.cookiesFile) {
                 crawlOptions.cookiesFile = options.cookiesFile;
             }
+            const { fetch } = await loadCrawlModule();
             const results = await fetch(currentUrl, crawlOptions);
             if (results && results.length > 0) {
                 const result = results[0];

package/dist/internal/turndownPluginCompat.d.ts

@@ -0,0 +1,12 @@
+interface DualPackageManifest {
+    module?: unknown;
+    main?: unknown;
+}
+export declare function applyCrawlMarkdownInteropPatch(source: string, turndownPluginUrl: string): string;
+export declare function selectPreferredPackageEntry(manifest: DualPackageManifest, packageName: string): string;
+export declare function resolvePackageModuleUrlFromImporter(packageName: string, importerPackageJsonPath: string): string;
+export declare function resolveTurndownPluginEsmUrl(): string;
+export declare function resolveTurndownPluginCompatUrl(): string;
+export declare function patchCrawlMarkdownInterop(): void;
+export declare function ensureTurndownPluginCompat(): Promise<void>;
+export {};

package/dist/internal/turndownPluginCompat.js

@@ -0,0 +1,76 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { createRequire, register } from 'node:module';
+import { pathToFileURL } from 'node:url';
+let loaderRegistered = false;
+const CRAWL_GFM_IMPORT = "import { gfm } from 'turndown-plugin-gfm';";
+const PATCHED_GFM_IMPORT_PATTERN = /\/\/ Patched by @just-every\/mcp-read-website-fast for Node ESM\/CJS interop\.\nimport \{ gfm \} from [^\n]+;/;
+function getCrawlPackageJsonPath() {
+    const requireFromHere = createRequire(import.meta.url);
+    return requireFromHere.resolve('@just-every/crawl/package.json');
+}
+function getPatchedGfmImport(turndownPluginUrl) {
+    return [
+        '// Patched by @just-every/mcp-read-website-fast for Node ESM/CJS interop.',
+        `import { gfm } from ${JSON.stringify(turndownPluginUrl)};`,
+    ].join('\n');
+}
+export function applyCrawlMarkdownInteropPatch(source, turndownPluginUrl) {
+    const patchedImport = getPatchedGfmImport(turndownPluginUrl);
+    if (source.includes(patchedImport) || source.includes(turndownPluginUrl)) {
+        return source;
+    }
+    if (PATCHED_GFM_IMPORT_PATTERN.test(source)) {
+        return source.replace(PATCHED_GFM_IMPORT_PATTERN, patchedImport);
+    }
+    if (!source.includes(CRAWL_GFM_IMPORT)) {
+        return source;
+    }
+    return source.replace(CRAWL_GFM_IMPORT, patchedImport);
+}
+export function selectPreferredPackageEntry(manifest, packageName) {
+    if (typeof manifest.module === 'string' && manifest.module.length > 0) {
+        return manifest.module;
+    }
+    if (typeof manifest.main === 'string' && manifest.main.length > 0) {
+        return manifest.main;
+    }
+    throw new Error(`Could not determine entry file for ${packageName}`);
+}
+export function resolvePackageModuleUrlFromImporter(packageName, importerPackageJsonPath) {
+    const importerRequire = createRequire(importerPackageJsonPath);
+    const packageJsonPath = importerRequire.resolve(`${packageName}/package.json`);
+    const manifest = JSON.parse(readFileSync(packageJsonPath, 'utf8'));
+    const entry = selectPreferredPackageEntry(manifest, packageName);
+    return pathToFileURL(join(dirname(packageJsonPath), entry)).href;
+}
+export function resolveTurndownPluginEsmUrl() {
+    const crawlPackageJsonPath = getCrawlPackageJsonPath();
+    return resolvePackageModuleUrlFromImporter('turndown-plugin-gfm', crawlPackageJsonPath);
+}
+export function resolveTurndownPluginCompatUrl() {
+    return new URL('./turndownPluginGfmCompat.js', import.meta.url).href;
+}
+export function patchCrawlMarkdownInterop() {
+    const crawlPackageJsonPath = getCrawlPackageJsonPath();
+    const crawlMarkdownPath = join(dirname(crawlPackageJsonPath), 'dist', 'parser', 'markdown.js');
+    const compatModuleUrl = resolveTurndownPluginCompatUrl();
+    const source = readFileSync(crawlMarkdownPath, 'utf8');
+    const patchedSource = applyCrawlMarkdownInteropPatch(source, compatModuleUrl);
+    if (patchedSource === source) {
+        return;
+    }
+    try {
+        writeFileSync(crawlMarkdownPath, patchedSource, 'utf8');
+    }
+    catch {
+    }
+}
+export async function ensureTurndownPluginCompat() {
+    if (loaderRegistered) {
+        return;
+    }
+    patchCrawlMarkdownInterop();
+    register(new URL('./turndownPluginLoader.js', import.meta.url), import.meta.url);
+    loaderRegistered = true;
+}

package/dist/internal/turndownPluginGfmCompat.d.ts

@@ -0,0 +1,3 @@
+type GfmPlugin = (service: unknown) => void;
+export declare const gfm: GfmPlugin | undefined;
+export {};

package/dist/internal/turndownPluginGfmCompat.js

@@ -0,0 +1,7 @@
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+const turndownPluginGfmModule = require('turndown-plugin-gfm');
+export const gfm = turndownPluginGfmModule.gfm ?? turndownPluginGfmModule.default?.gfm;
+if (typeof gfm !== 'function') {
+    throw new Error('turndown-plugin-gfm did not provide a usable gfm export');
+}

package/dist/internal/turndownPluginLoader.d.ts

@@ -0,0 +1,10 @@
+export declare function resolve(specifier: string, context: {
+    parentURL?: string;
+}, defaultResolve: (specifier: string, context: {
+    parentURL?: string;
+}) => Promise<{
+    url: string;
+}>): Promise<{
+    shortCircuit?: boolean;
+    url: string;
+}>;

package/dist/internal/turndownPluginLoader.js

@@ -0,0 +1,11 @@
+import { resolveTurndownPluginCompatUrl } from './turndownPluginCompat.js';
+const turndownPluginCompatUrl = resolveTurndownPluginCompatUrl();
+export async function resolve(specifier, context, defaultResolve) {
+    if (specifier === 'turndown-plugin-gfm') {
+        return {
+            shortCircuit: true,
+            url: turndownPluginCompatUrl,
+        };
+    }
+    return defaultResolve(specifier, context);
+}

package/dist/mcp/definitions.d.ts

@@ -0,0 +1,3 @@
+import type { Resource, Tool } from '@modelcontextprotocol/sdk/types.js';
+export declare const READ_WEBSITE_TOOL: Tool;
+export declare const RESOURCES: Resource[];

package/dist/mcp/definitions.js

@@ -0,0 +1,46 @@
+export const READ_WEBSITE_TOOL = {
+    name: 'read_website',
+    description: 'Fast, token-efficient web content extraction - ideal for reading documentation, analyzing content, and gathering information from websites. Converts to clean Markdown while preserving links and structure.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            url: {
+                type: 'string',
+                description: 'HTTP/HTTPS URL to fetch and convert to markdown',
+            },
+            pages: {
+                type: 'number',
+                description: 'Maximum number of pages to crawl (default: 1)',
+                default: 1,
+                minimum: 1,
+                maximum: 100,
+            },
+            cookiesFile: {
+                type: 'string',
+                description: 'Path to Netscape cookie file for authenticated pages',
+            },
+        },
+        required: ['url'],
+    },
+    annotations: {
+        title: 'Read Website',
+        readOnlyHint: true,
+        destructiveHint: false,
+        idempotentHint: true,
+        openWorldHint: true,
+    },
+};
+export const RESOURCES = [
+    {
+        uri: 'read-website-fast://status',
+        name: 'Cache Status',
+        mimeType: 'application/json',
+        description: 'Get cache status information',
+    },
+    {
+        uri: 'read-website-fast://clear-cache',
+        name: 'Clear Cache',
+        mimeType: 'application/json',
+        description: 'Clear the cache directory',
+    },
+];

package/dist/serve.js

@@ -2,6 +2,7 @@
 import { Server } from '@modelcontextprotocol/sdk/server/index.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { CallToolRequestSchema, ListToolsRequestSchema, ListResourcesRequestSchema, ReadResourceRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
+import { READ_WEBSITE_TOOL, RESOURCES } from './mcp/definitions.js';
 import { logger } from './utils/logger.js';
 logger.info('MCP Server starting up...');
 logger.debug('Node version:', process.version);
@@ -27,53 +28,6 @@ logger.info('MCP server instance created successfully');
 server.onerror = error => {
     logger.error('MCP Server Error:', error);
 };
-const READ_WEBSITE_TOOL = {
-    name: 'read_website',
-    description: 'Fast, token-efficient web content extraction - ideal for reading documentation, analyzing content, and gathering information from websites. Converts to clean Markdown while preserving links and structure.',
-    inputSchema: {
-        type: 'object',
-        properties: {
-            url: {
-                type: 'string',
-                description: 'HTTP/HTTPS URL to fetch and convert to markdown',
-            },
-            pages: {
-                type: 'number',
-                description: 'Maximum number of pages to crawl (default: 1)',
-                default: 1,
-                minimum: 1,
-                maximum: 100,
-            },
-            cookiesFile: {
-                type: 'string',
-                description: 'Path to Netscape cookie file for authenticated pages',
-                optional: true,
-            },
-        },
-        required: ['url'],
-    },
-    annotations: {
-        title: 'Read Website',
-        readOnlyHint: true,
-        destructiveHint: false,
-        idempotentHint: true,
-        openWorldHint: true,
-    },
-};
-const RESOURCES = [
-    {
-        uri: 'read-website-fast://status',
-        name: 'Cache Status',
-        mimeType: 'application/json',
-        description: 'Get cache status information',
-    },
-    {
-        uri: 'read-website-fast://clear-cache',
-        name: 'Clear Cache',
-        mimeType: 'application/json',
-        description: 'Clear the cache directory',
-    },
-];
 server.setRequestHandler(ListToolsRequestSchema, async () => {
     logger.debug('Received ListTools request');
     const response = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@just-every/mcp-read-website-fast",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "description": "Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown",
   "main": "dist/index.js",
   "bin": {
@@ -51,23 +51,23 @@
   "license": "MIT",
   "dependencies": {
     "@just-every/crawl": "^1.0.8",
-    "@modelcontextprotocol/sdk": "^1.17.4",
-    "commander": "^14.0.0",
-    "uuid": "^11.1.0"
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "commander": "^14.0.3",
+    "uuid": "^13.0.0"
   },
   "devDependencies": {
-    "@types/jsdom": "^21.1.7",
-    "@types/node": "^24.3.0",
-    "@types/turndown": "^5.0.5",
-    "@typescript-eslint/eslint-plugin": "^8.41.0",
-    "@typescript-eslint/parser": "^8.41.0",
-    "eslint": "^9.34.0",
+    "@types/jsdom": "^28.0.1",
+    "@types/node": "^25.5.2",
+    "@types/turndown": "^5.0.6",
+    "@typescript-eslint/eslint-plugin": "^8.58.0",
+    "@typescript-eslint/parser": "^8.58.0",
+    "eslint": "^9.39.4",
     "eslint-config-prettier": "^10.1.8",
-    "eslint-plugin-prettier": "^5.5.4",
-    "tsx": "^4.20.5",
-    "typescript": "^5.9.2",
-    "typescript-eslint": "^8.41.0",
-    "vitest": "^3.2.4"
+    "eslint-plugin-prettier": "^5.5.5",
+    "tsx": "^4.21.0",
+    "typescript": "^6.0.2",
+    "typescript-eslint": "^8.58.0",
+    "vitest": "^4.1.2"
   },
   "engines": {
     "node": ">=20.0.0"