@just-every/mcp-read-website-fast 0.1.1

package/dist/utils/chunker.js

@@ -0,0 +1,146 @@
+export class MarkdownChunker {
+    options;
+    constructor(options = {}) {
+        this.options = {
+            maxTokens: options.maxTokens ?? 0,
+            maxChars: options.maxChars ?? 4000,
+            splitOn: options.splitOn ?? 'heading',
+            overlap: options.overlap ?? 200
+        };
+    }
+    chunk(markdown) {
+        switch (this.options.splitOn) {
+            case 'heading':
+                return this.chunkByHeading(markdown);
+            case 'paragraph':
+                return this.chunkByParagraph(markdown);
+            case 'sentence':
+                return this.chunkBySentence(markdown);
+            default:
+                return this.chunkByHeading(markdown);
+        }
+    }
+    chunkByHeading(markdown) {
+        const chunks = [];
+        const lines = markdown.split('\n');
+        let currentChunk = [];
+        let currentHeadings = [];
+        let startLine = 0;
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            const isHeading = /^#+\s/.test(line);
+            if (isHeading && currentChunk.length > 0) {
+                chunks.push({
+                    content: currentChunk.join('\n').trim(),
+                    index: chunks.length,
+                    metadata: {
+                        headings: [...currentHeadings],
+                        startLine,
+                        endLine: i - 1
+                    }
+                });
+                const overlapLines = this.getOverlapLines(currentChunk);
+                currentChunk = [...overlapLines, line];
+                currentHeadings = [line];
+                startLine = i - overlapLines.length;
+            }
+            else {
+                currentChunk.push(line);
+                if (isHeading) {
+                    currentHeadings.push(line);
+                }
+            }
+            const currentSize = currentChunk.join('\n').length;
+            if (currentSize > this.options.maxChars) {
+                chunks.push({
+                    content: currentChunk.join('\n').trim(),
+                    index: chunks.length,
+                    metadata: {
+                        headings: [...currentHeadings],
+                        startLine,
+                        endLine: i
+                    }
+                });
+                const overlapLines = this.getOverlapLines(currentChunk);
+                currentChunk = [...overlapLines];
+                currentHeadings = [];
+                startLine = i - overlapLines.length + 1;
+            }
+        }
+        if (currentChunk.length > 0) {
+            chunks.push({
+                content: currentChunk.join('\n').trim(),
+                index: chunks.length,
+                metadata: {
+                    headings: currentHeadings,
+                    startLine,
+                    endLine: lines.length - 1
+                }
+            });
+        }
+        return chunks;
+    }
+    chunkByParagraph(markdown) {
+        const chunks = [];
+        const paragraphs = markdown.split(/\n\n+/);
+        let currentChunk = [];
+        for (const paragraph of paragraphs) {
+            const wouldExceedLimit = currentChunk.join('\n\n').length + paragraph.length > this.options.maxChars;
+            if (wouldExceedLimit && currentChunk.length > 0) {
+                chunks.push({
+                    content: currentChunk.join('\n\n').trim(),
+                    index: chunks.length
+                });
+                currentChunk = [];
+            }
+            currentChunk.push(paragraph);
+        }
+        if (currentChunk.length > 0) {
+            chunks.push({
+                content: currentChunk.join('\n\n').trim(),
+                index: chunks.length
+            });
+        }
+        return chunks;
+    }
+    chunkBySentence(markdown) {
+        const chunks = [];
+        const sentences = markdown.match(/[^.!?]+[.!?]+/g) || [markdown];
+        let currentChunk = [];
+        for (const sentence of sentences) {
+            const wouldExceedLimit = currentChunk.join(' ').length + sentence.length > this.options.maxChars;
+            if (wouldExceedLimit && currentChunk.length > 0) {
+                chunks.push({
+                    content: currentChunk.join(' ').trim(),
+                    index: chunks.length
+                });
+                currentChunk = [];
+            }
+            currentChunk.push(sentence.trim());
+        }
+        if (currentChunk.length > 0) {
+            chunks.push({
+                content: currentChunk.join(' ').trim(),
+                index: chunks.length
+            });
+        }
+        return chunks;
+    }
+    getOverlapLines(lines) {
+        if (this.options.overlap <= 0)
+            return [];
+        let overlapChars = 0;
+        const overlapLines = [];
+        for (let i = lines.length - 1; i >= 0; i--) {
+            overlapLines.unshift(lines[i]);
+            overlapChars += lines[i].length + 1;
+            if (overlapChars >= this.options.overlap) {
+                break;
+            }
+        }
+        return overlapLines;
+    }
+    estimateTokens(text) {
+        return Math.ceil(text.length / 4);
+    }
+}

package/dist/utils/logger.d.ts

@@ -0,0 +1,18 @@
+export declare enum LogLevel {
+    ERROR = 0,
+    WARN = 1,
+    INFO = 2,
+    DEBUG = 3
+}
+export declare class Logger {
+    private level;
+    private name;
+    constructor(name: string, level?: LogLevel);
+    private log;
+    error(message: string, ...args: any[]): void;
+    warn(message: string, ...args: any[]): void;
+    info(message: string, ...args: any[]): void;
+    debug(message: string, ...args: any[]): void;
+    setLevel(level: LogLevel): void;
+}
+export declare const logger: Logger;

package/dist/utils/logger.js

@@ -0,0 +1,52 @@
+export var LogLevel;
+(function (LogLevel) {
+    LogLevel[LogLevel["ERROR"] = 0] = "ERROR";
+    LogLevel[LogLevel["WARN"] = 1] = "WARN";
+    LogLevel[LogLevel["INFO"] = 2] = "INFO";
+    LogLevel[LogLevel["DEBUG"] = 3] = "DEBUG";
+})(LogLevel || (LogLevel = {}));
+export class Logger {
+    level;
+    name;
+    constructor(name, level = LogLevel.INFO) {
+        this.name = name;
+        this.level = level;
+    }
+    log(level, message, ...args) {
+        if (level > this.level)
+            return;
+        const timestamp = new Date().toISOString();
+        const levelName = LogLevel[level];
+        const prefix = `[${timestamp}] [${levelName}] [${this.name}]`;
+        switch (level) {
+            case LogLevel.ERROR:
+                console.error(prefix, message, ...args);
+                break;
+            case LogLevel.WARN:
+                console.warn(prefix, message, ...args);
+                break;
+            default:
+                console.log(prefix, message, ...args);
+        }
+    }
+    error(message, ...args) {
+        this.log(LogLevel.ERROR, message, ...args);
+    }
+    warn(message, ...args) {
+        this.log(LogLevel.WARN, message, ...args);
+    }
+    info(message, ...args) {
+        this.log(LogLevel.INFO, message, ...args);
+    }
+    debug(message, ...args) {
+        this.log(LogLevel.DEBUG, message, ...args);
+    }
+    setLevel(level) {
+        this.level = level;
+    }
+}
+export const logger = new Logger('MCP');
+const envLevel = process.env.LOG_LEVEL?.toUpperCase();
+if (envLevel && envLevel in LogLevel) {
+    logger.setLevel(LogLevel[envLevel]);
+}

package/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@just-every/mcp-read-website-fast",
+  "version": "0.1.1",
+  "description": "Markdown Content Preprocessor - Fetch web pages, extract content, convert to clean Markdown",
+  "main": "dist/index.js",
+  "bin": {
+    "mcp-read-website-fast": "bin/mcp-read-website.js"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.prod.json",
+    "build:dev": "tsc",
+    "dev": "tsx src/index.ts",
+    "start": "node dist/index.js",
+    "serve": "tsx src/serve.ts",
+    "serve:dev": "tsx src/serve.ts",
+    "test": "vitest",
+    "test:deploy": "vitest run test/deployment.test.ts",
+    "lint": "eslint src",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [
+    "mcp",
+    "mcp-server",
+    "model-context-protocol",
+    "markdown",
+    "web-scraper",
+    "readability",
+    "content-extraction",
+    "rag",
+    "llm",
+    "claude",
+    "cursor",
+    "vscode"
+  ],
+  "author": "Just Every",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/just-every/mcp-read-website-fast.git"
+  },
+  "bugs": {
+    "url": "https://github.com/just-every/mcp-read-website-fast/issues"
+  },
+  "homepage": "https://github.com/just-every/mcp-read-website-fast#readme",
+  "license": "MIT",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.12.1",
+    "@mozilla/readability": "^0.6.0",
+    "commander": "^14.0.0",
+    "jsdom": "^26.1.0",
+    "p-limit": "^6.2.0",
+    "robots-parser": "^3.0.1",
+    "turndown": "^7.1.3",
+    "turndown-plugin-gfm": "^1.0.2",
+    "undici": "^7.10.0"
+  },
+  "devDependencies": {
+    "@types/jsdom": "^21.1.6",
+    "@types/node": "^24.0.0",
+    "@types/turndown": "^5.0.4",
+    "@typescript-eslint/eslint-plugin": "^8.34.0",
+    "@typescript-eslint/parser": "^8.34.0",
+    "eslint": "^9.28.0",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3",
+    "vitest": "^3.2.3"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "type": "module"
+}

package/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "src/test-*.ts", "src/serve-*.ts", "src/trace-*.ts"]
+}