@j0hanz/superfetch 1.2.4 → 2.0.0

package/dist/utils/url-validator.js

@@ -1,4 +1,3 @@
-import { lookup } from 'node:dns/promises';
 import { BlockList, isIP } from 'node:net';
 import { config } from '../config/index.js';
 import { createErrorWithCode } from './error-utils.js';
@@ -31,36 +30,6 @@ for (const entry of BLOCKED_IPV4_SUBNETS) {
 for (const entry of BLOCKED_IPV6_SUBNETS) {
     BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv6');
 }
-const DNS_LOOKUP_TIMEOUT_MS = 5000;
-const DNS_DECISION_TTL_MS = 60000;
-const DNS_DECISION_MAX = 1000;
-const dnsDecisionCache = new Map();
-function getCachedDnsDecision(hostname) {
-    const cached = dnsDecisionCache.get(hostname);
-    if (!cached)
-        return null;
-    if (cached.expiresAt <= Date.now()) {
-        dnsDecisionCache.delete(hostname);
-        return null;
-    }
-    return cached;
-}
-function setCachedDnsDecision(hostname, ok) {
-    dnsDecisionCache.set(hostname, {
-        ok,
-        expiresAt: Date.now() + DNS_DECISION_TTL_MS,
-    });
-    if (dnsDecisionCache.size <= DNS_DECISION_MAX)
-        return;
-    const evictCount = Math.ceil(DNS_DECISION_MAX * 0.05);
-    const iterator = dnsDecisionCache.keys();
-    for (let i = 0; i < evictCount; i++) {
-        const { value, done } = iterator.next();
-        if (done)
-            break;
-        dnsDecisionCache.delete(value);
-    }
-}
 function matchesBlockedIpPatterns(resolvedIp) {
     for (const pattern of config.security.blockedIpPatterns) {
         if (pattern.test(resolvedIp)) {
@@ -91,55 +60,6 @@ function isBlockedByList(ip, ipType) {
     }
     return BLOCK_LIST.check(ip, 'ipv6');
 }
-function lookupWithTimeout(hostname) {
-    return new Promise((resolve, reject) => {
-        const timer = setTimeout(() => {
-            reject(createValidationError(`DNS lookup timed out for ${hostname}`));
-        }, DNS_LOOKUP_TIMEOUT_MS);
-        lookup(hostname, { all: true })
-            .then((result) => {
-            clearTimeout(timer);
-            resolve(result);
-        })
-            .catch((error) => {
-            clearTimeout(timer);
-            reject(error instanceof Error ? error : createValidationError(String(error)));
-        });
-    });
-}
-export async function assertResolvedAddressesAllowed(hostname) {
-    const cached = getCachedDnsDecision(hostname);
-    if (cached) {
-        if (!cached.ok) {
-            throw createValidationError(`Blocked IP range resolved from hostname: ${hostname}`);
-        }
-        return;
-    }
-    try {
-        const result = await lookupWithTimeout(hostname);
-        const addresses = Array.isArray(result) ? result : [result];
-        if (addresses.length === 0) {
-            throw createValidationError(`Unable to resolve hostname: ${hostname}`);
-        }
-        for (const { address } of addresses) {
-            if (isBlockedIp(address.toLowerCase())) {
-                setCachedDnsDecision(hostname, false);
-                throw createValidationError(`Blocked IP range resolved from hostname: ${hostname}`);
-            }
-        }
-        setCachedDnsDecision(hostname, true);
-    }
-    catch (error) {
-        const code = error?.code;
-        if (code === 'ENOTFOUND' || code === 'EAI_AGAIN') {
-            throw createValidationError(`Unable to resolve hostname: ${hostname}`);
-        }
-        if (error instanceof Error) {
-            throw error;
-        }
-        throw createValidationError(String(error));
-    }
-}
 export function normalizeUrl(urlString) {
     const trimmedUrl = requireTrimmedUrl(urlString);
     assertUrlLength(trimmedUrl);
@@ -150,10 +70,8 @@ export function normalizeUrl(urlString) {
     assertHostnameAllowed(hostname);
     return { normalizedUrl: url.href, hostname };
 }
-export async function validateAndNormalizeUrl(urlString) {
-    const { normalizedUrl, hostname } = normalizeUrl(urlString);
-    await assertResolvedAddressesAllowed(hostname);
-    return normalizedUrl;
+export function validateAndNormalizeUrl(urlString) {
+    return normalizeUrl(urlString).normalizedUrl;
 }
 const VALIDATION_ERROR_CODE = 'VALIDATION_ERROR';
 function createValidationError(message) {
@@ -199,15 +117,24 @@ function normalizeHostname(url) {
 }
 const BLOCKED_HOST_SUFFIXES = ['.local', '.internal'];
 function assertHostnameAllowed(hostname) {
-    if (config.security.blockedHosts.has(hostname)) {
-        throw createValidationError(`Blocked host: ${hostname}. Internal hosts are not allowed`);
-    }
-    if (isBlockedIp(hostname)) {
-        throw createValidationError(`Blocked IP range: ${hostname}. Private IPs are not allowed`);
-    }
-    if (matchesBlockedSuffix(hostname)) {
-        throw createValidationError(`Blocked hostname pattern: ${hostname}. Internal domain suffixes are not allowed`);
-    }
+    assertNotBlockedHost(hostname);
+    assertNotBlockedIp(hostname);
+    assertNotBlockedHostnameSuffix(hostname);
+}
+function assertNotBlockedHost(hostname) {
+    if (!config.security.blockedHosts.has(hostname))
+        return;
+    throw createValidationError(`Blocked host: ${hostname}. Internal hosts are not allowed`);
+}
+function assertNotBlockedIp(hostname) {
+    if (!isBlockedIp(hostname))
+        return;
+    throw createValidationError(`Blocked IP range: ${hostname}. Private IPs are not allowed`);
+}
+function assertNotBlockedHostnameSuffix(hostname) {
+    if (!matchesBlockedSuffix(hostname))
+        return;
+    throw createValidationError(`Blocked hostname pattern: ${hostname}. Internal domain suffixes are not allowed`);
 }
 function matchesBlockedSuffix(hostname) {
     return BLOCKED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix));

package/dist/workers/content-transform.worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/content-transform.worker.js

@@ -0,0 +1,40 @@
+import { parentPort } from 'node:worker_threads';
+import { isRecord } from '../utils/guards.js';
+import { transformHtmlToMarkdownSync } from '../tools/utils/content-transform.js';
+const port = parentPort;
+function isWorkerTransformRequest(value) {
+    if (!isRecord(value))
+        return false;
+    return (typeof value.id === 'number' &&
+        typeof value.html === 'string' &&
+        typeof value.url === 'string' &&
+        typeof value.options === 'object');
+}
+function handleMessage(value) {
+    if (!port)
+        return;
+    if (!isWorkerTransformRequest(value))
+        return;
+    const { id, html, url, options } = value;
+    try {
+        const result = transformHtmlToMarkdownSync(html, url, options);
+        const response = {
+            id,
+            ok: true,
+            result,
+        };
+        port.postMessage(response);
+    }
+    catch (error) {
+        const response = {
+            id,
+            ok: false,
+            error: error instanceof Error ? error.message : String(error),
+        };
+        port.postMessage(response);
+    }
+}
+if (!port) {
+    process.exit(1);
+}
+port.on('message', handleMessage);

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@j0hanz/superfetch",
-  "version": "1.2.4",
+  "version": "2.0.0",
   "mcpName": "io.github.j0hanz/superfetch",
-  "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
+  "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",
   "main": "dist/index.js",
   "bin": {
@@ -28,54 +28,51 @@
     "web-fetching",
     "content-extraction",
     "readability",
-    "jsonl",
+    "markdown",
     "ai-tools",
     "model-context-protocol",
     "superfetch"
   ],
   "scripts": {
-    "dev": "tsx watch src/index.ts",
-    "build": "tsc -p tsconfig.build.json && shx chmod +x dist/*.js",
+    "build": "tsc -p tsconfig.build.json && node -e \"require('fs').chmodSync('dist/index.js', '755')\"",
     "prepare": "npm run build",
-    "prepublishOnly": "npm run build && npm run lint",
+    "dev": "tsx watch src/index.ts",
     "start": "node dist/index.js",
-    "release": "node scripts/release.js",
     "format": "prettier --write .",
     "type-check": "tsc --noEmit",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "test": "npm run build --silent && node --test --experimental-transform-types",
     "test:coverage": "npm run build --silent && node --test --experimental-transform-types --experimental-test-coverage",
-    "bench": "npm run build && node scripts/bench.mjs",
     "knip": "knip",
-    "knip:fix": "knip --fix"
+    "knip:fix": "knip --fix",
+    "inspector": "npx @modelcontextprotocol/inspector",
+    "prepublishOnly": "npm run lint && npm run type-check && npm run build"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.25.1",
     "@mozilla/readability": "^0.6.0",
-    "cheerio": "^1.1.2",
-    "domhandler": "^5.0.3",
     "express": "^5.2.1",
     "linkedom": "^0.18.12",
     "turndown": "^7.2.2",
-    "undici": "^6.22.0",
-    "zod": "^4.3.4"
+    "undici": "^6.23.0",
+    "zod": "^4.3.5"
   },
   "devDependencies": {
     "@eslint/js": "^9.39.2",
-    "@trivago/prettier-plugin-sort-imports": "^6.0.0",
+    "@trivago/prettier-plugin-sort-imports": "^6.0.1",
     "@types/express": "^5.0.6",
     "@types/node": "^22.19.3",
     "@types/turndown": "^5.0.6",
     "eslint": "^9.23.2",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-unused-imports": "^4.3.0",
-    "knip": "^5.78.0",
+    "knip": "^5.80.0",
     "prettier": "^3.7.4",
     "shx": "^0.4.0",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",
-    "typescript-eslint": "^8.51.0"
+    "typescript-eslint": "^8.52.0"
   },
   "engines": {
     "node": ">=20.12.0"