@j0hanz/fetch-url-mcp 0.0.1

package/dist/cache.d.ts

@@ -0,0 +1,47 @@
+import type { ServerResponse } from 'node:http';
+import { z } from 'zod';
+declare const CachedPayloadSchema: z.ZodObject<{
+    content: z.ZodOptional<z.ZodString>;
+    markdown: z.ZodOptional<z.ZodString>;
+    title: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export type CachedPayload = z.infer<typeof CachedPayloadSchema>;
+export interface CacheEntry {
+    url: string;
+    title?: string;
+    content: string;
+    fetchedAt: string;
+    expiresAt: string;
+}
+export interface CacheKeyParts {
+    namespace: string;
+    urlHash: string;
+}
+export interface CacheSetOptions {
+    force?: boolean;
+}
+export interface CacheGetOptions {
+    force?: boolean;
+}
+interface CacheEntryMetadata {
+    url: string;
+    title?: string;
+}
+interface CacheUpdateEvent {
+    cacheKey: string;
+    namespace: string;
+    urlHash: string;
+}
+type CacheUpdateListener = (event: CacheUpdateEvent) => unknown;
+export declare function parseCachedPayload(raw: string): CachedPayload | null;
+export declare function resolveCachedPayloadContent(payload: CachedPayload): string | null;
+export declare function createCacheKey(namespace: string, url: string, vary?: Record<string, unknown> | string): string | null;
+export declare function parseCacheKey(cacheKey: string): CacheKeyParts | null;
+export declare function onCacheUpdate(listener: CacheUpdateListener): () => void;
+export declare function get(cacheKey: string | null, options?: CacheGetOptions): CacheEntry | undefined;
+export declare function set(cacheKey: string | null, content: string, metadata: CacheEntryMetadata, options?: CacheSetOptions): void;
+export declare function keys(): readonly string[];
+export declare function isEnabled(): boolean;
+export declare function generateSafeFilename(url: string, title?: string, hashFallback?: string, extension?: string): string;
+export declare function handleDownload(res: ServerResponse, namespace: string, hash: string): void;
+export {};

package/dist/cache.js

@@ -0,0 +1,316 @@
+import { EventEmitter } from 'node:events';
+import { posix as pathPosix } from 'node:path';
+import { z } from 'zod';
+import { config } from './config.js';
+import { sha256Hex } from './crypto.js';
+import { getErrorMessage } from './errors.js';
+import { stableStringify as stableJsonStringify } from './json.js';
+import { logWarn } from './observability.js';
+/* -------------------------------------------------------------------------------------------------
+ * Schemas & Types
+ * ------------------------------------------------------------------------------------------------- */
+const CacheNamespace = z.literal('markdown');
+const HashString = z
+    .string()
+    .regex(/^[a-f0-9.]+$/i)
+    .min(8)
+    .max(64);
+const CachedPayloadSchema = z.strictObject({
+    content: z.string().optional(),
+    markdown: z.string().optional(),
+    title: z.string().optional(),
+});
+/* -------------------------------------------------------------------------------------------------
+ * Core: Cache Key Logic
+ * ------------------------------------------------------------------------------------------------- */
+const CACHE_CONSTANTS = {
+    URL_HASH_LENGTH: 32,
+    VARY_HASH_LENGTH: 16,
+};
+export function parseCachedPayload(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        return CachedPayloadSchema.parse(parsed);
+    }
+    catch {
+        return null;
+    }
+}
+export function resolveCachedPayloadContent(payload) {
+    return payload.markdown ?? payload.content ?? null;
+}
+function createHashFragment(input, length) {
+    return sha256Hex(input).substring(0, length);
+}
+function buildCacheKey(namespace, urlHash, varyHash) {
+    return varyHash
+        ? `${namespace}:${urlHash}.${varyHash}`
+        : `${namespace}:${urlHash}`;
+}
+export function createCacheKey(namespace, url, vary) {
+    if (!namespace || !url)
+        return null;
+    const urlHash = createHashFragment(url, CACHE_CONSTANTS.URL_HASH_LENGTH);
+    let varyHash;
+    if (vary) {
+        let varyString;
+        if (typeof vary === 'string') {
+            varyString = vary;
+        }
+        else {
+            try {
+                varyString = stableJsonStringify(vary);
+            }
+            catch {
+                return null;
+            }
+        }
+        if (varyString) {
+            varyHash = createHashFragment(varyString, CACHE_CONSTANTS.VARY_HASH_LENGTH);
+        }
+    }
+    return buildCacheKey(namespace, urlHash, varyHash);
+}
+export function parseCacheKey(cacheKey) {
+    if (!cacheKey)
+        return null;
+    const [namespace, ...rest] = cacheKey.split(':');
+    const urlHash = rest.join(':');
+    if (!namespace || !urlHash)
+        return null;
+    return { namespace, urlHash };
+}
+/* -------------------------------------------------------------------------------------------------
+ * Core: In-Memory Store
+ * ------------------------------------------------------------------------------------------------- */
+class InMemoryCacheStore {
+    max = config.cache.maxKeys;
+    maxBytes = config.cache.maxSizeBytes;
+    ttlMs = config.cache.ttl * 1000;
+    entries = new Map();
+    updateEmitter = new EventEmitter();
+    currentBytes = 0;
+    isEnabled() {
+        return config.cache.enabled;
+    }
+    keys() {
+        if (!this.isEnabled())
+            return [];
+        const now = Date.now();
+        const result = [];
+        for (const [key, entry] of this.entries) {
+            if (entry.expiresAtMs > now)
+                result.push(key);
+        }
+        return result;
+    }
+    onUpdate(listener) {
+        const wrapped = (event) => {
+            try {
+                const result = listener(event);
+                if (result instanceof Promise) {
+                    void result.catch((error) => {
+                        this.logError('Cache update listener failed (async)', event.cacheKey, error);
+                    });
+                }
+            }
+            catch (error) {
+                this.logError('Cache update listener failed', event.cacheKey, error);
+            }
+        };
+        this.updateEmitter.on('update', wrapped);
+        return () => {
+            this.updateEmitter.off('update', wrapped);
+        };
+    }
+    get(cacheKey, options) {
+        if (!cacheKey || (!this.isEnabled() && !options?.force))
+            return undefined;
+        const entry = this.entries.get(cacheKey);
+        if (!entry)
+            return undefined;
+        const now = Date.now();
+        if (entry.expiresAtMs <= now) {
+            this.delete(cacheKey);
+            return undefined;
+        }
+        // Refresh LRU position
+        this.entries.delete(cacheKey);
+        this.entries.set(cacheKey, entry);
+        return entry;
+    }
+    delete(cacheKey) {
+        const entry = this.entries.get(cacheKey);
+        if (entry) {
+            this.currentBytes -= entry.content.length;
+            this.entries.delete(cacheKey);
+        }
+    }
+    set(cacheKey, content, metadata, options) {
+        if (!cacheKey || !content)
+            return;
+        if (!this.isEnabled() && !options?.force)
+            return;
+        const now = Date.now();
+        const expiresAtMs = now + this.ttlMs;
+        // Check size limit before insertion
+        const entrySize = content.length;
+        if (entrySize > this.maxBytes) {
+            logWarn('Cache entry exceeds max size', {
+                key: cacheKey,
+                size: entrySize,
+                max: this.maxBytes,
+            });
+            return;
+        }
+        // Evict if needed (size-based)
+        while (this.currentBytes + entrySize > this.maxBytes) {
+            const firstKey = this.entries.keys().next();
+            if (firstKey.done)
+                break;
+            this.delete(firstKey.value);
+        }
+        const entry = {
+            url: metadata.url,
+            content,
+            fetchedAt: new Date(now).toISOString(),
+            expiresAt: new Date(expiresAtMs).toISOString(),
+            expiresAtMs,
+            ...(metadata.title ? { title: metadata.title } : {}),
+        };
+        if (this.entries.has(cacheKey)) {
+            this.delete(cacheKey);
+        }
+        this.entries.set(cacheKey, entry);
+        this.currentBytes += entrySize;
+        // Eviction (LRU: first insertion-order key) - Count based
+        if (this.entries.size > this.max) {
+            const firstKey = this.entries.keys().next();
+            if (!firstKey.done)
+                this.delete(firstKey.value);
+        }
+        this.notify(cacheKey);
+    }
+    notify(cacheKey) {
+        if (this.updateEmitter.listenerCount('update') === 0)
+            return;
+        const parts = parseCacheKey(cacheKey);
+        if (!parts)
+            return;
+        this.updateEmitter.emit('update', { cacheKey, ...parts });
+    }
+    logError(message, cacheKey, error) {
+        logWarn(message, {
+            key: cacheKey.length > 100 ? cacheKey.slice(0, 100) : cacheKey,
+            error: getErrorMessage(error),
+        });
+    }
+}
+// Singleton Instance
+const store = new InMemoryCacheStore();
+// Public Proxy API
+export function onCacheUpdate(listener) {
+    return store.onUpdate(listener);
+}
+export function get(cacheKey, options) {
+    return store.get(cacheKey, options);
+}
+export function set(cacheKey, content, metadata, options) {
+    store.set(cacheKey, content, metadata, options);
+}
+export function keys() {
+    return store.keys();
+}
+export function isEnabled() {
+    return store.isEnabled();
+}
+/* -------------------------------------------------------------------------------------------------
+ * Utils: Filename Logic
+ * ------------------------------------------------------------------------------------------------- */
+const FILENAME_RULES = {
+    MAX_LEN: 200,
+    UNSAFE_CHARS: /[<>:"/\\|?*\p{C}]/gu,
+    WHITESPACE: /\s+/g,
+    EXTENSIONS: /\.(html?|php|aspx?|jsp)$/i,
+};
+function sanitizeString(input) {
+    return input
+        .toLowerCase()
+        .replace(FILENAME_RULES.UNSAFE_CHARS, '')
+        .replace(FILENAME_RULES.WHITESPACE, '-')
+        .replace(/-+/g, '-')
+        .replace(/(?:^-|-$)/g, '');
+}
+export function generateSafeFilename(url, title, hashFallback, extension = '.md') {
+    const tryUrl = () => {
+        try {
+            if (!URL.canParse(url))
+                return null;
+            const parsed = new URL(url);
+            if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:')
+                return null;
+            const { pathname } = parsed;
+            const basename = pathPosix.basename(pathname);
+            if (!basename || basename === 'index')
+                return null;
+            const cleaned = basename.replace(FILENAME_RULES.EXTENSIONS, '');
+            const sanitized = sanitizeString(cleaned);
+            if (sanitized === 'index')
+                return null;
+            return sanitized || null;
+        }
+        catch {
+            return null;
+        }
+    };
+    const tryTitle = () => {
+        if (!title)
+            return null;
+        return sanitizeString(title) || null;
+    };
+    const name = tryUrl() ??
+        tryTitle() ??
+        hashFallback?.substring(0, 16) ??
+        `download-${Date.now()}`;
+    const maxBase = FILENAME_RULES.MAX_LEN - extension.length;
+    const truncated = name.length > maxBase ? name.substring(0, maxBase) : name;
+    return `${truncated}${extension}`;
+}
+/* -------------------------------------------------------------------------------------------------
+ * Adapter: Download Handler
+ * ------------------------------------------------------------------------------------------------- */
+const DownloadParamsSchema = z.strictObject({
+    namespace: CacheNamespace,
+    hash: HashString,
+});
+export function handleDownload(res, namespace, hash) {
+    const respond = (status, msg, code) => {
+        res.writeHead(status, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ error: msg, code }));
+    };
+    const parsed = DownloadParamsSchema.safeParse({ namespace, hash });
+    if (!parsed.success) {
+        respond(400, 'Invalid namespace or hash', 'BAD_REQUEST');
+        return;
+    }
+    const cacheKey = `${parsed.data.namespace}:${parsed.data.hash}`;
+    const entry = store.get(cacheKey, { force: true });
+    if (!entry) {
+        respond(404, 'Not found or expired', 'NOT_FOUND');
+        return;
+    }
+    const payload = parseCachedPayload(entry.content);
+    const content = payload ? resolveCachedPayloadContent(payload) : null;
+    if (!content) {
+        respond(404, 'Content missing', 'NOT_FOUND');
+        return;
+    }
+    const fileName = generateSafeFilename(entry.url, payload?.title, parsed.data.hash);
+    // Safe header generation
+    const encoded = encodeURIComponent(fileName).replace(/'/g, '%27');
+    res.setHeader('Content-Type', 'text/markdown; charset=utf-8');
+    res.setHeader('Content-Disposition', `attachment; filename="${fileName}"; filename*=UTF-8''${encoded}`);
+    res.setHeader('Cache-Control', `private, max-age=${config.cache.ttl}`);
+    res.setHeader('X-Content-Type-Options', 'nosniff');
+    res.end(content);
+}

package/dist/cli.d.ts

@@ -0,0 +1,17 @@
+export interface CliValues {
+    readonly stdio: boolean;
+    readonly help: boolean;
+    readonly version: boolean;
+}
+interface CliParseSuccess {
+    readonly ok: true;
+    readonly values: CliValues;
+}
+interface CliParseFailure {
+    readonly ok: false;
+    readonly message: string;
+}
+export type CliParseResult = CliParseSuccess | CliParseFailure;
+export declare function renderCliUsage(): string;
+export declare function parseCliArgs(args: readonly string[]): CliParseResult;
+export {};

package/dist/cli.js

@@ -0,0 +1,48 @@
+import { parseArgs } from 'node:util';
+const usageLines = [
+    'Fetch URL MCP server',
+    '',
+    'Usage:',
+    '  fetch-url-mcp [--stdio|-s] [--help|-h] [--version|-v]',
+    '',
+    'Options:',
+    '  --stdio, -s   Run in stdio mode (no HTTP server).',
+    '  --help, -h    Show this help message.',
+    '  --version, -v Show server version.',
+    '',
+];
+const optionSchema = {
+    stdio: { type: 'boolean', short: 's', default: false },
+    help: { type: 'boolean', short: 'h', default: false },
+    version: { type: 'boolean', short: 'v', default: false },
+};
+function toErrorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+export function renderCliUsage() {
+    return `${usageLines.join('\n')}\n`;
+}
+export function parseCliArgs(args) {
+    try {
+        const { values } = parseArgs({
+            args: [...args],
+            options: optionSchema,
+            strict: true,
+            allowPositionals: false,
+        });
+        return {
+            ok: true,
+            values: {
+                stdio: values.stdio,
+                help: values.help,
+                version: values.version,
+            },
+        };
+    }
+    catch (error) {
+        return {
+            ok: false,
+            message: toErrorMessage(error),
+        };
+    }
+}

package/dist/config.d.ts

@@ -0,0 +1,142 @@
+export declare const serverVersion: string;
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+export type TransformWorkerMode = 'threads' | 'process';
+type AuthMode = 'oauth' | 'static';
+interface WorkerResourceLimits {
+    maxOldGenerationSizeMb?: number;
+    maxYoungGenerationSizeMb?: number;
+    codeRangeSizeMb?: number;
+    stackSizeMb?: number;
+}
+interface AuthConfig {
+    mode: AuthMode;
+    issuerUrl: URL | undefined;
+    authorizationUrl: URL | undefined;
+    tokenUrl: URL | undefined;
+    revocationUrl: URL | undefined;
+    registrationUrl: URL | undefined;
+    introspectionUrl: URL | undefined;
+    resourceUrl: URL;
+    requiredScopes: string[];
+    clientId: string | undefined;
+    clientSecret: string | undefined;
+    introspectionTimeoutMs: number;
+    staticTokens: string[];
+}
+interface HttpsConfig {
+    enabled: boolean;
+    keyFile: string | undefined;
+    certFile: string | undefined;
+    caFile: string | undefined;
+}
+interface RuntimeState {
+    httpMode: boolean;
+}
+export declare const config: {
+    server: {
+        name: string;
+        version: string;
+        port: number;
+        host: string;
+        https: HttpsConfig;
+        sessionTtlMs: number;
+        sessionInitTimeoutMs: number;
+        maxSessions: number;
+        http: {
+            headersTimeoutMs: number | undefined;
+            requestTimeoutMs: number | undefined;
+            keepAliveTimeoutMs: number | undefined;
+            keepAliveTimeoutBufferMs: number | undefined;
+            maxHeadersCount: number | undefined;
+            maxConnections: number;
+            blockPrivateConnections: boolean;
+            shutdownCloseIdleConnections: boolean;
+            shutdownCloseAllConnections: boolean;
+        };
+    };
+    fetcher: {
+        timeout: number;
+        maxRedirects: number;
+        userAgent: string;
+        maxContentLength: number;
+    };
+    transform: {
+        timeoutMs: number;
+        stageWarnRatio: number;
+        metadataFormat: string;
+        maxWorkerScale: number;
+        workerMode: TransformWorkerMode;
+        workerResourceLimits: WorkerResourceLimits | undefined;
+    };
+    tools: {
+        enabled: string[];
+        timeoutMs: number;
+    };
+    tasks: {
+        maxTotal: number;
+        maxPerOwner: number;
+    };
+    cache: {
+        enabled: boolean;
+        ttl: number;
+        maxKeys: number;
+        maxSizeBytes: number;
+    };
+    extraction: {
+        maxBlockLength: number;
+        minParagraphLength: number;
+    };
+    noiseRemoval: {
+        extraTokens: string[];
+        extraSelectors: string[];
+        enabledCategories: string[];
+        debug: boolean;
+        aggressiveMode: boolean;
+        preserveSvgCanvas: boolean;
+        weights: {
+            hidden: number;
+            structural: number;
+            promo: number;
+            stickyFixed: number;
+            threshold: number;
+        };
+    };
+    markdownCleanup: {
+        promoteOrphanHeadings: boolean;
+        removeSkipLinks: boolean;
+        removeTocBlocks: boolean;
+        removeTypeDocComments: boolean;
+        headingKeywords: string[];
+    };
+    i18n: {
+        locale: string | undefined;
+    };
+    logging: {
+        level: LogLevel;
+        format: string;
+    };
+    constants: {
+        maxHtmlSize: number;
+        maxUrlLength: number;
+        maxInlineContentChars: number;
+    };
+    security: {
+        blockedHosts: Set<string>;
+        blockedIpPatterns: readonly RegExp[];
+        blockedIpPattern: RegExp;
+        blockedIpv4MappedPattern: RegExp;
+        allowedHosts: Set<string>;
+        apiKey: string | undefined;
+        allowRemote: boolean;
+    };
+    auth: AuthConfig;
+    rateLimit: {
+        enabled: boolean;
+        maxRequests: number;
+        windowMs: number;
+        cleanupIntervalMs: number;
+    };
+    runtime: RuntimeState;
+};
+export declare function enableHttpMode(): void;
+export {};