@j0hanz/fetch-url-mcp 1.9.3 → 1.9.5

package/dist/lib/code-lang.d.ts

@@ -0,0 +1,4 @@
+export declare function extractLanguageFromClassName(className: string): string | undefined;
+export declare function resolveLanguageFromAttributes(className: string, dataLang: string): string | undefined;
+export declare function detectLanguageFromCode(code: string): string | undefined;
+//# sourceMappingURL=code-lang.d.ts.map

package/dist/lib/code-lang.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"code-lang.d.ts","sourceRoot":"","sources":["../../src/lib/code-lang.ts"],"names":[],"mappings":"AAgRA,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,GAChB,MAAM,GAAG,SAAS,CAuBpB;AAqBD,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,MAAM,GAAG,SAAS,CAKpB;AACD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAqBvE"}

package/dist/lib/code-lang.js

@@ -0,0 +1,317 @@
+const ASCII_SPACE = 32;
+const ASCII_TAB = 9;
+const ASCII_LT = 60;
+const ASCII_DIGIT_0 = 48;
+const ASCII_DIGIT_9 = 57;
+const ASCII_UPPER_A = 65;
+const ASCII_UPPER_Z = 90;
+const ASCII_LOWER_A = 97;
+const ASCII_LOWER_Z = 122;
+const ASCII_UNDERSCORE = 95;
+class DetectionContext {
+    code;
+    _lower;
+    _lines;
+    _trimmedStart;
+    constructor(code) {
+        this.code = code;
+    }
+    get lower() {
+        this._lower ??= this.code.toLowerCase();
+        return this._lower;
+    }
+    get lines() {
+        this._lines ??= this.code.split(/\r?\n/);
+        return this._lines;
+    }
+    get trimmedStart() {
+        this._trimmedStart ??= this.code.trimStart();
+        return this._trimmedStart;
+    }
+}
+const BASH_COMMANDS = new Set([
+    'sudo',
+    'chmod',
+    'mkdir',
+    'cd',
+    'ls',
+    'cat',
+    'echo',
+]);
+const BASH_PACKAGE_MANAGERS = [
+    'npm',
+    'yarn',
+    'pnpm',
+    'npx',
+    'brew',
+    'apt',
+    'pip',
+    'cargo',
+    'go',
+];
+const BASH_VERBS = new Set(['install', 'add', 'run', 'build', 'start']);
+const TYPESCRIPT_HINTS = [
+    ': string',
+    ':string',
+    ': number',
+    ':number',
+    ': boolean',
+    ':boolean',
+    ': void',
+    ':void',
+    ': any',
+    ':any',
+    ': unknown',
+    ':unknown',
+    ': never',
+    ':never',
+];
+const HTML_TAGS = [
+    '<!doctype',
+    '<html',
+    '<head',
+    '<body',
+    '<div',
+    '<span',
+    '<p',
+    '<a',
+    '<script',
+    '<style',
+];
+const RUST_REGEX = /\b(?:fn|impl|struct|enum)\b/;
+const JS_REGEX = /\b(?:const|let|var|function|class|async|await|export|import)\b/;
+const PYTHON_UNIQUE_REGEX = /\b(?:def |elif |except |finally:|yield |lambda |raise |pass$)/m;
+const JS_SIGNAL_REGEX = /\b(?:const |let |var |function |require\(|=>|===|!==|console\.)/;
+const CSS_REGEX = /@media|@import|@keyframes|@theme\b|@utility\b|@layer\b|@apply\b|@variant\b|@custom-variant\b|@reference\b|@source\b/;
+const CSS_PROPERTY_REGEX = /^\s*[a-z][\w-]*\s*:/;
+function containsJsxTag(code) {
+    const len = code.length;
+    for (let i = 0; i < len - 1; i++) {
+        if (code.charCodeAt(i) === ASCII_LT) {
+            const next = code.charCodeAt(i + 1);
+            if (next >= ASCII_UPPER_A && next <= ASCII_UPPER_Z)
+                return true;
+        }
+    }
+    return false;
+}
+function isBashLine(line) {
+    const trimmed = line.trimStart();
+    if (trimmed.length === 0)
+        return false;
+    // Shell Prefix
+    if (trimmed.startsWith('#!') ||
+        trimmed.startsWith('$ ') ||
+        trimmed.startsWith('# ')) {
+        return true;
+    }
+    const spaceIdx = trimmed.indexOf(' ');
+    const firstWord = spaceIdx === -1 ? trimmed : trimmed.slice(0, spaceIdx);
+    if (BASH_COMMANDS.has(firstWord))
+        return true;
+    // Package Managers
+    const isPkgMgr = BASH_PACKAGE_MANAGERS.includes(firstWord);
+    if (isPkgMgr && spaceIdx !== -1) {
+        const rest = trimmed.slice(spaceIdx + 1);
+        const secondSpaceIdx = rest.indexOf(' ');
+        const secondWord = secondSpaceIdx === -1 ? rest : rest.slice(0, secondSpaceIdx);
+        if (BASH_VERBS.has(secondWord))
+            return true;
+    }
+    return false;
+}
+function detectBashIndicators(lines) {
+    return lines.some((line) => isBashLine(line));
+}
+function detectCssStructure(lines) {
+    for (const line of lines) {
+        const trimmed = line.trimStart();
+        if (trimmed.length === 0)
+            continue;
+        const hasSelector = (trimmed.startsWith('.') || trimmed.startsWith('#')) &&
+            trimmed.includes('{');
+        if (hasSelector)
+            return true;
+        if (trimmed.includes(';') &&
+            CSS_PROPERTY_REGEX.test(trimmed) &&
+            !trimmed.includes('(')) {
+            return true;
+        }
+    }
+    return false;
+}
+function detectYamlStructure(lines) {
+    for (const line of lines) {
+        const trimmed = line.trim();
+        if (trimmed.length === 0)
+            continue;
+        const colonIdx = trimmed.indexOf(':');
+        if (colonIdx <= 0)
+            continue;
+        const after = trimmed.charCodeAt(colonIdx + 1);
+        if (after === ASCII_SPACE || after === ASCII_TAB)
+            return true;
+    }
+    return false;
+}
+function matchRust(ctx) {
+    if (ctx.lower.includes('let mut'))
+        return true;
+    if (RUST_REGEX.test(ctx.lower))
+        return true;
+    return ctx.lower.includes('use ') && ctx.lower.includes('::');
+}
+function matchGo(ctx) {
+    if (ctx.lower.includes('import "'))
+        return true;
+    return /\b(?:package|func)\b/.test(ctx.lower);
+}
+function matchJsx(ctx) {
+    const l = ctx.lower;
+    if (l.includes('classname=') ||
+        l.includes('jsx:') ||
+        l.includes("from 'react'") ||
+        l.includes('from "react"')) {
+        return true;
+    }
+    return containsJsxTag(ctx.code);
+}
+function matchTypeScript(ctx) {
+    if (/\b(?:interface|type)\b/.test(ctx.lower))
+        return true;
+    const l = ctx.lower;
+    for (const hint of TYPESCRIPT_HINTS) {
+        if (l.includes(hint))
+            return true;
+    }
+    return false;
+}
+function matchSql(ctx) {
+    return /\b(?:select\s+(?:.+?\s+from|[\d*@])|insert\s+into|update\s+.+?\s+set|delete\s+from|create\s+(?:table|database|index|view|function|procedure|trigger|user|role)|alter\s+(?:table|database|index|view))\b/.test(ctx.lower);
+}
+function hasJsSignals(lowerCode) {
+    return (JS_SIGNAL_REGEX.test(lowerCode) ||
+        lowerCode.includes('{') ||
+        lowerCode.includes("from '"));
+}
+function matchPython(ctx) {
+    if (matchHtml(ctx))
+        return false;
+    const l = ctx.lower;
+    if (l.includes('print(') || l.includes('__name__'))
+        return true;
+    if (l.includes('self.') || l.includes('elif '))
+        return true;
+    // Check for Python's None/True/False using original case (they are capitalized in Python)
+    if (ctx.code.includes('None') ||
+        ctx.code.includes('True') ||
+        ctx.code.includes('False')) {
+        return true;
+    }
+    if (PYTHON_UNIQUE_REGEX.test(l))
+        return true;
+    // Shared keywords (import, from, class) — only match if no JS signals present
+    if (/\b(?:import|from|class)\b/.test(l) && !hasJsSignals(l)) {
+        return true;
+    }
+    return false;
+}
+function matchHtml(ctx) {
+    const l = ctx.lower;
+    for (const tag of HTML_TAGS) {
+        if (l.includes(tag))
+            return true;
+    }
+    return false;
+}
+// Pre-sorted by weight descending — first match wins in detectLanguageFromCode
+const LANGUAGES = [
+    { lang: 'rust', weight: 25, match: matchRust },
+    { lang: 'go', weight: 22, match: matchGo },
+    { lang: 'jsx', weight: 22, match: matchJsx },
+    { lang: 'typescript', weight: 20, match: matchTypeScript },
+    { lang: 'sql', weight: 20, match: matchSql },
+    { lang: 'html', weight: 19, match: matchHtml },
+    { lang: 'python', weight: 18, match: matchPython },
+    {
+        lang: 'css',
+        weight: 18,
+        match: (ctx) => CSS_REGEX.test(ctx.lower) || detectCssStructure(ctx.lines),
+    },
+    { lang: 'bash', weight: 15, match: (ctx) => detectBashIndicators(ctx.lines) },
+    { lang: 'yaml', weight: 15, match: (ctx) => detectYamlStructure(ctx.lines) },
+    { lang: 'javascript', weight: 15, match: (ctx) => JS_REGEX.test(ctx.lower) },
+    {
+        lang: 'json',
+        weight: 10,
+        match: (ctx) => ctx.trimmedStart.startsWith('{') || ctx.trimmedStart.startsWith('['),
+    },
+];
+export function extractLanguageFromClassName(className) {
+    if (!className)
+        return undefined;
+    // Split by whitespace and check for language indicators
+    const tokens = className.match(/\S+/g);
+    if (!tokens)
+        return undefined;
+    // Fast path: check for prefixes
+    for (const token of tokens) {
+        const lower = token.toLowerCase();
+        if (lower.startsWith('language-'))
+            return token.slice(9);
+        if (lower.startsWith('lang-'))
+            return token.slice(5);
+        if (lower.startsWith('highlight-'))
+            return token.slice(10);
+    }
+    // Fallback: check for hljs context
+    if (!tokens.includes('hljs'))
+        return undefined;
+    const langClass = tokens.find((t) => {
+        const l = t.toLowerCase();
+        return l !== 'hljs' && !l.startsWith('hljs-');
+    });
+    return langClass;
+}
+function resolveLanguageFromDataAttribute(dataLang) {
+    const trimmed = dataLang.trim();
+    if (!trimmed)
+        return undefined;
+    // Check if \w+
+    for (let i = 0; i < trimmed.length; i++) {
+        const c = trimmed.charCodeAt(i);
+        const isUpper = c >= ASCII_UPPER_A && c <= ASCII_UPPER_Z;
+        const isLower = c >= ASCII_LOWER_A && c <= ASCII_LOWER_Z;
+        const isDigit = c >= ASCII_DIGIT_0 && c <= ASCII_DIGIT_9;
+        const isUnder = c === ASCII_UNDERSCORE;
+        if (!isUpper && !isLower && !isDigit && !isUnder) {
+            return undefined;
+        }
+    }
+    return trimmed;
+}
+export function resolveLanguageFromAttributes(className, dataLang) {
+    return (extractLanguageFromClassName(className) ??
+        resolveLanguageFromDataAttribute(dataLang));
+}
+export function detectLanguageFromCode(code) {
+    if (!code)
+        return undefined;
+    // Fast path for empty/whitespace only
+    let empty = true;
+    for (let i = 0; i < code.length; i++) {
+        if (code.charCodeAt(i) > ASCII_SPACE) {
+            empty = false;
+            break;
+        }
+    }
+    if (empty)
+        return undefined;
+    const ctx = new DetectionContext(code);
+    // LANGUAGES is pre-sorted by weight descending — first match is highest confidence
+    for (const def of LANGUAGES) {
+        if (def.match(ctx))
+            return def.lang;
+    }
+    return undefined;
+}

package/dist/lib/dom-prep.d.ts

@@ -0,0 +1,4 @@
+export declare function serializeDocumentForMarkdown(document: Document, fallback: string): string;
+export declare function prepareDocumentForMarkdown(document: Document, baseUrl?: string, signal?: AbortSignal): void;
+export declare function removeNoiseFromHtml(html: string, document?: Document, baseUrl?: string, signal?: AbortSignal): string;
+//# sourceMappingURL=dom-prep.d.ts.map

package/dist/lib/dom-prep.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dom-prep.d.ts","sourceRoot":"","sources":["../../src/lib/dom-prep.ts"],"names":[],"mappings":"AAwlBA,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CAQR;AAmPD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,IAAI,CAkBN;AAiED,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,MAAM,CAcR"}