@j0hanz/superfetch 2.5.3 → 2.6.0

package/dist/language-detection.js

@@ -1,331 +1,340 @@
-/**
- * Language detection for code blocks.
- * Detects programming languages from code content and HTML attributes.
- */
-function createCodeSample(code) {
-    return {
-        code,
-        lower: code.toLowerCase(),
-        lines: code.split('\n'),
-        trimmedStart: code.trimStart(),
-    };
-}
-/* -------------------------------------------------------------------------------------------------
- * Word boundary matcher (cached)
- * ------------------------------------------------------------------------------------------------- */
-class WordBoundaryMatcher {
-    cache = new Map();
-    containsWord(source, word) {
-        return this.getRegex(word).test(source);
+// This module provides a heuristic-based language detection mechanism for code snippets.
+class DetectionContext {
+    code;
+    _lower;
+    _lines;
+    _trimmedStart;
+    constructor(code) {
+        this.code = code;
     }
-    getRegex(word) {
-        const cached = this.cache.get(word);
-        if (cached)
-            return cached;
-        // Keep behavior: compile `\b${word}\b` without escaping (words are controlled by patterns).
-        const compiled = new RegExp(`\\b${word}\\b`);
-        this.cache.set(word, compiled);
-        return compiled;
+    get lower() {
+        this._lower ??= this.code.toLowerCase();
+        return this._lower;
     }
-}
-const wordMatcher = new WordBoundaryMatcher();
-/* -------------------------------------------------------------------------------------------------
- * Attribute-based language resolution
- * ------------------------------------------------------------------------------------------------- */
-class LanguageAttributeResolver {
-    resolve(className, dataLang) {
-        const classMatch = this.extractFromClassName(className);
-        return classMatch ?? this.resolveFromDataAttribute(dataLang);
+    get lines() {
+        this._lines ??= this.code.split(/\r?\n/);
+        return this._lines;
     }
-    /**
-     * Extract language from class name (e.g., "language-typescript", "lang-js", "hljs javascript").
-     * Note: preserves current behavior by returning the sliced original token casing.
-     */
-    extractFromClassName(className) {
-        const tokens = className.match(/\S+/g);
-        if (!tokens)
-            return undefined;
-        for (const token of tokens) {
-            const lower = token.toLowerCase();
-            if (lower.startsWith('language-'))
-                return token.slice('language-'.length);
-            if (lower.startsWith('lang-'))
-                return token.slice('lang-'.length);
-            if (lower.startsWith('highlight-'))
-                return token.slice('highlight-'.length);
+    get trimmedStart() {
+        this._trimmedStart ??= this.code.trimStart();
+        return this._trimmedStart;
+    }
+}
+const BASH_COMMANDS = new Set([
+    'sudo',
+    'chmod',
+    'mkdir',
+    'cd',
+    'ls',
+    'cat',
+    'echo',
+]);
+const BASH_PACKAGE_MANAGERS = [
+    'npm',
+    'yarn',
+    'pnpm',
+    'npx',
+    'brew',
+    'apt',
+    'pip',
+    'cargo',
+    'go',
+];
+const BASH_VERBS = new Set(['install', 'add', 'run', 'build', 'start']);
+const TYPESCRIPT_HINTS = [
+    ': string',
+    ':string',
+    ': number',
+    ':number',
+    ': boolean',
+    ':boolean',
+    ': void',
+    ':void',
+    ': any',
+    ':any',
+    ': unknown',
+    ':unknown',
+    ': never',
+    ':never',
+];
+const HTML_TAGS = [
+    '<!doctype',
+    '<html',
+    '<head',
+    '<body',
+    '<div',
+    '<span',
+    '<p',
+    '<a',
+    '<script',
+    '<style',
+];
+const RUST_REGEX = /\b(?:fn|impl|struct|enum)\b/;
+const JS_REGEX = /\b(?:const|let|var|function|class|async|await|export|import)\b/;
+const PYTHON_REGEX = /\b(?:def|class|import|from)\b/;
+const CSS_REGEX = /@media|@import|@keyframes/;
+function containsJsxTag(code) {
+    const len = code.length;
+    for (let i = 0; i < len - 1; i++) {
+        if (code.charCodeAt(i) === 60 /* < */) {
+            const next = code.charCodeAt(i + 1);
+            if (next >= 65 && next <= 90)
+                return true; // A-Z
         }
-        if (tokens.includes('hljs')) {
-            const langClass = tokens.find((t) => t !== 'hljs' && !t.startsWith('hljs-'));
-            if (langClass)
-                return langClass;
+    }
+    return false;
+}
+function isBashLine(line) {
+    const trimmed = line.trimStart();
+    if (trimmed.length === 0)
+        return false;
+    // Shell Prefix
+    if (trimmed.startsWith('#!') ||
+        trimmed.startsWith('$ ') ||
+        trimmed.startsWith('# ')) {
+        return true;
+    }
+    const spaceIdx = trimmed.indexOf(' ');
+    const firstWord = spaceIdx === -1 ? trimmed : trimmed.slice(0, spaceIdx);
+    if (BASH_COMMANDS.has(firstWord))
+        return true;
+    // Package Managers
+    let isPkgMgr = false;
+    for (const mgr of BASH_PACKAGE_MANAGERS) {
+        if (firstWord === mgr) {
+            isPkgMgr = true;
+            break;
         }
-        return undefined;
     }
-    /**
-     * Resolve language from data-language attribute.
-     * Only allows word characters (alphanumeric + underscore).
-     */
-    resolveFromDataAttribute(dataLang) {
-        const trimmed = dataLang.trim();
-        if (!trimmed)
-            return undefined;
-        return /^\w+$/.test(trimmed) ? trimmed : undefined;
+    if (isPkgMgr && spaceIdx !== -1) {
+        const rest = trimmed.slice(spaceIdx + 1);
+        const secondSpaceIdx = rest.indexOf(' ');
+        const secondWord = secondSpaceIdx === -1 ? rest : rest.slice(0, secondSpaceIdx);
+        if (BASH_VERBS.has(secondWord))
+            return true;
     }
+    return false;
 }
-const attributeResolver = new LanguageAttributeResolver();
-/* -------------------------------------------------------------------------------------------------
- * Heuristics
- * ------------------------------------------------------------------------------------------------- */
-const Heuristics = {
-    containsJsxTag(code) {
-        // Preserve original behavior (scan for `<` followed by A-Z).
-        for (let i = 0; i < code.length - 1; i += 1) {
-            if (code[i] !== '<')
-                continue;
-            const next = code[i + 1];
-            if (!next)
-                continue;
-            if (next >= 'A' && next <= 'Z')
-                return true;
+function detectBashIndicators(lines) {
+    for (const line of lines) {
+        if (isBashLine(line))
+            return true;
+    }
+    return false;
+}
+function detectCssStructure(lines) {
+    for (const line of lines) {
+        const trimmed = line.trimStart();
+        if (trimmed.length === 0)
+            continue;
+        const hasSelector = (trimmed.startsWith('.') || trimmed.startsWith('#')) &&
+            trimmed.includes('{');
+        if (hasSelector || (trimmed.includes(':') && trimmed.includes(';'))) {
+            return true;
         }
-        return false;
-    },
-    bash: {
-        commands: ['sudo', 'chmod', 'mkdir', 'cd', 'ls', 'cat', 'echo'],
-        pkgManagers: [
-            'npm',
-            'yarn',
-            'pnpm',
-            'npx',
-            'brew',
-            'apt',
-            'pip',
-            'cargo',
-            'go',
-        ],
-        verbs: ['install', 'add', 'run', 'build', 'start'],
-        isShellPrefix(line) {
-            return (line.startsWith('#!') || line.startsWith('$ ') || line.startsWith('# '));
-        },
-        matchesCommand(line) {
-            return Heuristics.bash.commands.some((cmd) => line === cmd || line.startsWith(`${cmd} `));
-        },
-        matchesPackageManagerVerb(line) {
-            for (const mgr of Heuristics.bash.pkgManagers) {
-                if (!line.startsWith(`${mgr} `))
-                    continue;
-                const rest = line.slice(mgr.length + 1);
-                if (Heuristics.bash.verbs.some((v) => rest === v || rest.startsWith(`${v} `))) {
-                    return true;
-                }
-            }
-            return false;
-        },
-        detectIndicators(lines) {
-            for (const line of lines) {
-                const trimmed = line.trimStart();
-                if (trimmed &&
-                    (Heuristics.bash.isShellPrefix(trimmed) ||
-                        Heuristics.bash.matchesCommand(trimmed) ||
-                        Heuristics.bash.matchesPackageManagerVerb(trimmed))) {
-                    return true;
-                }
-            }
-            return false;
-        },
-    },
-    css: {
-        detectStructure(lines) {
-            for (const line of lines) {
-                const trimmed = line.trimStart();
-                if (!trimmed)
-                    continue;
-                const hasSelector = (trimmed.startsWith('.') || trimmed.startsWith('#')) &&
-                    trimmed.includes('{');
-                if (hasSelector || (trimmed.includes(':') && trimmed.includes(';'))) {
-                    return true;
-                }
-            }
-            return false;
-        },
-    },
-    yaml: {
-        detectStructure(lines) {
-            for (const line of lines) {
-                const trimmed = line.trim();
-                if (!trimmed)
-                    continue;
-                const colonIdx = trimmed.indexOf(':');
-                if (colonIdx > 0) {
-                    const after = trimmed[colonIdx + 1];
-                    if (after === ' ' || after === '\t')
-                        return true;
-                }
-            }
-            return false;
-        },
-    },
-};
-/* -------------------------------------------------------------------------------------------------
- * Pattern engine
- * ------------------------------------------------------------------------------------------------- */
-const LANGUAGE_PATTERNS = [
+    }
+    return false;
+}
+function detectYamlStructure(lines) {
+    for (const line of lines) {
+        const trimmed = line.trim();
+        if (trimmed.length === 0)
+            continue;
+        const colonIdx = trimmed.indexOf(':');
+        if (colonIdx <= 0)
+            continue;
+        const after = trimmed.charCodeAt(colonIdx + 1);
+        // space (32) or tab (9)
+        if (after === 32 || after === 9)
+            return true;
+    }
+    return false;
+}
+const LANGUAGES = [
     {
-        language: 'jsx',
-        pattern: {
-            keywords: ['classname=', 'jsx:', "from 'react'", 'from "react"'],
-            custom: (code) => Heuristics.containsJsxTag(code),
+        lang: 'rust',
+        weight: 25,
+        match: (ctx) => {
+            if (ctx.lower.includes('let mut'))
+                return true;
+            if (RUST_REGEX.test(ctx.lower))
+                return true;
+            return ctx.lower.includes('use ') && ctx.lower.includes('::');
         },
     },
     {
-        language: 'typescript',
-        pattern: {
-            wordBoundary: ['interface', 'type'],
-            custom: (_code, lower) => [
-                ': string',
-                ':string',
-                ': number',
-                ':number',
-                ': boolean',
-                ':boolean',
-                ': void',
-                ':void',
-                ': any',
-                ':any',
-                ': unknown',
-                ':unknown',
-                ': never',
-                ':never',
-            ].some((hint) => lower.includes(hint)),
+        lang: 'go',
+        weight: 22,
+        match: (ctx) => {
+            if (ctx.lower.includes('import "'))
+                return true;
+            return /\b(?:package|func)\b/.test(ctx.lower);
         },
     },
     {
-        language: 'rust',
-        pattern: {
-            regex: /\b(?:fn|impl|struct|enum)\b/,
-            keywords: ['let mut'],
-            custom: (_code, lower) => lower.includes('use ') && lower.includes('::'),
+        lang: 'jsx',
+        weight: 22,
+        match: (ctx) => {
+            const l = ctx.lower;
+            if (l.includes('classname=') ||
+                l.includes('jsx:') ||
+                l.includes("from 'react'") ||
+                l.includes('from "react"')) {
+                return true;
+            }
+            return containsJsxTag(ctx.code);
         },
     },
     {
-        language: 'javascript',
-        pattern: {
-            regex: /\b(?:const|let|var|function|class|async|await|export|import)\b/,
+        lang: 'typescript',
+        weight: 20,
+        match: (ctx) => {
+            if (/\b(?:interface|type)\b/.test(ctx.lower))
+                return true;
+            const l = ctx.lower;
+            for (const hint of TYPESCRIPT_HINTS) {
+                if (l.includes(hint))
+                    return true;
+            }
+            return false;
         },
     },
     {
-        language: 'python',
-        pattern: {
-            regex: /\b(?:def|class|import|from)\b/,
-            keywords: ['print(', '__name__'],
+        lang: 'sql',
+        weight: 20,
+        match: (ctx) => {
+            const l = ctx.lower;
+            return /\b(?:select|insert|update|delete|create|alter|drop)\b/.test(l);
         },
     },
     {
-        language: 'bash',
-        pattern: {
-            custom: (_code, _lower, lines) => Heuristics.bash.detectIndicators(lines),
+        lang: 'python',
+        weight: 18,
+        match: (ctx) => {
+            const l = ctx.lower;
+            if (l.includes('print(') || l.includes('__name__'))
+                return true;
+            return PYTHON_REGEX.test(l);
         },
     },
     {
-        language: 'css',
-        pattern: {
-            regex: /@media|@import|@keyframes/,
-            custom: (_code, _lower, lines) => Heuristics.css.detectStructure(lines),
+        lang: 'css',
+        weight: 18,
+        match: (ctx) => {
+            if (CSS_REGEX.test(ctx.lower))
+                return true;
+            return detectCssStructure(ctx.lines);
         },
     },
     {
-        language: 'html',
-        pattern: {
-            keywords: [
-                '<!doctype',
-                '<html',
-                '<head',
-                '<body',
-                '<div',
-                '<span',
-                '<p',
-                '<a',
-                '<script',
-                '<style',
-            ],
-        },
+        lang: 'bash',
+        weight: 15,
+        match: (ctx) => detectBashIndicators(ctx.lines),
     },
     {
-        language: 'json',
-        pattern: {
-            startsWith: ['{', '['],
-        },
+        lang: 'yaml',
+        weight: 15,
+        match: (ctx) => detectYamlStructure(ctx.lines),
     },
     {
-        language: 'yaml',
-        pattern: {
-            custom: (_code, _lower, lines) => Heuristics.yaml.detectStructure(lines),
-        },
+        lang: 'javascript',
+        weight: 12,
+        match: (ctx) => JS_REGEX.test(ctx.lower),
     },
     {
-        language: 'sql',
-        pattern: {
-            wordBoundary: [
-                'select',
-                'insert',
-                'update',
-                'delete',
-                'create',
-                'alter',
-                'drop',
-            ],
+        lang: 'html',
+        weight: 12,
+        match: (ctx) => {
+            const l = ctx.lower;
+            for (const tag of HTML_TAGS) {
+                if (l.includes(tag))
+                    return true;
+            }
+            return false;
         },
     },
     {
-        language: 'go',
-        pattern: {
-            wordBoundary: ['package', 'func'],
-            keywords: ['import "'],
+        lang: 'json',
+        weight: 10,
+        match: (ctx) => {
+            const s = ctx.trimmedStart;
+            return s.startsWith('{') || s.startsWith('[');
         },
     },
 ];
-class PatternEngine {
-    matches(sample, pattern) {
-        if (pattern.keywords?.some((kw) => sample.lower.includes(kw)))
-            return true;
-        if (pattern.wordBoundary?.some((w) => wordMatcher.containsWord(sample.lower, w)))
-            return true;
-        if (pattern.regex?.test(sample.lower))
-            return true;
-        if (pattern.startsWith?.some((prefix) => sample.trimmedStart.startsWith(prefix))) {
-            return true;
-        }
-        if (pattern.custom?.(sample.code, sample.lower, sample.lines))
-            return true;
-        return false;
+export function extractLanguageFromClassName(className) {
+    if (!className)
+        return undefined;
+    // Split by whitespace and check for language indicators
+    const tokens = className.match(/\S+/g);
+    if (!tokens)
+        return undefined;
+    // Fast path: check for prefixes
+    for (const token of tokens) {
+        const lower = token.toLowerCase();
+        if (lower.startsWith('language-'))
+            return token.slice(9);
+        if (lower.startsWith('lang-'))
+            return token.slice(5);
+        if (lower.startsWith('highlight-'))
+            return token.slice(10);
     }
+    // Fallback: check for hljs context
+    if (!tokens.includes('hljs'))
+        return undefined;
+    const langClass = tokens.find((t) => {
+        const l = t.toLowerCase();
+        return l !== 'hljs' && !l.startsWith('hljs-');
+    });
+    return langClass;
 }
-class LanguageDetector {
-    engine = new PatternEngine();
-    detect(code) {
-        const sample = createCodeSample(code);
-        for (const { language, pattern } of LANGUAGE_PATTERNS) {
-            if (this.engine.matches(sample, pattern))
-                return language;
-        }
+function resolveLanguageFromDataAttribute(dataLang) {
+    const trimmed = dataLang.trim();
+    if (!trimmed)
         return undefined;
+    // Check if \w+
+    for (let i = 0; i < trimmed.length; i++) {
+        const c = trimmed.charCodeAt(i);
+        // valid: A-Z, a-z, 0-9, _
+        const isUpper = c >= 65 && c <= 90;
+        const isLower = c >= 97 && c <= 122;
+        const isDigit = c >= 48 && c <= 57;
+        const isUnder = c === 95;
+        if (!isUpper && !isLower && !isDigit && !isUnder) {
+            return undefined;
+        }
     }
+    return trimmed;
+}
+export function resolveLanguageFromAttributes(className, dataLang) {
+    return (extractLanguageFromClassName(className) ??
+        resolveLanguageFromDataAttribute(dataLang));
 }
-const detector = new LanguageDetector();
-/* -------------------------------------------------------------------------------------------------
- * Public API
- * ------------------------------------------------------------------------------------------------- */
-/**
- * Detect programming language from code content using heuristics.
- */
 export function detectLanguageFromCode(code) {
-    if (!code || code.trim().length === 0)
+    if (!code)
         return undefined;
-    return detector.detect(code);
-}
-/**
- * Resolve language from HTML attributes (class name and data-language).
- */
-export function resolveLanguageFromAttributes(className, dataLang) {
-    return attributeResolver.resolve(className, dataLang);
+    // Fast path for empty/whitespace only
+    let empty = true;
+    for (let i = 0; i < code.length; i++) {
+        if (code.charCodeAt(i) > 32) {
+            empty = false;
+            break;
+        }
+    }
+    if (empty)
+        return undefined;
+    const ctx = new DetectionContext(code);
+    let bestLang;
+    let bestScore = -1;
+    for (const def of LANGUAGES) {
+        if (def.match(ctx)) {
+            if (def.weight > bestScore) {
+                bestScore = def.weight;
+                bestLang = def.lang;
+                if (bestScore >= 25)
+                    break;
+            }
+        }
+    }
+    return bestLang;
 }

package/dist/markdown-cleanup.d.ts

@@ -3,5 +3,4 @@ export declare function cleanupMarkdownArtifacts(content: string): string;
 export declare function extractTitleFromRawMarkdown(content: string): string | undefined;
 export declare function addSourceToMarkdown(content: string, url: string): string;
 export declare function isRawTextContent(content: string): boolean;
-export declare function isLikelyHtmlContent(content: string): boolean;
 export declare function buildMetadataFooter(metadata?: MetadataBlock, fallbackUrl?: string): string;