@j0hanz/superfetch 2.0.0 → 2.1.0

package/dist/utils/code-language.js

@@ -1,56 +1,260 @@
-const CODE_PATTERNS = [
-    [
-        /^\s*import\s+.*\s+from\s+['"]react['"]|<[A-Z][a-zA-Z]*[\s/>]|jsx\s*:|className=/m,
-        'jsx',
-    ],
-    [
-        /:\s*(string|number|boolean|void|any|unknown|never)\b|interface\s+\w+|type\s+\w+\s*=/m,
-        'typescript',
-    ],
-    [/^\s*(fn|let\s+mut|impl|struct|enum|use\s+\w+::)/m, 'rust'],
-    [
-        /^\s*(export|const|let|var|function|class|async|await)\b|^\s*import\s+.*['"]]/m,
-        'javascript',
-    ],
-    [/^\s*(def|class|import|from|if __name__|print\()/m, 'python'],
-    [
-        /^\s*(npm|yarn|pnpm|npx|brew|apt|pip|cargo|go )\s+(install|add|run|build|start)/m,
-        'bash',
-    ],
-    [/^\s*[$#]\s+\w+|^\s*#!|^\s*(sudo|chmod|mkdir|cd|ls|cat|echo)\s+/m, 'bash'],
-    [/^\s*[.#@]?[\w-]+\s*\{[^}]*\}|@media|@import|@keyframes/m, 'css'],
-    [/^\s*<(!DOCTYPE|html|head|body|div|span|p|a|script|style)\b/im, 'html'],
-    [/^\s*\{\s*"|^\s*\[\s*("|\d|true|false|null)/m, 'json'],
-    [/^\s*[\w-]+:\s*.+$/m, 'yaml'],
-    [/^\s*(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER|DROP)\s+/im, 'sql'],
-    [/^\s*(func|package|import\s+")/m, 'go'],
+function containsJsxTag(code) {
+    for (let index = 0; index < code.length - 1; index += 1) {
+        if (code[index] !== '<')
+            continue;
+        const next = code[index + 1];
+        if (!next)
+            continue;
+        if (next >= 'A' && next <= 'Z')
+            return true;
+    }
+    return false;
+}
+function containsWord(source, word) {
+    let startIndex = source.indexOf(word);
+    while (startIndex !== -1) {
+        const before = startIndex === 0 ? '' : source[startIndex - 1];
+        const afterIndex = startIndex + word.length;
+        const after = afterIndex >= source.length ? '' : source[afterIndex];
+        if (!isWordChar(before) && !isWordChar(after))
+            return true;
+        startIndex = source.indexOf(word, startIndex + word.length);
+    }
+    return false;
+}
+function splitLines(content) {
+    return content.split('\n');
+}
+function extractLanguageFromClassName(className) {
+    const tokens = className.match(/\S+/g);
+    if (!tokens)
+        return undefined;
+    for (const token of tokens) {
+        const lower = token.toLowerCase();
+        if (lower.startsWith('language-'))
+            return token.slice('language-'.length);
+        if (lower.startsWith('lang-'))
+            return token.slice('lang-'.length);
+        if (lower.startsWith('highlight-')) {
+            return token.slice('highlight-'.length);
+        }
+    }
+    return undefined;
+}
+function resolveLanguageFromDataAttribute(dataLang) {
+    const trimmed = dataLang.trim();
+    if (!trimmed)
+        return undefined;
+    for (const char of trimmed) {
+        if (!isWordChar(char))
+            return undefined;
+    }
+    return trimmed;
+}
+function isWordChar(char) {
+    if (!char)
+        return false;
+    const code = char.charCodeAt(0);
+    return ((code >= 48 && code <= 57) ||
+        (code >= 65 && code <= 90) ||
+        (code >= 97 && code <= 122) ||
+        char === '_');
+}
+const BASH_PACKAGE_MANAGERS = [
+    'npm',
+    'yarn',
+    'pnpm',
+    'npx',
+    'brew',
+    'apt',
+    'pip',
+    'cargo',
+    'go',
+];
+const BASH_VERBS = ['install', 'add', 'run', 'build', 'start'];
+const BASH_COMMANDS = ['sudo', 'chmod', 'mkdir', 'cd', 'ls', 'cat', 'echo'];
+function detectBash(code) {
+    const lines = splitLines(code);
+    for (const line of lines) {
+        const trimmed = line.trimStart();
+        if (!trimmed)
+            continue;
+        if (isBashIndicator(trimmed))
+            return true;
+    }
+    return false;
+}
+function startsWithCommand(line, commands) {
+    return commands.some((command) => line === command || line.startsWith(`${command} `));
+}
+function isBashIndicator(line) {
+    return (isShebang(line) ||
+        isPromptLine(line) ||
+        startsWithCommand(line, BASH_COMMANDS) ||
+        startsWithPackageManagerCommand(line));
+}
+function isShebang(line) {
+    return line.startsWith('#!');
+}
+function isPromptLine(line) {
+    return line.startsWith('$ ') || line.startsWith('# ');
+}
+function startsWithPackageManagerCommand(line) {
+    return BASH_PACKAGE_MANAGERS.some((manager) => {
+        if (!line.startsWith(`${manager} `))
+            return false;
+        const rest = line.slice(manager.length + 1);
+        return BASH_VERBS.some((verb) => rest === verb || rest.startsWith(`${verb} `));
+    });
+}
+const TYPE_HINTS = [
+    'string',
+    'number',
+    'boolean',
+    'void',
+    'any',
+    'unknown',
+    'never',
+];
+const HTML_TAGS = [
+    '<!doctype',
+    '<html',
+    '<head',
+    '<body',
+    '<div',
+    '<span',
+    '<p',
+    '<a',
+    '<script',
+    '<style',
+];
+const SQL_KEYWORDS = [
+    'select',
+    'insert',
+    'update',
+    'delete',
+    'create',
+    'alter',
+    'drop',
 ];
-const CLASS_PATTERNS = [
-    /language-(\w+)/,
-    /lang-(\w+)/,
-    /highlight-(\w+)/,
+const JS_WORD_REGEX = /\b(?:const|let|var|function|class|async|await|export|import)\b/;
+const PYTHON_WORD_REGEX = /\b(?:def|class|import|from)\b/;
+const RUST_WORD_REGEX = /\b(?:fn|impl|struct|enum)\b/;
+const CSS_DIRECTIVE_REGEX = /@media|@import|@keyframes/;
+const CODE_DETECTORS = [
+    { language: 'jsx', detect: detectJsx },
+    { language: 'typescript', detect: detectTypescript },
+    { language: 'rust', detect: detectRust },
+    { language: 'javascript', detect: detectJavascript },
+    { language: 'python', detect: detectPython },
+    { language: 'bash', detect: detectBash },
+    { language: 'css', detect: detectCss },
+    { language: 'html', detect: detectHtml },
+    { language: 'json', detect: detectJson },
+    { language: 'yaml', detect: detectYaml },
+    { language: 'sql', detect: detectSql },
+    { language: 'go', detect: detectGo },
 ];
+function detectJsx(code) {
+    const lower = code.toLowerCase();
+    if (lower.includes('classname='))
+        return true;
+    if (lower.includes('jsx:'))
+        return true;
+    if (lower.includes("from 'react'") || lower.includes('from "react"')) {
+        return true;
+    }
+    return containsJsxTag(code);
+}
+function detectTypescript(code) {
+    const lower = code.toLowerCase();
+    if (containsWord(lower, 'interface'))
+        return true;
+    if (containsWord(lower, 'type'))
+        return true;
+    return TYPE_HINTS.some((hint) => lower.includes(`: ${hint}`) || lower.includes(`:${hint}`));
+}
+function detectRust(code) {
+    const lower = code.toLowerCase();
+    return (RUST_WORD_REGEX.test(lower) ||
+        lower.includes('let mut') ||
+        (lower.includes('use ') && lower.includes('::')));
+}
+function detectJavascript(code) {
+    const lower = code.toLowerCase();
+    return JS_WORD_REGEX.test(lower);
+}
+function detectPython(code) {
+    const lower = code.toLowerCase();
+    return (PYTHON_WORD_REGEX.test(lower) ||
+        lower.includes('print(') ||
+        lower.includes('__name__'));
+}
+function detectCss(code) {
+    const lower = code.toLowerCase();
+    if (CSS_DIRECTIVE_REGEX.test(lower))
+        return true;
+    const lines = splitLines(code);
+    for (const line of lines) {
+        const trimmed = line.trimStart();
+        if (!trimmed)
+            continue;
+        if (isCssSelectorLine(trimmed) || isCssPropertyLine(trimmed))
+            return true;
+    }
+    return false;
+}
+function detectHtml(code) {
+    const lower = code.toLowerCase();
+    return HTML_TAGS.some((tag) => lower.includes(tag));
+}
+function detectJson(code) {
+    const trimmed = code.trimStart();
+    if (!trimmed)
+        return false;
+    return trimmed.startsWith('{') || trimmed.startsWith('[');
+}
+function detectYaml(code) {
+    const lines = splitLines(code);
+    for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        const colonIndex = trimmed.indexOf(':');
+        if (colonIndex <= 0)
+            continue;
+        const after = trimmed[colonIndex + 1];
+        if (after === ' ' || after === '\t')
+            return true;
+    }
+    return false;
+}
+function detectSql(code) {
+    const lower = code.toLowerCase();
+    return SQL_KEYWORDS.some((keyword) => containsWord(lower, keyword));
+}
+function detectGo(code) {
+    const lower = code.toLowerCase();
+    return (containsWord(lower, 'package') ||
+        containsWord(lower, 'func') ||
+        lower.includes('import "'));
+}
+function isCssSelectorLine(line) {
+    if (!line.startsWith('.') && !line.startsWith('#'))
+        return false;
+    return line.includes('{');
+}
+function isCssPropertyLine(line) {
+    return line.includes(':') && line.includes(';');
+}
 export function detectLanguageFromCode(code) {
-    for (const [pattern, language] of CODE_PATTERNS) {
-        if (pattern.test(code)) {
+    for (const { language, detect } of CODE_DETECTORS) {
+        if (detect(code))
             return language;
-        }
     }
     return undefined;
 }
 export function resolveLanguageFromAttributes(className, dataLang) {
-    const classMatch = matchFirstCapture(className, CLASS_PATTERNS);
+    const classMatch = extractLanguageFromClassName(className);
     return classMatch ?? resolveLanguageFromDataAttribute(dataLang);
 }
-function matchFirstCapture(value, patterns) {
-    for (const pattern of patterns) {
-        const match = pattern.exec(value);
-        if (match?.[1])
-            return match[1];
-    }
-    return undefined;
-}
-function resolveLanguageFromDataAttribute(dataLang) {
-    const match = /^(\w+)$/.exec(dataLang);
-    return match?.[1];
-}

package/dist/utils/error-details.d.ts

@@ -0,0 +1,3 @@
+export declare function getErrorMessage(error: unknown): string;
+export declare function createErrorWithCode(message: string, code: string): NodeJS.ErrnoException;
+export declare function isSystemError(error: unknown): error is NodeJS.ErrnoException;

package/dist/utils/error-details.js

@@ -0,0 +1,12 @@
+export function getErrorMessage(error) {
+    return error instanceof Error ? error.message : 'Unknown error';
+}
+export function createErrorWithCode(message, code) {
+    const error = new Error(message);
+    return Object.assign(error, { code });
+}
+export function isSystemError(error) {
+    return (error instanceof Error &&
+        'code' in error &&
+        typeof Reflect.get(error, 'code') === 'string');
+}

package/dist/utils/filename-generator.js

@@ -2,6 +2,17 @@ const MAX_FILENAME_LENGTH = 200;
 const UNSAFE_CHARS_REGEX = /[<>:"/\\|?*]|\p{C}/gu;
 const WHITESPACE_REGEX = /\s+/g;
 const DEFAULT_EXTENSION = '.md';
+function trimHyphens(value) {
+    let start = 0;
+    let end = value.length;
+    while (start < end && value[start] === '-') {
+        start += 1;
+    }
+    while (end > start && value[end - 1] === '-') {
+        end -= 1;
+    }
+    return value.slice(start, end);
+}
 export function generateSafeFilename(url, title, hashFallback, extension = DEFAULT_EXTENSION) {
     const fromUrl = extractFilenameFromUrl(url);
     if (fromUrl)
@@ -52,9 +63,9 @@ function slugifyTitle(title) {
         .trim()
         .replace(UNSAFE_CHARS_REGEX, '')
         .replace(WHITESPACE_REGEX, '-')
-        .replace(/-+/g, '-')
-        .replace(/^-|-$/g, '');
-    return slug || null;
+        .replace(/-+/g, '-');
+    const trimmed = trimHyphens(slug);
+    return trimmed || null;
 }
 function sanitizeFilename(name, extension) {
     let sanitized = name

package/dist/utils/host-normalizer.d.ts

@@ -0,0 +1 @@
+export declare function normalizeHost(value: string): string | null;

package/dist/utils/host-normalizer.js

@@ -0,0 +1,37 @@
+import { isIP } from 'node:net';
+function takeFirstHostValue(value) {
+    const first = value.split(',')[0];
+    if (!first)
+        return null;
+    const trimmed = first.trim();
+    return trimmed ? trimmed : null;
+}
+function stripIpv6Brackets(value) {
+    if (!value.startsWith('['))
+        return null;
+    const end = value.indexOf(']');
+    if (end === -1)
+        return null;
+    return value.slice(1, end);
+}
+function stripPortIfPresent(value) {
+    const colonIndex = value.indexOf(':');
+    if (colonIndex === -1)
+        return value;
+    return value.slice(0, colonIndex);
+}
+export function normalizeHost(value) {
+    const trimmed = value.trim().toLowerCase();
+    if (!trimmed)
+        return null;
+    const first = takeFirstHostValue(trimmed);
+    if (!first)
+        return null;
+    const ipv6 = stripIpv6Brackets(first);
+    if (ipv6)
+        return ipv6;
+    if (isIP(first) === 6) {
+        return first;
+    }
+    return stripPortIfPresent(first);
+}

package/dist/utils/ip-address.d.ts

@@ -0,0 +1,4 @@
+type IpSegment = number | string;
+export declare function buildIpv4(parts: readonly [number, number, number, number]): string;
+export declare function buildIpv6(parts: readonly IpSegment[]): string;
+export {};

package/dist/utils/ip-address.js

@@ -0,0 +1,6 @@
+export function buildIpv4(parts) {
+    return parts.join('.');
+}
+export function buildIpv6(parts) {
+    return parts.map(String).join(':');
+}

package/dist/utils/tool-error-handler.js

@@ -1,11 +1,5 @@
 import { FetchError } from '../errors/app-error.js';
-import { isSystemError } from './error-utils.js';
-function createFallbackErrorResponse(fallbackMessage, url, error) {
-    return createToolErrorResponse(`${fallbackMessage}: ${error.message}`, url);
-}
-function createUnknownErrorResponse(fallbackMessage, url) {
-    return createToolErrorResponse(`${fallbackMessage}: Unknown error`, url);
-}
+import { isSystemError } from './error-details.js';
 export function createToolErrorResponse(message, url) {
     const structuredContent = {
         error: message,
@@ -18,19 +12,20 @@ export function createToolErrorResponse(message, url) {
     };
 }
 export function handleToolError(error, url, fallbackMessage = 'Operation failed') {
-    if (isValidationError(error)) {
-        return createToolErrorResponse(error.message, url);
-    }
-    if (error instanceof FetchError) {
-        return createToolErrorResponse(error.message, url);
-    }
-    if (error instanceof Error) {
-        return createFallbackErrorResponse(fallbackMessage, url, error);
-    }
-    return createUnknownErrorResponse(fallbackMessage, url);
+    const message = resolveToolErrorMessage(error, fallbackMessage);
+    return createToolErrorResponse(message, url);
 }
 function isValidationError(error) {
     return (error instanceof Error &&
         isSystemError(error) &&
         error.code === 'VALIDATION_ERROR');
 }
+function resolveToolErrorMessage(error, fallbackMessage) {
+    if (isValidationError(error) || error instanceof FetchError) {
+        return error.message;
+    }
+    if (error instanceof Error) {
+        return `${fallbackMessage}: ${error.message}`;
+    }
+    return `${fallbackMessage}: Unknown error`;
+}

package/dist/utils/url-redactor.d.ts

@@ -0,0 +1 @@
+export declare function redactUrl(rawUrl: string): string;

package/dist/utils/url-redactor.js

@@ -0,0 +1,13 @@
+export function redactUrl(rawUrl) {
+    try {
+        const url = new URL(rawUrl);
+        url.username = '';
+        url.password = '';
+        url.hash = '';
+        url.search = '';
+        return url.toString();
+    }
+    catch {
+        return rawUrl;
+    }
+}

package/dist/utils/url-validator.js

@@ -1,28 +1,38 @@
 import { BlockList, isIP } from 'node:net';
 import { config } from '../config/index.js';
-import { createErrorWithCode } from './error-utils.js';
+import { createErrorWithCode } from './error-details.js';
+import { buildIpv4, buildIpv6 } from './ip-address.js';
 const BLOCK_LIST = new BlockList();
+const IPV6_ZERO = buildIpv6([0, 0, 0, 0, 0, 0, 0, 0]);
+const IPV6_LOOPBACK = buildIpv6([0, 0, 0, 0, 0, 0, 0, 1]);
+const IPV6_64_FF9B = buildIpv6(['64', 'ff9b', 0, 0, 0, 0, 0, 0]);
+const IPV6_64_FF9B_1 = buildIpv6(['64', 'ff9b', 1, 0, 0, 0, 0, 0]);
+const IPV6_2001 = buildIpv6(['2001', 0, 0, 0, 0, 0, 0, 0]);
+const IPV6_2002 = buildIpv6(['2002', 0, 0, 0, 0, 0, 0, 0]);
+const IPV6_FC00 = buildIpv6(['fc00', 0, 0, 0, 0, 0, 0, 0]);
+const IPV6_FE80 = buildIpv6(['fe80', 0, 0, 0, 0, 0, 0, 0]);
+const IPV6_FF00 = buildIpv6(['ff00', 0, 0, 0, 0, 0, 0, 0]);
 const BLOCKED_IPV4_SUBNETS = [
-    { subnet: '0.0.0.0', prefix: 8 },
-    { subnet: '10.0.0.0', prefix: 8 },
-    { subnet: '100.64.0.0', prefix: 10 },
-    { subnet: '127.0.0.0', prefix: 8 },
-    { subnet: '169.254.0.0', prefix: 16 },
-    { subnet: '172.16.0.0', prefix: 12 },
-    { subnet: '192.168.0.0', prefix: 16 },
-    { subnet: '224.0.0.0', prefix: 4 },
-    { subnet: '240.0.0.0', prefix: 4 },
+    { subnet: buildIpv4([0, 0, 0, 0]), prefix: 8 },
+    { subnet: buildIpv4([10, 0, 0, 0]), prefix: 8 },
+    { subnet: buildIpv4([100, 64, 0, 0]), prefix: 10 },
+    { subnet: buildIpv4([127, 0, 0, 0]), prefix: 8 },
+    { subnet: buildIpv4([169, 254, 0, 0]), prefix: 16 },
+    { subnet: buildIpv4([172, 16, 0, 0]), prefix: 12 },
+    { subnet: buildIpv4([192, 168, 0, 0]), prefix: 16 },
+    { subnet: buildIpv4([224, 0, 0, 0]), prefix: 4 },
+    { subnet: buildIpv4([240, 0, 0, 0]), prefix: 4 },
 ];
 const BLOCKED_IPV6_SUBNETS = [
-    { subnet: '::', prefix: 128 },
-    { subnet: '::1', prefix: 128 },
-    { subnet: '64:ff9b::', prefix: 96 },
-    { subnet: '64:ff9b:1::', prefix: 48 },
-    { subnet: '2001::', prefix: 32 },
-    { subnet: '2002::', prefix: 16 },
-    { subnet: 'fc00::', prefix: 7 },
-    { subnet: 'fe80::', prefix: 10 },
-    { subnet: 'ff00::', prefix: 8 },
+    { subnet: IPV6_ZERO, prefix: 128 },
+    { subnet: IPV6_LOOPBACK, prefix: 128 },
+    { subnet: IPV6_64_FF9B, prefix: 96 },
+    { subnet: IPV6_64_FF9B_1, prefix: 48 },
+    { subnet: IPV6_2001, prefix: 32 },
+    { subnet: IPV6_2002, prefix: 16 },
+    { subnet: IPV6_FC00, prefix: 7 },
+    { subnet: IPV6_FE80, prefix: 10 },
+    { subnet: IPV6_FF00, prefix: 8 },
 ];
 for (const entry of BLOCKED_IPV4_SUBNETS) {
     BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv4');
@@ -68,6 +78,8 @@ export function normalizeUrl(urlString) {
     assertNoCredentials(url);
     const hostname = normalizeHostname(url);
     assertHostnameAllowed(hostname);
+    // Canonicalize hostname to avoid trailing-dot variants and keep url.href consistent.
+    url.hostname = hostname;
     return { normalizedUrl: url.href, hostname };
 }
 export function validateAndNormalizeUrl(urlString) {
@@ -109,7 +121,10 @@ function assertNoCredentials(url) {
     throw createValidationError('URLs with embedded credentials are not allowed');
 }
 function normalizeHostname(url) {
-    const hostname = url.hostname.toLowerCase();
+    let hostname = url.hostname.toLowerCase();
+    while (hostname.endsWith('.')) {
+        hostname = hostname.slice(0, -1);
+    }
     if (!hostname) {
         throw createValidationError('URL must have a valid hostname');
     }