@j0hanz/superfetch 1.2.4 → 2.0.0

package/dist/transformers/markdown.transformer.js

@@ -1,13 +1,8 @@
 import TurndownService from 'turndown';
 import { CODE_BLOCK, FRONTMATTER_DELIMITER, joinLines, } from '../config/formatting.js';
 import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../utils/code-language.js';
+import { isRecord } from '../utils/guards.js';
 let turndownInstance = null;
-function getTurndown() {
-    if (turndownInstance)
-        return turndownInstance;
-    turndownInstance = createTurndownInstance();
-    return turndownInstance;
-}
 function createTurndownInstance() {
     const instance = new TurndownService({
         headingStyle: 'atx',
@@ -19,12 +14,97 @@ function createTurndownInstance() {
     addFencedCodeRule(instance);
     return instance;
 }
+function getTurndown() {
+    turndownInstance ??= createTurndownInstance();
+    return turndownInstance;
+}
+function isElement(node) {
+    if (!isRecord(node))
+        return false;
+    return 'getAttribute' in node && typeof node.getAttribute === 'function';
+}
+const STRUCTURAL_TAGS = new Set([
+    'script',
+    'style',
+    'noscript',
+    'iframe',
+    'nav',
+    'footer',
+    'aside',
+    'header',
+    'form',
+    'button',
+    'input',
+    'select',
+    'textarea',
+]);
+const NAVIGATION_ROLES = new Set([
+    'navigation',
+    'banner',
+    'complementary',
+    'contentinfo',
+    'tree',
+    'menubar',
+    'menu',
+]);
+const PROMO_PATTERN = /banner|promo|announcement|cta|callout|advert|newsletter|subscribe|cookie|consent|popup|modal|overlay|toast/;
+const FIXED_PATTERN = /\b(fixed|sticky)\b/;
+const HIGH_Z_PATTERN = /\bz-(?:4[0-9]|50)\b/;
+const ISOLATE_PATTERN = /\bisolate\b/;
+function isStructuralNoiseTag(tagName) {
+    return (STRUCTURAL_TAGS.has(tagName) || tagName === 'svg' || tagName === 'canvas');
+}
+function isElementHidden(element) {
+    return (element.getAttribute('hidden') !== null ||
+        element.getAttribute('aria-hidden') === 'true');
+}
+function hasNoiseRole(role) {
+    return role ? NAVIGATION_ROLES.has(role) : false;
+}
+function matchesPromoIdOrClass(className, id) {
+    const combined = `${className} ${id}`.toLowerCase();
+    return PROMO_PATTERN.test(combined);
+}
+function matchesHighZIsolate(className) {
+    return HIGH_Z_PATTERN.test(className) && ISOLATE_PATTERN.test(className);
+}
+function matchesFixedOrHighZIsolate(className) {
+    if (FIXED_PATTERN.test(className))
+        return true;
+    return matchesHighZIsolate(className);
+}
 function addNoiseRule(instance) {
     instance.addRule('removeNoise', {
-        filter: ['script', 'style', 'noscript', 'nav', 'footer', 'aside', 'iframe'],
+        filter: (node) => isNoiseNode(node),
         replacement: () => '',
     });
 }
+function isNoiseNode(node) {
+    if (!isElement(node))
+        return false;
+    return isNoiseElement(node);
+}
+function readElementMetadata(element) {
+    return {
+        tagName: element.tagName.toLowerCase(),
+        className: element.getAttribute('class') ?? '',
+        id: element.getAttribute('id') ?? '',
+        role: element.getAttribute('role'),
+        isHidden: isElementHidden(element),
+    };
+}
+function isNoiseElement(node) {
+    const metadata = readElementMetadata(node);
+    if (isStructuralNoiseTag(metadata.tagName))
+        return true;
+    if (metadata.isHidden)
+        return true;
+    if (hasNoiseRole(metadata.role))
+        return true;
+    if (matchesFixedOrHighZIsolate(metadata.className))
+        return true;
+    return matchesPromoIdOrClass(metadata.className, metadata.id);
+}
 function addFencedCodeRule(instance) {
     instance.addRule('fencedCodeBlockWithLanguage', {
         filter: (node, options) => isFencedCodeBlock(node, options),
@@ -41,12 +121,6 @@ function isFencedCodeBlock(node, options) {
         return false;
     return firstChild.nodeName === 'CODE';
 }
-function isElement(node) {
-    return (node !== null &&
-        typeof node === 'object' &&
-        'getAttribute' in node &&
-        typeof node.getAttribute === 'function');
-}
 function formatFencedCodeBlock(node) {
     const codeNode = node.firstChild;
     if (!isElement(codeNode))
@@ -56,11 +130,16 @@ function formatFencedCodeBlock(node) {
     return CODE_BLOCK.format(code, language);
 }
 function resolveCodeLanguage(codeNode, code) {
-    const className = codeNode.getAttribute('class') ?? '';
-    const dataLang = codeNode.getAttribute('data-language') ?? '';
-    const attributeLanguage = resolveLanguageFromAttributes(className, dataLang);
+    const { className, dataLanguage } = readCodeAttributes(codeNode);
+    const attributeLanguage = resolveLanguageFromAttributes(className, dataLanguage);
     return attributeLanguage ?? detectLanguageFromCode(code) ?? '';
 }
+function readCodeAttributes(codeNode) {
+    return {
+        className: codeNode.getAttribute('class') ?? '',
+        dataLanguage: codeNode.getAttribute('data-language') ?? '',
+    };
+}
 const YAML_SPECIAL_CHARS = /[:[\]{}"\r\t'|>&*!?,#]|\n/;
 const YAML_NUMERIC = /^[\d.]+$/;
 const YAML_RESERVED_WORDS = /^(true|false|null|yes|no|on|off)$/i;
@@ -91,36 +170,32 @@ function escapeYamlValue(value) {
         .replace(ESCAPE_PATTERNS.tab, '\\t');
     return `"${escaped}"`;
 }
+function appendFrontmatterField(lines, key, value) {
+    if (!value)
+        return;
+    lines.push(`${key}: ${escapeYamlValue(value)}`);
+}
 function createFrontmatter(metadata) {
     const lines = [FRONTMATTER_DELIMITER];
-    if (metadata.title) {
-        lines.push(`title: ${escapeYamlValue(metadata.title)}`);
-    }
-    if (metadata.url) {
-        lines.push(`source: ${escapeYamlValue(metadata.url)}`);
-    }
+    appendFrontmatterField(lines, 'title', metadata.title);
+    appendFrontmatterField(lines, 'source', metadata.url);
     lines.push(FRONTMATTER_DELIMITER);
     return joinLines(lines);
 }
-function convertHtmlToMarkdown(html) {
-    return getTurndown().turndown(html).trim();
-}
-function buildFrontmatterBlock(metadata) {
-    return metadata ? createFrontmatter(metadata) : '';
-}
 export function htmlToMarkdown(html, metadata) {
-    const frontmatter = buildFrontmatterBlock(metadata);
-    if (!isValidHtmlInput(html)) {
+    const frontmatter = buildFrontmatter(metadata);
+    if (!html)
         return frontmatter;
-    }
     try {
-        const content = convertHtmlToMarkdown(html);
+        const content = getTurndown().turndown(html).trim();
         return frontmatter ? `${frontmatter}\n${content}` : content;
     }
     catch {
         return frontmatter;
     }
 }
-function isValidHtmlInput(html) {
-    return Boolean(html && typeof html === 'string');
+function buildFrontmatter(metadata) {
+    if (!metadata)
+        return '';
+    return createFrontmatter(metadata);
 }

package/dist/utils/cached-payload.d.ts

@@ -0,0 +1,7 @@
+export interface CachedPayload {
+    content?: string;
+    markdown?: string;
+    title?: string;
+}
+export declare function parseCachedPayload(raw: string): CachedPayload | null;
+export declare function resolveCachedPayloadContent(payload: CachedPayload): string | null;

package/dist/utils/cached-payload.js

@@ -0,0 +1,36 @@
+import { isRecord } from './guards.js';
+export function parseCachedPayload(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        return isCachedPayload(parsed) ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function resolveCachedPayloadContent(payload) {
+    if (typeof payload.markdown === 'string') {
+        return payload.markdown;
+    }
+    if (typeof payload.content === 'string') {
+        return payload.content;
+    }
+    return null;
+}
+function hasOptionalStringProperty(value, key) {
+    const prop = value[key];
+    if (prop === undefined)
+        return true;
+    return typeof prop === 'string';
+}
+function isCachedPayload(value) {
+    if (!isRecord(value))
+        return false;
+    if (!hasOptionalStringProperty(value, 'content'))
+        return false;
+    if (!hasOptionalStringProperty(value, 'markdown'))
+        return false;
+    if (!hasOptionalStringProperty(value, 'title'))
+        return false;
+    return true;
+}

package/dist/utils/error-utils.js

@@ -8,5 +8,5 @@ export function createErrorWithCode(message, code) {
 export function isSystemError(error) {
     return (error instanceof Error &&
         'code' in error &&
-        typeof error.code === 'string');
+        typeof Reflect.get(error, 'code') === 'string');
 }

package/dist/utils/filename-generator.js

@@ -16,20 +16,31 @@ export function generateSafeFilename(url, title, hashFallback, extension = DEFAU
     }
     return `download-${Date.now()}${extension}`;
 }
+function getLastPathSegment(url) {
+    const segments = url.pathname.split('/').filter(Boolean);
+    if (segments.length === 0)
+        return null;
+    const lastSegment = segments[segments.length - 1];
+    return lastSegment ?? null;
+}
+function stripCommonPageExtension(segment) {
+    return segment.replace(/\.(html?|php|aspx?|jsp)$/i, '');
+}
+function normalizeUrlFilenameSegment(segment) {
+    const cleaned = stripCommonPageExtension(segment);
+    if (!cleaned)
+        return null;
+    if (cleaned === 'index')
+        return null;
+    return cleaned;
+}
 function extractFilenameFromUrl(url) {
     try {
         const urlObj = new URL(url);
-        const { pathname } = urlObj;
-        const segments = pathname.split('/').filter(Boolean);
-        if (segments.length === 0)
-            return null;
-        const lastSegment = segments[segments.length - 1];
+        const lastSegment = getLastPathSegment(urlObj);
         if (!lastSegment)
             return null;
-        const cleaned = lastSegment.replace(/\.(html?|php|aspx?|jsp)$/i, '');
-        if (!cleaned || cleaned === 'index')
-            return null;
-        return cleaned;
+        return normalizeUrlFilenameSegment(lastSegment);
     }
     catch {
         return null;
@@ -43,7 +54,7 @@ function slugifyTitle(title) {
         .replace(WHITESPACE_REGEX, '-')
         .replace(/-+/g, '-')
         .replace(/^-|-$/g, '');
-    return slug.length > 0 ? slug : null;
+    return slug || null;
 }
 function sanitizeFilename(name, extension) {
     let sanitized = name

package/dist/utils/guards.d.ts

@@ -0,0 +1 @@
+export declare function isRecord(value: unknown): value is Record<string, unknown>;

package/dist/utils/guards.js

@@ -0,0 +1,3 @@
+export function isRecord(value) {
+    return typeof value === 'object' && value !== null;
+}

package/dist/utils/header-normalizer.d.ts

@@ -2,7 +2,4 @@ interface NormalizeOptions {
     readonly trimValues?: boolean;
 }
 export declare function normalizeHeaderRecord(headers: Record<string, string> | undefined, blockedHeaders: Set<string>, options?: NormalizeOptions): Record<string, string> | undefined;
-export declare function normalizeHeaderEntries(headers: Record<string, string>, blockedHeaders: Set<string>, options?: NormalizeOptions): Headers;
-export declare function hasHeaderEntries(headers: Headers): boolean;
-export declare function headersToRecord(headers: Headers): Record<string, string>;
 export {};

package/dist/utils/header-normalizer.js

@@ -6,7 +6,7 @@ export function normalizeHeaderRecord(headers, blockedHeaders, options = {}) {
         return undefined;
     return headersToRecord(normalized);
 }
-export function normalizeHeaderEntries(headers, blockedHeaders, options = {}) {
+function normalizeHeaderEntries(headers, blockedHeaders, options = {}) {
     const normalized = new Headers();
     for (const [key, value] of Object.entries(headers)) {
         if (blockedHeaders.has(key.toLowerCase()))
@@ -15,10 +15,10 @@ export function normalizeHeaderEntries(headers, blockedHeaders, options = {}) {
     }
     return normalized;
 }
-export function hasHeaderEntries(headers) {
+function hasHeaderEntries(headers) {
     return !headers.keys().next().done;
 }
-export function headersToRecord(headers) {
+function headersToRecord(headers) {
     return Object.fromEntries(headers.entries());
 }
 function setHeaderValue(headers, key, value, trimValue) {

package/dist/utils/tool-error-handler.d.ts

@@ -1,3 +1,3 @@
 import type { ToolErrorResponse } from '../config/types/tools.js';
-export declare function createToolErrorResponse(message: string, url: string, code: string, details?: Record<string, unknown>): ToolErrorResponse;
-export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string, details?: Record<string, unknown>): ToolErrorResponse;
+export declare function createToolErrorResponse(message: string, url: string): ToolErrorResponse;
+export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string): ToolErrorResponse;

package/dist/utils/tool-error-handler.js

@@ -1,33 +1,15 @@
-import { ErrorCode } from '@modelcontextprotocol/sdk/types.js';
 import { FetchError } from '../errors/app-error.js';
 import { isSystemError } from './error-utils.js';
-const IS_DEVELOPMENT_WITH_STACK_TRACES = process.env.NODE_ENV === 'development' &&
-    process.env.EXPOSE_STACK_TRACES === 'true';
-const MCP_ERROR_CODE_MAP = {
-    VALIDATION_ERROR: String(ErrorCode.InvalidParams),
-    INVALID_PARAMS: String(ErrorCode.InvalidParams),
-    INTERNAL_ERROR: String(ErrorCode.InternalError),
-    FETCH_ERROR: String(ErrorCode.InternalError),
-    BATCH_ERROR: String(ErrorCode.InternalError),
-    PROMISE_REJECTED: String(ErrorCode.InternalError),
-    UNKNOWN_ERROR: String(ErrorCode.InternalError),
-};
-const NUMERIC_ERROR_CODE = /^-?\d+$/;
-function normalizeToolErrorCode(code) {
-    if (!code)
-        return String(ErrorCode.InternalError);
-    if (NUMERIC_ERROR_CODE.test(code))
-        return code;
-    if (code.startsWith('HTTP_'))
-        return String(ErrorCode.InternalError);
-    return MCP_ERROR_CODE_MAP[code] ?? code;
+function createFallbackErrorResponse(fallbackMessage, url, error) {
+    return createToolErrorResponse(`${fallbackMessage}: ${error.message}`, url);
 }
-export function createToolErrorResponse(message, url, code, details = {}) {
+function createUnknownErrorResponse(fallbackMessage, url) {
+    return createToolErrorResponse(`${fallbackMessage}: Unknown error`, url);
+}
+export function createToolErrorResponse(message, url) {
     const structuredContent = {
-        ...details,
         error: message,
         url,
-        errorCode: normalizeToolErrorCode(code),
     };
     return {
         content: [{ type: 'text', text: JSON.stringify(structuredContent) }],
@@ -35,26 +17,17 @@ export function createToolErrorResponse(message, url, code, details = {}) {
         isError: true,
     };
 }
-function formatErrorMessage(baseMessage, error, fallback) {
-    const message = fallback ? `${fallback}: ${error.message}` : error.message;
-    if (IS_DEVELOPMENT_WITH_STACK_TRACES && error.stack) {
-        return `${message}\n${error.stack}`;
-    }
-    return message;
-}
-export function handleToolError(error, url, fallbackMessage = 'Operation failed', details = {}) {
+export function handleToolError(error, url, fallbackMessage = 'Operation failed') {
     if (isValidationError(error)) {
-        return createToolErrorResponse(error.message, url, 'VALIDATION_ERROR', details);
+        return createToolErrorResponse(error.message, url);
     }
     if (error instanceof FetchError) {
-        const message = formatErrorMessage(error.message, error);
-        return createToolErrorResponse(message, url, error.code, details);
+        return createToolErrorResponse(error.message, url);
     }
     if (error instanceof Error) {
-        const message = formatErrorMessage(error.message, error, fallbackMessage);
-        return createToolErrorResponse(message, url, 'UNKNOWN_ERROR', details);
+        return createFallbackErrorResponse(fallbackMessage, url, error);
     }
-    return createToolErrorResponse(`${fallbackMessage}: Unknown error`, url, 'UNKNOWN_ERROR', details);
+    return createUnknownErrorResponse(fallbackMessage, url);
 }
 function isValidationError(error) {
     return (error instanceof Error &&

package/dist/utils/url-transformer.d.ts

@@ -0,0 +1,7 @@
+export interface TransformResult {
+    readonly url: string;
+    readonly transformed: boolean;
+    readonly platform?: string;
+}
+export declare function transformToRawUrl(url: string): TransformResult;
+export declare function isRawTextContentUrl(url: string): boolean;

package/dist/utils/url-transformer.js

@@ -0,0 +1,147 @@
+import { logDebug } from '../services/logger.js';
+const GITHUB_BLOB_RULE = {
+    name: 'github',
+    pattern: /^https?:\/\/(?:www\.)?github\.com\/([^/]+)\/([^/]+)\/blob\/([^/]+)\/(.+)$/i,
+    transform: (match) => {
+        const owner = match[1] ?? '';
+        const repo = match[2] ?? '';
+        const branch = match[3] ?? '';
+        const path = match[4] ?? '';
+        return `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${path}`;
+    },
+};
+const GITHUB_GIST_RULE = {
+    name: 'github-gist',
+    pattern: /^https?:\/\/gist\.github\.com\/([^/]+)\/([a-f0-9]+)(?:#file-(.+)|\/raw\/([^/]+))?$/i,
+    transform: (match) => {
+        const user = match[1] ?? '';
+        const gistId = match[2] ?? '';
+        const hashFile = match[3];
+        const rawFile = match[4];
+        const filename = rawFile ?? hashFile?.replace(/-/g, '.');
+        const filePath = filename ? `/${filename}` : '';
+        return `https://gist.githubusercontent.com/${user}/${gistId}/raw${filePath}`;
+    },
+};
+const GITLAB_BLOB_RULE = {
+    name: 'gitlab',
+    pattern: /^(https?:\/\/(?:[^/]+\.)?gitlab\.com\/[^/]+\/[^/]+)\/-\/blob\/([^/]+)\/(.+)$/i,
+    transform: (match) => {
+        const baseUrl = match[1] ?? '';
+        const branch = match[2] ?? '';
+        const path = match[3] ?? '';
+        return `${baseUrl}/-/raw/${branch}/${path}`;
+    },
+};
+const BITBUCKET_SRC_RULE = {
+    name: 'bitbucket',
+    pattern: /^(https?:\/\/(?:www\.)?bitbucket\.org\/[^/]+\/[^/]+)\/src\/([^/]+)\/(.+)$/i,
+    transform: (match) => {
+        const baseUrl = match[1] ?? '';
+        const branch = match[2] ?? '';
+        const path = match[3] ?? '';
+        return `${baseUrl}/raw/${branch}/${path}`;
+    },
+};
+const TRANSFORM_RULES = [
+    GITHUB_BLOB_RULE,
+    GITHUB_GIST_RULE,
+    GITLAB_BLOB_RULE,
+    BITBUCKET_SRC_RULE,
+];
+function isRawUrl(url) {
+    const lowerUrl = url.toLowerCase();
+    return (lowerUrl.includes('raw.githubusercontent.com') ||
+        lowerUrl.includes('gist.githubusercontent.com') ||
+        lowerUrl.includes('/-/raw/') ||
+        /bitbucket\.org\/[^/]+\/[^/]+\/raw\//.test(lowerUrl));
+}
+function getUrlWithoutParams(url) {
+    const hashIndex = url.indexOf('#');
+    const queryIndex = url.indexOf('?');
+    let endIndex = url.length;
+    if (queryIndex !== -1) {
+        if (hashIndex !== -1) {
+            endIndex = Math.min(queryIndex, hashIndex);
+        }
+        else {
+            endIndex = queryIndex;
+        }
+    }
+    else if (hashIndex !== -1) {
+        endIndex = hashIndex;
+    }
+    const hash = hashIndex !== -1 ? url.slice(hashIndex) : '';
+    return {
+        base: url.slice(0, endIndex),
+        hash,
+    };
+}
+function resolveUrlToMatch(rule, base, hash) {
+    if (rule.name !== 'github-gist')
+        return base;
+    if (!hash.startsWith('#file-'))
+        return base;
+    return base + hash;
+}
+function applyTransformRules(base, hash) {
+    for (const rule of TRANSFORM_RULES) {
+        const urlToMatch = resolveUrlToMatch(rule, base, hash);
+        const match = rule.pattern.exec(urlToMatch);
+        if (match) {
+            return { url: rule.transform(match), platform: rule.name };
+        }
+    }
+    return null;
+}
+export function transformToRawUrl(url) {
+    if (!url)
+        return { url, transformed: false };
+    if (isRawUrl(url)) {
+        return { url, transformed: false };
+    }
+    const { base, hash } = getUrlWithoutParams(url);
+    const result = applyTransformRules(base, hash);
+    if (!result)
+        return { url, transformed: false };
+    logDebug('URL transformed to raw content URL', {
+        platform: result.platform,
+        original: url.substring(0, 100),
+        transformed: result.url.substring(0, 100),
+    });
+    return {
+        url: result.url,
+        transformed: true,
+        platform: result.platform,
+    };
+}
+const RAW_TEXT_EXTENSIONS = new Set([
+    '.md',
+    '.markdown',
+    '.txt',
+    '.json',
+    '.yaml',
+    '.yml',
+    '.toml',
+    '.xml',
+    '.csv',
+    '.rst',
+    '.adoc',
+    '.org',
+]);
+export function isRawTextContentUrl(url) {
+    if (!url)
+        return false;
+    if (isRawUrl(url))
+        return true;
+    const { base } = getUrlWithoutParams(url);
+    const lowerBase = base.toLowerCase();
+    return hasKnownRawTextExtension(lowerBase);
+}
+function hasKnownRawTextExtension(urlBaseLower) {
+    for (const ext of RAW_TEXT_EXTENSIONS) {
+        if (urlBaseLower.endsWith(ext))
+            return true;
+    }
+    return false;
+}

package/dist/utils/url-validator.d.ts

@@ -1,7 +1,6 @@
 export declare function isBlockedIp(ip: string): boolean;
-export declare function assertResolvedAddressesAllowed(hostname: string): Promise<void>;
 export declare function normalizeUrl(urlString: string): {
     normalizedUrl: string;
     hostname: string;
 };
-export declare function validateAndNormalizeUrl(urlString: string): Promise<string>;
+export declare function validateAndNormalizeUrl(urlString: string): string;