@j0hanz/superfetch 1.0.2 → 1.0.5

package/dist/tools/handlers/fetch-links.tool.js

@@ -1,104 +1,129 @@
-import { validateAndNormalizeUrl, isInternalUrl, } from '../../utils/url-validator.js';
-import { fetchUrlWithRetry } from '../../services/fetcher.js';
-import * as cache from '../../services/cache.js';
 import * as cheerio from 'cheerio';
-import { logError } from '../../services/logger.js';
+import { logDebug, logError } from '../../services/logger.js';
 import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
+import { isInternalUrl } from '../../utils/url-validator.js';
+import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
 export const FETCH_LINKS_TOOL_NAME = 'fetch-links';
-export const FETCH_LINKS_TOOL_DESCRIPTION = 'Extracts all hyperlinks from a webpage with anchor text and type classification';
-/**
- * Extracts links from HTML, filtering by type and deduplicating
- */
-function extractLinksFromHtml(html, baseUrl, options) {
+export const FETCH_LINKS_TOOL_DESCRIPTION = 'Extracts all hyperlinks from a webpage with anchor text and type classification. Supports filtering, image links, and link limits.';
+function tryResolveUrl(href, baseUrl) {
+    try {
+        return new URL(href, baseUrl).href;
+    }
+    catch {
+        return null;
+    }
+}
+function shouldIncludeLink(type, url, options, seen) {
+    if (seen.has(url))
+        return false;
+    if (options.filterPattern && !options.filterPattern.test(url))
+        return false;
+    if (type === 'internal' && !options.includeInternal)
+        return false;
+    if (type === 'external' && !options.includeExternal)
+        return false;
+    return true;
+}
+function extractLinks(html, baseUrl, options) {
     const $ = cheerio.load(html);
     const links = [];
-    const seenUrls = new Set();
-    $('a[href]').each((_, element) => {
-        const href = $(element).attr('href');
-        const text = $(element).text().trim();
-        // Skip invalid hrefs
-        if (!href || href.startsWith('#') || href.startsWith('javascript:')) {
+    const seen = new Set();
+    let filtered = 0;
+    $('a[href]').each((_, el) => {
+        const href = $(el).attr('href');
+        if (!href || href.startsWith('#') || href.startsWith('javascript:'))
+            return;
+        const url = tryResolveUrl(href, baseUrl);
+        if (!url)
+            return;
+        const type = isInternalUrl(url, baseUrl)
+            ? 'internal'
+            : 'external';
+        if (!shouldIncludeLink(type, url, options, seen)) {
+            if (!seen.has(url))
+                filtered++;
             return;
         }
-        try {
-            const absoluteUrl = new URL(href, baseUrl).href;
-            // Skip duplicates
-            if (seenUrls.has(absoluteUrl)) {
+        seen.add(url);
+        links.push({ href: url, text: $(el).text().trim() || url, type });
+    });
+    if (options.includeImages) {
+        $('img[src]').each((_, el) => {
+            const src = $(el).attr('src');
+            if (!src || src.startsWith('data:'))
                 return;
-            }
-            seenUrls.add(absoluteUrl);
-            const type = isInternalUrl(absoluteUrl, baseUrl)
-                ? 'internal'
-                : 'external';
-            // Filter based on options
-            if (type === 'internal' && !options.includeInternal)
+            const url = tryResolveUrl(src, baseUrl);
+            if (!url)
                 return;
-            if (type === 'external' && !options.includeExternal)
+            if (!shouldIncludeLink('image', url, options, seen)) {
+                if (!seen.has(url))
+                    filtered++;
                 return;
+            }
+            seen.add(url);
             links.push({
-                href: absoluteUrl,
-                text: text || absoluteUrl,
-                type,
+                href: url,
+                text: $(el).attr('alt')?.trim() ?? url,
+                type: 'image',
             });
-        }
-        catch {
-            // Skip invalid URLs silently
-        }
-    });
-    return links;
+        });
+    }
+    const truncated = options.maxLinks ? links.length > options.maxLinks : false;
+    const resultLinks = truncated ? links.slice(0, options.maxLinks) : links;
+    return {
+        links: resultLinks,
+        linkCount: resultLinks.length,
+        filtered,
+        truncated,
+    };
 }
-/**
- * Tool handler for extracting links from a URL
- */
 export async function fetchLinksToolHandler(input) {
-    try {
-        // Validate URL input
-        if (!input.url) {
-            return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
-        }
-        const url = validateAndNormalizeUrl(input.url);
-        const cacheKey = cache.createCacheKey('links', url);
-        // Check cache first
-        if (cacheKey) {
-            const cached = cache.get(cacheKey);
-            if (cached) {
-                // Parse the cached content to return as structuredContent
-                try {
-                    const structuredContent = JSON.parse(cached.content);
-                    return {
-                        content: [{ type: 'text', text: cached.content }],
-                        structuredContent,
-                    };
-                }
-                catch {
-                    return {
-                        content: [{ type: 'text', text: cached.content }],
-                    };
-                }
-            }
+    if (!input.url) {
+        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
+    }
+    let filterPattern;
+    if (input.filterPattern) {
+        try {
+            filterPattern = new RegExp(input.filterPattern, 'i');
         }
-        const html = await fetchUrlWithRetry(url);
-        // Validate HTML content was received
-        if (!html) {
-            return createToolErrorResponse('No content received from URL', url, 'EMPTY_CONTENT');
+        catch {
+            return createToolErrorResponse(`Invalid filter pattern: ${input.filterPattern}`, input.url, 'VALIDATION_ERROR');
         }
-        // Extract links
-        const links = extractLinksFromHtml(html, url, {
+    }
+    try {
+        const options = {
             includeInternal: input.includeInternal ?? true,
             includeExternal: input.includeExternal ?? true,
+            includeImages: input.includeImages ?? false,
+            maxLinks: input.maxLinks,
+            filterPattern,
+        };
+        logDebug('Extracting links', {
+            url: input.url,
+            ...options,
+            filterPattern: input.filterPattern,
+        });
+        const result = await executeFetchPipeline({
+            url: input.url,
+            cacheNamespace: 'links',
+            customHeaders: input.customHeaders,
+            retries: input.retries,
+            transform: (html, url) => extractLinks(html, url, options),
         });
         const structuredContent = {
-            url,
-            linkCount: links.length,
-            links,
+            url: result.url,
+            linkCount: result.data.linkCount,
+            links: result.data.links,
+            ...(result.data.filtered > 0 && { filtered: result.data.filtered }),
+            ...(result.data.truncated && { truncated: result.data.truncated }),
         };
-        const outputText = JSON.stringify(structuredContent, null, 2);
-        // Cache the result
-        if (cacheKey) {
-            cache.set(cacheKey, outputText);
-        }
         return {
-            content: [{ type: 'text', text: outputText }],
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify(structuredContent, null, 2),
+                },
+            ],
             structuredContent,
         };
     }

package/dist/tools/handlers/fetch-links.tool.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,aAAa,GACd,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAG3C,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC;AACnD,MAAM,CAAC,MAAM,4BAA4B,GACvC,iFAAiF,CAAC;AAEpF;;GAEG;AACH,SAAS,oBAAoB,CAC3B,IAAY,EACZ,OAAe,EACf,OAA+D;IAE/D,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAEtC,qBAAqB;QACrB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YACpE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAEhD,kBAAkB;YAClB,IAAI,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC9B,OAAO;YACT,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAE1B,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,OAAO,CAAC;gBAC9C,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,UAAU,CAAC;YAEf,0BAA0B;YAC1B,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;gBAAE,OAAO;YAC5D,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;gBAAE,OAAO;YAE5D,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,IAAI,IAAI,WAAW;gBACzB,IAAI;aACL,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,6BAA6B;QAC/B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAAsB;IAChE,IAAI,CAAC;QACH,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,GAAG,GAAG,uBAAuB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAEpD,oBAAoB;QACpB,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,0DAA0D;gBAC1D,IAAI,CAAC;oBACH,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAIlD,CAAC;oBACF,OAAO;wBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;wBAC1D,iBAAiB;qBAClB,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;wBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;qBAC3D,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAE1C,qCAAqC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,uBAAuB,CAC5B,8BAA8B,EAC9B,GAAG,EACH,eAAe,CAChB,CAAC;QACJ,CAAC;QAED,gBAAgB;QAChB,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE;YAC5C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;SAC/C,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG;YACxB,GAAG;YACH,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,KAAK;SACN,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAE9D,mBAAmB;QACnB,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAClC,CAAC;QAED,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;YACtD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,wBAAwB,EACxB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IACtE,CAAC;AACH,CAAC"}
+{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AASnC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC;AACnD,MAAM,CAAC,MAAM,4BAA4B,GACvC,oIAAoI,CAAC;AAIvI,SAAS,aAAa,CAAC,IAAY,EAAE,OAAe;IAClD,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,IAAc,EACd,GAAW,EACX,OAA4B,EAC5B,IAAiB;IAEjB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAChC,IAAI,OAAO,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5E,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CACnB,IAAY,EACZ,OAAe,EACf,OAA4B;IAE5B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;YAAE,OAAO;QAE5E,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,MAAM,IAAI,GAAa,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC;YAChD,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,UAAU,CAAC;QACf,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,QAAQ,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,OAAO;YAE5C,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEjB,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;gBACpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,QAAQ,EAAE,CAAC;gBAC/B,OAAO;YACT,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG;gBACtC,IAAI,EAAE,OAAO;aACd,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7E,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAEzE,OAAO;QACL,KAAK,EAAE,WAAW;QAClB,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,QAAQ;QACR,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAAsB;IAKhE,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,aAAiC,CAAC;IACtC,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,aAAa,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,uBAAuB,CAC5B,2BAA2B,KAAK,CAAC,aAAa,EAAE,EAChD,KAAK,CAAC,GAAG,EACT,kBAAkB,CACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAwB;YACnC,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,KAAK;YAC3C,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,aAAa;SACd,CAAC;QAEF,QAAQ,CAAC,kBAAkB,EAAE;YAC3B,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,GAAG,OAAO;YACV,aAAa,EAAE,KAAK,CAAC,aAAa;SACnC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAuB;YAC9D,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,cAAc,EAAE,OAAO;YACvB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;SAC3D,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG;YACxB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS;YAChC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;YACxB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;SACnE,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;iBACjD;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,wBAAwB,EACxB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IACtE,CAAC;AACH,CAAC"}

package/dist/tools/handlers/fetch-markdown.tool.d.ts

@@ -1,16 +1,12 @@
-import type { FetchMarkdownInput } from '../../types/index.js';
+import type { FetchMarkdownInput } from '../../config/types.js';
 export declare const FETCH_MARKDOWN_TOOL_NAME = "fetch-markdown";
-export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter";
-export declare function fetchMarkdownToolHandler(input: FetchMarkdownInput): Promise<import("../../utils/tool-error-handler.js").ToolErrorResponse | {
+export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter, table of contents, and content length limits";
+export declare function fetchMarkdownToolHandler(input: FetchMarkdownInput): Promise<{
     content: {
-        type: "text";
+        type: 'text';
         text: string;
     }[];
-    structuredContent: {
-        url: string;
-        cached: boolean;
-        fetchedAt: string;
-        markdown: string;
-    };
+    structuredContent?: Record<string, unknown>;
+    isError?: boolean;
 }>;
 //# sourceMappingURL=fetch-markdown.tool.d.ts.map

package/dist/tools/handlers/fetch-markdown.tool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE/D,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AACzD,eAAO,MAAM,+BAA+B,yFAC4C,CAAC;AAmDzF,wBAAsB,wBAAwB,CAAC,KAAK,EAAE,kBAAkB;;;;;;;;;;;GA6EvE"}
+{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAInB,MAAM,uBAAuB,CAAC;AAe/B,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AACzD,eAAO,MAAM,+BAA+B,uIAC0F,CAAC;AAsEvI,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC;IACT,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC,CA4DD"}

package/dist/tools/handlers/fetch-markdown.tool.js

@@ -1,103 +1,102 @@
-import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
-import { fetchUrlWithRetry } from '../../services/fetcher.js';
 import { extractContent } from '../../services/extractor.js';
-import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
-import * as cache from '../../services/cache.js';
-import { config } from '../../config/index.js';
-import { logError } from '../../services/logger.js';
+import { logDebug, logError } from '../../services/logger.js';
+import { stripMarkdownLinks } from '../../utils/content-cleaner.js';
 import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
+import { buildMetadata, shouldUseArticle } from '../utils/common.js';
+import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
+import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
 export const FETCH_MARKDOWN_TOOL_NAME = 'fetch-markdown';
-export const FETCH_MARKDOWN_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format with optional frontmatter';
-function extractAndConvertToMarkdown(html, url, options) {
-    // Use the optimized extractContent that parses JSDOM only once
-    const { article, metadata: extractedMeta } = extractContent(html, url);
-    if (options.extractMainContent &&
-        config.extraction.extractMainContent &&
-        article) {
-        const metadata = options.includeMetadata && config.extraction.includeMetadata
-            ? {
-                type: 'metadata',
-                title: article.title,
-                author: article.byline,
-                url,
-                fetchedAt: new Date().toISOString(),
-            }
-            : undefined;
-        return {
-            markdown: htmlToMarkdown(article.content, metadata),
-            title: article.title,
-        };
+export const FETCH_MARKDOWN_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format with optional frontmatter, table of contents, and content length limits';
+/**
+ * Generate URL-friendly slug from text
+ * Strips markdown link syntax before slugifying
+ */
+function slugify(text) {
+    // First strip markdown links: [Text](#anchor) -> Text
+    const cleanText = stripMarkdownLinks(text);
+    return cleanText
+        .toLowerCase()
+        .replace(/[^\w\s-]/g, '')
+        .replace(/\s+/g, '-')
+        .replace(/--+/g, '-')
+        .trim();
+}
+/**
+ * Extract table of contents from markdown headings
+ * Returns clean text without markdown link syntax
+ */
+function extractToc(markdown) {
+    const headingRegex = /^(#{1,6})\s+(.+)$/gm;
+    const toc = [];
+    let match;
+    while ((match = headingRegex.exec(markdown)) !== null) {
+        if (!match[1] || !match[2])
+            continue;
+        const rawText = match[2].trim();
+        // Clean markdown links from TOC text: [Usage](#usage) -> Usage
+        const text = stripMarkdownLinks(rawText);
+        toc.push({ level: match[1].length, text, slug: slugify(rawText) });
     }
-    // Fallback: convert full HTML
-    const metadata = options.includeMetadata && config.extraction.includeMetadata
-        ? {
-            type: 'metadata',
-            title: extractedMeta.title,
-            description: extractedMeta.description,
-            author: extractedMeta.author,
-            url,
-            fetchedAt: new Date().toISOString(),
-        }
-        : undefined;
-    return {
-        markdown: htmlToMarkdown(html, metadata),
-        title: extractedMeta.title,
-    };
+    return toc;
+}
+function transformToMarkdown(html, url, options) {
+    // Only invoke JSDOM when extractMainContent is true (lazy loading optimization)
+    const { article, metadata: extractedMeta } = extractContent(html, url, {
+        extractArticle: options.extractMainContent,
+    });
+    const useArticle = shouldUseArticle(options.extractMainContent, article);
+    const metadata = buildMetadata(url, article, extractedMeta, useArticle, options.includeMetadata);
+    const sourceHtml = useArticle ? article.content : html;
+    const title = useArticle ? article.title : extractedMeta.title;
+    let markdown = htmlToMarkdown(sourceHtml, metadata);
+    const toc = options.generateToc ? extractToc(markdown) : undefined;
+    let truncated = false;
+    if (options.maxContentLength && markdown.length > options.maxContentLength) {
+        markdown = `${markdown.substring(0, options.maxContentLength)}\n\n...[truncated]`;
+        truncated = true;
+    }
+    return { markdown, title, toc, truncated };
 }
 export async function fetchMarkdownToolHandler(input) {
+    if (!input.url) {
+        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
+    }
     try {
-        // Validate URL input
-        if (!input.url) {
-            return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
-        }
-        const url = validateAndNormalizeUrl(input.url);
-        const cacheKey = cache.createCacheKey('markdown', url);
-        // Check cache first
-        if (cacheKey) {
-            const cached = cache.get(cacheKey);
-            if (cached) {
-                const structuredContent = {
-                    url,
-                    cached: true,
-                    fetchedAt: cached.fetchedAt,
-                    markdown: cached.content,
-                };
-                return {
-                    content: [
-                        {
-                            type: 'text',
-                            text: JSON.stringify(structuredContent),
-                        },
-                    ],
-                    structuredContent,
-                };
-            }
-        }
-        const html = await fetchUrlWithRetry(url);
-        // Validate HTML content was received
-        if (!html) {
-            return createToolErrorResponse('No content received from URL', url, 'EMPTY_CONTENT');
-        }
-        const { markdown, title } = extractAndConvertToMarkdown(html, url, {
+        const options = {
             extractMainContent: input.extractMainContent ?? true,
             includeMetadata: input.includeMetadata ?? true,
+            generateToc: input.generateToc ?? false,
+            maxContentLength: input.maxContentLength,
+        };
+        logDebug('Fetching markdown', { url: input.url, ...options });
+        const result = await executeFetchPipeline({
+            url: input.url,
+            cacheNamespace: 'markdown',
+            customHeaders: input.customHeaders,
+            retries: input.retries,
+            transform: (html, url) => transformToMarkdown(html, url, options),
+            serialize: (data) => data.markdown,
+            deserialize: (cached) => ({
+                markdown: cached,
+                title: undefined,
+                toc: undefined,
+                truncated: false,
+            }),
         });
-        // Cache the result
-        if (cacheKey) {
-            cache.set(cacheKey, markdown);
-        }
         const structuredContent = {
-            url,
-            title,
-            fetchedAt: new Date().toISOString(),
-            markdown,
-            cached: false,
+            url: result.url,
+            title: result.data.title,
+            fetchedAt: result.fetchedAt,
+            markdown: result.data.markdown,
+            ...(result.data.toc && { toc: result.data.toc }),
+            cached: result.fromCache,
+            ...(result.data.truncated && { truncated: result.data.truncated }),
         };
         return {
             content: [
                 {
                     type: 'text',
-                    text: JSON.stringify(structuredContent, null, 2),
+                    text: JSON.stringify(structuredContent, result.fromCache ? undefined : null, result.fromCache ? undefined : 2),
                 },
             ],
             structuredContent,

package/dist/tools/handlers/fetch-markdown.tool.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-markdown.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAC5E,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAG3C,MAAM,CAAC,MAAM,wBAAwB,GAAG,gBAAgB,CAAC;AACzD,MAAM,CAAC,MAAM,+BAA+B,GAC1C,sFAAsF,CAAC;AAEzF,SAAS,2BAA2B,CAClC,IAAY,EACZ,GAAW,EACX,OAAkE;IAElE,+DAA+D;IAC/D,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAEvE,IACE,OAAO,CAAC,kBAAkB;QAC1B,MAAM,CAAC,UAAU,CAAC,kBAAkB;QACpC,OAAO,EACP,CAAC;QACD,MAAM,QAAQ,GACZ,OAAO,CAAC,eAAe,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe;YAC1D,CAAC,CAAC;gBACE,IAAI,EAAE,UAAmB;gBACzB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG;gBACH,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC;YACH,CAAC,CAAC,SAAS,CAAC;QAEhB,OAAO;YACL,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC;YACnD,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC;IACJ,CAAC;IAED,8BAA8B;IAC9B,MAAM,QAAQ,GACZ,OAAO,CAAC,eAAe,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe;QAC1D,CAAC,CAAC;YACE,IAAI,EAAE,UAAmB;YACzB,KAAK,EAAE,aAAa,CAAC,KAAK;YAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;YACtC,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,GAAG;YACH,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC;QACH,CAAC,CAAC,SAAS,CAAC;IAEhB,OAAO;QACL,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC;QACxC,KAAK,EAAE,aAAa,CAAC,KAAK;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,KAAyB;IACtE,IAAI,CAAC;QACH,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,GAAG,GAAG,uBAAuB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAEvD,oBAAoB;QACpB,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,iBAAiB,GAAG;oBACxB,GAAG;oBACH,MAAM,EAAE,IAAI;oBACZ,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,QAAQ,EAAE,MAAM,CAAC,OAAO;iBACzB,CAAC;gBACF,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC;yBACxC;qBACF;oBACD,iBAAiB;iBAClB,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAE1C,qCAAqC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,uBAAuB,CAC5B,8BAA8B,EAC9B,GAAG,EACH,eAAe,CAChB,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,2BAA2B,CAAC,IAAI,EAAE,GAAG,EAAE;YACjE,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;YACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;SAC/C,CAAC,CAAC;QAEH,mBAAmB;QACnB,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,iBAAiB,GAAG;YACxB,GAAG;YACH,KAAK;YACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,QAAQ;YACR,MAAM,EAAE,KAAK;SACd,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;iBACjD;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
+{"version":3,"file":"fetch-markdown.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,MAAM,CAAC,MAAM,wBAAwB,GAAG,gBAAgB,CAAC;AACzD,MAAM,CAAC,MAAM,+BAA+B,GAC1C,oIAAoI,CAAC;AAEvI;;;GAGG;AACH,SAAS,OAAO,CAAC,IAAY;IAC3B,sDAAsD;IACtD,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAE3C,OAAO,SAAS;SACb,WAAW,EAAE;SACb,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,QAAgB;IAClC,MAAM,YAAY,GAAG,qBAAqB,CAAC;IAC3C,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAAE,SAAS;QACrC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,+DAA+D;QAC/D,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACzC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,mBAAmB,CAC1B,IAAY,EACZ,GAAW,EACX,OAAyB;IAEzB,gFAAgF;IAChF,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,aAAa,CAC5B,GAAG,EACH,OAAO,EACP,aAAa,EACb,UAAU,EACV,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE/D,IAAI,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACpD,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEnE,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC3E,QAAQ,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,oBAAoB,CAAC;QAClF,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,KAAyB;IAMzB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAqB;YAChC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;YACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,KAAK;YACvC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;SACzC,CAAC;QAEF,QAAQ,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAA0B;YACjE,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,cAAc,EAAE,UAAU;YAC1B,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;YACjE,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ;YAClC,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACxB,QAAQ,EAAE,MAAM;gBAChB,KAAK,EAAE,SAAS;gBAChB,GAAG,EAAE,SAAS;gBACd,SAAS,EAAE,KAAK;aACjB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG;YACxB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;YACxB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;YAC9B,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YAChD,MAAM,EAAE,MAAM,CAAC,SAAS;YACxB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;SACnE,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB,iBAAiB,EACjB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EACnC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CACjC;iBACF;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}

package/dist/tools/handlers/fetch-url.tool.d.ts

@@ -1,18 +1,12 @@
-import type { FetchUrlInput } from '../../types/index.js';
+import type { FetchUrlInput } from '../../config/types.js';
 export declare const FETCH_URL_TOOL_NAME = "fetch-url";
-export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks";
-export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<import("../../utils/tool-error-handler.js").ToolErrorResponse | {
+export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks. Supports custom headers, retries, and content length limits.";
+export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<{
     content: {
-        type: "text";
+        type: 'text';
         text: string;
     }[];
-    structuredContent: {
-        url: string;
-        cached: boolean;
-        fetchedAt: string;
-        content: string;
-        format: "jsonl";
-        contentBlocks: number;
-    };
+    structuredContent?: Record<string, unknown>;
+    isError?: boolean;
 }>;
 //# sourceMappingURL=fetch-url.tool.d.ts.map

package/dist/tools/handlers/fetch-url.tool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EACV,aAAa,EAGd,MAAM,sBAAsB,CAAC;AAE9B,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,+FACuD,CAAC;AAsD/F,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,aAAa;;;;;;;;;;;;;GAgG7D"}
+{"version":3,"file":"fetch-url.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EAEd,MAAM,uBAAuB,CAAC;AAmB/B,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,6JACqH,CAAC;AA8B7J,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC;IACvE,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC,CAkED"}