@j0hanz/superfetch 1.0.2 → 1.0.3

package/dist/services/parser.d.ts

@@ -1,8 +1,3 @@
-import type { ContentBlockUnion } from '../types/index.js';
-/**
- * Parses HTML content and extracts semantic blocks
- * @param html - HTML string to parse
- * @returns Array of content blocks (empty array if parsing fails)
- */
+import type { ContentBlockUnion } from '../config/types.js';
 export declare function parseHtml(html: string): ContentBlockUnion[];
 //# sourceMappingURL=parser.d.ts.map

package/dist/services/parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAOV,iBAAiB,EAClB,MAAM,mBAAmB,CAAC;AA+J3B;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAwC3D"}
+{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAGV,iBAAiB,EAOlB,MAAM,oBAAoB,CAAC;AAsM5B,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,EAAE,CA0C3D"}

package/dist/services/parser.js

@@ -1,11 +1,13 @@
 import * as cheerio from 'cheerio';
-import { sanitizeText } from '../utils/sanitizer.js';
 import { config } from '../config/index.js';
+import { cleanCodeBlock, cleanHeading, cleanListItems, cleanParagraph, removeInlineTimestamps, } from '../utils/content-cleaner.js';
+import { detectLanguage } from '../utils/language-detector.js';
+import { sanitizeText } from '../utils/sanitizer.js';
 import { logWarn } from './logger.js';
-// Maximum HTML size to parse (10MB)
 const MAX_HTML_SIZE = 10 * 1024 * 1024;
 function parseHeading($, element) {
-    const text = sanitizeText($(element).text());
+    const rawText = sanitizeText($(element).text());
+    const text = cleanHeading(rawText);
     if (!text)
         return null;
     return {
@@ -15,20 +17,25 @@ function parseHeading($, element) {
     };
 }
 function parseParagraph($, element) {
-    const text = sanitizeText($(element).text());
+    let rawText = sanitizeText($(element).text());
+    // Remove inline timestamps like "13 days ago" from paragraphs
+    rawText = removeInlineTimestamps(rawText);
+    const text = cleanParagraph(rawText);
     if (!text || text.length < config.extraction.minParagraphLength)
         return null;
     return { type: 'paragraph', text };
 }
 function parseList($, element) {
-    const items = [];
-    $(element)
-        .find('li')
-        .each((_, li) => {
+    const listItems = $(element).find('li').toArray();
+    const rawItems = [];
+    // Use for...of instead of .each() to avoid callback overhead
+    for (const li of listItems) {
         const text = sanitizeText($(li).text());
         if (text)
-            items.push(text);
-    });
+            rawItems.push(text);
+    }
+    // Clean list items to remove noise
+    const items = cleanListItems(rawItems);
     if (items.length === 0)
         return null;
     return {
@@ -38,14 +45,23 @@ function parseList($, element) {
     };
 }
 function parseCode($, element) {
-    const text = $(element).text().trim();
+    const rawText = $(element).text().trim();
+    const text = cleanCodeBlock(rawText);
     if (!text)
         return null;
-    const className = $(element).attr('class') || '';
-    const languageMatch = className.match(/language-(\w+)/);
+    // Try to get language from class attribute first
+    const className = $(element).attr('class') ?? '';
+    const dataLang = $(element).attr('data-language') ?? '';
+    // Check multiple possible class patterns for language
+    const languageMatch = /language-(\w+)/.exec(className) ??
+        /lang-(\w+)/.exec(className) ??
+        /highlight-(\w+)/.exec(className) ??
+        /^(\w+)$/.exec(dataLang);
+    // Use detected language from class, or try to detect from content
+    const language = languageMatch?.[1] ?? detectLanguage(text);
     return {
         type: 'code',
-        language: languageMatch?.[1],
+        language,
         text,
     };
 }
@@ -53,31 +69,28 @@ function parseTable($, element) {
     const headers = [];
     const rows = [];
     const $table = $(element);
-    // Extract headers from thead or first row
-    $table.find('thead th, thead td').each((_, cell) => {
+    // Use toArray() + for...of instead of .each() callbacks
+    const headerCells = $table.find('thead th, thead td').toArray();
+    for (const cell of headerCells) {
         headers.push(sanitizeText($(cell).text()));
-    });
+    }
     if (headers.length === 0) {
-        $table
-            .find('tr')
-            .first()
-            .find('th, td')
-            .each((_, cell) => {
+        const firstRowCells = $table.find('tr').first().find('th, td').toArray();
+        for (const cell of firstRowCells) {
             headers.push(sanitizeText($(cell).text()));
-        });
+        }
     }
-    // Extract body rows
     const rowsSelector = headers.length > 0 ? 'tbody tr, tr:not(:first)' : 'tbody tr, tr';
-    $table.find(rowsSelector).each((_, row) => {
+    const tableRows = $table.find(rowsSelector).toArray();
+    for (const row of tableRows) {
+        const rowCells = $(row).find('td, th').toArray();
         const cells = [];
-        $(row)
-            .find('td, th')
-            .each((_, cell) => {
+        for (const cell of rowCells) {
             cells.push(sanitizeText($(cell).text()));
-        });
+        }
         if (cells.length > 0)
             rows.push(cells);
-    });
+    }
     if (rows.length === 0)
         return null;
     return {
@@ -93,9 +106,16 @@ function parseImage($, element) {
     return {
         type: 'image',
         src,
-        alt: $(element).attr('alt') || undefined,
+        alt: $(element).attr('alt') ?? undefined,
     };
 }
+function parseBlockquote($, element) {
+    const rawText = sanitizeText($(element).text());
+    const text = cleanParagraph(rawText);
+    if (!text || text.length < config.extraction.minParagraphLength)
+        return null;
+    return { type: 'blockquote', text };
+}
 const ELEMENT_PARSERS = {
     h1: parseHeading,
     h2: parseHeading,
@@ -110,6 +130,7 @@ const ELEMENT_PARSERS = {
     code: parseCode,
     table: parseTable,
     img: parseImage,
+    blockquote: parseBlockquote,
 };
 function isParseableTag(tag) {
     return tag in ELEMENT_PARSERS;
@@ -128,6 +149,7 @@ function filterBlocks(blocks) {
             case 'paragraph':
             case 'heading':
             case 'code':
+            case 'blockquote':
                 return block.text.length > 0;
             case 'list':
                 return block.items.length > 0;
@@ -136,40 +158,35 @@ function filterBlocks(blocks) {
         }
     });
 }
-/**
- * Parses HTML content and extracts semantic blocks
- * @param html - HTML string to parse
- * @returns Array of content blocks (empty array if parsing fails)
- */
 export function parseHtml(html) {
-    // Input validation
-    if (!html || typeof html !== 'string') {
+    if (!html || typeof html !== 'string')
         return [];
-    }
-    // Size validation to prevent memory issues
+    let processedHtml = html;
     if (html.length > MAX_HTML_SIZE) {
         logWarn('HTML content exceeds maximum size, truncating', {
             size: html.length,
             maxSize: MAX_HTML_SIZE,
         });
-        html = html.substring(0, MAX_HTML_SIZE);
+        processedHtml = html.substring(0, MAX_HTML_SIZE);
     }
     try {
-        const $ = cheerio.load(html);
+        const $ = cheerio.load(processedHtml);
         const blocks = [];
         $('script, style, noscript, iframe, svg').remove();
-        $('body')
-            .find('h1, h2, h3, h4, h5, h6, p, ul, ol, pre, code, table, img')
-            .each((_, element) => {
+        // Use toArray() + for...of instead of .each() to avoid callback overhead
+        const elements = $('body')
+            .find('h1, h2, h3, h4, h5, h6, p, ul, ol, pre, code:not(pre code), table, img, blockquote')
+            .toArray();
+        for (const element of elements) {
             try {
                 const block = parseElement($, element);
                 if (block)
                     blocks.push(block);
             }
             catch {
-                // Skip individual element parsing errors
+                // Skip element errors
             }
-        });
+        }
         return filterBlocks(blocks);
     }
     catch (error) {

package/dist/services/parser.js.map

@@ -1 +1 @@
-{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAWtC,oCAAoC;AACpC,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvC,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7C,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,CAAC,CAAC,OAAO,CAAC;SACP,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEL,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAExD,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC;QAC5B,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,0CAA0C;IAC1C,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACjD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM;aACH,IAAI,CAAC,IAAI,CAAC;aACV,KAAK,EAAE;aACP,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,oBAAoB;IACpB,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IACnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;QACxC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,GAAG,CAAC;aACH,IAAI,CAAC,QAAQ,CAAC;aACd,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACL,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;CAIhB,CAAC;AAIF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,WAAW,CAAC;YACjB,KAAK,SAAS,CAAC;YACf,KAAK,MAAM;gBACT,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,KAAK,MAAM;gBACT,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAChC;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,mBAAmB;IACnB,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,2CAA2C;IAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QAChC,OAAO,CAAC,+CAA+C,EAAE;YACvD,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,OAAO,EAAE,aAAa;SACvB,CAAC,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAwB,EAAE,CAAC;QAEvC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;QAEnD,CAAC,CAAC,MAAM,CAAC;aACN,IAAI,CAAC,0DAA0D,CAAC;aAChE,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACnB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACvC,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,yCAAyC;YAC3C,CAAC;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;YAC/D,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
+{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/services/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAInC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAa5C,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,sBAAsB,GACvB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvC,SAAS,YAAY,CAAC,CAAa,EAAE,OAAgB;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,CAAa,EACb,OAAgB;IAEhB,IAAI,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,8DAA8D;IAC9D,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;IAClD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,6DAA6D;IAC7D,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxC,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,mCAAmC;IACnC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,IAAI;QAC/C,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,CAAa,EAAE,OAAgB;IAChD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAExD,sDAAsD;IACtD,MAAM,aAAa,GACjB,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5B,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC;QACjC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAE3B,kEAAkE;IAClE,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;IAE5D,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,QAAQ;QACR,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE1B,wDAAwD;IACxD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,OAAO,EAAE,CAAC;IAChE,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;QACzE,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,cAAc,CAAC;IACnE,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;IAEtD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;QACjD,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,OAAO;QACL,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAa,EAAE,OAAgB;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,GAAG;QACH,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,CAAa,EACb,OAAgB;IAEhB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE7E,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,eAAe,GAAG;IACtB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,YAAY;IAChB,CAAC,EAAE,cAAc;IACjB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,UAAU;IACjB,GAAG,EAAE,UAAU;IACf,UAAU,EAAE,eAAe;CAI5B,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,eAAe,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,IAAa;IAChD,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,YAAY,CAAC,MAA2B;IAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,WAAW,CAAC;YACjB,KAAK,SAAS,CAAC;YACf,KAAK,MAAM,CAAC;YACZ,KAAK,YAAY;gBACf,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,KAAK,MAAM;gBACT,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAChC;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEjD,IAAI,aAAa,GAAG,IAAI,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QAChC,OAAO,CAAC,+CAA+C,EAAE;YACvD,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,OAAO,EAAE,aAAa;SACvB,CAAC,CAAC;QACH,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACtC,MAAM,MAAM,GAAwB,EAAE,CAAC;QAEvC,CAAC,CAAC,sCAAsC,CAAC,CAAC,MAAM,EAAE,CAAC;QAEnD,yEAAyE;QACzE,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC;aACvB,IAAI,CACH,oFAAoF,CACrF;aACA,OAAO,EAAE,CAAC;QAEb,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACvC,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;QACH,CAAC;QAED,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,sBAAsB,EAAE;YAC9B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;YAC/D,UAAU,EAAE,IAAI,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}

package/dist/tools/handlers/fetch-links.tool.d.ts

@@ -1,24 +1,17 @@
-import type { FetchLinksInput, ExtractedLink } from '../../types/index.js';
+import type { ExtractedLink, FetchLinksInput } from '../../config/types.js';
 export declare const FETCH_LINKS_TOOL_NAME = "fetch-links";
-export declare const FETCH_LINKS_TOOL_DESCRIPTION = "Extracts all hyperlinks from a webpage with anchor text and type classification";
-/**
- * Tool handler for extracting links from a URL
- */
-export declare function fetchLinksToolHandler(input: FetchLinksInput): Promise<import("../../utils/tool-error-handler.js").ToolErrorResponse | {
+export declare const FETCH_LINKS_TOOL_DESCRIPTION = "Extracts all hyperlinks from a webpage with anchor text and type classification. Supports filtering, image links, and link limits.";
+export declare function fetchLinksToolHandler(input: FetchLinksInput): Promise<import("../../config/types.js").ToolErrorResponse | {
     content: {
         type: "text";
         text: string;
     }[];
     structuredContent: {
+        truncated?: true;
+        filtered?: number;
         url: string;
         linkCount: number;
         links: ExtractedLink[];
     };
-} | {
-    content: {
-        type: "text";
-        text: string;
-    }[];
-    structuredContent?: undefined;
 }>;
 //# sourceMappingURL=fetch-links.tool.d.ts.map

package/dist/tools/handlers/fetch-links.tool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-links.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE3E,eAAO,MAAM,qBAAqB,gBAAgB,CAAC;AACnD,eAAO,MAAM,4BAA4B,oFAC0C,CAAC;AAqDpF;;GAEG;AACH,wBAAsB,qBAAqB,CAAC,KAAK,EAAE,eAAe;;;;;;aAiBjD,MAAM;mBACA,MAAM;eACV,aAAa,EAAE;;;;;;;;GAuDjC"}
+{"version":3,"file":"fetch-links.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,aAAa,EAEb,eAAe,EAEhB,MAAM,uBAAuB,CAAC;AAW/B,eAAO,MAAM,qBAAqB,gBAAgB,CAAC;AACnD,eAAO,MAAM,4BAA4B,uIAC6F,CAAC;AAuFvI,wBAAsB,qBAAqB,CAAC,KAAK,EAAE,eAAe;;;;;;;;;;;;GAiEjE"}

package/dist/tools/handlers/fetch-links.tool.js

@@ -1,104 +1,129 @@
-import { validateAndNormalizeUrl, isInternalUrl, } from '../../utils/url-validator.js';
-import { fetchUrlWithRetry } from '../../services/fetcher.js';
-import * as cache from '../../services/cache.js';
 import * as cheerio from 'cheerio';
-import { logError } from '../../services/logger.js';
+import { logDebug, logError } from '../../services/logger.js';
 import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
+import { isInternalUrl } from '../../utils/url-validator.js';
+import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
 export const FETCH_LINKS_TOOL_NAME = 'fetch-links';
-export const FETCH_LINKS_TOOL_DESCRIPTION = 'Extracts all hyperlinks from a webpage with anchor text and type classification';
-/**
- * Extracts links from HTML, filtering by type and deduplicating
- */
-function extractLinksFromHtml(html, baseUrl, options) {
+export const FETCH_LINKS_TOOL_DESCRIPTION = 'Extracts all hyperlinks from a webpage with anchor text and type classification. Supports filtering, image links, and link limits.';
+function tryResolveUrl(href, baseUrl) {
+    try {
+        return new URL(href, baseUrl).href;
+    }
+    catch {
+        return null;
+    }
+}
+function shouldIncludeLink(type, url, options, seen) {
+    if (seen.has(url))
+        return false;
+    if (options.filterPattern && !options.filterPattern.test(url))
+        return false;
+    if (type === 'internal' && !options.includeInternal)
+        return false;
+    if (type === 'external' && !options.includeExternal)
+        return false;
+    return true;
+}
+function extractLinks(html, baseUrl, options) {
     const $ = cheerio.load(html);
     const links = [];
-    const seenUrls = new Set();
-    $('a[href]').each((_, element) => {
-        const href = $(element).attr('href');
-        const text = $(element).text().trim();
-        // Skip invalid hrefs
-        if (!href || href.startsWith('#') || href.startsWith('javascript:')) {
+    const seen = new Set();
+    let filtered = 0;
+    $('a[href]').each((_, el) => {
+        const href = $(el).attr('href');
+        if (!href || href.startsWith('#') || href.startsWith('javascript:'))
+            return;
+        const url = tryResolveUrl(href, baseUrl);
+        if (!url)
+            return;
+        const type = isInternalUrl(url, baseUrl)
+            ? 'internal'
+            : 'external';
+        if (!shouldIncludeLink(type, url, options, seen)) {
+            if (!seen.has(url))
+                filtered++;
             return;
         }
-        try {
-            const absoluteUrl = new URL(href, baseUrl).href;
-            // Skip duplicates
-            if (seenUrls.has(absoluteUrl)) {
+        seen.add(url);
+        links.push({ href: url, text: $(el).text().trim() || url, type });
+    });
+    if (options.includeImages) {
+        $('img[src]').each((_, el) => {
+            const src = $(el).attr('src');
+            if (!src || src.startsWith('data:'))
                 return;
-            }
-            seenUrls.add(absoluteUrl);
-            const type = isInternalUrl(absoluteUrl, baseUrl)
-                ? 'internal'
-                : 'external';
-            // Filter based on options
-            if (type === 'internal' && !options.includeInternal)
+            const url = tryResolveUrl(src, baseUrl);
+            if (!url)
                 return;
-            if (type === 'external' && !options.includeExternal)
+            if (!shouldIncludeLink('image', url, options, seen)) {
+                if (!seen.has(url))
+                    filtered++;
                 return;
+            }
+            seen.add(url);
             links.push({
-                href: absoluteUrl,
-                text: text || absoluteUrl,
-                type,
+                href: url,
+                text: $(el).attr('alt')?.trim() ?? url,
+                type: 'image',
             });
-        }
-        catch {
-            // Skip invalid URLs silently
-        }
-    });
-    return links;
+        });
+    }
+    const truncated = options.maxLinks ? links.length > options.maxLinks : false;
+    const resultLinks = truncated ? links.slice(0, options.maxLinks) : links;
+    return {
+        links: resultLinks,
+        linkCount: resultLinks.length,
+        filtered,
+        truncated,
+    };
 }
-/**
- * Tool handler for extracting links from a URL
- */
 export async function fetchLinksToolHandler(input) {
-    try {
-        // Validate URL input
-        if (!input.url) {
-            return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
-        }
-        const url = validateAndNormalizeUrl(input.url);
-        const cacheKey = cache.createCacheKey('links', url);
-        // Check cache first
-        if (cacheKey) {
-            const cached = cache.get(cacheKey);
-            if (cached) {
-                // Parse the cached content to return as structuredContent
-                try {
-                    const structuredContent = JSON.parse(cached.content);
-                    return {
-                        content: [{ type: 'text', text: cached.content }],
-                        structuredContent,
-                    };
-                }
-                catch {
-                    return {
-                        content: [{ type: 'text', text: cached.content }],
-                    };
-                }
-            }
+    if (!input.url) {
+        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
+    }
+    let filterPattern;
+    if (input.filterPattern) {
+        try {
+            filterPattern = new RegExp(input.filterPattern, 'i');
         }
-        const html = await fetchUrlWithRetry(url);
-        // Validate HTML content was received
-        if (!html) {
-            return createToolErrorResponse('No content received from URL', url, 'EMPTY_CONTENT');
+        catch {
+            return createToolErrorResponse(`Invalid filter pattern: ${input.filterPattern}`, input.url, 'VALIDATION_ERROR');
         }
-        // Extract links
-        const links = extractLinksFromHtml(html, url, {
+    }
+    try {
+        const options = {
             includeInternal: input.includeInternal ?? true,
             includeExternal: input.includeExternal ?? true,
+            includeImages: input.includeImages ?? false,
+            maxLinks: input.maxLinks,
+            filterPattern,
+        };
+        logDebug('Extracting links', {
+            url: input.url,
+            ...options,
+            filterPattern: input.filterPattern,
+        });
+        const result = await executeFetchPipeline({
+            url: input.url,
+            cacheNamespace: 'links',
+            customHeaders: input.customHeaders,
+            retries: input.retries,
+            transform: (html, url) => extractLinks(html, url, options),
         });
         const structuredContent = {
-            url,
-            linkCount: links.length,
-            links,
+            url: result.url,
+            linkCount: result.data.linkCount,
+            links: result.data.links,
+            ...(result.data.filtered > 0 && { filtered: result.data.filtered }),
+            ...(result.data.truncated && { truncated: result.data.truncated }),
         };
-        const outputText = JSON.stringify(structuredContent, null, 2);
-        // Cache the result
-        if (cacheKey) {
-            cache.set(cacheKey, outputText);
-        }
         return {
-            content: [{ type: 'text', text: outputText }],
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify(structuredContent, null, 2),
+                },
+            ],
             structuredContent,
         };
     }

package/dist/tools/handlers/fetch-links.tool.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,aAAa,GACd,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAG3C,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC;AACnD,MAAM,CAAC,MAAM,4BAA4B,GACvC,iFAAiF,CAAC;AAEpF;;GAEG;AACH,SAAS,oBAAoB,CAC3B,IAAY,EACZ,OAAe,EACf,OAA+D;IAE/D,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAEtC,qBAAqB;QACrB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YACpE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAEhD,kBAAkB;YAClB,IAAI,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC9B,OAAO;YACT,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAE1B,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,OAAO,CAAC;gBAC9C,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,UAAU,CAAC;YAEf,0BAA0B;YAC1B,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;gBAAE,OAAO;YAC5D,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;gBAAE,OAAO;YAE5D,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,IAAI,IAAI,WAAW;gBACzB,IAAI;aACL,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,6BAA6B;QAC/B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAAsB;IAChE,IAAI,CAAC;QACH,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,GAAG,GAAG,uBAAuB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAEpD,oBAAoB;QACpB,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,0DAA0D;gBAC1D,IAAI,CAAC;oBACH,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAIlD,CAAC;oBACF,OAAO;wBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;wBAC1D,iBAAiB;qBAClB,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;wBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;qBAC3D,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAE1C,qCAAqC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,uBAAuB,CAC5B,8BAA8B,EAC9B,GAAG,EACH,eAAe,CAChB,CAAC;QACJ,CAAC;QAED,gBAAgB;QAChB,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE;YAC5C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;SAC/C,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG;YACxB,GAAG;YACH,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,KAAK;SACN,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAE9D,mBAAmB;QACnB,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAClC,CAAC;QAED,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;YACtD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,wBAAwB,EACxB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IACtE,CAAC;AACH,CAAC"}
+{"version":3,"file":"fetch-links.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-links.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AASnC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,CAAC;AACnD,MAAM,CAAC,MAAM,4BAA4B,GACvC,oIAAoI,CAAC;AAIvI,SAAS,aAAa,CAAC,IAAY,EAAE,OAAe;IAClD,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,IAAc,EACd,GAAW,EACX,OAA4B,EAC5B,IAAiB;IAEjB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAChC,IAAI,OAAO,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5E,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CACnB,IAAY,EACZ,OAAe,EACf,OAA4B;IAE5B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;YAAE,OAAO;QAE5E,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,MAAM,IAAI,GAAa,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC;YAChD,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,UAAU,CAAC;QACf,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,QAAQ,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,OAAO;YAE5C,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEjB,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;gBACpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,QAAQ,EAAE,CAAC;gBAC/B,OAAO;YACT,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG;gBACtC,IAAI,EAAE,OAAO;aACd,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7E,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAEzE,OAAO;QACL,KAAK,EAAE,WAAW;QAClB,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,QAAQ;QACR,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAAsB;IAChE,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,aAAiC,CAAC;IACtC,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,aAAa,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,uBAAuB,CAC5B,2BAA2B,KAAK,CAAC,aAAa,EAAE,EAChD,KAAK,CAAC,GAAG,EACT,kBAAkB,CACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAwB;YACnC,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,KAAK;YAC3C,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,aAAa;SACd,CAAC;QAEF,QAAQ,CAAC,kBAAkB,EAAE;YAC3B,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,GAAG,OAAO;YACV,aAAa,EAAE,KAAK,CAAC,aAAa;SACnC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAuB;YAC9D,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,cAAc,EAAE,OAAO;YACvB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC;SAC3D,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG;YACxB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS;YAChC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;YACxB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;SACnE,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;iBACjD;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,wBAAwB,EACxB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IACtE,CAAC;AACH,CAAC"}

package/dist/tools/handlers/fetch-markdown.tool.d.ts

@@ -1,14 +1,17 @@
-import type { FetchMarkdownInput } from '../../types/index.js';
+import type { FetchMarkdownInput, TocEntry } from '../../config/types.js';
 export declare const FETCH_MARKDOWN_TOOL_NAME = "fetch-markdown";
-export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter";
-export declare function fetchMarkdownToolHandler(input: FetchMarkdownInput): Promise<import("../../utils/tool-error-handler.js").ToolErrorResponse | {
+export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter, table of contents, and content length limits";
+export declare function fetchMarkdownToolHandler(input: FetchMarkdownInput): Promise<import("../../config/types.js").ToolErrorResponse | {
     content: {
         type: "text";
         text: string;
     }[];
     structuredContent: {
-        url: string;
+        truncated?: true;
         cached: boolean;
+        toc?: TocEntry[];
+        url: string;
+        title: string | undefined;
         fetchedAt: string;
         markdown: string;
     };

package/dist/tools/handlers/fetch-markdown.tool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE/D,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AACzD,eAAO,MAAM,+BAA+B,yFAC4C,CAAC;AAmDzF,wBAAsB,wBAAwB,CAAC,KAAK,EAAE,kBAAkB;;;;;;;;;;;GA6EvE"}
+{"version":3,"file":"fetch-markdown.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-markdown.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAElB,QAAQ,EAET,MAAM,uBAAuB,CAAC;AAe/B,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AACzD,eAAO,MAAM,+BAA+B,uIAC0F,CAAC;AAuEvI,wBAAsB,wBAAwB,CAAC,KAAK,EAAE,kBAAkB;;;;;;;;;;;;;;GA4DvE"}