@j0hanz/superfetch 1.0.2 → 1.0.5

package/dist/tools/handlers/fetch-url.tool.js

@@ -1,111 +1,69 @@
-import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
-import { fetchUrlWithRetry } from '../../services/fetcher.js';
 import { extractContent } from '../../services/extractor.js';
+import { logDebug, logError } from '../../services/logger.js';
 import { parseHtml } from '../../services/parser.js';
-import { toJsonl } from '../../transformers/jsonl.transformer.js';
-import * as cache from '../../services/cache.js';
-import { config } from '../../config/index.js';
-import { logError } from '../../services/logger.js';
 import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
+import { buildMetadata, shouldUseArticle, truncateContent, } from '../utils/common.js';
+import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
+import { toJsonl } from '../../transformers/jsonl.transformer.js';
 export const FETCH_URL_TOOL_NAME = 'fetch-url';
-export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks';
-function extractContentFromHtml(html, url, options) {
-    // Use the optimized extractContent that parses JSDOM only once
-    const { article, metadata: extractedMeta } = extractContent(html, url);
-    if (options.extractMainContent &&
-        config.extraction.extractMainContent &&
-        article) {
-        const contentBlocks = parseHtml(article.content);
-        const metadata = options.includeMetadata && config.extraction.includeMetadata
-            ? {
-                type: 'metadata',
-                title: article.title,
-                author: article.byline,
-                url,
-                fetchedAt: new Date().toISOString(),
-            }
-            : undefined;
-        return { contentBlocks, metadata, title: article.title };
-    }
-    // Fallback: use parsed HTML directly
-    const contentBlocks = parseHtml(html);
-    const metadata = options.includeMetadata && config.extraction.includeMetadata
-        ? {
-            type: 'metadata',
-            title: extractedMeta.title,
-            description: extractedMeta.description,
-            author: extractedMeta.author,
-            url,
-            fetchedAt: new Date().toISOString(),
-        }
-        : undefined;
-    return { contentBlocks, metadata, title: extractedMeta.title };
+export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks. Supports custom headers, retries, and content length limits.';
+function transformToJsonl(html, url, options) {
+    // Only invoke JSDOM when extractMainContent is true (lazy loading optimization)
+    const { article, metadata: extractedMeta } = extractContent(html, url, {
+        extractArticle: options.extractMainContent,
+    });
+    const useArticle = shouldUseArticle(options.extractMainContent, article);
+    const sourceHtml = useArticle ? article.content : html;
+    const contentBlocks = parseHtml(sourceHtml);
+    const metadata = buildMetadata(url, article, extractedMeta, useArticle, options.includeMetadata);
+    const title = useArticle ? article.title : extractedMeta.title;
+    return {
+        content: toJsonl(contentBlocks, metadata),
+        contentBlocks: contentBlocks.length,
+        title,
+    };
 }
 export async function fetchUrlToolHandler(input) {
+    if (!input.url) {
+        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
+    }
     try {
-        // Validate URL input
-        if (!input.url) {
-            return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
-        }
-        const url = validateAndNormalizeUrl(input.url);
-        const cacheKey = cache.createCacheKey('url', url);
-        // Check cache first
-        if (cacheKey) {
-            const cached = cache.get(cacheKey);
-            if (cached) {
-                const structuredContent = {
-                    url,
-                    cached: true,
-                    fetchedAt: cached.fetchedAt,
-                    content: cached.content,
-                    format: 'jsonl',
-                    contentBlocks: 0, // Unknown from cache
-                };
-                return {
-                    content: [
-                        {
-                            type: 'text',
-                            text: JSON.stringify(structuredContent),
-                        },
-                    ],
-                    structuredContent,
-                };
-            }
-        }
-        const html = await fetchUrlWithRetry(url, input.customHeaders);
-        // Validate HTML content was received
-        if (!html) {
-            return createToolErrorResponse('No content received from URL', url, 'EMPTY_CONTENT');
-        }
-        const { contentBlocks, metadata, title } = extractContentFromHtml(html, url, {
-            extractMainContent: input.extractMainContent ?? true,
-            includeMetadata: input.includeMetadata ?? true,
+        const extractMainContent = input.extractMainContent ?? true;
+        const includeMetadata = input.includeMetadata ?? true;
+        logDebug('Fetching URL', {
+            url: input.url,
+            extractMainContent,
+            includeMetadata,
+        });
+        const result = await executeFetchPipeline({
+            url: input.url,
+            cacheNamespace: 'url',
+            customHeaders: input.customHeaders,
+            retries: input.retries,
+            transform: (html, url) => transformToJsonl(html, url, { extractMainContent, includeMetadata }),
+            serialize: (data) => data.content,
+            deserialize: (cached) => ({
+                content: cached,
+                contentBlocks: 0,
+                title: undefined,
+            }),
         });
-        let jsonlContent = toJsonl(contentBlocks, metadata);
-        if (input.maxContentLength &&
-            input.maxContentLength > 0 &&
-            jsonlContent.length > input.maxContentLength) {
-            jsonlContent =
-                jsonlContent.substring(0, input.maxContentLength) + '\n...[truncated]';
-        }
-        // Cache the result
-        if (cacheKey) {
-            cache.set(cacheKey, jsonlContent);
-        }
+        const { content, truncated } = truncateContent(result.data.content, input.maxContentLength);
         const structuredContent = {
-            url,
-            title,
-            contentBlocks: contentBlocks.length,
-            fetchedAt: new Date().toISOString(),
+            url: result.url,
+            title: result.data.title,
+            contentBlocks: result.data.contentBlocks,
+            fetchedAt: result.fetchedAt,
             format: 'jsonl',
-            content: jsonlContent,
-            cached: false,
+            content,
+            cached: result.fromCache,
+            ...(truncated && { truncated }),
         };
         return {
             content: [
                 {
                     type: 'text',
-                    text: JSON.stringify(structuredContent, null, 2),
+                    text: JSON.stringify(structuredContent, result.fromCache ? undefined : null, result.fromCache ? undefined : 2),
                 },
             ],
             structuredContent,

package/dist/tools/handlers/fetch-url.tool.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-url.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAO3C,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAC/C,MAAM,CAAC,MAAM,0BAA0B,GACrC,4FAA4F,CAAC;AAQ/F,SAAS,sBAAsB,CAC7B,IAAY,EACZ,GAAW,EACX,OAAkE;IAElE,+DAA+D;IAC/D,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAEvE,IACE,OAAO,CAAC,kBAAkB;QAC1B,MAAM,CAAC,UAAU,CAAC,kBAAkB;QACpC,OAAO,EACP,CAAC;QACD,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,GACZ,OAAO,CAAC,eAAe,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe;YAC1D,CAAC,CAAC;gBACE,IAAI,EAAE,UAAmB;gBACzB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG;gBACH,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC;YACH,CAAC,CAAC,SAAS,CAAC;QAEhB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;IAC3D,CAAC;IAED,qCAAqC;IACrC,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAEtC,MAAM,QAAQ,GACZ,OAAO,CAAC,eAAe,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe;QAC1D,CAAC,CAAC;YACE,IAAI,EAAE,UAAmB;YACzB,KAAK,EAAE,aAAa,CAAC,KAAK;YAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;YACtC,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,GAAG;YACH,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC;QACH,CAAC,CAAC,SAAS,CAAC;IAEhB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,CAAC,KAAK,EAAE,CAAC;AACjE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAoB;IAC5D,IAAI,CAAC;QACH,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,GAAG,GAAG,uBAAuB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAElD,oBAAoB;QACpB,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,iBAAiB,GAAG;oBACxB,GAAG;oBACH,MAAM,EAAE,IAAI;oBACZ,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,MAAM,EAAE,OAAgB;oBACxB,aAAa,EAAE,CAAC,EAAE,qBAAqB;iBACxC,CAAC;gBACF,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC;yBACxC;qBACF;oBACD,iBAAiB;iBAClB,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QAE/D,qCAAqC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,uBAAuB,CAC5B,8BAA8B,EAC9B,GAAG,EACH,eAAe,CAChB,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,sBAAsB,CAC/D,IAAI,EACJ,GAAG,EACH;YACE,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;YACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;SAC/C,CACF,CAAC;QAEF,IAAI,YAAY,GAAG,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QAEpD,IACE,KAAK,CAAC,gBAAgB;YACtB,KAAK,CAAC,gBAAgB,GAAG,CAAC;YAC1B,YAAY,CAAC,MAAM,GAAG,KAAK,CAAC,gBAAgB,EAC5C,CAAC;YACD,YAAY;gBACV,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,kBAAkB,CAAC;QAC3E,CAAC;QAED,mBAAmB;QACnB,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACpC,CAAC;QAED,MAAM,iBAAiB,GAAG;YACxB,GAAG;YACH,KAAK;YACL,aAAa,EAAE,aAAa,CAAC,MAAM;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,OAAgB;YACxB,OAAO,EAAE,YAAY;YACrB,MAAM,EAAE,KAAK;SACd,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;iBACjD;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,sBAAsB,EACtB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;IAClE,CAAC;AACH,CAAC"}
+{"version":3,"file":"fetch-url.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-url.tool.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EACL,uBAAuB,EACvB,eAAe,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,eAAe,GAChB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAElE,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAElE,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAC/C,MAAM,CAAC,MAAM,0BAA0B,GACrC,0JAA0J,CAAC;AAE7J,SAAS,gBAAgB,CACvB,IAAY,EACZ,GAAW,EACX,OAAkE;IAElE,gFAAgF;IAChF,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,aAAa,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,aAAa,CAC5B,GAAG,EACH,OAAO,EACP,aAAa,EACb,UAAU,EACV,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE/D,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC;QACzC,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAoB;IAK5D,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACf,OAAO,uBAAuB,CAAC,iBAAiB,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC;QACH,MAAM,kBAAkB,GAAG,KAAK,CAAC,kBAAkB,IAAI,IAAI,CAAC;QAC5D,MAAM,eAAe,GAAG,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC;QAEtD,QAAQ,CAAC,cAAc,EAAE;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,kBAAkB;YAClB,eAAe;SAChB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAuB;YAC9D,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,cAAc,EAAE,KAAK;YACrB,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CACvB,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,kBAAkB,EAAE,eAAe,EAAE,CAAC;YACtE,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO;YACjC,WAAW,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACxB,OAAO,EAAE,MAAM;gBACf,aAAa,EAAE,CAAC;gBAChB,KAAK,EAAE,SAAS;aACjB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,MAAM,CAAC,IAAI,CAAC,OAAO,EACnB,KAAK,CAAC,gBAAgB,CACvB,CAAC;QAEF,MAAM,iBAAiB,GAAG;YACxB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;YACxB,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa;YACxC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,OAAgB;YACxB,OAAO;YACP,MAAM,EAAE,MAAM,CAAC,SAAS;YACxB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;SAChC,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB,iBAAiB,EACjB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EACnC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CACjC;iBACF;aACF;YACD,iBAAiB;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,sBAAsB,EACtB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;IAClE,CAAC;AACH,CAAC"}

package/dist/tools/handlers/fetch-urls.tool.d.ts

@@ -0,0 +1,12 @@
+import type { FetchUrlsInput } from '../../config/types.js';
+export declare const FETCH_URLS_TOOL_NAME = "fetch-urls";
+export declare const FETCH_URLS_TOOL_DESCRIPTION = "Fetches multiple URLs in parallel and converts them to AI-readable format (JSONL or Markdown). Supports concurrency control and continues on individual failures.";
+export declare function fetchUrlsToolHandler(input: FetchUrlsInput): Promise<{
+    content: {
+        type: 'text';
+        text: string;
+    }[];
+    structuredContent?: Record<string, unknown>;
+    isError?: boolean;
+}>;
+//# sourceMappingURL=fetch-urls.tool.d.ts.map

package/dist/tools/handlers/fetch-urls.tool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-urls.tool.d.ts","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-urls.tool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAEV,cAAc,EAGf,MAAM,uBAAuB,CAAC;AAwB/B,eAAO,MAAM,oBAAoB,eAAe,CAAC;AACjD,eAAO,MAAM,2BAA2B,sKAC6H,CAAC;AA6HtK,wBAAsB,oBAAoB,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC;IACzE,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC,CAsHD"}

package/dist/tools/handlers/fetch-urls.tool.js

@@ -0,0 +1,184 @@
+import * as cheerio from 'cheerio';
+import * as cache from '../../services/cache.js';
+import { extractContent, extractMetadataWithCheerio, } from '../../services/extractor.js';
+import { fetchUrlWithRetry } from '../../services/fetcher.js';
+import { logDebug, logError, logWarn } from '../../services/logger.js';
+import { parseHtml } from '../../services/parser.js';
+import { runWithConcurrency } from '../../utils/concurrency.js';
+import { createToolErrorResponse } from '../../utils/tool-error-handler.js';
+import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
+import { buildMetadata, shouldUseArticle, truncateContent, } from '../utils/common.js';
+import { createBatchResponse } from '../utils/response-builder.js';
+import { toJsonl } from '../../transformers/jsonl.transformer.js';
+import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
+export const FETCH_URLS_TOOL_NAME = 'fetch-urls';
+export const FETCH_URLS_TOOL_DESCRIPTION = 'Fetches multiple URLs in parallel and converts them to AI-readable format (JSONL or Markdown). Supports concurrency control and continues on individual failures.';
+const MAX_URLS = 10;
+const DEFAULT_CONCURRENCY = 3;
+async function processSingleUrl(url, options) {
+    try {
+        const normalizedUrl = validateAndNormalizeUrl(url);
+        const cacheNamespace = options.format === 'markdown' ? 'markdown' : 'url';
+        const cacheKey = cache.createCacheKey(cacheNamespace, normalizedUrl);
+        if (cacheKey) {
+            const cached = cache.get(cacheKey);
+            if (cached) {
+                logDebug('Batch cache hit', { url: normalizedUrl });
+                return {
+                    url: normalizedUrl,
+                    success: true,
+                    content: cached.content,
+                    cached: true,
+                };
+            }
+        }
+        const fetchResult = await fetchUrlWithRetry(normalizedUrl);
+        let sourceHtml;
+        let title;
+        let metadata;
+        // Fast path: Skip JSDOM entirely when extractMainContent is false
+        if (!options.extractMainContent) {
+            sourceHtml = fetchResult.html;
+            const $ = cheerio.load(fetchResult.html);
+            const extractedMeta = extractMetadataWithCheerio($);
+            ({ title } = extractedMeta);
+            if (options.includeMetadata) {
+                metadata = {
+                    type: 'metadata',
+                    url: normalizedUrl,
+                    fetchedAt: new Date().toISOString(),
+                    title: extractedMeta.title,
+                    description: extractedMeta.description,
+                    author: extractedMeta.author,
+                };
+            }
+        }
+        else {
+            // Slow path: Use JSDOM only when article extraction is needed
+            const { article, metadata: extractedMeta } = extractContent(fetchResult.html, normalizedUrl, {
+                extractArticle: true,
+            });
+            const useArticle = shouldUseArticle(true, article);
+            metadata = buildMetadata(normalizedUrl, article, extractedMeta, useArticle, options.includeMetadata);
+            sourceHtml = useArticle ? article.content : fetchResult.html;
+            title = useArticle ? article.title : extractedMeta.title;
+        }
+        let content;
+        let contentBlocks;
+        if (options.format === 'markdown') {
+            content = htmlToMarkdown(sourceHtml, metadata);
+        }
+        else {
+            const blocks = parseHtml(sourceHtml);
+            contentBlocks = blocks.length;
+            content = toJsonl(blocks, metadata);
+        }
+        const { content: truncatedContent } = truncateContent(content, options.maxContentLength);
+        content = truncatedContent;
+        if (cacheKey)
+            cache.set(cacheKey, content);
+        return {
+            url: normalizedUrl,
+            success: true,
+            title,
+            content,
+            contentBlocks,
+            cached: false,
+        };
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+        const errorCode = error instanceof Error &&
+            'code' in error &&
+            typeof error.code === 'string'
+            ? error.code
+            : 'FETCH_ERROR';
+        logWarn('Batch URL processing failed', { url, error: errorMessage });
+        return {
+            url,
+            success: false,
+            cached: false,
+            error: errorMessage,
+            errorCode,
+        };
+    }
+}
+export async function fetchUrlsToolHandler(input) {
+    try {
+        // Validate input - urls array is guaranteed by Zod schema but check for empty
+        if (input.urls.length === 0) {
+            return createToolErrorResponse('At least one URL is required', '', 'VALIDATION_ERROR');
+        }
+        // Enforce max URLs limit
+        if (input.urls.length > MAX_URLS) {
+            return createToolErrorResponse(`Maximum ${MAX_URLS} URLs allowed per batch`, '', 'VALIDATION_ERROR');
+        }
+        // Filter out empty URLs
+        const validUrls = input.urls.filter((url) => typeof url === 'string' && url.trim().length > 0);
+        if (validUrls.length === 0) {
+            return createToolErrorResponse('No valid URLs provided', '', 'VALIDATION_ERROR');
+        }
+        const concurrency = Math.min(Math.max(1, input.concurrency ?? DEFAULT_CONCURRENCY), 5);
+        const continueOnError = input.continueOnError ?? true;
+        const format = input.format ?? 'jsonl';
+        logDebug('Starting batch URL fetch', {
+            urlCount: validUrls.length,
+            concurrency,
+            format,
+        });
+        // Create tasks for each URL
+        const tasks = validUrls.map((url) => async () => processSingleUrl(url, {
+            extractMainContent: input.extractMainContent ?? true,
+            includeMetadata: input.includeMetadata ?? true,
+            maxContentLength: input.maxContentLength,
+            format,
+        }));
+        // Execute with concurrency control
+        const settledResults = await runWithConcurrency(concurrency, tasks, {
+            onProgress: (completed, total) => {
+                logDebug('Batch progress', {
+                    completed,
+                    total,
+                    percentage: Math.round((completed / total) * 100),
+                });
+            },
+        });
+        // Helper to safely extract error message from rejected promise
+        const getErrorMessage = ({ reason }) => {
+            const typedReason = reason;
+            return typedReason instanceof Error
+                ? typedReason.message
+                : String(typedReason);
+        };
+        // Process results
+        const results = settledResults.map((result, index) => {
+            if (result.status === 'fulfilled') {
+                return result.value;
+            }
+            else {
+                // Promise rejection (shouldn't happen as processSingleUrl catches errors)
+                return {
+                    url: validUrls[index] ?? 'unknown',
+                    success: false,
+                    cached: false,
+                    error: getErrorMessage(result),
+                    errorCode: 'PROMISE_REJECTED',
+                };
+            }
+        });
+        // Check if we should fail fast on errors
+        if (!continueOnError) {
+            const firstError = results.find((r) => !r.success);
+            if (firstError && !firstError.success) {
+                const errorMsg = firstError.error ?? 'Unknown error';
+                return createToolErrorResponse(`Batch failed: ${errorMsg}`, firstError.url, firstError.errorCode ?? 'BATCH_ERROR');
+            }
+        }
+        return createBatchResponse(results);
+    }
+    catch (error) {
+        logError('fetch-urls tool error', error instanceof Error ? error : undefined);
+        return createToolErrorResponse(error instanceof Error ? error.message : 'Failed to fetch URLs', '', 'BATCH_ERROR');
+    }
+}
+//# sourceMappingURL=fetch-urls.tool.js.map

package/dist/tools/handlers/fetch-urls.tool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-urls.tool.js","sourceRoot":"","sources":["../../../src/tools/handlers/fetch-urls.tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AASnC,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,EACL,cAAc,EACd,0BAA0B,GAC3B,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,eAAe,GAChB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAEnE,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,MAAM,CAAC,MAAM,oBAAoB,GAAG,YAAY,CAAC;AACjD,MAAM,CAAC,MAAM,2BAA2B,GACtC,mKAAmK,CAAC;AAEtK,MAAM,QAAQ,GAAG,EAAE,CAAC;AACpB,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAS9B,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,OAAuB;IAEvB,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;QACnD,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;QAC1E,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;QAErE,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,QAAQ,CAAC,iBAAiB,EAAE,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,CAAC;gBACpD,OAAO;oBACL,GAAG,EAAE,aAAa;oBAClB,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,MAAM,EAAE,IAAI;iBACb,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,aAAa,CAAC,CAAC;QAE3D,IAAI,UAAkB,CAAC;QACvB,IAAI,KAAyB,CAAC;QAC9B,IAAI,QAAmC,CAAC;QAExC,kEAAkE;QAClE,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC;YAChC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC;YAC9B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,aAAa,GAAG,0BAA0B,CAAC,CAAC,CAAC,CAAC;YACpD,CAAC,EAAE,KAAK,EAAE,GAAG,aAAa,CAAC,CAAC;YAE5B,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC5B,QAAQ,GAAG;oBACT,IAAI,EAAE,UAAmB;oBACzB,GAAG,EAAE,aAAa;oBAClB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;oBACtC,MAAM,EAAE,aAAa,CAAC,MAAM;iBAC7B,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,8DAA8D;YAC9D,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CACzD,WAAW,CAAC,IAAI,EAChB,aAAa,EACb;gBACE,cAAc,EAAE,IAAI;aACrB,CACF,CAAC;YACF,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACnD,QAAQ,GAAG,aAAa,CACtB,aAAa,EACb,OAAO,EACP,aAAa,EACb,UAAU,EACV,OAAO,CAAC,eAAe,CACxB,CAAC;YACF,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC;YAC7D,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;QAC3D,CAAC;QAED,IAAI,OAAe,CAAC;QACpB,IAAI,aAAiC,CAAC;QAEtC,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAClC,OAAO,GAAG,cAAc,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;YACrC,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC;YAC9B,OAAO,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,eAAe,CACnD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;QACF,OAAO,GAAG,gBAAgB,CAAC;QAC3B,IAAI,QAAQ;YAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE3C,OAAO;YACL,GAAG,EAAE,aAAa;YAClB,OAAO,EAAE,IAAI;YACb,KAAK;YACL,OAAO;YACP,aAAa;YACb,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QAC3D,MAAM,SAAS,GACb,KAAK,YAAY,KAAK;YACtB,MAAM,IAAI,KAAK;YACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;YAC5B,CAAC,CAAC,KAAK,CAAC,IAAI;YACZ,CAAC,CAAC,aAAa,CAAC;QAEpB,OAAO,CAAC,6BAA6B,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;QACrE,OAAO;YACL,GAAG;YACH,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,YAAY;YACnB,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAAqB;IAK9D,IAAI,CAAC;QACH,8EAA8E;QAC9E,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,uBAAuB,CAC5B,8BAA8B,EAC9B,EAAE,EACF,kBAAkB,CACnB,CAAC;QACJ,CAAC;QAED,yBAAyB;QACzB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACjC,OAAO,uBAAuB,CAC5B,WAAW,QAAQ,yBAAyB,EAC5C,EAAE,EACF,kBAAkB,CACnB,CAAC;QACJ,CAAC;QAED,wBAAwB;QACxB,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CACjC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAC1D,CAAC;QAEF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,uBAAuB,CAC5B,wBAAwB,EACxB,EAAE,EACF,kBAAkB,CACnB,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,IAAI,mBAAmB,CAAC,EACrD,CAAC,CACF,CAAC;QACF,MAAM,eAAe,GAAG,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC;QACtD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC;QAEvC,QAAQ,CAAC,0BAA0B,EAAE;YACnC,QAAQ,EAAE,SAAS,CAAC,MAAM;YAC1B,WAAW;YACX,MAAM;SACP,CAAC,CAAC;QAEH,4BAA4B;QAC5B,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CACzB,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,CAClB,gBAAgB,CAAC,GAAG,EAAE;YACpB,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,IAAI,IAAI;YACpD,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;YAC9C,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,MAAM;SACP,CAAC,CACL,CAAC;QAEF,mCAAmC;QACnC,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,WAAW,EAAE,KAAK,EAAE;YAClE,UAAU,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;gBAC/B,QAAQ,CAAC,gBAAgB,EAAE;oBACzB,SAAS;oBACT,KAAK;oBACL,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;iBAClD,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QAEH,+DAA+D;QAC/D,MAAM,eAAe,GAAG,CAAC,EAAE,MAAM,EAAyB,EAAU,EAAE;YACpE,MAAM,WAAW,GAAY,MAAM,CAAC;YACpC,OAAO,WAAW,YAAY,KAAK;gBACjC,CAAC,CAAC,WAAW,CAAC,OAAO;gBACrB,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC1B,CAAC,CAAC;QAEF,kBAAkB;QAClB,MAAM,OAAO,GAAqB,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACrE,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,MAAM,CAAC,KAAK,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,0EAA0E;gBAC1E,OAAO;oBACL,GAAG,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI,SAAS;oBAClC,OAAO,EAAE,KAAc;oBACvB,MAAM,EAAE,KAAc;oBACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC;oBAC9B,SAAS,EAAE,kBAAkB;iBAC9B,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,yCAAyC;QACzC,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACnD,IAAI,UAAU,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;gBACtC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,IAAI,eAAe,CAAC;gBACrD,OAAO,uBAAuB,CAC5B,iBAAiB,QAAQ,EAAE,EAC3B,UAAU,CAAC,GAAG,EACd,UAAU,CAAC,SAAS,IAAI,aAAa,CACtC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,mBAAmB,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,uBAAuB,EACvB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QAEF,OAAO,uBAAuB,CAC5B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,EAC/D,EAAE,EACF,aAAa,CACd,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/tools/index.d.ts

@@ -1,7 +1,3 @@
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-/**
- * Registers all tools with the MCP server using the modern McpServer API
- * Tools are registered with Zod schemas for automatic validation
- */
 export declare function registerTools(server: McpServer): void;
 //# sourceMappingURL=index.d.ts.map

package/dist/tools/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAwHzE;;;GAGG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAoCrD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAsQzE,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CA4CrD"}

package/dist/tools/index.js

@@ -1,8 +1,28 @@
 import { z } from 'zod';
-import { fetchUrlToolHandler, FETCH_URL_TOOL_NAME, FETCH_URL_TOOL_DESCRIPTION, } from './handlers/fetch-url.tool.js';
-import { fetchLinksToolHandler, FETCH_LINKS_TOOL_NAME, FETCH_LINKS_TOOL_DESCRIPTION, } from './handlers/fetch-links.tool.js';
-import { fetchMarkdownToolHandler, FETCH_MARKDOWN_TOOL_NAME, FETCH_MARKDOWN_TOOL_DESCRIPTION, } from './handlers/fetch-markdown.tool.js';
+import { FETCH_LINKS_TOOL_DESCRIPTION, FETCH_LINKS_TOOL_NAME, fetchLinksToolHandler, } from './handlers/fetch-links.tool.js';
+import { FETCH_MARKDOWN_TOOL_DESCRIPTION, FETCH_MARKDOWN_TOOL_NAME, fetchMarkdownToolHandler, } from './handlers/fetch-markdown.tool.js';
+import { FETCH_URL_TOOL_DESCRIPTION, FETCH_URL_TOOL_NAME, fetchUrlToolHandler, } from './handlers/fetch-url.tool.js';
+import { FETCH_URLS_TOOL_DESCRIPTION, FETCH_URLS_TOOL_NAME, fetchUrlsToolHandler, } from './handlers/fetch-urls.tool.js';
 // Zod schemas for runtime validation - single source of truth
+// Common request options shared across tools
+const RequestOptionsSchema = {
+    customHeaders: z
+        .record(z.string())
+        .optional()
+        .describe('Custom HTTP headers for the request'),
+    timeout: z
+        .number()
+        .min(1000)
+        .max(60000)
+        .optional()
+        .describe('Request timeout in milliseconds (1000-60000)'),
+    retries: z
+        .number()
+        .min(1)
+        .max(10)
+        .optional()
+        .describe('Number of retry attempts (1-10)'),
+};
 // Input schemas
 const FetchUrlInputSchema = {
     url: z.string().min(1).describe('The URL to fetch'),
@@ -26,10 +46,7 @@ const FetchUrlInputSchema = {
         .optional()
         .default('jsonl')
         .describe('Output format'),
-    customHeaders: z
-        .record(z.string())
-        .optional()
-        .describe('Custom HTTP headers for the request'),
+    ...RequestOptionsSchema,
 };
 const FetchLinksInputSchema = {
     url: z.string().min(1).describe('The URL to extract links from'),
@@ -43,6 +60,22 @@ const FetchLinksInputSchema = {
         .optional()
         .default(true)
         .describe('Include internal links'),
+    maxLinks: z
+        .number()
+        .positive()
+        .max(1000)
+        .optional()
+        .describe('Maximum number of links to return (1-1000)'),
+    filterPattern: z
+        .string()
+        .optional()
+        .describe('Regex pattern to filter links (matches against href)'),
+    includeImages: z
+        .boolean()
+        .optional()
+        .default(false)
+        .describe('Include image links (img src attributes)'),
+    ...RequestOptionsSchema,
 };
 const FetchMarkdownInputSchema = {
     url: z.string().min(1).describe('The URL to fetch'),
@@ -56,6 +89,57 @@ const FetchMarkdownInputSchema = {
         .optional()
         .default(true)
         .describe('Include YAML frontmatter metadata'),
+    maxContentLength: z
+        .number()
+        .positive()
+        .optional()
+        .describe('Maximum content length in characters'),
+    generateToc: z
+        .boolean()
+        .optional()
+        .default(false)
+        .describe('Generate table of contents from headings'),
+    ...RequestOptionsSchema,
+};
+const FetchUrlsInputSchema = {
+    urls: z
+        .array(z.string().min(1))
+        .min(1)
+        .max(10)
+        .describe('Array of URLs to fetch (1-10 URLs)'),
+    extractMainContent: z
+        .boolean()
+        .optional()
+        .default(true)
+        .describe('Use Readability to extract main article content'),
+    includeMetadata: z
+        .boolean()
+        .optional()
+        .default(true)
+        .describe('Include page metadata (title, description, etc.)'),
+    maxContentLength: z
+        .number()
+        .positive()
+        .optional()
+        .describe('Maximum content length per URL in characters'),
+    format: z
+        .enum(['jsonl', 'markdown'])
+        .optional()
+        .default('jsonl')
+        .describe('Output format for all URLs'),
+    concurrency: z
+        .number()
+        .min(1)
+        .max(5)
+        .optional()
+        .default(3)
+        .describe('Maximum concurrent requests (1-5)'),
+    continueOnError: z
+        .boolean()
+        .optional()
+        .default(true)
+        .describe('Continue processing if some URLs fail'),
+    ...RequestOptionsSchema,
 };
 // Output schemas for structured content validation
 const FetchUrlOutputSchema = {
@@ -78,9 +162,17 @@ const FetchLinksOutputSchema = {
         .array(z.object({
         href: z.string().describe('The link URL'),
         text: z.string().describe('The link anchor text'),
-        type: z.enum(['internal', 'external']).describe('Link type'),
+        type: z.enum(['internal', 'external', 'image']).describe('Link type'),
     }))
         .describe('Array of extracted links'),
+    filtered: z
+        .number()
+        .optional()
+        .describe('Number of links filtered out by pattern'),
+    truncated: z
+        .boolean()
+        .optional()
+        .describe('Whether results were truncated by maxLinks'),
     error: z.string().optional().describe('Error message if the request failed'),
     errorCode: z.string().optional().describe('Error code if the request failed'),
 };
@@ -91,35 +183,73 @@ const FetchMarkdownOutputSchema = {
         .string()
         .describe('ISO timestamp of when the content was fetched'),
     markdown: z.string().describe('The extracted content in Markdown format'),
+    toc: z
+        .array(z.object({
+        level: z.number().describe('Heading level (1-6)'),
+        text: z.string().describe('Heading text'),
+        slug: z.string().describe('URL-friendly anchor slug'),
+    }))
+        .optional()
+        .describe('Table of contents (if generateToc is true)'),
     cached: z.boolean().describe('Whether the result was served from cache'),
+    truncated: z
+        .boolean()
+        .optional()
+        .describe('Whether content was truncated by maxContentLength'),
     error: z.string().optional().describe('Error message if the request failed'),
     errorCode: z.string().optional().describe('Error code if the request failed'),
 };
-/**
- * Registers all tools with the MCP server using the modern McpServer API
- * Tools are registered with Zod schemas for automatic validation
- */
+const FetchUrlsOutputSchema = {
+    results: z
+        .array(z.object({
+        url: z.string().describe('The fetched URL'),
+        success: z.boolean().describe('Whether the fetch was successful'),
+        title: z.string().optional().describe('Page title'),
+        content: z.string().optional().describe('The extracted content'),
+        contentBlocks: z
+            .number()
+            .optional()
+            .describe('Number of content blocks (JSONL only)'),
+        cached: z.boolean().optional().describe('Whether served from cache'),
+        error: z.string().optional().describe('Error message if failed'),
+        errorCode: z.string().optional().describe('Error code if failed'),
+    }))
+        .describe('Array of results for each URL'),
+    summary: z
+        .object({
+        total: z.number().describe('Total URLs processed'),
+        successful: z.number().describe('Number of successful fetches'),
+        failed: z.number().describe('Number of failed fetches'),
+        cached: z.number().describe('Number served from cache'),
+        totalContentBlocks: z.number().describe('Total content blocks extracted'),
+    })
+        .describe('Summary statistics'),
+    fetchedAt: z.string().describe('ISO timestamp of batch completion'),
+};
 export function registerTools(server) {
-    // Register fetch-url tool
     server.registerTool(FETCH_URL_TOOL_NAME, {
         title: 'Fetch URL',
         description: FETCH_URL_TOOL_DESCRIPTION,
         inputSchema: FetchUrlInputSchema,
         outputSchema: FetchUrlOutputSchema,
     }, async (args) => fetchUrlToolHandler(args));
-    // Register fetch-links tool
     server.registerTool(FETCH_LINKS_TOOL_NAME, {
         title: 'Fetch Links',
         description: FETCH_LINKS_TOOL_DESCRIPTION,
         inputSchema: FetchLinksInputSchema,
         outputSchema: FetchLinksOutputSchema,
     }, async (args) => fetchLinksToolHandler(args));
-    // Register fetch-markdown tool
     server.registerTool(FETCH_MARKDOWN_TOOL_NAME, {
         title: 'Fetch Markdown',
         description: FETCH_MARKDOWN_TOOL_DESCRIPTION,
         inputSchema: FetchMarkdownInputSchema,
         outputSchema: FetchMarkdownOutputSchema,
     }, async (args) => fetchMarkdownToolHandler(args));
+    server.registerTool(FETCH_URLS_TOOL_NAME, {
+        title: 'Fetch URLs (Batch)',
+        description: FETCH_URLS_TOOL_DESCRIPTION,
+        inputSchema: FetchUrlsInputSchema,
+        outputSchema: FetchUrlsOutputSchema,
+    }, async (args) => fetchUrlsToolHandler(args));
 }
 //# sourceMappingURL=index.js.map