@j0hanz/superfetch 1.2.2 → 1.2.3

package/dist/tools/handlers/fetch-single.shared.d.ts

@@ -1,5 +1,6 @@
 import type { PipelineResult, ToolContentBlock } from '../../config/types/runtime.js';
 import type { FileDownloadInfo, ToolResponseBase } from '../../config/types/tools.js';
+import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
 import { applyInlineContentLimit } from '../utils/inline-content.js';
 type SharedFetchFormat = 'jsonl' | 'markdown';
 interface SharedFetchOptions<T extends {
@@ -10,17 +11,21 @@ interface SharedFetchOptions<T extends {
     readonly extractMainContent: boolean;
     readonly includeMetadata: boolean;
     readonly maxContentLength?: number;
+    readonly includeContentBlocks?: boolean;
     readonly cacheVariant?: string;
     readonly customHeaders?: Record<string, string>;
     readonly retries?: number;
     readonly timeout?: number;
-    readonly transform: (html: string, normalizedUrl: string) => T;
+    readonly transform: (html: string, normalizedUrl: string) => T | Promise<T>;
     readonly serialize?: (result: T) => string;
     readonly deserialize?: (cached: string) => T | undefined;
 }
+interface SharedFetchDeps {
+    readonly executeFetchPipeline?: typeof executeFetchPipeline;
+}
 export declare function performSharedFetch<T extends {
     content: string;
-}>(options: SharedFetchOptions<T>): Promise<{
+}>(options: SharedFetchOptions<T>, deps?: SharedFetchDeps): Promise<{
     pipeline: PipelineResult<T>;
     inlineResult: ReturnType<typeof applyInlineContentLimit>;
 }>;
@@ -31,7 +36,7 @@ interface DownloadContext {
     title?: string;
 }
 export declare function getFileDownloadInfo(context: DownloadContext): FileDownloadInfo | null;
-export declare function getInlineErrorResponse(inlineResult: InlineResult, url: string): ToolResponseBase | null;
+export declare function getInlineErrorResponse(inlineResult: InlineResult, url: string, details?: Record<string, unknown>): ToolResponseBase | null;
 export declare function applyInlineResultToStructuredContent(structuredContent: Record<string, unknown>, inlineResult: InlineResult, contentKey: string): void;
 export declare function buildToolContentBlocks(structuredContent: Record<string, unknown>, fromCache: boolean, inlineResult: InlineResult, resourceName: string, cacheKey?: string | null, fullContent?: string, format?: SharedFetchFormat, url?: string, title?: string): ToolContentBlock[];
 export {};

package/dist/tools/handlers/fetch-single.shared.js

@@ -5,7 +5,8 @@ import { createToolErrorResponse } from '../../utils/tool-error-handler.js';
 import { appendHeaderVary } from '../utils/cache-vary.js';
 import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
 import { applyInlineContentLimit } from '../utils/inline-content.js';
-export async function performSharedFetch(options) {
+export async function performSharedFetch(options, deps = {}) {
+    const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
     const cacheNamespace = options.format === 'markdown' ? 'markdown' : 'url';
     const cacheVary = appendHeaderVary({
         format: options.format,
@@ -13,33 +14,54 @@ export async function performSharedFetch(options) {
         includeMetadata: options.includeMetadata,
         maxContentLength: options.maxContentLength,
         ...(options.cacheVariant ? { variant: options.cacheVariant } : {}),
-        ...(options.format === 'markdown' ? {} : { contentBlocks: true }),
+        ...(options.format === 'markdown'
+            ? { includeContentBlocks: options.includeContentBlocks }
+            : { contentBlocks: true }),
     }, options.customHeaders);
-    const pipeline = await executeFetchPipeline({
+    const pipelineOptions = {
         url: options.url,
         cacheNamespace,
-        customHeaders: options.customHeaders,
-        retries: options.retries,
-        timeout: options.timeout,
-        cacheVary,
         transform: options.transform,
-        serialize: options.serialize,
-        deserialize: options.deserialize,
-    });
+    };
+    if (options.customHeaders !== undefined) {
+        pipelineOptions.customHeaders = options.customHeaders;
+    }
+    if (options.retries !== undefined) {
+        pipelineOptions.retries = options.retries;
+    }
+    if (options.timeout !== undefined) {
+        pipelineOptions.timeout = options.timeout;
+    }
+    if (cacheVary !== undefined) {
+        pipelineOptions.cacheVary = cacheVary;
+    }
+    if (options.serialize !== undefined) {
+        pipelineOptions.serialize = options.serialize;
+    }
+    if (options.deserialize !== undefined) {
+        pipelineOptions.deserialize = options.deserialize;
+    }
+    const pipeline = await executePipeline(pipelineOptions);
     const inlineResult = applyInlineContentLimit(pipeline.data.content, pipeline.cacheKey ?? null, options.format);
     return { pipeline, inlineResult };
 }
 export function getFileDownloadInfo(context) {
-    return buildFileDownloadInfo({
+    const infoOptions = {
         cacheKey: context.cacheKey,
         url: context.url,
-        title: context.title,
-    });
+    };
+    if (context.title !== undefined) {
+        return buildFileDownloadInfo({
+            ...infoOptions,
+            title: context.title,
+        });
+    }
+    return buildFileDownloadInfo(infoOptions);
 }
-export function getInlineErrorResponse(inlineResult, url) {
+export function getInlineErrorResponse(inlineResult, url, details) {
     if (!inlineResult.error)
         return null;
-    return createToolErrorResponse(inlineResult.error, url, 'INTERNAL_ERROR');
+    return createToolErrorResponse(inlineResult.error, url, 'INTERNAL_ERROR', details);
 }
 export function applyInlineResultToStructuredContent(structuredContent, inlineResult, contentKey) {
     if (inlineResult.truncated) {
@@ -60,13 +82,16 @@ function buildResourceLink(inlineResult, name) {
     if (!inlineResult.resourceUri) {
         return null;
     }
-    return {
+    const block = {
         type: 'resource_link',
         uri: inlineResult.resourceUri,
         name,
-        mimeType: inlineResult.resourceMimeType,
         description: `Content exceeds inline limit (${config.constants.maxInlineContentChars} chars)`,
     };
+    if (inlineResult.resourceMimeType !== undefined) {
+        block.mimeType = inlineResult.resourceMimeType;
+    }
+    return block;
 }
 function buildEmbeddedResource(content, mimeType, url, title) {
     if (!content) {

package/dist/tools/handlers/fetch-url.tool.js

@@ -1,7 +1,7 @@
 import { config } from '../../config/index.js';
 import { logDebug, logError } from '../../services/logger.js';
 import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
-import { transformHtmlToJsonl, transformHtmlToMarkdownWithBlocks, } from '../utils/content-transform.js';
+import { transformHtmlToJsonlAsync, transformHtmlToMarkdownWithBlocksAsync, } from '../utils/content-transform-async.js';
 import { applyInlineResultToStructuredContent, buildToolContentBlocks, getInlineErrorResponse, performSharedFetch, } from './fetch-single.shared.js';
 export const FETCH_URL_TOOL_NAME = 'fetch-url';
 export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks. Supports custom headers, retries, and content length limits.';
@@ -37,17 +37,36 @@ function deserializeJsonlTransformResult(cached) {
     }
 }
 function resolveFetchUrlOptions(input) {
+    const format = input.format ?? 'jsonl';
     return {
         extractMainContent: input.extractMainContent ?? config.extraction.extractMainContent,
         includeMetadata: input.includeMetadata ?? config.extraction.includeMetadata,
-        maxContentLength: input.maxContentLength,
-        format: input.format ?? 'jsonl',
+        format,
+        includeContentBlocks: input.includeContentBlocks ?? (format === 'markdown' ? false : true),
+        ...(input.maxContentLength !== undefined && {
+            maxContentLength: input.maxContentLength,
+        }),
+    };
+}
+function buildFetchUrlErrorDetails(format) {
+    return {
+        contentBlocks: 0,
+        fetchedAt: new Date().toISOString(),
+        format,
+        cached: false,
     };
 }
 function buildFetchUrlTransform(options) {
-    return (html, url) => options.format === 'markdown'
-        ? transformHtmlToMarkdownWithBlocks(html, url, options)
-        : transformHtmlToJsonl(html, url, options);
+    return async (html, url) => options.format === 'markdown'
+        ? transformHtmlToMarkdownWithBlocksAsync(html, url, {
+            extractMainContent: options.extractMainContent,
+            includeMetadata: options.includeMetadata,
+            ...(options.maxContentLength !== undefined && {
+                maxContentLength: options.maxContentLength,
+            }),
+            includeContentBlocks: options.includeContentBlocks,
+        })
+        : transformHtmlToJsonlAsync(html, url, options);
 }
 function buildFetchUrlStructuredContent(format, pipeline, inlineResult) {
     const structuredContent = {
@@ -74,22 +93,31 @@ function logFetchUrlStart(url, options) {
         extractMainContent: options.extractMainContent,
         includeMetadata: options.includeMetadata,
         format: options.format,
+        includeContentBlocks: options.includeContentBlocks,
     });
 }
 async function fetchUrlPipeline(url, input, options) {
-    return performSharedFetch({
+    const sharedOptions = {
         url,
         format: options.format,
         extractMainContent: options.extractMainContent,
         includeMetadata: options.includeMetadata,
-        maxContentLength: options.maxContentLength,
-        customHeaders: input.customHeaders,
-        retries: input.retries,
-        timeout: input.timeout,
-        cacheVariant: options.format === 'markdown' ? 'markdown-with-blocks' : undefined,
+        includeContentBlocks: options.includeContentBlocks,
+        ...(options.maxContentLength !== undefined && {
+            maxContentLength: options.maxContentLength,
+        }),
+        ...(input.customHeaders !== undefined && {
+            customHeaders: input.customHeaders,
+        }),
+        ...(input.retries !== undefined && { retries: input.retries }),
+        ...(input.timeout !== undefined && { timeout: input.timeout }),
+        ...(options.format === 'markdown' && {
+            cacheVariant: 'markdown-with-blocks',
+        }),
         transform: buildFetchUrlTransform(options),
         deserialize: deserializeJsonlTransformResult,
-    });
+    };
+    return performSharedFetch(sharedOptions);
 }
 function buildFetchUrlResponse(pipeline, inlineResult, format) {
     const structuredContent = buildFetchUrlStructuredContent(format, pipeline, inlineResult);
@@ -104,18 +132,20 @@ export async function fetchUrlToolHandler(input) {
     }
     catch (error) {
         logError('fetch-url tool error', error instanceof Error ? error : undefined);
-        return handleToolError(error, input.url, 'Failed to fetch URL');
+        const errorDetails = buildFetchUrlErrorDetails(input.format ?? 'jsonl');
+        return handleToolError(error, input.url, 'Failed to fetch URL', errorDetails);
     }
 }
 async function executeFetchUrl(input) {
     const { url } = input;
+    const format = input.format ?? 'jsonl';
     if (!url) {
-        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
+        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR', buildFetchUrlErrorDetails(format));
     }
     const options = resolveFetchUrlOptions(input);
     logFetchUrlStart(url, options);
     const { pipeline, inlineResult } = await fetchUrlPipeline(url, input, options);
-    const inlineError = getInlineErrorResponse(inlineResult, url);
+    const inlineError = getInlineErrorResponse(inlineResult, url, buildFetchUrlErrorDetails(options.format));
     if (inlineError)
         return inlineError;
     return buildFetchUrlResponse(pipeline, inlineResult, options.format);

package/dist/tools/index.js

@@ -9,6 +9,12 @@ const TOOL_DEFINITIONS = [
         inputSchema: fetchUrlInputSchema,
         outputSchema: fetchUrlOutputSchema,
         handler: fetchUrlToolHandler,
+        annotations: {
+            readOnlyHint: true,
+            destructiveHint: false,
+            idempotentHint: true,
+            openWorldHint: true,
+        },
     },
     {
         name: FETCH_MARKDOWN_TOOL_NAME,
@@ -17,6 +23,12 @@ const TOOL_DEFINITIONS = [
         inputSchema: fetchMarkdownInputSchema,
         outputSchema: fetchMarkdownOutputSchema,
         handler: fetchMarkdownToolHandler,
+        annotations: {
+            readOnlyHint: true,
+            destructiveHint: false,
+            idempotentHint: true,
+            openWorldHint: true,
+        },
     },
 ];
 export function registerTools(server) {
@@ -26,6 +38,7 @@ export function registerTools(server) {
             description: tool.description,
             inputSchema: tool.inputSchema,
             outputSchema: tool.outputSchema,
+            annotations: tool.annotations,
         }, tool.handler);
     }
 }

package/dist/tools/schemas.d.ts

@@ -11,24 +11,27 @@ export declare const fetchUrlInputSchema: z.ZodObject<{
     maxContentLength: z.ZodOptional<z.ZodNumber>;
 } & {
     format: z.ZodDefault<z.ZodEnum<["jsonl", "markdown"]>>;
+    includeContentBlocks: z.ZodOptional<z.ZodBoolean>;
 }, "strict", z.ZodTypeAny, {
     url: string;
-    timeout: number;
-    retries: number;
     extractMainContent: boolean;
     includeMetadata: boolean;
+    retries: number;
     format: "jsonl" | "markdown";
-    customHeaders?: Record<string, string> | undefined;
+    timeout: number;
     maxContentLength?: number | undefined;
+    includeContentBlocks?: boolean | undefined;
+    customHeaders?: Record<string, string> | undefined;
 }, {
     url: string;
-    customHeaders?: Record<string, string> | undefined;
-    timeout?: number | undefined;
-    retries?: number | undefined;
     extractMainContent?: boolean | undefined;
     includeMetadata?: boolean | undefined;
     maxContentLength?: number | undefined;
+    retries?: number | undefined;
     format?: "jsonl" | "markdown" | undefined;
+    includeContentBlocks?: boolean | undefined;
+    timeout?: number | undefined;
+    customHeaders?: Record<string, string> | undefined;
 }>;
 export declare const fetchMarkdownInputSchema: z.ZodObject<{
     customHeaders: z.ZodOptional<z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodString>, Record<string, string>, Record<string, string>>>;
@@ -42,20 +45,20 @@ export declare const fetchMarkdownInputSchema: z.ZodObject<{
     maxContentLength: z.ZodOptional<z.ZodNumber>;
 }, "strict", z.ZodTypeAny, {
     url: string;
-    timeout: number;
-    retries: number;
     extractMainContent: boolean;
     includeMetadata: boolean;
-    customHeaders?: Record<string, string> | undefined;
+    retries: number;
+    timeout: number;
     maxContentLength?: number | undefined;
+    customHeaders?: Record<string, string> | undefined;
 }, {
     url: string;
-    customHeaders?: Record<string, string> | undefined;
-    timeout?: number | undefined;
-    retries?: number | undefined;
     extractMainContent?: boolean | undefined;
     includeMetadata?: boolean | undefined;
     maxContentLength?: number | undefined;
+    retries?: number | undefined;
+    timeout?: number | undefined;
+    customHeaders?: Record<string, string> | undefined;
 }>;
 export declare const fetchUrlOutputSchema: z.ZodObject<{
     url: z.ZodString;
@@ -74,31 +77,31 @@ export declare const fetchUrlOutputSchema: z.ZodObject<{
     errorCode: z.ZodOptional<z.ZodString>;
 }, "strict", z.ZodTypeAny, {
     url: string;
+    fetchedAt: string;
     format: "jsonl" | "markdown";
     contentBlocks: number;
-    fetchedAt: string;
     cached: boolean;
     error?: string | undefined;
     title?: string | undefined;
-    content?: string | undefined;
-    contentSize?: number | undefined;
+    truncated?: boolean | undefined;
     resourceUri?: string | undefined;
     resourceMimeType?: string | undefined;
-    truncated?: boolean | undefined;
+    content?: string | undefined;
+    contentSize?: number | undefined;
     errorCode?: string | undefined;
 }, {
     url: string;
+    fetchedAt: string;
     format: "jsonl" | "markdown";
     contentBlocks: number;
-    fetchedAt: string;
     cached: boolean;
     error?: string | undefined;
     title?: string | undefined;
-    content?: string | undefined;
-    contentSize?: number | undefined;
+    truncated?: boolean | undefined;
     resourceUri?: string | undefined;
     resourceMimeType?: string | undefined;
-    truncated?: boolean | undefined;
+    content?: string | undefined;
+    contentSize?: number | undefined;
     errorCode?: string | undefined;
 }>;
 export declare const fetchMarkdownOutputSchema: z.ZodObject<{
@@ -111,13 +114,13 @@ export declare const fetchMarkdownOutputSchema: z.ZodObject<{
         fileName: z.ZodString;
         expiresAt: z.ZodString;
     }, "strip", z.ZodTypeAny, {
+        expiresAt: string;
         downloadUrl: string;
         fileName: string;
-        expiresAt: string;
     }, {
+        expiresAt: string;
         downloadUrl: string;
         fileName: string;
-        expiresAt: string;
     }>>;
 } & {
     contentSize: z.ZodOptional<z.ZodNumber>;
@@ -134,16 +137,16 @@ export declare const fetchMarkdownOutputSchema: z.ZodObject<{
     error?: string | undefined;
     markdown?: string | undefined;
     title?: string | undefined;
-    contentSize?: number | undefined;
+    truncated?: boolean | undefined;
     resourceUri?: string | undefined;
     resourceMimeType?: string | undefined;
-    truncated?: boolean | undefined;
-    errorCode?: string | undefined;
+    contentSize?: number | undefined;
     file?: {
+        expiresAt: string;
         downloadUrl: string;
         fileName: string;
-        expiresAt: string;
     } | undefined;
+    errorCode?: string | undefined;
 }, {
     url: string;
     fetchedAt: string;
@@ -151,14 +154,14 @@ export declare const fetchMarkdownOutputSchema: z.ZodObject<{
     error?: string | undefined;
     markdown?: string | undefined;
     title?: string | undefined;
-    contentSize?: number | undefined;
+    truncated?: boolean | undefined;
     resourceUri?: string | undefined;
     resourceMimeType?: string | undefined;
-    truncated?: boolean | undefined;
-    errorCode?: string | undefined;
+    contentSize?: number | undefined;
     file?: {
+        expiresAt: string;
         downloadUrl: string;
         fileName: string;
-        expiresAt: string;
     } | undefined;
+    errorCode?: string | undefined;
 }>;

package/dist/tools/schemas.js

@@ -47,6 +47,10 @@ const formatOptionsSchema = z.object({
         .enum(['jsonl', 'markdown'])
         .default('jsonl')
         .describe('Output format'),
+    includeContentBlocks: z
+        .boolean()
+        .optional()
+        .describe('Include content block counts when format=markdown'),
 });
 const resourceFieldsSchema = z.object({
     contentSize: z.number().optional().describe('Content length in characters'),

package/dist/tools/utils/common.js

@@ -6,22 +6,26 @@ export function createContentMetadataBlock(url, article, extractedMeta, shouldEx
     if (!includeMetadata)
         return undefined;
     const now = new Date().toISOString();
-    return shouldExtractFromArticle && article
-        ? {
-            type: 'metadata',
-            title: article.title,
-            author: article.byline,
-            url,
-            fetchedAt: now,
-        }
-        : {
-            type: 'metadata',
-            title: extractedMeta.title,
-            description: extractedMeta.description,
-            author: extractedMeta.author,
-            url,
-            fetchedAt: now,
-        };
+    const metadata = {
+        type: 'metadata',
+        url,
+        fetchedAt: now,
+    };
+    if (shouldExtractFromArticle && article) {
+        if (article.title !== undefined)
+            metadata.title = article.title;
+        if (article.byline !== undefined)
+            metadata.author = article.byline;
+        return metadata;
+    }
+    if (extractedMeta.title !== undefined)
+        metadata.title = extractedMeta.title;
+    if (extractedMeta.description !== undefined) {
+        metadata.description = extractedMeta.description;
+    }
+    if (extractedMeta.author !== undefined)
+        metadata.author = extractedMeta.author;
+    return metadata;
 }
 export function truncateContent(content, maxLength, suffix = TRUNCATION_MARKER) {
     if (maxLength === undefined ||

package/dist/tools/utils/content-transform-async.d.ts

@@ -0,0 +1,6 @@
+import type { JsonlTransformResult, MarkdownTransformResult, TransformOptions } from '../../config/types/content.js';
+export declare function transformHtmlToJsonlAsync(html: string, url: string, options: TransformOptions): Promise<JsonlTransformResult>;
+export declare function transformHtmlToMarkdownAsync(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;
+export declare function transformHtmlToMarkdownWithBlocksAsync(html: string, url: string, options: TransformOptions & {
+    includeContentBlocks?: boolean;
+}): Promise<JsonlTransformResult>;

package/dist/tools/utils/content-transform-async.js

@@ -0,0 +1,33 @@
+import { logWarn } from '../../services/logger.js';
+import { runTransformInWorker, } from '../../services/transform-worker-pool.js';
+import { transformHtmlToJsonl, transformHtmlToMarkdown, transformHtmlToMarkdownWithBlocks, } from './content-transform.js';
+async function runOrFallback(job, fallback) {
+    try {
+        const result = await runTransformInWorker(job);
+        if (result)
+            return result;
+    }
+    catch (error) {
+        logWarn('Transform worker unavailable; using main thread', {
+            error: error instanceof Error ? error.message : String(error),
+        });
+    }
+    return fallback();
+}
+export async function transformHtmlToJsonlAsync(html, url, options) {
+    const result = await runOrFallback({ mode: 'jsonl', html, url, options }, () => transformHtmlToJsonl(html, url, options));
+    return result;
+}
+export async function transformHtmlToMarkdownAsync(html, url, options) {
+    const result = await runOrFallback({ mode: 'markdown', html, url, options }, () => transformHtmlToMarkdown(html, url, options));
+    return result;
+}
+export async function transformHtmlToMarkdownWithBlocksAsync(html, url, options) {
+    const result = await runOrFallback({
+        mode: 'markdown-blocks',
+        html,
+        url,
+        options,
+    }, () => transformHtmlToMarkdownWithBlocks(html, url, options));
+    return result;
+}

package/dist/tools/utils/content-transform.d.ts

@@ -8,7 +8,10 @@ interface ContentLengthOptions {
 }
 interface MarkdownOptions extends ExtractionOptions, ContentLengthOptions {
 }
+interface MarkdownWithBlocksOptions extends ExtractionOptions, ContentLengthOptions {
+    readonly includeContentBlocks?: boolean;
+}
 export declare function transformHtmlToJsonl(html: string, url: string, options: ExtractionOptions & ContentLengthOptions): JsonlTransformResult;
 export declare function transformHtmlToMarkdown(html: string, url: string, options: MarkdownOptions): MarkdownTransformResult;
-export declare function transformHtmlToMarkdownWithBlocks(html: string, url: string, options: ExtractionOptions & ContentLengthOptions): JsonlTransformResult;
+export declare function transformHtmlToMarkdownWithBlocks(html: string, url: string, options: MarkdownWithBlocksOptions): JsonlTransformResult;
 export {};

package/dist/tools/utils/content-transform.js

@@ -102,7 +102,10 @@ export function transformHtmlToMarkdown(html, url, options) {
     };
 }
 export function transformHtmlToMarkdownWithBlocks(html, url, options) {
-    if (!options.extractMainContent && options.includeMetadata) {
+    const includeContentBlocks = options.includeContentBlocks ?? true;
+    if (includeContentBlocks &&
+        !options.extractMainContent &&
+        options.includeMetadata) {
         const parsed = parseHtmlWithMetadata(html);
         const context = {
             sourceHtml: html,
@@ -118,7 +121,9 @@ export function transformHtmlToMarkdownWithBlocks(html, url, options) {
         };
     }
     const context = resolveContentSource(html, url, options);
-    const contentBlocks = parseHtml(context.sourceHtml);
+    const contentBlocks = includeContentBlocks
+        ? parseHtml(context.sourceHtml)
+        : [];
     const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);
     return {
         content,

package/dist/tools/utils/fetch-pipeline.js

@@ -51,7 +51,7 @@ export async function executeFetchPipeline(options) {
     const fetchOptions = buildFetchOptions(options);
     logDebug('Fetching URL', { url: normalizedUrl, retries: options.retries });
     const html = await fetchNormalizedUrlWithRetry(normalizedUrl, fetchOptions, options.retries);
-    const data = options.transform(html, normalizedUrl);
+    const data = await options.transform(html, normalizedUrl);
     if (cache.isEnabled()) {
         persistCache(cacheKey, data, options.serialize, normalizedUrl);
     }
@@ -62,20 +62,28 @@ function resolveCacheKey(options, normalizedUrl) {
     return cache.createCacheKey(options.cacheNamespace, normalizedUrl, cacheVary);
 }
 function buildFetchOptions(options) {
-    return {
-        customHeaders: options.customHeaders,
-        signal: options.signal,
-        timeout: options.timeout,
-    };
+    const fetchOptions = {};
+    if (options.customHeaders !== undefined) {
+        fetchOptions.customHeaders = options.customHeaders;
+    }
+    if (options.signal !== undefined) {
+        fetchOptions.signal = options.signal;
+    }
+    if (options.timeout !== undefined) {
+        fetchOptions.timeout = options.timeout;
+    }
+    return fetchOptions;
 }
 function persistCache(cacheKey, data, serialize, normalizedUrl) {
     if (!cacheKey)
         return;
     const serializer = serialize ?? JSON.stringify;
-    cache.set(cacheKey, serializer(data), {
-        url: normalizedUrl,
-        title: extractTitle(data),
-    });
+    const metadata = { url: normalizedUrl };
+    const title = extractTitle(data);
+    if (title !== undefined) {
+        metadata.title = title;
+    }
+    cache.set(cacheKey, serializer(data), metadata);
 }
 function extractTitle(value) {
     if (!value || typeof value !== 'object')

package/dist/utils/content-cleaner.d.ts

@@ -1,5 +1,5 @@
 export declare function cleanParagraph(text: string): string | null;
 export declare function cleanHeading(text: string): string | null;
-export declare function cleanListItems(items: string[]): string[];
+export declare function cleanListItems(items: readonly string[]): readonly string[];
 export declare function cleanCodeBlock(code: string): string | null;
 export declare function removeInlineTimestamps(text: string): string;

package/dist/utils/download-url.d.ts

@@ -1,8 +1,16 @@
+import { config } from '../config/index.js';
 import type { FileDownloadInfo } from '../config/types/tools.js';
+import * as cache from '../services/cache.js';
+import { generateSafeFilename } from './filename-generator.js';
 interface DownloadInfoOptions {
     cacheKey: string | null;
     url: string;
     title?: string;
 }
-export declare function buildFileDownloadInfo(options: DownloadInfoOptions): FileDownloadInfo | null;
+interface DownloadInfoDeps {
+    readonly config?: typeof config;
+    readonly cache?: Pick<typeof cache, 'get' | 'parseCacheKey'>;
+    readonly generateSafeFilename?: typeof generateSafeFilename;
+}
+export declare function buildFileDownloadInfo(options: DownloadInfoOptions, deps?: DownloadInfoDeps): FileDownloadInfo | null;
 export {};