@j0hanz/superfetch 1.2.1 → 1.2.2

package/dist/http/download-routes.js

@@ -80,7 +80,8 @@ export function resolveDownloadPayload(params, cacheEntry) {
     const content = resolvePayloadContent(payload, params.namespace);
     if (!content)
         return null;
-    const fileName = generateSafeFilename(cacheEntry.url, cacheEntry.title ?? payload.title, params.hash, resolveExtension(params.namespace));
+    const safeTitle = typeof payload.title === 'string' ? payload.title : undefined;
+    const fileName = generateSafeFilename(cacheEntry.url, cacheEntry.title ?? safeTitle, params.hash, resolveExtension(params.namespace));
     return {
         content,
         contentType: resolveContentType(params.namespace),

package/dist/http/server-middleware.js

@@ -1,6 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import { config } from '../config/index.js';
-import { requestContext } from '../services/context.js';
+import { bindToRequestContext, runWithRequestContext, } from '../services/context.js';
 import { getSessionId } from './sessions.js';
 const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1']);
 function normalizeHost(value) {
@@ -84,8 +84,9 @@ export function createContextMiddleware() {
     return (req, _res, next) => {
         const requestId = randomUUID();
         const sessionId = getSessionId(req);
-        requestContext.run({ requestId, sessionId }, () => {
-            next();
+        runWithRequestContext({ requestId, sessionId }, () => {
+            const boundNext = bindToRequestContext(next);
+            boundNext();
         });
     };
 }

package/dist/services/cache.d.ts

@@ -18,4 +18,5 @@ export declare function onCacheUpdate(listener: CacheUpdateListener): () => void
 export declare function get(cacheKey: string | null): CacheEntry | undefined;
 export declare function set(cacheKey: string | null, content: string, metadata: CacheEntryMetadata): void;
 export declare function keys(): string[];
+export declare function isEnabled(): boolean;
 export {};

package/dist/services/cache.js

@@ -175,6 +175,9 @@ export function set(cacheKey, content, metadata) {
 export function keys() {
     return Array.from(contentCache.keys());
 }
+export function isEnabled() {
+    return config.cache.enabled;
+}
 function buildCacheEntry(cacheKey, content, metadata) {
     return {
         url: metadata.url,

package/dist/services/context.d.ts

@@ -4,6 +4,8 @@ interface RequestContext {
     sessionId?: string;
 }
 export declare const requestContext: AsyncLocalStorage<RequestContext>;
+export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
+export declare function bindToRequestContext<T extends (...args: unknown[]) => unknown>(fn: T): T;
 export declare function getRequestId(): string | undefined;
 export declare function getSessionId(): string | undefined;
 export {};

package/dist/services/context.js

@@ -1,5 +1,15 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
 export const requestContext = new AsyncLocalStorage();
+export function runWithRequestContext(context, fn) {
+    return requestContext.run(context, fn);
+}
+export function bindToRequestContext(fn) {
+    const store = requestContext.getStore();
+    if (!store) {
+        return fn;
+    }
+    return ((...args) => requestContext.run(store, () => fn(...args)));
+}
 export function getRequestId() {
     return requestContext.getStore()?.requestId;
 }

package/dist/services/fetcher/agents.js

@@ -3,9 +3,26 @@ import os from 'node:os';
 import { Agent } from 'undici';
 import { createErrorWithCode } from '../../utils/error-utils.js';
 import { isBlockedIp } from '../../utils/url-validator.js';
+const DNS_LOOKUP_TIMEOUT_MS = 5000;
 function resolveDns(hostname, options, callback) {
     const { normalizedOptions, useAll, resolvedFamily } = buildLookupContext(options);
-    dns.lookup(hostname, { ...normalizedOptions, all: true }, createLookupCallback(hostname, resolvedFamily, useAll, callback));
+    const lookupOptions = buildLookupOptions(normalizedOptions);
+    let done = false;
+    const timer = setTimeout(() => {
+        if (done)
+            return;
+        done = true;
+        callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
+    }, DNS_LOOKUP_TIMEOUT_MS);
+    timer.unref();
+    const safeCallback = (err, address, family) => {
+        if (done)
+            return;
+        done = true;
+        clearTimeout(timer);
+        callback(err, address, family);
+    };
+    dns.lookup(hostname, lookupOptions, createLookupCallback(hostname, resolvedFamily, useAll, safeCallback));
 }
 function normalizeLookupOptions(options) {
     return typeof options === 'number' ? { family: options } : options;
@@ -18,6 +35,29 @@ function buildLookupContext(options) {
         resolvedFamily: resolveFamily(normalizedOptions.family),
     };
 }
+const DEFAULT_DNS_ORDER = 'verbatim';
+function resolveResultOrder(options) {
+    if (options.order)
+        return options.order;
+    const legacyVerbatim = getLegacyVerbatim(options);
+    if (legacyVerbatim !== undefined) {
+        return legacyVerbatim ? 'verbatim' : 'ipv4first';
+    }
+    return DEFAULT_DNS_ORDER;
+}
+function getLegacyVerbatim(options) {
+    const legacy = options.verbatim;
+    return typeof legacy === 'boolean' ? legacy : undefined;
+}
+function buildLookupOptions(normalizedOptions) {
+    const options = {
+        ...normalizedOptions,
+        order: resolveResultOrder(normalizedOptions),
+        all: true,
+    };
+    delete options.verbatim;
+    return options;
+}
 function createLookupCallback(hostname, resolvedFamily, useAll, callback) {
     return (err, addresses) => {
         handleLookupResult(err, addresses, hostname, resolvedFamily, useAll, callback);
@@ -42,6 +82,11 @@ function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll,
         return;
     }
     const list = normalizeLookupResults(addresses, resolvedFamily);
+    const invalidFamilyError = findInvalidFamilyError(list, hostname);
+    if (invalidFamilyError) {
+        callback(invalidFamilyError, list);
+        return;
+    }
     const blockedError = findBlockedIpError(list, hostname);
     if (blockedError) {
         callback(blockedError, list);
@@ -89,6 +134,15 @@ function findBlockedIpError(list, hostname) {
     }
     return null;
 }
+function findInvalidFamilyError(list, hostname) {
+    for (const addr of list) {
+        const family = typeof addr === 'string' ? 0 : addr.family;
+        if (family === 4 || family === 6)
+            continue;
+        return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
+    }
+    return null;
+}
 function createNoDnsResultsError(hostname) {
     return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
 }

package/dist/services/fetcher/interceptors.js

@@ -4,21 +4,44 @@ import { performance } from 'node:perf_hooks';
 import { isSystemError } from '../../utils/error-utils.js';
 import { logDebug, logError, logWarn } from '../logger.js';
 const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
+function redactUrl(rawUrl) {
+    try {
+        const url = new URL(rawUrl);
+        url.username = '';
+        url.password = '';
+        url.hash = '';
+        url.search = '';
+        return url.toString();
+    }
+    catch {
+        return rawUrl;
+    }
+}
+function publishFetchEvent(event) {
+    if (!fetchChannel.hasSubscribers)
+        return;
+    try {
+        fetchChannel.publish(event);
+    }
+    catch {
+        // Avoid crashing the publisher if a subscriber throws.
+    }
+}
 export function startFetchTelemetry(url, method) {
+    const safeUrl = redactUrl(url);
     const context = {
         requestId: randomUUID(),
         startTime: performance.now(),
-        url,
+        url: safeUrl,
         method: method.toUpperCase(),
     };
-    if (fetchChannel.hasSubscribers) {
-        fetchChannel.publish({
-            type: 'start',
-            requestId: context.requestId,
-            method: context.method,
-            url: context.url,
-        });
-    }
+    publishFetchEvent({
+        v: 1,
+        type: 'start',
+        requestId: context.requestId,
+        method: context.method,
+        url: context.url,
+    });
     logDebug('HTTP Request', {
         requestId: context.requestId,
         method: context.method,
@@ -38,9 +61,8 @@ export function recordFetchResponse(context, response, contentSize) {
     logSlowRequestIfNeeded(context, duration);
 }
 function publishFetchEnd(context, status, duration) {
-    if (!fetchChannel.hasSubscribers)
-        return;
-    fetchChannel.publish({
+    publishFetchEvent({
+        v: 1,
         type: 'end',
         requestId: context.requestId,
         status,
@@ -69,17 +91,16 @@ export function recordFetchError(context, error, status) {
     const duration = performance.now() - context.startTime;
     const err = error instanceof Error ? error : new Error(String(error));
     const code = isSystemError(err) ? err.code : undefined;
-    if (fetchChannel.hasSubscribers) {
-        fetchChannel.publish({
-            type: 'error',
-            requestId: context.requestId,
-            url: context.url,
-            error: err.message,
-            code,
-            status,
-            duration,
-        });
-    }
+    publishFetchEvent({
+        v: 1,
+        type: 'error',
+        requestId: context.requestId,
+        url: context.url,
+        error: err.message,
+        code,
+        status,
+        duration,
+    });
     const log = status === 429 ? logWarn : logError;
     log('HTTP Request Error', {
         requestId: context.requestId,

package/dist/services/fetcher/response.d.ts

@@ -1,4 +1,4 @@
-export declare function readResponseText(response: Response, url: string, maxBytes: number): Promise<{
+export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal): Promise<{
     text: string;
     size: number;
 }>;

package/dist/services/fetcher/response.js

@@ -9,30 +9,43 @@ function assertContentLengthWithinLimit(response, url, maxBytes) {
     }
     throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
 }
-async function readStreamWithLimit(stream, url, maxBytes) {
+function throwIfReadAborted(url, signal) {
+    if (!signal?.aborted)
+        return;
+    throw new FetchError('Request was aborted during response read', url, 499, {
+        reason: 'aborted',
+    });
+}
+async function readStreamWithLimit(stream, url, maxBytes, signal) {
     const reader = stream.getReader();
     const decoder = new TextDecoder();
     let total = 0;
     const chunks = [];
-    for (;;) {
-        const { value, done } = await reader.read();
-        if (done)
-            break;
-        total += value.byteLength;
-        if (total > maxBytes) {
-            await reader.cancel();
-            throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+    try {
+        for (;;) {
+            throwIfReadAborted(url, signal);
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            total += value.byteLength;
+            if (total > maxBytes) {
+                await reader.cancel();
+                throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+            }
+            chunks.push(decoder.decode(value, { stream: true }));
         }
-        chunks.push(decoder.decode(value, { stream: true }));
+        chunks.push(decoder.decode());
+        return { text: chunks.join(''), size: total };
+    }
+    finally {
+        reader.releaseLock();
     }
-    chunks.push(decoder.decode());
-    return { text: chunks.join(''), size: total };
 }
-export async function readResponseText(response, url, maxBytes) {
+export async function readResponseText(response, url, maxBytes, signal) {
     assertContentLengthWithinLimit(response, url, maxBytes);
     if (!response.body) {
         const text = await response.text();
         return { text, size: Buffer.byteLength(text) };
     }
-    return readStreamWithLimit(response.body, url, maxBytes);
+    return readStreamWithLimit(response.body, url, maxBytes, signal);
 }

package/dist/services/fetcher.d.ts

@@ -2,3 +2,4 @@ import type { FetchOptions } from '../config/types/runtime.js';
 import { destroyAgents } from './fetcher/agents.js';
 export { destroyAgents };
 export declare function fetchUrlWithRetry(url: string, options?: FetchOptions, maxRetries?: number): Promise<string>;
+export declare function fetchNormalizedUrlWithRetry(normalizedUrl: string, options?: FetchOptions, maxRetries?: number): Promise<string>;

package/dist/services/fetcher.js

@@ -39,7 +39,7 @@ function buildRequestInit(headers, signal) {
         dispatcher,
     };
 }
-async function handleFetchResponse(response, finalUrl, telemetry) {
+async function handleFetchResponse(response, finalUrl, telemetry, signal) {
     if (response.status === 429) {
         void response.body?.cancel();
         throw createRateLimitError(finalUrl, response.headers.get('retry-after'));
@@ -48,7 +48,7 @@ async function handleFetchResponse(response, finalUrl, telemetry) {
         void response.body?.cancel();
         throw createHttpError(finalUrl, response.status, response.statusText);
     }
-    const { text, size } = await readResponseText(response, finalUrl, config.fetcher.maxContentLength);
+    const { text, size } = await readResponseText(response, finalUrl, config.fetcher.maxContentLength, signal);
     recordFetchResponse(telemetry, response, size);
     return text;
 }
@@ -57,7 +57,7 @@ async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
     try {
         const { response, url: finalUrl } = await fetchWithRedirects(normalizedUrl, requestInit, config.fetcher.maxRedirects);
         telemetry.url = finalUrl;
-        return await handleFetchResponse(response, finalUrl, telemetry);
+        return await handleFetchResponse(response, finalUrl, telemetry, requestInit.signal ?? undefined);
     }
     catch (error) {
         const mapped = mapFetchError(error, normalizedUrl, timeoutMs);
@@ -68,6 +68,9 @@ async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
 }
 export async function fetchUrlWithRetry(url, options, maxRetries = 3) {
     const normalizedUrl = await validateAndNormalizeUrl(url);
+    return fetchNormalizedUrlWithRetry(normalizedUrl, options, maxRetries);
+}
+export async function fetchNormalizedUrlWithRetry(normalizedUrl, options, maxRetries = 3) {
     const context = buildRequestContext(options);
     return executeWithRetry(normalizedUrl, maxRetries, async () => runFetch(normalizedUrl, context), context.signal);
 }

package/dist/services/parser.d.ts

@@ -1,2 +1,6 @@
-import type { ContentBlockUnion } from '../config/types/content.js';
+import type { ContentBlockUnion, ExtractedMetadata } from '../config/types/content.js';
 export declare function parseHtml(html: string): ContentBlockUnion[];
+export declare function parseHtmlWithMetadata(html: string): {
+    blocks: ContentBlockUnion[];
+    metadata: ExtractedMetadata;
+};

package/dist/services/parser.js

@@ -7,6 +7,60 @@ import { truncateHtml } from '../utils/html-truncator.js';
 import { sanitizeText } from '../utils/sanitizer.js';
 import { logWarn } from './logger.js';
 const CONTENT_SELECTOR = 'h1, h2, h3, h4, h5, h6, p, ul, ol, pre, code:not(pre code), table, img, blockquote';
+function createMetaCollectorState() {
+    return {
+        title: {},
+        description: {},
+        author: {},
+    };
+}
+function resolveMetaField(state, field) {
+    const sources = state[field];
+    return sources.og ?? sources.twitter ?? sources.standard;
+}
+function extractMetadata($) {
+    const state = createMetaCollectorState();
+    $('meta').each((_, element) => {
+        const content = $(element).attr('content')?.trim();
+        if (!content)
+            return;
+        const property = $(element).attr('property');
+        if (property?.startsWith('og:')) {
+            const key = property.replace('og:', '');
+            if (key === 'title')
+                state.title.og = content;
+            if (key === 'description')
+                state.description.og = content;
+            return;
+        }
+        const name = $(element).attr('name');
+        if (name?.startsWith('twitter:')) {
+            const key = name.replace('twitter:', '');
+            if (key === 'title')
+                state.title.twitter = content;
+            if (key === 'description')
+                state.description.twitter = content;
+            return;
+        }
+        if (name === 'description') {
+            state.description.standard = content;
+        }
+        if (name === 'author') {
+            state.author.standard = content;
+        }
+    });
+    if (!state.title.standard) {
+        const titleText = $('title').first().text().trim();
+        if (titleText) {
+            state.title.standard = titleText;
+        }
+    }
+    return {
+        title: resolveMetaField(state, 'title'),
+        description: resolveMetaField(state, 'description'),
+        author: resolveMetaField(state, 'author'),
+    };
+}
 function parseHeading($, element) {
     const rawText = sanitizeText($(element).text());
     const text = cleanHeading(rawText);
@@ -173,6 +227,10 @@ function loadHtml(html) {
         return null;
     }
 }
+function prepareCheerio(html) {
+    const processedHtml = truncateHtml(html);
+    return loadHtml(processedHtml);
+}
 function removeNoiseElements($) {
     $('script, style, noscript, iframe, svg').remove();
 }
@@ -198,10 +256,22 @@ function safeParseElement($, element) {
 export function parseHtml(html) {
     if (!html || typeof html !== 'string')
         return [];
-    const processedHtml = truncateHtml(html);
-    const $ = loadHtml(processedHtml);
+    const $ = prepareCheerio(html);
     if (!$)
         return [];
     removeNoiseElements($);
     return filterBlocks(collectBlocks($));
 }
+export function parseHtmlWithMetadata(html) {
+    if (!html || typeof html !== 'string') {
+        return { blocks: [], metadata: {} };
+    }
+    const $ = prepareCheerio(html);
+    if (!$) {
+        return { blocks: [], metadata: {} };
+    }
+    const metadata = extractMetadata($);
+    removeNoiseElements($);
+    const blocks = filterBlocks(collectBlocks($));
+    return { blocks, metadata };
+}

package/dist/tools/schemas.d.ts

@@ -1,6 +1,6 @@
 import { z } from 'zod';
 export declare const fetchUrlInputSchema: z.ZodObject<{
-    customHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    customHeaders: z.ZodOptional<z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodString>, Record<string, string>, Record<string, string>>>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
 } & {
@@ -13,25 +13,25 @@ export declare const fetchUrlInputSchema: z.ZodObject<{
     format: z.ZodDefault<z.ZodEnum<["jsonl", "markdown"]>>;
 }, "strict", z.ZodTypeAny, {
     url: string;
+    timeout: number;
+    retries: number;
     extractMainContent: boolean;
     includeMetadata: boolean;
-    retries: number;
     format: "jsonl" | "markdown";
-    timeout: number;
     customHeaders?: Record<string, string> | undefined;
     maxContentLength?: number | undefined;
 }, {
     url: string;
     customHeaders?: Record<string, string> | undefined;
+    timeout?: number | undefined;
+    retries?: number | undefined;
     extractMainContent?: boolean | undefined;
     includeMetadata?: boolean | undefined;
-    retries?: number | undefined;
-    format?: "jsonl" | "markdown" | undefined;
     maxContentLength?: number | undefined;
-    timeout?: number | undefined;
+    format?: "jsonl" | "markdown" | undefined;
 }>;
 export declare const fetchMarkdownInputSchema: z.ZodObject<{
-    customHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    customHeaders: z.ZodOptional<z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodString>, Record<string, string>, Record<string, string>>>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
 } & {
@@ -42,20 +42,20 @@ export declare const fetchMarkdownInputSchema: z.ZodObject<{
     maxContentLength: z.ZodOptional<z.ZodNumber>;
 }, "strict", z.ZodTypeAny, {
     url: string;
+    timeout: number;
+    retries: number;
     extractMainContent: boolean;
     includeMetadata: boolean;
-    retries: number;
-    timeout: number;
     customHeaders?: Record<string, string> | undefined;
     maxContentLength?: number | undefined;
 }, {
     url: string;
     customHeaders?: Record<string, string> | undefined;
+    timeout?: number | undefined;
+    retries?: number | undefined;
     extractMainContent?: boolean | undefined;
     includeMetadata?: boolean | undefined;
-    retries?: number | undefined;
     maxContentLength?: number | undefined;
-    timeout?: number | undefined;
 }>;
 export declare const fetchUrlOutputSchema: z.ZodObject<{
     url: z.ZodString;
@@ -74,31 +74,31 @@ export declare const fetchUrlOutputSchema: z.ZodObject<{
     errorCode: z.ZodOptional<z.ZodString>;
 }, "strict", z.ZodTypeAny, {
     url: string;
+    format: "jsonl" | "markdown";
     contentBlocks: number;
     fetchedAt: string;
-    format: "jsonl" | "markdown";
     cached: boolean;
     error?: string | undefined;
-    content?: string | undefined;
     title?: string | undefined;
-    truncated?: boolean | undefined;
+    content?: string | undefined;
+    contentSize?: number | undefined;
     resourceUri?: string | undefined;
     resourceMimeType?: string | undefined;
-    contentSize?: number | undefined;
+    truncated?: boolean | undefined;
     errorCode?: string | undefined;
 }, {
     url: string;
+    format: "jsonl" | "markdown";
     contentBlocks: number;
     fetchedAt: string;
-    format: "jsonl" | "markdown";
     cached: boolean;
     error?: string | undefined;
-    content?: string | undefined;
     title?: string | undefined;
-    truncated?: boolean | undefined;
+    content?: string | undefined;
+    contentSize?: number | undefined;
     resourceUri?: string | undefined;
     resourceMimeType?: string | undefined;
-    contentSize?: number | undefined;
+    truncated?: boolean | undefined;
     errorCode?: string | undefined;
 }>;
 export declare const fetchMarkdownOutputSchema: z.ZodObject<{
@@ -111,13 +111,13 @@ export declare const fetchMarkdownOutputSchema: z.ZodObject<{
         fileName: z.ZodString;
         expiresAt: z.ZodString;
     }, "strip", z.ZodTypeAny, {
+        downloadUrl: string;
         fileName: string;
         expiresAt: string;
-        downloadUrl: string;
     }, {
+        downloadUrl: string;
         fileName: string;
         expiresAt: string;
-        downloadUrl: string;
     }>>;
 } & {
     contentSize: z.ZodOptional<z.ZodNumber>;
@@ -134,16 +134,16 @@ export declare const fetchMarkdownOutputSchema: z.ZodObject<{
     error?: string | undefined;
     markdown?: string | undefined;
     title?: string | undefined;
-    truncated?: boolean | undefined;
+    contentSize?: number | undefined;
     resourceUri?: string | undefined;
     resourceMimeType?: string | undefined;
-    contentSize?: number | undefined;
+    truncated?: boolean | undefined;
+    errorCode?: string | undefined;
     file?: {
+        downloadUrl: string;
         fileName: string;
         expiresAt: string;
-        downloadUrl: string;
     } | undefined;
-    errorCode?: string | undefined;
 }, {
     url: string;
     fetchedAt: string;
@@ -151,14 +151,14 @@ export declare const fetchMarkdownOutputSchema: z.ZodObject<{
     error?: string | undefined;
     markdown?: string | undefined;
     title?: string | undefined;
-    truncated?: boolean | undefined;
+    contentSize?: number | undefined;
     resourceUri?: string | undefined;
     resourceMimeType?: string | undefined;
-    contentSize?: number | undefined;
+    truncated?: boolean | undefined;
+    errorCode?: string | undefined;
     file?: {
+        downloadUrl: string;
         fileName: string;
         expiresAt: string;
-        downloadUrl: string;
     } | undefined;
-    errorCode?: string | undefined;
 }>;

package/dist/tools/schemas.js

@@ -1,8 +1,16 @@
 import { z } from 'zod';
 import { config } from '../config/index.js';
+const MAX_HEADER_NAME_LENGTH = 128;
+const MAX_HEADER_VALUE_LENGTH = 2048;
+const MAX_HEADER_COUNT = 50;
+const MAX_CONTENT_LENGTH = config.constants.maxContentSize;
+const customHeadersSchema = z
+    .record(z.string().max(MAX_HEADER_NAME_LENGTH), z.string().max(MAX_HEADER_VALUE_LENGTH))
+    .refine((headers) => Object.keys(headers).length <= MAX_HEADER_COUNT, {
+    message: `customHeaders must have at most ${MAX_HEADER_COUNT} entries`,
+});
 const requestOptionsSchema = z.object({
-    customHeaders: z
-        .record(z.string())
+    customHeaders: customHeadersSchema
         .optional()
         .describe('Custom HTTP headers for the request'),
     timeout: z
@@ -30,6 +38,7 @@ const extractionOptionsSchema = z.object({
     maxContentLength: z
         .number()
         .positive()
+        .max(MAX_CONTENT_LENGTH)
         .optional()
         .describe('Maximum content length in characters'),
 });
@@ -64,14 +73,22 @@ const fileDownloadSchema = z.object({
 });
 export const fetchUrlInputSchema = requestOptionsSchema
     .extend({
-    url: z.string().min(1).describe('The URL to fetch'),
+    url: z
+        .string()
+        .min(1)
+        .max(config.constants.maxUrlLength)
+        .describe('The URL to fetch'),
 })
     .merge(extractionOptionsSchema)
     .merge(formatOptionsSchema)
     .strict();
 export const fetchMarkdownInputSchema = requestOptionsSchema
     .extend({
-    url: z.string().min(1).describe('The URL to fetch'),
+    url: z
+        .string()
+        .min(1)
+        .max(config.constants.maxUrlLength)
+        .describe('The URL to fetch'),
 })
     .merge(extractionOptionsSchema)
     .strict();

package/dist/tools/utils/content-transform.js

@@ -1,6 +1,6 @@
 import { TRUNCATION_MARKER } from '../../config/formatting.js';
 import { extractContent } from '../../services/extractor.js';
-import { parseHtml } from '../../services/parser.js';
+import { parseHtml, parseHtmlWithMetadata } from '../../services/parser.js';
 import { sanitizeText } from '../../utils/sanitizer.js';
 import { toJsonl } from '../../transformers/jsonl.transformer.js';
 import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
@@ -56,7 +56,10 @@ function decodeHtmlEntities(value) {
 }
 function buildJsonlPayload(context, maxContentLength) {
     const contentBlocks = parseHtml(context.sourceHtml);
-    const { content, truncated } = truncateContent(toJsonl(contentBlocks, context.metadata), maxContentLength);
+    return buildJsonlPayloadFromBlocks(contentBlocks, context.metadata, maxContentLength);
+}
+function buildJsonlPayloadFromBlocks(contentBlocks, metadata, maxContentLength) {
+    const { content, truncated } = truncateContent(toJsonl(contentBlocks, metadata), maxContentLength);
     return {
         content,
         contentBlocks: contentBlocks.length,
@@ -69,6 +72,17 @@ function buildMarkdownPayload(context, maxContentLength) {
     return { content, truncated };
 }
 export function transformHtmlToJsonl(html, url, options) {
+    if (!options.extractMainContent && options.includeMetadata) {
+        const parsed = parseHtmlWithMetadata(html);
+        const metadataBlock = createContentMetadataBlock(url, null, parsed.metadata, false, true);
+        const { content, contentBlocks, truncated } = buildJsonlPayloadFromBlocks(parsed.blocks, metadataBlock, options.maxContentLength);
+        return {
+            content,
+            contentBlocks,
+            title: parsed.metadata.title,
+            ...(truncated && { truncated }),
+        };
+    }
     const context = resolveContentSource(html, url, options);
     const { content, contentBlocks, truncated } = buildJsonlPayload(context, options.maxContentLength);
     return {
@@ -88,6 +102,21 @@ export function transformHtmlToMarkdown(html, url, options) {
     };
 }
 export function transformHtmlToMarkdownWithBlocks(html, url, options) {
+    if (!options.extractMainContent && options.includeMetadata) {
+        const parsed = parseHtmlWithMetadata(html);
+        const context = {
+            sourceHtml: html,
+            title: parsed.metadata.title,
+            metadata: createContentMetadataBlock(url, null, parsed.metadata, false, true),
+        };
+        const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);
+        return {
+            content,
+            contentBlocks: parsed.blocks.length,
+            title: context.title,
+            ...(truncated && { truncated }),
+        };
+    }
     const context = resolveContentSource(html, url, options);
     const contentBlocks = parseHtml(context.sourceHtml);
     const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);

package/dist/tools/utils/fetch-pipeline.js

@@ -1,7 +1,7 @@
 import * as cache from '../../services/cache.js';
-import { fetchUrlWithRetry } from '../../services/fetcher.js';
+import { fetchNormalizedUrlWithRetry } from '../../services/fetcher.js';
 import { logDebug } from '../../services/logger.js';
-import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
+import { assertResolvedAddressesAllowed, normalizeUrl, } from '../../utils/url-validator.js';
 import { appendHeaderVary } from './cache-vary.js';
 function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalizedUrl) {
     if (!cacheKey)
@@ -42,16 +42,19 @@ function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalized
  * @returns Promise resolving to the pipeline result
  */
 export async function executeFetchPipeline(options) {
-    const normalizedUrl = await validateAndNormalizeUrl(options.url);
+    const { normalizedUrl, hostname } = normalizeUrl(options.url);
     const cacheKey = resolveCacheKey(options, normalizedUrl);
     const cachedResult = attemptCacheRetrieval(cacheKey, options.deserialize, options.cacheNamespace, normalizedUrl);
     if (cachedResult)
         return cachedResult;
+    await assertResolvedAddressesAllowed(hostname);
     const fetchOptions = buildFetchOptions(options);
     logDebug('Fetching URL', { url: normalizedUrl, retries: options.retries });
-    const html = await fetchUrlWithRetry(normalizedUrl, fetchOptions, options.retries);
+    const html = await fetchNormalizedUrlWithRetry(normalizedUrl, fetchOptions, options.retries);
     const data = options.transform(html, normalizedUrl);
-    persistCache(cacheKey, data, options.serialize, normalizedUrl);
+    if (cache.isEnabled()) {
+        persistCache(cacheKey, data, options.serialize, normalizedUrl);
+    }
     return buildPipelineResult(normalizedUrl, data, cacheKey);
 }
 function resolveCacheKey(options, normalizedUrl) {

package/dist/utils/url-validator.d.ts

@@ -1,2 +1,7 @@
 export declare function isBlockedIp(ip: string): boolean;
+export declare function assertResolvedAddressesAllowed(hostname: string): Promise<void>;
+export declare function normalizeUrl(urlString: string): {
+    normalizedUrl: string;
+    hostname: string;
+};
 export declare function validateAndNormalizeUrl(urlString: string): Promise<string>;

package/dist/utils/url-validator.js

@@ -78,7 +78,7 @@ function lookupWithTimeout(hostname) {
         });
     });
 }
-async function assertResolvedAddressesAllowed(hostname) {
+export async function assertResolvedAddressesAllowed(hostname) {
     try {
         const result = await lookupWithTimeout(hostname);
         const addresses = Array.isArray(result) ? result : [result];
@@ -102,7 +102,7 @@ async function assertResolvedAddressesAllowed(hostname) {
         throw createValidationError(String(error));
     }
 }
-export async function validateAndNormalizeUrl(urlString) {
+export function normalizeUrl(urlString) {
     const trimmedUrl = requireTrimmedUrl(urlString);
     assertUrlLength(trimmedUrl);
     const url = parseUrl(trimmedUrl);
@@ -110,8 +110,12 @@ export async function validateAndNormalizeUrl(urlString) {
     assertNoCredentials(url);
     const hostname = normalizeHostname(url);
     assertHostnameAllowed(hostname);
+    return { normalizedUrl: url.href, hostname };
+}
+export async function validateAndNormalizeUrl(urlString) {
+    const { normalizedUrl, hostname } = normalizeUrl(urlString);
     await assertResolvedAddressesAllowed(hostname);
-    return url.href;
+    return normalizedUrl;
 }
 const VALIDATION_ERROR_CODE = 'VALIDATION_ERROR';
 function createValidationError(message) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/superfetch",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "mcpName": "io.github.j0hanz/superfetch",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
   "type": "module",