@j0hanz/superfetch 2.0.0 → 2.1.0

package/dist/tools/handlers/fetch-url.tool.js

@@ -1,16 +1,61 @@
 import { logDebug, logError } from '../../services/logger.js';
+import { isRecord } from '../../utils/guards.js';
 import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
-import { parseCachedMarkdownResult } from '../utils/cached-markdown.js';
 import { transformHtmlToMarkdown } from '../utils/content-transform.js';
 import { buildToolContentBlocks, performSharedFetch, } from './fetch-single.shared.js';
 export const FETCH_URL_TOOL_NAME = 'fetch-url';
 export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format';
+function parseJsonRecord(input) {
+    try {
+        const parsed = JSON.parse(input);
+        return isRecord(parsed) ? parsed : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function resolveMarkdownContent(parsed) {
+    const { markdown } = parsed;
+    if (typeof markdown === 'string')
+        return markdown;
+    const { content } = parsed;
+    if (typeof content === 'string')
+        return content;
+    return undefined;
+}
+function resolveOptionalTitle(parsed) {
+    const { title } = parsed;
+    if (title === undefined)
+        return undefined;
+    return typeof title === 'string' ? title : undefined;
+}
+function resolveTruncatedFlag(parsed) {
+    const { truncated } = parsed;
+    return typeof truncated === 'boolean' ? truncated : false;
+}
+export function parseCachedMarkdownResult(cached) {
+    const parsed = parseJsonRecord(cached);
+    if (!parsed)
+        return undefined;
+    const resolvedContent = resolveMarkdownContent(parsed);
+    if (resolvedContent === undefined)
+        return undefined;
+    const title = resolveOptionalTitle(parsed);
+    if (parsed.title !== undefined && title === undefined)
+        return undefined;
+    return {
+        content: resolvedContent,
+        markdown: resolvedContent,
+        title,
+        truncated: resolveTruncatedFlag(parsed),
+    };
+}
 function deserializeMarkdownResult(cached) {
     return parseCachedMarkdownResult(cached);
 }
 function buildMarkdownTransform() {
-    return (html, url) => {
-        const result = transformHtmlToMarkdown(html, url, {
+    return async (html, url) => {
+        const result = await transformHtmlToMarkdown(html, url, {
             includeMetadata: true,
         });
         return { ...result, content: result.markdown };
@@ -23,9 +68,11 @@ function serializeMarkdownResult(result) {
         truncated: result.truncated,
     });
 }
-function buildStructuredContent(pipeline, inlineResult) {
+function buildStructuredContent(pipeline, inlineResult, inputUrl) {
     return {
         url: pipeline.url,
+        resolvedUrl: pipeline.url,
+        inputUrl,
         title: pipeline.data.title,
         markdown: inlineResult.content,
     };
@@ -44,8 +91,8 @@ async function fetchPipeline(url) {
         deserialize: deserializeMarkdownResult,
     });
 }
-function buildResponse(pipeline, inlineResult) {
-    const structuredContent = buildStructuredContent(pipeline, inlineResult);
+function buildResponse(pipeline, inlineResult, inputUrl) {
+    const structuredContent = buildStructuredContent(pipeline, inlineResult, inputUrl);
     const content = buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult);
     return {
         content,
@@ -53,13 +100,10 @@ function buildResponse(pipeline, inlineResult) {
     };
 }
 export async function fetchUrlToolHandler(input) {
-    try {
-        return await executeFetch(input);
-    }
-    catch (error) {
+    return executeFetch(input).catch((error) => {
         logError('fetch-url tool error', error instanceof Error ? error : undefined);
         return handleToolError(error, input.url, 'Failed to fetch URL');
-    }
+    });
 }
 async function executeFetch(input) {
     const { url } = input;
@@ -71,5 +115,5 @@ async function executeFetch(input) {
     if (inlineResult.error) {
         return createToolErrorResponse(inlineResult.error, url);
     }
-    return buildResponse(pipeline, inlineResult);
+    return buildResponse(pipeline, inlineResult, url);
 }

package/dist/tools/index.d.ts

@@ -1,2 +1,3 @@
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+export declare function withRequestContextIfMissing<TParams, TResult>(handler: (params: TParams) => Promise<TResult>): (params: TParams) => Promise<TResult>;
 export declare function registerTools(server: McpServer): void;

package/dist/tools/index.js

@@ -1,3 +1,5 @@
+import { randomUUID } from 'node:crypto';
+import { getRequestId, runWithRequestContext } from '../services/context.js';
 import { FETCH_URL_TOOL_DESCRIPTION, FETCH_URL_TOOL_NAME, fetchUrlToolHandler, } from './handlers/fetch-url.tool.js';
 import { fetchUrlInputSchema, fetchUrlOutputSchema } from './schemas.js';
 const TOOL_DEFINITION = {
@@ -14,6 +16,16 @@ const TOOL_DEFINITION = {
         openWorldHint: true,
     },
 };
+export function withRequestContextIfMissing(handler) {
+    return async (params) => {
+        const existingRequestId = getRequestId();
+        if (existingRequestId) {
+            return handler(params);
+        }
+        const requestId = randomUUID();
+        return runWithRequestContext({ requestId, operationId: requestId }, () => handler(params));
+    };
+}
 export function registerTools(server) {
     server.registerTool(TOOL_DEFINITION.name, {
         title: TOOL_DEFINITION.title,
@@ -21,5 +33,5 @@ export function registerTools(server) {
         inputSchema: TOOL_DEFINITION.inputSchema,
         outputSchema: TOOL_DEFINITION.outputSchema,
         annotations: TOOL_DEFINITION.annotations,
-    }, TOOL_DEFINITION.handler);
+    }, withRequestContextIfMissing(TOOL_DEFINITION.handler));
 }

package/dist/tools/schemas.d.ts

@@ -4,6 +4,8 @@ export declare const fetchUrlInputSchema: z.ZodObject<{
 }, z.core.$strict>;
 export declare const fetchUrlOutputSchema: z.ZodObject<{
     url: z.ZodString;
+    inputUrl: z.ZodOptional<z.ZodString>;
+    resolvedUrl: z.ZodOptional<z.ZodString>;
     title: z.ZodOptional<z.ZodString>;
     markdown: z.ZodOptional<z.ZodString>;
     error: z.ZodOptional<z.ZodString>;

package/dist/tools/schemas.js

@@ -4,6 +4,14 @@ export const fetchUrlInputSchema = z.strictObject({
 });
 export const fetchUrlOutputSchema = z.strictObject({
     url: z.string().describe('The fetched URL'),
+    inputUrl: z
+        .string()
+        .optional()
+        .describe('The original URL provided by the caller'),
+    resolvedUrl: z
+        .string()
+        .optional()
+        .describe('The normalized or transformed URL that was fetched'),
     title: z.string().optional().describe('Page title'),
     markdown: z
         .string()

package/dist/tools/utils/content-shaping.js

@@ -1,10 +1,25 @@
 const MIN_CONTENT_RATIO = 0.3;
 const MIN_HTML_LENGTH_FOR_GATE = 100;
+function stripHtmlTags(html) {
+    const parts = [];
+    let inTag = false;
+    for (const char of html) {
+        if (char === '<') {
+            inTag = true;
+            continue;
+        }
+        if (char === '>') {
+            inTag = false;
+            continue;
+        }
+        if (!inTag) {
+            parts.push(char);
+        }
+    }
+    return parts.join('');
+}
 function estimateTextLength(html) {
-    return html
-        .replace(/<[^>]*>/g, '')
-        .replace(/\s+/g, ' ')
-        .trim().length;
+    return stripHtmlTags(html).replace(/\s+/g, ' ').trim().length;
 }
 export function isExtractionSufficient(article, originalHtml) {
     if (!article)

package/dist/tools/utils/content-transform-core.d.ts

@@ -0,0 +1,5 @@
+import type { ExtractedArticle, ExtractedMetadata, MarkdownTransformResult, MetadataBlock, TransformOptions } from '../../config/types/content.js';
+export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtml: string): boolean;
+export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
+export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
+export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;

package/dist/tools/utils/content-transform-core.js

@@ -0,0 +1,180 @@
+import { extractContent } from '../../services/extractor.js';
+import { logDebug } from '../../services/logger.js';
+import { endTransformStage, startTransformStage, } from '../../services/telemetry.js';
+import { throwIfAborted } from '../../utils/cancellation.js';
+import { htmlToMarkdown } from '../../transformers/markdown.js';
+import { tryTransformRawContent } from './raw-markdown.js';
+const MIN_CONTENT_RATIO = 0.3;
+const MIN_HTML_LENGTH_FOR_GATE = 100;
+function stripHtmlTags(html) {
+    const parts = [];
+    let inTag = false;
+    for (const char of html) {
+        if (char === '<') {
+            inTag = true;
+            continue;
+        }
+        if (char === '>') {
+            inTag = false;
+            continue;
+        }
+        if (!inTag) {
+            parts.push(char);
+        }
+    }
+    return parts.join('');
+}
+function estimateTextLength(html) {
+    return stripHtmlTags(html).replace(/\s+/g, ' ').trim().length;
+}
+export function isExtractionSufficient(article, originalHtml) {
+    if (!article)
+        return false;
+    const articleLength = article.textContent.length;
+    const originalLength = estimateTextLength(originalHtml);
+    if (originalLength < MIN_HTML_LENGTH_FOR_GATE)
+        return true;
+    return articleLength / originalLength >= MIN_CONTENT_RATIO;
+}
+export function determineContentExtractionSource(article) {
+    return !!article;
+}
+function applyArticleMetadata(metadata, article) {
+    if (article.title !== undefined)
+        metadata.title = article.title;
+    if (article.byline !== undefined)
+        metadata.author = article.byline;
+}
+function applyExtractedMetadata(metadata, extractedMeta) {
+    if (extractedMeta.title !== undefined)
+        metadata.title = extractedMeta.title;
+    if (extractedMeta.description !== undefined) {
+        metadata.description = extractedMeta.description;
+    }
+    if (extractedMeta.author !== undefined) {
+        metadata.author = extractedMeta.author;
+    }
+}
+export function createContentMetadataBlock(url, article, extractedMeta, shouldExtractFromArticle, includeMetadata) {
+    if (!includeMetadata)
+        return undefined;
+    const now = new Date().toISOString();
+    const metadata = {
+        type: 'metadata',
+        url,
+        fetchedAt: now,
+    };
+    if (shouldExtractFromArticle && article) {
+        applyArticleMetadata(metadata, article);
+        return metadata;
+    }
+    applyExtractedMetadata(metadata, extractedMeta);
+    return metadata;
+}
+function buildArticleContentSource({ url, article, extractedMeta, includeMetadata, }) {
+    const metadata = createContentMetadataBlock(url, article, extractedMeta, true, includeMetadata);
+    return {
+        sourceHtml: article.content,
+        title: article.title,
+        metadata,
+    };
+}
+function buildFullHtmlContentSource({ html, url, article, extractedMeta, includeMetadata, }) {
+    const metadata = createContentMetadataBlock(url, article, extractedMeta, false, includeMetadata);
+    return {
+        sourceHtml: html,
+        title: extractedMeta.title,
+        metadata,
+    };
+}
+function logQualityGateFallback({ url, articleLength, }) {
+    logDebug('Quality gate: Readability extraction below threshold, using full HTML', {
+        url: url.substring(0, 80),
+        articleLength,
+    });
+}
+function tryBuildExtractedArticleContentSource({ html, url, article, extractedMeta, includeMetadata, }) {
+    if (!article)
+        return null;
+    const shouldExtractFromArticle = determineContentExtractionSource(article);
+    if (shouldExtractFromArticle && isExtractionSufficient(article, html)) {
+        return buildArticleContentSource({
+            url,
+            article,
+            extractedMeta,
+            includeMetadata,
+        });
+    }
+    if (shouldExtractFromArticle) {
+        logQualityGateFallback({
+            url,
+            articleLength: article.textContent.length,
+        });
+    }
+    return null;
+}
+function resolveContentSource({ html, url, includeMetadata, signal, }) {
+    const { article, metadata: extractedMeta } = extractContent(html, url, {
+        extractArticle: true,
+        ...(signal ? { signal } : {}),
+    });
+    const extracted = tryBuildExtractedArticleContentSource({
+        html,
+        url,
+        article,
+        extractedMeta,
+        includeMetadata,
+    });
+    if (extracted)
+        return extracted;
+    return buildFullHtmlContentSource({
+        html,
+        url,
+        article,
+        extractedMeta,
+        includeMetadata,
+    });
+}
+export function transformHtmlToMarkdownInProcess(html, url, options) {
+    const totalStage = startTransformStage(url, 'transform:total');
+    let success = false;
+    try {
+        throwIfAborted(options.signal, url, 'transform:begin');
+        const rawStage = startTransformStage(url, 'transform:raw');
+        const raw = tryTransformRawContent({
+            html,
+            url,
+            includeMetadata: options.includeMetadata,
+        });
+        endTransformStage(rawStage);
+        if (raw) {
+            success = true;
+            return raw;
+        }
+        const extractStage = startTransformStage(url, 'transform:extract');
+        const context = resolveContentSource({
+            html,
+            url,
+            includeMetadata: options.includeMetadata,
+            ...(options.signal ? { signal: options.signal } : {}),
+        });
+        endTransformStage(extractStage);
+        const markdownStage = startTransformStage(url, 'transform:markdown');
+        const content = htmlToMarkdown(context.sourceHtml, context.metadata, {
+            url,
+            ...(options.signal ? { signal: options.signal } : {}),
+        });
+        endTransformStage(markdownStage);
+        success = true;
+        return {
+            markdown: content,
+            title: context.title,
+            truncated: false,
+        };
+    }
+    finally {
+        if (success) {
+            endTransformStage(totalStage, { truncated: false });
+        }
+    }
+}

package/dist/tools/utils/content-transform-workers.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/tools/utils/content-transform-workers.js

@@ -0,0 +1 @@
+export {};

package/dist/tools/utils/content-transform.d.ts

@@ -1,2 +1,3 @@
 import type { MarkdownTransformResult, TransformOptions } from '../../config/types/content.js';
-export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): MarkdownTransformResult;
+export { createContentMetadataBlock, determineContentExtractionSource, isExtractionSufficient, } from './content-transform-core.js';
+export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;

package/dist/tools/utils/content-transform.js

@@ -1,140 +1,41 @@
-import { extractContent } from '../../services/extractor.js';
-import { logDebug } from '../../services/logger.js';
-import { isRawTextContentUrl } from '../../utils/url-transformer.js';
-import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
-import { createContentMetadataBlock, determineContentExtractionSource, isExtractionSufficient, } from './content-shaping.js';
-function buildArticleContentSource(url, article, extractedMeta, includeMetadata) {
-    const metadata = createContentMetadataBlock(url, article, extractedMeta, true, includeMetadata);
-    return {
-        sourceHtml: article.content,
-        title: article.title,
-        metadata,
-    };
-}
-function buildFullHtmlContentSource(html, url, article, extractedMeta, includeMetadata) {
-    const metadata = createContentMetadataBlock(url, article, extractedMeta, false, includeMetadata);
-    return {
-        sourceHtml: html,
-        title: extractedMeta.title,
-        metadata,
-    };
-}
-function logQualityGateFallback(url, article) {
-    logDebug('Quality gate: Readability extraction below threshold, using full HTML', {
-        url: url.substring(0, 80),
-        articleLength: article.textContent.length,
-    });
-}
-function tryBuildExtractedArticleContentSource(html, url, article, extractedMeta, options) {
-    if (!article)
-        return null;
-    const shouldExtractFromArticle = determineContentExtractionSource(article);
-    if (shouldExtractFromArticle && isExtractionSufficient(article, html)) {
-        return buildArticleContentSource(url, article, extractedMeta, options.includeMetadata);
-    }
-    if (shouldExtractFromArticle) {
-        logQualityGateFallback(url, article);
-    }
-    return null;
-}
-function resolveContentSource(html, url, options) {
-    const { article, metadata: extractedMeta } = extractContent(html, url, {
-        extractArticle: true,
-    });
-    const extracted = tryBuildExtractedArticleContentSource(html, url, article, extractedMeta, options);
-    if (extracted)
-        return extracted;
-    return buildFullHtmlContentSource(html, url, article, extractedMeta, options.includeMetadata);
-}
-function buildMarkdownPayload(context) {
-    return htmlToMarkdown(context.sourceHtml, context.metadata);
-}
-function buildRawMarkdownPayload(rawContent, url, includeMetadata) {
-    const title = extractTitleFromRawMarkdown(rawContent);
-    const content = includeMetadata
-        ? addSourceToMarkdown(rawContent, url)
-        : rawContent;
-    return { content, title };
-}
-function extractTitleFromRawMarkdown(content) {
-    const frontmatterMatch = /^---\r?\n([\s\S]*?)\r?\n---/.exec(content);
-    if (!frontmatterMatch)
-        return undefined;
-    const frontmatter = frontmatterMatch[1] ?? '';
-    const titleMatch = /^(?:title|name):\s*["']?(.+?)["']?\s*$/im.exec(frontmatter);
-    return titleMatch?.[1]?.trim();
-}
-function addSourceToMarkdown(content, url) {
-    const frontmatterMatch = /^(---\r?\n)([\s\S]*?)(\r?\n---)/.exec(content);
-    if (frontmatterMatch) {
-        const start = frontmatterMatch[1] ?? '---\n';
-        const existingFields = frontmatterMatch[2] ?? '';
-        const end = frontmatterMatch[3] ?? '\n---';
-        const rest = content.slice(frontmatterMatch[0].length);
-        if (/^source:/im.test(existingFields)) {
-            return content;
+import { FetchError } from '../../errors/app-error.js';
+import { endTransformStage, startTransformStage, } from '../../services/telemetry.js';
+import { getOrCreateTransformWorkerPool } from '../../services/transform-worker-pool.js';
+import { throwIfAborted } from '../../utils/cancellation.js';
+import { transformHtmlToMarkdownInProcess } from './content-transform-core.js';
+export { createContentMetadataBlock, determineContentExtractionSource, isExtractionSufficient, } from './content-transform-core.js';
+export async function transformHtmlToMarkdown(html, url, options) {
+    const totalStage = startTransformStage(url, 'transform:total');
+    let success = false;
+    try {
+        throwIfAborted(options.signal, url, 'transform:begin');
+        const workerStage = startTransformStage(url, 'transform:worker');
+        try {
+            const pool = getOrCreateTransformWorkerPool();
+            const result = await pool.transform(html, url, {
+                includeMetadata: options.includeMetadata,
+                ...(options.signal ? { signal: options.signal } : {}),
+            });
+            success = true;
+            return result;
+        }
+        catch (error) {
+            if (error instanceof FetchError) {
+                throw error;
+            }
+            // Stability-first: if worker infrastructure fails, fall back to in-process.
+            throwIfAborted(options.signal, url, 'transform:worker-fallback');
+            const fallback = transformHtmlToMarkdownInProcess(html, url, options);
+            success = true;
+            return fallback;
+        }
+        finally {
+            endTransformStage(workerStage);
         }
-        return `${start}${existingFields}\nsource: "${url}"${end}${rest}`;
-    }
-    return `---\nsource: "${url}"\n---\n\n${content}`;
-}
-function looksLikeHtmlDocument(trimmed) {
-    return (trimmed.startsWith('<!DOCTYPE') ||
-        trimmed.startsWith('<!doctype') ||
-        trimmed.startsWith('<html') ||
-        trimmed.startsWith('<HTML'));
-}
-function hasFrontmatter(trimmed) {
-    return /^---\r?\n/.test(trimmed);
-}
-function countCommonHtmlTags(content) {
-    const matches = content.match(/<(html|head|body|div|span|script|style|meta|link)\b/gi) ??
-        [];
-    return matches.length;
-}
-function looksLikeMarkdown(content) {
-    const hasMarkdownHeadings = /^#{1,6}\s+/m.test(content);
-    const hasMarkdownLists = /^[\s]*[-*+]\s+/m.test(content);
-    const hasMarkdownCodeBlocks = /```[\s\S]*?```/.test(content);
-    return hasMarkdownHeadings || hasMarkdownLists || hasMarkdownCodeBlocks;
-}
-function isRawTextContent(content) {
-    const trimmed = content.trim();
-    if (looksLikeHtmlDocument(trimmed)) {
-        return false;
-    }
-    if (hasFrontmatter(trimmed)) {
-        return true;
-    }
-    if (countCommonHtmlTags(content) > 2) {
-        return false;
-    }
-    if (looksLikeMarkdown(content)) {
-        return true;
     }
-    return false;
-}
-function tryTransformRawContent(html, url, options) {
-    if (!isRawTextContentUrl(url) && !isRawTextContent(html)) {
-        return null;
+    finally {
+        if (success) {
+            endTransformStage(totalStage, { truncated: false });
+        }
     }
-    logDebug('Preserving raw markdown content', { url: url.substring(0, 80) });
-    const { content, title } = buildRawMarkdownPayload(html, url, options.includeMetadata);
-    return {
-        markdown: content,
-        title,
-        truncated: false,
-    };
-}
-export function transformHtmlToMarkdown(html, url, options) {
-    const raw = tryTransformRawContent(html, url, options);
-    if (raw)
-        return raw;
-    const context = resolveContentSource(html, url, options);
-    const content = buildMarkdownPayload(context);
-    return {
-        markdown: content,
-        title: context.title,
-        truncated: false,
-    };
 }