@j0hanz/superfetch 2.0.1 → 2.1.0

package/dist/tools/utils/content-transform-core.js

@@ -0,0 +1,180 @@
+import { extractContent } from '../../services/extractor.js';
+import { logDebug } from '../../services/logger.js';
+import { endTransformStage, startTransformStage, } from '../../services/telemetry.js';
+import { throwIfAborted } from '../../utils/cancellation.js';
+import { htmlToMarkdown } from '../../transformers/markdown.js';
+import { tryTransformRawContent } from './raw-markdown.js';
+const MIN_CONTENT_RATIO = 0.3;
+const MIN_HTML_LENGTH_FOR_GATE = 100;
+function stripHtmlTags(html) {
+    const parts = [];
+    let inTag = false;
+    for (const char of html) {
+        if (char === '<') {
+            inTag = true;
+            continue;
+        }
+        if (char === '>') {
+            inTag = false;
+            continue;
+        }
+        if (!inTag) {
+            parts.push(char);
+        }
+    }
+    return parts.join('');
+}
+function estimateTextLength(html) {
+    return stripHtmlTags(html).replace(/\s+/g, ' ').trim().length;
+}
+export function isExtractionSufficient(article, originalHtml) {
+    if (!article)
+        return false;
+    const articleLength = article.textContent.length;
+    const originalLength = estimateTextLength(originalHtml);
+    if (originalLength < MIN_HTML_LENGTH_FOR_GATE)
+        return true;
+    return articleLength / originalLength >= MIN_CONTENT_RATIO;
+}
+export function determineContentExtractionSource(article) {
+    return !!article;
+}
+function applyArticleMetadata(metadata, article) {
+    if (article.title !== undefined)
+        metadata.title = article.title;
+    if (article.byline !== undefined)
+        metadata.author = article.byline;
+}
+function applyExtractedMetadata(metadata, extractedMeta) {
+    if (extractedMeta.title !== undefined)
+        metadata.title = extractedMeta.title;
+    if (extractedMeta.description !== undefined) {
+        metadata.description = extractedMeta.description;
+    }
+    if (extractedMeta.author !== undefined) {
+        metadata.author = extractedMeta.author;
+    }
+}
+export function createContentMetadataBlock(url, article, extractedMeta, shouldExtractFromArticle, includeMetadata) {
+    if (!includeMetadata)
+        return undefined;
+    const now = new Date().toISOString();
+    const metadata = {
+        type: 'metadata',
+        url,
+        fetchedAt: now,
+    };
+    if (shouldExtractFromArticle && article) {
+        applyArticleMetadata(metadata, article);
+        return metadata;
+    }
+    applyExtractedMetadata(metadata, extractedMeta);
+    return metadata;
+}
+function buildArticleContentSource({ url, article, extractedMeta, includeMetadata, }) {
+    const metadata = createContentMetadataBlock(url, article, extractedMeta, true, includeMetadata);
+    return {
+        sourceHtml: article.content,
+        title: article.title,
+        metadata,
+    };
+}
+function buildFullHtmlContentSource({ html, url, article, extractedMeta, includeMetadata, }) {
+    const metadata = createContentMetadataBlock(url, article, extractedMeta, false, includeMetadata);
+    return {
+        sourceHtml: html,
+        title: extractedMeta.title,
+        metadata,
+    };
+}
+function logQualityGateFallback({ url, articleLength, }) {
+    logDebug('Quality gate: Readability extraction below threshold, using full HTML', {
+        url: url.substring(0, 80),
+        articleLength,
+    });
+}
+function tryBuildExtractedArticleContentSource({ html, url, article, extractedMeta, includeMetadata, }) {
+    if (!article)
+        return null;
+    const shouldExtractFromArticle = determineContentExtractionSource(article);
+    if (shouldExtractFromArticle && isExtractionSufficient(article, html)) {
+        return buildArticleContentSource({
+            url,
+            article,
+            extractedMeta,
+            includeMetadata,
+        });
+    }
+    if (shouldExtractFromArticle) {
+        logQualityGateFallback({
+            url,
+            articleLength: article.textContent.length,
+        });
+    }
+    return null;
+}
+function resolveContentSource({ html, url, includeMetadata, signal, }) {
+    const { article, metadata: extractedMeta } = extractContent(html, url, {
+        extractArticle: true,
+        ...(signal ? { signal } : {}),
+    });
+    const extracted = tryBuildExtractedArticleContentSource({
+        html,
+        url,
+        article,
+        extractedMeta,
+        includeMetadata,
+    });
+    if (extracted)
+        return extracted;
+    return buildFullHtmlContentSource({
+        html,
+        url,
+        article,
+        extractedMeta,
+        includeMetadata,
+    });
+}
+export function transformHtmlToMarkdownInProcess(html, url, options) {
+    const totalStage = startTransformStage(url, 'transform:total');
+    let success = false;
+    try {
+        throwIfAborted(options.signal, url, 'transform:begin');
+        const rawStage = startTransformStage(url, 'transform:raw');
+        const raw = tryTransformRawContent({
+            html,
+            url,
+            includeMetadata: options.includeMetadata,
+        });
+        endTransformStage(rawStage);
+        if (raw) {
+            success = true;
+            return raw;
+        }
+        const extractStage = startTransformStage(url, 'transform:extract');
+        const context = resolveContentSource({
+            html,
+            url,
+            includeMetadata: options.includeMetadata,
+            ...(options.signal ? { signal: options.signal } : {}),
+        });
+        endTransformStage(extractStage);
+        const markdownStage = startTransformStage(url, 'transform:markdown');
+        const content = htmlToMarkdown(context.sourceHtml, context.metadata, {
+            url,
+            ...(options.signal ? { signal: options.signal } : {}),
+        });
+        endTransformStage(markdownStage);
+        success = true;
+        return {
+            markdown: content,
+            title: context.title,
+            truncated: false,
+        };
+    }
+    finally {
+        if (success) {
+            endTransformStage(totalStage, { truncated: false });
+        }
+    }
+}

package/dist/tools/utils/content-transform-workers.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/tools/utils/content-transform-workers.js

@@ -0,0 +1 @@
+export {};

package/dist/tools/utils/content-transform.d.ts

@@ -1,5 +1,3 @@
-import type { ExtractedArticle, ExtractedMetadata, MarkdownTransformResult, MetadataBlock, TransformOptions } from '../../config/types/content.js';
-export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtml: string): boolean;
-export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
-export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
-export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): MarkdownTransformResult;
+import type { MarkdownTransformResult, TransformOptions } from '../../config/types/content.js';
+export { createContentMetadataBlock, determineContentExtractionSource, isExtractionSufficient, } from './content-transform-core.js';
+export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;

package/dist/tools/utils/content-transform.js

@@ -1,154 +1,41 @@
-import { extractContent } from '../../services/extractor.js';
-import { logDebug } from '../../services/logger.js';
-import { htmlToMarkdown } from '../../transformers/markdown.js';
-import { tryTransformRawContent } from './raw-markdown.js';
-const MIN_CONTENT_RATIO = 0.3;
-const MIN_HTML_LENGTH_FOR_GATE = 100;
-function stripHtmlTags(html) {
-    const parts = [];
-    let inTag = false;
-    for (const char of html) {
-        if (char === '<') {
-            inTag = true;
-            continue;
+import { FetchError } from '../../errors/app-error.js';
+import { endTransformStage, startTransformStage, } from '../../services/telemetry.js';
+import { getOrCreateTransformWorkerPool } from '../../services/transform-worker-pool.js';
+import { throwIfAborted } from '../../utils/cancellation.js';
+import { transformHtmlToMarkdownInProcess } from './content-transform-core.js';
+export { createContentMetadataBlock, determineContentExtractionSource, isExtractionSufficient, } from './content-transform-core.js';
+export async function transformHtmlToMarkdown(html, url, options) {
+    const totalStage = startTransformStage(url, 'transform:total');
+    let success = false;
+    try {
+        throwIfAborted(options.signal, url, 'transform:begin');
+        const workerStage = startTransformStage(url, 'transform:worker');
+        try {
+            const pool = getOrCreateTransformWorkerPool();
+            const result = await pool.transform(html, url, {
+                includeMetadata: options.includeMetadata,
+                ...(options.signal ? { signal: options.signal } : {}),
+            });
+            success = true;
+            return result;
         }
-        if (char === '>') {
-            inTag = false;
-            continue;
+        catch (error) {
+            if (error instanceof FetchError) {
+                throw error;
+            }
+            // Stability-first: if worker infrastructure fails, fall back to in-process.
+            throwIfAborted(options.signal, url, 'transform:worker-fallback');
+            const fallback = transformHtmlToMarkdownInProcess(html, url, options);
+            success = true;
+            return fallback;
         }
-        if (!inTag) {
-            parts.push(char);
+        finally {
+            endTransformStage(workerStage);
         }
     }
-    return parts.join('');
-}
-function estimateTextLength(html) {
-    return stripHtmlTags(html).replace(/\s+/g, ' ').trim().length;
-}
-export function isExtractionSufficient(article, originalHtml) {
-    if (!article)
-        return false;
-    const articleLength = article.textContent.length;
-    const originalLength = estimateTextLength(originalHtml);
-    if (originalLength < MIN_HTML_LENGTH_FOR_GATE)
-        return true;
-    return articleLength / originalLength >= MIN_CONTENT_RATIO;
-}
-export function determineContentExtractionSource(article) {
-    return !!article;
-}
-function applyArticleMetadata(metadata, article) {
-    if (article.title !== undefined)
-        metadata.title = article.title;
-    if (article.byline !== undefined)
-        metadata.author = article.byline;
-}
-function applyExtractedMetadata(metadata, extractedMeta) {
-    if (extractedMeta.title !== undefined)
-        metadata.title = extractedMeta.title;
-    if (extractedMeta.description !== undefined) {
-        metadata.description = extractedMeta.description;
-    }
-    if (extractedMeta.author !== undefined) {
-        metadata.author = extractedMeta.author;
-    }
-}
-export function createContentMetadataBlock(url, article, extractedMeta, shouldExtractFromArticle, includeMetadata) {
-    if (!includeMetadata)
-        return undefined;
-    const now = new Date().toISOString();
-    const metadata = {
-        type: 'metadata',
-        url,
-        fetchedAt: now,
-    };
-    if (shouldExtractFromArticle && article) {
-        applyArticleMetadata(metadata, article);
-        return metadata;
-    }
-    applyExtractedMetadata(metadata, extractedMeta);
-    return metadata;
-}
-function buildArticleContentSource({ url, article, extractedMeta, includeMetadata, }) {
-    const metadata = createContentMetadataBlock(url, article, extractedMeta, true, includeMetadata);
-    return {
-        sourceHtml: article.content,
-        title: article.title,
-        metadata,
-    };
-}
-function buildFullHtmlContentSource({ html, url, article, extractedMeta, includeMetadata, }) {
-    const metadata = createContentMetadataBlock(url, article, extractedMeta, false, includeMetadata);
-    return {
-        sourceHtml: html,
-        title: extractedMeta.title,
-        metadata,
-    };
-}
-function logQualityGateFallback({ url, articleLength, }) {
-    logDebug('Quality gate: Readability extraction below threshold, using full HTML', {
-        url: url.substring(0, 80),
-        articleLength,
-    });
-}
-function tryBuildExtractedArticleContentSource({ html, url, article, extractedMeta, includeMetadata, }) {
-    if (!article)
-        return null;
-    const shouldExtractFromArticle = determineContentExtractionSource(article);
-    if (shouldExtractFromArticle && isExtractionSufficient(article, html)) {
-        return buildArticleContentSource({
-            url,
-            article,
-            extractedMeta,
-            includeMetadata,
-        });
-    }
-    if (shouldExtractFromArticle) {
-        logQualityGateFallback({
-            url,
-            articleLength: article.textContent.length,
-        });
+    finally {
+        if (success) {
+            endTransformStage(totalStage, { truncated: false });
+        }
     }
-    return null;
-}
-function resolveContentSource({ html, url, includeMetadata, }) {
-    const { article, metadata: extractedMeta } = extractContent(html, url, {
-        extractArticle: true,
-    });
-    const extracted = tryBuildExtractedArticleContentSource({
-        html,
-        url,
-        article,
-        extractedMeta,
-        includeMetadata,
-    });
-    if (extracted)
-        return extracted;
-    return buildFullHtmlContentSource({
-        html,
-        url,
-        article,
-        extractedMeta,
-        includeMetadata,
-    });
-}
-export function transformHtmlToMarkdown(html, url, options) {
-    const raw = tryTransformRawContent({
-        html,
-        url,
-        includeMetadata: options.includeMetadata,
-    });
-    if (raw)
-        return raw;
-    const context = resolveContentSource({
-        html,
-        url,
-        includeMetadata: options.includeMetadata,
-    });
-    const content = htmlToMarkdown(context.sourceHtml, context.metadata);
-    return {
-        markdown: content,
-        title: context.title,
-        truncated: false,
-    };
 }

package/dist/tools/utils/raw-markdown.js

@@ -110,6 +110,20 @@ function isRawTextContent(content) {
     return (!isHtmlDocument &&
         (hasMarkdownFrontmatter || (!hasTooManyHtmlTags && isMarkdown)));
 }
+function isLikelyHtmlContent(content) {
+    const trimmed = content.trim();
+    if (!trimmed)
+        return false;
+    if (looksLikeHtmlDocument(trimmed))
+        return true;
+    return countCommonHtmlTags(content) > 2;
+}
+function shouldPreserveRawContent(url, content) {
+    if (isRawTextContentUrl(url)) {
+        return !isLikelyHtmlContent(content);
+    }
+    return isRawTextContent(content);
+}
 function buildRawMarkdownPayload({ rawContent, url, includeMetadata, }) {
     const title = extractTitleFromRawMarkdown(rawContent);
     const content = includeMetadata
@@ -118,7 +132,7 @@ function buildRawMarkdownPayload({ rawContent, url, includeMetadata, }) {
     return { content, title };
 }
 export function tryTransformRawContent({ html, url, includeMetadata, }) {
-    if (!isRawTextContentUrl(url) && !isRawTextContent(html)) {
+    if (!shouldPreserveRawContent(url, html)) {
         return null;
     }
     logDebug('Preserving raw markdown content', { url: url.substring(0, 80) });

package/dist/tools.d.ts

@@ -0,0 +1,104 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { type MarkdownTransformResult } from './transform.js';
+export interface FetchUrlInput {
+    url: string;
+}
+export interface ToolContentBlock {
+    type: 'text';
+    text: string;
+}
+export interface ToolContentResourceLinkBlock {
+    type: 'resource_link';
+    uri: string;
+    name: string;
+    title?: string;
+    description?: string;
+    mimeType?: string;
+}
+export interface ToolContentResourceBlock {
+    type: 'resource';
+    resource: {
+        uri: string;
+        mimeType?: string;
+        text: string;
+    };
+}
+export type ToolContentBlockUnion = ToolContentBlock | ToolContentResourceLinkBlock | ToolContentResourceBlock;
+export interface ToolErrorResponse {
+    [x: string]: unknown;
+    content: ToolContentBlockUnion[];
+    structuredContent: {
+        error: string;
+        url: string;
+    };
+    isError: true;
+}
+export interface ToolResponseBase {
+    [x: string]: unknown;
+    content: ToolContentBlockUnion[];
+    structuredContent?: Record<string, unknown>;
+    isError?: boolean;
+}
+export interface FetchPipelineOptions<T> {
+    /** URL to fetch */
+    url: string;
+    /** Cache namespace (e.g., 'markdown') */
+    cacheNamespace: string;
+    /** Optional: AbortSignal for request cancellation */
+    signal?: AbortSignal;
+    /** Optional: cache variation input for headers/flags */
+    cacheVary?: Record<string, unknown> | string;
+    /** Transform function to process HTML into desired format */
+    transform: (html: string, url: string) => T | Promise<T>;
+    /** Optional: serialize result for caching (defaults to JSON.stringify) */
+    serialize?: (result: T) => string;
+    /** Optional: deserialize cached content */
+    deserialize?: (cached: string) => T | undefined;
+}
+export interface PipelineResult<T> {
+    data: T;
+    fromCache: boolean;
+    url: string;
+    fetchedAt: string;
+    cacheKey?: string | null;
+}
+export declare const FETCH_URL_TOOL_NAME = "fetch-url";
+export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format";
+interface InlineContentResult {
+    content?: string;
+    contentSize: number;
+    resourceUri?: string;
+    resourceMimeType?: string;
+    error?: string;
+    truncated?: boolean;
+}
+declare function applyInlineContentLimit(content: string, cacheKey: string | null): InlineContentResult;
+export type InlineResult = ReturnType<typeof applyInlineContentLimit>;
+export declare function executeFetchPipeline<T>(options: FetchPipelineOptions<T>): Promise<PipelineResult<T>>;
+interface SharedFetchOptions<T extends {
+    content: string;
+}> {
+    readonly url: string;
+    readonly transform: (html: string, normalizedUrl: string) => T | Promise<T>;
+    readonly serialize?: (result: T) => string;
+    readonly deserialize?: (cached: string) => T | undefined;
+}
+interface SharedFetchDeps {
+    readonly executeFetchPipeline?: typeof executeFetchPipeline;
+}
+export declare function performSharedFetch<T extends {
+    content: string;
+}>(options: SharedFetchOptions<T>, deps?: SharedFetchDeps): Promise<{
+    pipeline: PipelineResult<T>;
+    inlineResult: InlineResult;
+}>;
+export declare function createToolErrorResponse(message: string, url: string): ToolErrorResponse;
+export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string): ToolErrorResponse;
+type MarkdownPipelineResult = MarkdownTransformResult & {
+    readonly content: string;
+};
+export declare function parseCachedMarkdownResult(cached: string): MarkdownPipelineResult | undefined;
+export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<ToolResponseBase>;
+export declare function withRequestContextIfMissing<TParams, TResult>(handler: (params: TParams) => Promise<TResult>): (params: TParams) => Promise<TResult>;
+export declare function registerTools(server: McpServer): void;
+export {};