@j0hanz/superfetch 2.0.1 → 2.1.0

package/dist/tools.js

@@ -0,0 +1,421 @@
+import { randomUUID } from 'node:crypto';
+import { z } from 'zod';
+import * as cache from './cache.js';
+import { config } from './config.js';
+import { FetchError, isSystemError } from './errors.js';
+import { fetchNormalizedUrl, normalizeUrl, transformToRawUrl, } from './fetch.js';
+import { getRequestId, logDebug, logError, runWithRequestContext, } from './observability.js';
+import { transformHtmlToMarkdown, } from './transform.js';
+const TRUNCATION_MARKER = '...[truncated]';
+const fetchUrlInputSchema = z.strictObject({
+    url: z.url({ protocol: /^https?$/i }).describe('The URL to fetch'),
+});
+const fetchUrlOutputSchema = z.strictObject({
+    url: z.string().describe('The fetched URL'),
+    inputUrl: z
+        .string()
+        .optional()
+        .describe('The original URL provided by the caller'),
+    resolvedUrl: z
+        .string()
+        .optional()
+        .describe('The normalized or transformed URL that was fetched'),
+    title: z.string().optional().describe('Page title'),
+    markdown: z
+        .string()
+        .optional()
+        .describe('The extracted content in Markdown format'),
+    error: z.string().optional().describe('Error message if the request failed'),
+});
+export const FETCH_URL_TOOL_NAME = 'fetch-url';
+export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
+}
+function serializeStructuredContent(structuredContent, fromCache) {
+    return JSON.stringify(structuredContent, fromCache ? undefined : null, fromCache ? undefined : 2);
+}
+function buildResourceLink(inlineResult, name) {
+    if (!inlineResult.resourceUri) {
+        return null;
+    }
+    const block = {
+        type: 'resource_link',
+        uri: inlineResult.resourceUri,
+        name,
+        description: `Content exceeds inline limit (${config.constants.maxInlineContentChars} chars)`,
+    };
+    if (inlineResult.resourceMimeType !== undefined) {
+        block.mimeType = inlineResult.resourceMimeType;
+    }
+    return block;
+}
+function buildEmbeddedResource(content, url, title) {
+    if (!content) {
+        return null;
+    }
+    const filename = cache.generateSafeFilename(url, title, undefined, '.md');
+    const uri = `file:///${filename}`;
+    return {
+        type: 'resource',
+        resource: {
+            uri,
+            mimeType: 'text/markdown',
+            text: content,
+        },
+    };
+}
+function resolveContentToEmbed(inlineResult, fullContent, useInlineInHttpMode) {
+    if (useInlineInHttpMode) {
+        return inlineResult.content;
+    }
+    return fullContent ?? inlineResult.content;
+}
+function maybeAppendEmbeddedResource(blocks, contentToEmbed, url, title) {
+    if (typeof contentToEmbed !== 'string')
+        return;
+    if (!url)
+        return;
+    const embeddedResource = buildEmbeddedResource(contentToEmbed, url, title);
+    if (embeddedResource) {
+        blocks.push(embeddedResource);
+    }
+}
+function maybeAppendResourceLink(blocks, inlineResult, resourceName) {
+    const resourceLink = buildResourceLink(inlineResult, resourceName);
+    if (resourceLink) {
+        blocks.push(resourceLink);
+    }
+}
+function buildTextBlock(structuredContent, fromCache) {
+    return {
+        type: 'text',
+        text: serializeStructuredContent(structuredContent, fromCache),
+    };
+}
+function buildToolContentBlocks(structuredContent, fromCache, inlineResult, resourceName, cacheKey, fullContent, url, title) {
+    const blocks = [
+        buildTextBlock(structuredContent, fromCache),
+    ];
+    const contentToEmbed = resolveContentToEmbed(inlineResult, fullContent, config.runtime.httpMode);
+    maybeAppendEmbeddedResource(blocks, contentToEmbed, url, title);
+    maybeAppendResourceLink(blocks, inlineResult, resourceName);
+    return blocks;
+}
+function applyInlineContentLimit(content, cacheKey) {
+    const contentSize = content.length;
+    const inlineLimit = config.constants.maxInlineContentChars;
+    if (contentSize <= inlineLimit) {
+        return { content, contentSize };
+    }
+    const resourceUri = resolveResourceUri(cacheKey);
+    if (!resourceUri) {
+        return buildTruncatedFallback(content, contentSize, inlineLimit);
+    }
+    return {
+        contentSize,
+        resourceUri,
+        resourceMimeType: 'text/markdown',
+    };
+}
+function resolveResourceUri(cacheKey) {
+    if (!cache.isEnabled() || !cacheKey)
+        return null;
+    return cache.toResourceUri(cacheKey);
+}
+function buildTruncatedFallback(content, contentSize, inlineLimit) {
+    const maxContentLength = Math.max(0, inlineLimit - TRUNCATION_MARKER.length);
+    const truncatedContent = content.length > inlineLimit
+        ? `${content.substring(0, maxContentLength)}${TRUNCATION_MARKER}`
+        : content;
+    return {
+        content: truncatedContent,
+        contentSize,
+        truncated: true,
+    };
+}
+function attemptCacheRetrieval({ cacheKey, deserialize, cacheNamespace, normalizedUrl, }) {
+    if (!cacheKey)
+        return null;
+    const cached = cache.get(cacheKey);
+    if (!cached)
+        return null;
+    if (!deserialize) {
+        logCacheMiss('missing deserializer', cacheNamespace, normalizedUrl);
+        return null;
+    }
+    const data = deserialize(cached.content);
+    if (data === undefined) {
+        logCacheMiss('deserialize failure', cacheNamespace, normalizedUrl);
+        return null;
+    }
+    logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
+    return {
+        data,
+        fromCache: true,
+        url: normalizedUrl,
+        fetchedAt: cached.fetchedAt,
+        cacheKey,
+    };
+}
+function resolveNormalizedUrl(url) {
+    const { normalizedUrl: validatedUrl } = normalizeUrl(url);
+    const { url: normalizedUrl, transformed } = transformToRawUrl(validatedUrl);
+    return { normalizedUrl, originalUrl: validatedUrl, transformed };
+}
+export async function executeFetchPipeline(options) {
+    const resolvedUrl = resolveNormalizedUrl(options.url);
+    logRawUrlTransformation(resolvedUrl);
+    const cacheKey = cache.createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
+    const cachedResult = attemptCacheRetrieval({
+        cacheKey,
+        deserialize: options.deserialize,
+        cacheNamespace: options.cacheNamespace,
+        normalizedUrl: resolvedUrl.normalizedUrl,
+    });
+    if (cachedResult)
+        return cachedResult;
+    logDebug('Fetching URL', { url: resolvedUrl.normalizedUrl });
+    const fetchOptions = options.signal === undefined ? {} : { signal: options.signal };
+    const html = await fetchNormalizedUrl(resolvedUrl.normalizedUrl, fetchOptions);
+    const data = await options.transform(html, resolvedUrl.normalizedUrl);
+    if (cache.isEnabled()) {
+        persistCache({
+            cacheKey,
+            data,
+            serialize: options.serialize,
+            normalizedUrl: resolvedUrl.normalizedUrl,
+        });
+    }
+    return {
+        data,
+        fromCache: false,
+        url: resolvedUrl.normalizedUrl,
+        fetchedAt: new Date().toISOString(),
+        cacheKey,
+    };
+}
+function persistCache({ cacheKey, data, serialize, normalizedUrl, }) {
+    if (!cacheKey)
+        return;
+    const serializer = serialize ?? JSON.stringify;
+    const title = extractTitle(data);
+    const metadata = {
+        url: normalizedUrl,
+        ...(title === undefined ? {} : { title }),
+    };
+    cache.set(cacheKey, serializer(data), metadata);
+}
+function extractTitle(value) {
+    if (!isRecord(value))
+        return undefined;
+    const { title } = value;
+    return typeof title === 'string' ? title : undefined;
+}
+function logCacheMiss(reason, cacheNamespace, normalizedUrl) {
+    logDebug(`Cache miss due to ${reason}`, {
+        namespace: cacheNamespace,
+        url: normalizedUrl,
+    });
+}
+function logRawUrlTransformation(resolvedUrl) {
+    if (!resolvedUrl.transformed)
+        return;
+    logDebug('Using transformed raw content URL', {
+        original: resolvedUrl.originalUrl,
+    });
+}
+function applyOptionalPipelineSerialization(pipelineOptions, options) {
+    if (options.serialize !== undefined) {
+        pipelineOptions.serialize = options.serialize;
+    }
+    if (options.deserialize !== undefined) {
+        pipelineOptions.deserialize = options.deserialize;
+    }
+}
+export async function performSharedFetch(options, deps = {}) {
+    const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
+    const pipelineOptions = {
+        url: options.url,
+        cacheNamespace: 'markdown',
+        transform: options.transform,
+    };
+    applyOptionalPipelineSerialization(pipelineOptions, options);
+    const pipeline = await executePipeline(pipelineOptions);
+    const inlineResult = applyInlineContentLimit(pipeline.data.content, pipeline.cacheKey ?? null);
+    return { pipeline, inlineResult };
+}
+export function createToolErrorResponse(message, url) {
+    const structuredContent = {
+        error: message,
+        url,
+    };
+    return {
+        content: [buildTextBlock(structuredContent, true)],
+        structuredContent,
+        isError: true,
+    };
+}
+export function handleToolError(error, url, fallbackMessage = 'Operation failed') {
+    const message = resolveToolErrorMessage(error, fallbackMessage);
+    return createToolErrorResponse(message, url);
+}
+function isValidationError(error) {
+    return (error instanceof Error &&
+        isSystemError(error) &&
+        error.code === 'VALIDATION_ERROR');
+}
+function resolveToolErrorMessage(error, fallbackMessage) {
+    if (isValidationError(error) || error instanceof FetchError) {
+        return error.message;
+    }
+    if (error instanceof Error) {
+        return `${fallbackMessage}: ${error.message}`;
+    }
+    return `${fallbackMessage}: Unknown error`;
+}
+function parseJsonRecord(input) {
+    try {
+        const parsed = JSON.parse(input);
+        return isRecord(parsed) ? parsed : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function resolveMarkdownContent(parsed) {
+    const { markdown } = parsed;
+    if (typeof markdown === 'string')
+        return markdown;
+    const { content } = parsed;
+    if (typeof content === 'string')
+        return content;
+    return undefined;
+}
+function resolveOptionalTitle(parsed) {
+    const { title } = parsed;
+    if (title === undefined)
+        return undefined;
+    return typeof title === 'string' ? title : undefined;
+}
+function resolveTruncatedFlag(parsed) {
+    const { truncated } = parsed;
+    return typeof truncated === 'boolean' ? truncated : false;
+}
+export function parseCachedMarkdownResult(cached) {
+    const parsed = parseJsonRecord(cached);
+    if (!parsed)
+        return undefined;
+    const resolvedContent = resolveMarkdownContent(parsed);
+    if (resolvedContent === undefined)
+        return undefined;
+    const title = resolveOptionalTitle(parsed);
+    if (parsed.title !== undefined && title === undefined)
+        return undefined;
+    return {
+        content: resolvedContent,
+        markdown: resolvedContent,
+        title,
+        truncated: resolveTruncatedFlag(parsed),
+    };
+}
+function deserializeMarkdownResult(cached) {
+    return parseCachedMarkdownResult(cached);
+}
+function buildMarkdownTransform() {
+    return async (html, url) => {
+        const result = await transformHtmlToMarkdown(html, url, {
+            includeMetadata: true,
+        });
+        return { ...result, content: result.markdown };
+    };
+}
+function serializeMarkdownResult(result) {
+    return JSON.stringify({
+        markdown: result.markdown,
+        title: result.title,
+        truncated: result.truncated,
+    });
+}
+function buildStructuredContent(pipeline, inlineResult, inputUrl) {
+    return {
+        url: pipeline.url,
+        resolvedUrl: pipeline.url,
+        inputUrl,
+        title: pipeline.data.title,
+        markdown: inlineResult.content,
+    };
+}
+function buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult) {
+    return buildToolContentBlocks(structuredContent, pipeline.fromCache, inlineResult, 'Fetched markdown', pipeline.cacheKey, pipeline.data.content, pipeline.url, pipeline.data.title);
+}
+function logFetchStart(url) {
+    logDebug('Fetching URL', { url });
+}
+async function fetchPipeline(url) {
+    return performSharedFetch({
+        url,
+        transform: buildMarkdownTransform(),
+        serialize: serializeMarkdownResult,
+        deserialize: deserializeMarkdownResult,
+    });
+}
+function buildResponse(pipeline, inlineResult, inputUrl) {
+    const structuredContent = buildStructuredContent(pipeline, inlineResult, inputUrl);
+    const content = buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult);
+    return {
+        content,
+        structuredContent,
+    };
+}
+async function executeFetch(input) {
+    const { url } = input;
+    if (!url) {
+        return createToolErrorResponse('URL is required', '');
+    }
+    logFetchStart(url);
+    const { pipeline, inlineResult } = await fetchPipeline(url);
+    if (inlineResult.error) {
+        return createToolErrorResponse(inlineResult.error, url);
+    }
+    return buildResponse(pipeline, inlineResult, url);
+}
+export async function fetchUrlToolHandler(input) {
+    return executeFetch(input).catch((error) => {
+        logError('fetch-url tool error', error instanceof Error ? error : undefined);
+        return handleToolError(error, input.url, 'Failed to fetch URL');
+    });
+}
+const TOOL_DEFINITION = {
+    name: FETCH_URL_TOOL_NAME,
+    title: 'Fetch URL',
+    description: FETCH_URL_TOOL_DESCRIPTION,
+    inputSchema: fetchUrlInputSchema,
+    outputSchema: fetchUrlOutputSchema,
+    handler: fetchUrlToolHandler,
+    annotations: {
+        readOnlyHint: true,
+        destructiveHint: false,
+        idempotentHint: true,
+        openWorldHint: true,
+    },
+};
+export function withRequestContextIfMissing(handler) {
+    return async (params) => {
+        const existingRequestId = getRequestId();
+        if (existingRequestId) {
+            return handler(params);
+        }
+        const requestId = randomUUID();
+        return runWithRequestContext({ requestId, operationId: requestId }, () => handler(params));
+    };
+}
+export function registerTools(server) {
+    server.registerTool(TOOL_DEFINITION.name, {
+        title: TOOL_DEFINITION.title,
+        description: TOOL_DEFINITION.description,
+        inputSchema: TOOL_DEFINITION.inputSchema,
+        outputSchema: TOOL_DEFINITION.outputSchema,
+        annotations: TOOL_DEFINITION.annotations,
+    }, withRequestContextIfMissing(TOOL_DEFINITION.handler));
+}

package/dist/transform.d.ts

@@ -0,0 +1,69 @@
+export interface MetadataBlock {
+    type: 'metadata';
+    title?: string;
+    description?: string;
+    author?: string;
+    url: string;
+    fetchedAt: string;
+}
+export interface ExtractedArticle {
+    title?: string;
+    byline?: string;
+    content: string;
+    textContent: string;
+    excerpt?: string;
+    siteName?: string;
+}
+export interface ExtractedMetadata {
+    title?: string;
+    description?: string;
+    author?: string;
+}
+export interface ExtractionResult {
+    article: ExtractedArticle | null;
+    metadata: ExtractedMetadata;
+}
+export interface MarkdownTransformResult {
+    markdown: string;
+    title: string | undefined;
+    truncated: boolean;
+}
+export interface TransformOptions {
+    includeMetadata: boolean;
+    signal?: AbortSignal;
+}
+export interface TransformStageEvent {
+    v: 1;
+    type: 'stage';
+    stage: string;
+    durationMs: number;
+    url: string;
+    requestId?: string;
+    operationId?: string;
+    truncated?: boolean;
+}
+export interface TransformStageContext {
+    readonly stage: string;
+    readonly startTime: number;
+    readonly url: string;
+}
+export declare function startTransformStage(url: string, stage: string): TransformStageContext | null;
+export declare function endTransformStage(context: TransformStageContext | null, options?: {
+    truncated?: boolean;
+}): void;
+export declare function extractContent(html: string, url: string, options?: {
+    extractArticle?: boolean;
+    signal?: AbortSignal;
+}): ExtractionResult;
+export declare function detectLanguageFromCode(code: string): string | undefined;
+export declare function resolveLanguageFromAttributes(className: string, dataLang: string): string | undefined;
+export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock, options?: {
+    url?: string;
+    signal?: AbortSignal;
+}): string;
+export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtml: string): boolean;
+export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
+export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
+export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;
+export declare function shutdownTransformWorkerPool(): Promise<void>;
+export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;