@j0hanz/fetch-url-mcp 1.3.1 → 1.5.0

package/dist/{tool-pipeline.d.ts → lib/fetch-pipeline.d.ts}

@@ -1,6 +1,5 @@
-import type { MarkdownTransformResult } from './transform/types.js';
+import { type MarkdownTransformResult } from '../transform/types.js';
 type JsonRecord = Record<string, unknown>;
-export declare function readUnknown(obj: unknown, key: string): unknown;
 export declare function readString(obj: unknown, key: string): string | undefined;
 export declare function readNestedRecord(obj: unknown, keys: readonly string[]): JsonRecord | undefined;
 export declare function withSignal(signal?: AbortSignal): {
@@ -69,4 +68,3 @@ export declare function performSharedFetch(options: SharedFetchOptions, deps?: S
     inlineResult: InlineContentResult;
 }>;
 export {};
-//# sourceMappingURL=tool-pipeline.d.ts.map

package/dist/{tool-pipeline.js → lib/fetch-pipeline.js}

@@ -1,15 +1,13 @@
 import { z } from 'zod';
-import * as cache from './cache.js';
-import { config } from './config.js';
-import { getErrorMessage } from './errors.js';
-import { fetchNormalizedUrlBuffer, normalizeUrl, transformToRawUrl, } from './fetch.js';
-import { logDebug, logWarn } from './observability.js';
-import { transformBufferToMarkdown } from './transform/transform.js';
-import { isObject } from './type-guards.js';
+import { transformBufferToMarkdown } from '../transform/transform.js';
+import {} from '../transform/types.js';
+import { config, createCacheKey, get, isEnabled, logDebug, logWarn, set, } from './core.js';
+import { fetchNormalizedUrlBuffer, normalizeUrl, transformToRawUrl, } from './http.js';
+import { getErrorMessage, isObject } from './utils.js';
 function asRecord(value) {
     return isObject(value) ? value : undefined;
 }
-export function readUnknown(obj, key) {
+function readUnknown(obj, key) {
     const record = asRecord(obj);
     return record ? record[key] : undefined;
 }
@@ -38,7 +36,7 @@ export function withSignal(signal) {
     return signal === undefined ? {} : { signal };
 }
 /* -------------------------------------------------------------------------------------------------
- * Inline content limiting
+ * Inline content truncation
  * ------------------------------------------------------------------------------------------------- */
 export const TRUNCATION_MARKER = '...[truncated]';
 function getOpenCodeFence(content) {
@@ -147,29 +145,22 @@ const inlineLimiter = new InlineContentLimiter();
 function applyInlineContentLimit(content, inlineLimitOverride) {
     return inlineLimiter.apply(content, inlineLimitOverride);
 }
-function createUrlResolution(params) {
-    return {
-        normalizedUrl: params.normalizedUrl,
-        originalUrl: params.originalUrl,
-        transformed: params.transformed,
-    };
-}
 function resolveNormalizedUrl(url) {
     const { normalizedUrl: validatedUrl } = normalizeUrl(url);
     const transformedResult = transformToRawUrl(validatedUrl);
     if (!transformedResult.transformed) {
-        return createUrlResolution({
+        return {
             normalizedUrl: validatedUrl,
             originalUrl: validatedUrl,
             transformed: false,
-        });
+        };
     }
     const { normalizedUrl: transformedUrl } = normalizeUrl(transformedResult.url);
-    return createUrlResolution({
+    return {
         normalizedUrl: transformedUrl,
         originalUrl: validatedUrl,
         transformed: true,
-    });
+    };
 }
 function logRawUrlTransformation(resolvedUrl) {
     if (!resolvedUrl.transformed)
@@ -178,9 +169,6 @@ function logRawUrlTransformation(resolvedUrl) {
         original: resolvedUrl.originalUrl,
     });
 }
-/* -------------------------------------------------------------------------------------------------
- * Cache helpers
- * ------------------------------------------------------------------------------------------------- */
 function extractTitle(value) {
     return readString(value, 'title');
 }
@@ -207,7 +195,7 @@ function attemptCacheRetrieval(params) {
     const { cacheKey, deserialize, cacheNamespace, normalizedUrl } = params;
     if (!cacheKey)
         return null;
-    const cached = cache.get(cacheKey);
+    const cached = get(cacheKey);
     if (!cached)
         return null;
     if (!deserialize) {
@@ -246,7 +234,7 @@ function persistCache(params) {
         ...(title === undefined ? {} : { title }),
     };
     try {
-        cache.set(cacheKey, serializer(data), metadata, force ? { force: true } : undefined);
+        set(cacheKey, serializer(data), metadata, force ? { force: true } : undefined);
     }
     catch (error) {
         logWarn('Failed to persist cache entry', {
@@ -256,13 +244,10 @@ function persistCache(params) {
         });
     }
 }
-/* -------------------------------------------------------------------------------------------------
- * Pipeline executor
- * ------------------------------------------------------------------------------------------------- */
 export async function executeFetchPipeline(options) {
     const resolvedUrl = resolveNormalizedUrl(options.url);
     logRawUrlTransformation(resolvedUrl);
-    const cacheKey = cache.createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
+    const cacheKey = createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
     if (!options.forceRefresh) {
         const cachedResult = attemptCacheRetrieval({
             cacheKey,
@@ -279,7 +264,7 @@ export async function executeFetchPipeline(options) {
     const resolvedFinalUrl = finalUrl || resolvedUrl.normalizedUrl;
     const transformUrl = resolvedFinalUrl;
     const data = await options.transform({ buffer, encoding, ...(truncated ? { truncated: true } : {}) }, transformUrl);
-    if (cache.isEnabled()) {
+    if (isEnabled()) {
         persistCache({
             cacheKey,
             data,
@@ -288,7 +273,7 @@ export async function executeFetchPipeline(options) {
             cacheNamespace: options.cacheNamespace,
         });
         if (finalUrl && finalUrl !== resolvedUrl.normalizedUrl) {
-            const finalCacheKey = cache.createCacheKey(options.cacheNamespace, finalUrl, options.cacheVary);
+            const finalCacheKey = createCacheKey(options.cacheNamespace, finalUrl, options.cacheVary);
             if (finalCacheKey && finalCacheKey !== cacheKey) {
                 persistCache({
                     cacheKey: finalCacheKey,
@@ -313,18 +298,8 @@ export async function executeFetchPipeline(options) {
 function normalizeExtractedMetadata(metadata) {
     if (!metadata)
         return undefined;
-    const normalized = {
-        ...(metadata.title ? { title: metadata.title } : {}),
-        ...(metadata.description ? { description: metadata.description } : {}),
-        ...(metadata.author ? { author: metadata.author } : {}),
-        ...(metadata.image ? { image: metadata.image } : {}),
-        ...(metadata.favicon ? { favicon: metadata.favicon } : {}),
-        ...(metadata.publishedAt ? { publishedAt: metadata.publishedAt } : {}),
-        ...(metadata.modifiedAt ? { modifiedAt: metadata.modifiedAt } : {}),
-    };
-    if (Object.keys(normalized).length === 0)
-        return undefined;
-    return normalized;
+    const normalized = Object.fromEntries(Object.entries(metadata).filter(([, v]) => Boolean(v)));
+    return Object.keys(normalized).length > 0 ? normalized : undefined;
 }
 const cachedMarkdownSchema = z
     .object({
@@ -405,4 +380,3 @@ export async function performSharedFetch(options, deps = {}) {
     const inlineResult = applyInlineContentLimit(pipeline.data.content, options.maxInlineChars);
     return { pipeline, inlineResult };
 }
-//# sourceMappingURL=tool-pipeline.js.map

package/dist/{fetch.d.ts → lib/http.d.ts}

@@ -1,11 +1,7 @@
-interface FetchOptions {
-    signal?: AbortSignal;
-}
-interface TransformResult {
-    readonly url: string;
-    readonly transformed: boolean;
-    readonly platform?: string;
-}
+import { type ServerResponse } from 'node:http';
+import { type TransformResult } from './url.js';
+export declare function generateSafeFilename(url: string, title?: string, hashFallback?: string, extension?: string): string;
+export declare function handleDownload(res: ServerResponse, namespace: string, hash: string): void;
 interface FetchTelemetryContext {
     requestId: string;
     startTime: number;
@@ -14,6 +10,9 @@ interface FetchTelemetryContext {
     contextRequestId?: string;
     operationId?: string;
 }
+interface FetchOptions {
+    signal?: AbortSignal;
+}
 export declare function isBlockedIp(ip: string): boolean;
 export declare function normalizeUrl(urlString: string): {
     normalizedUrl: string;
@@ -41,4 +40,3 @@ export declare function fetchNormalizedUrlBuffer(normalizedUrl: string, options?
     finalUrl: string;
 }>;
 export {};
-//# sourceMappingURL=fetch.d.ts.map