@j0hanz/fetch-url-mcp 1.4.0 → 1.5.0

package/dist/lib/core.d.ts

@@ -0,0 +1,253 @@
+import { type McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { type StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+import { z } from 'zod';
+export declare const serverVersion: string;
+type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+type TransformWorkerMode = 'threads' | 'process';
+type AuthMode = 'oauth' | 'static';
+interface WorkerResourceLimits {
+    maxOldGenerationSizeMb?: number;
+    maxYoungGenerationSizeMb?: number;
+    codeRangeSizeMb?: number;
+    stackSizeMb?: number;
+}
+interface AuthConfig {
+    mode: AuthMode;
+    issuerUrl: URL | undefined;
+    authorizationUrl: URL | undefined;
+    tokenUrl: URL | undefined;
+    revocationUrl: URL | undefined;
+    registrationUrl: URL | undefined;
+    introspectionUrl: URL | undefined;
+    resourceUrl: URL;
+    requiredScopes: string[];
+    clientId: string | undefined;
+    clientSecret: string | undefined;
+    introspectionTimeoutMs: number;
+    staticTokens: string[];
+}
+interface HttpsConfig {
+    enabled: boolean;
+    keyFile: string | undefined;
+    certFile: string | undefined;
+    caFile: string | undefined;
+}
+interface RuntimeState {
+    httpMode: boolean;
+}
+export declare const config: {
+    server: {
+        name: string;
+        version: string;
+        port: number;
+        host: string;
+        https: HttpsConfig;
+        sessionTtlMs: number;
+        sessionInitTimeoutMs: number;
+        maxSessions: number;
+        http: {
+            headersTimeoutMs: number | undefined;
+            requestTimeoutMs: number | undefined;
+            keepAliveTimeoutMs: number | undefined;
+            keepAliveTimeoutBufferMs: number | undefined;
+            maxHeadersCount: number | undefined;
+            maxConnections: number;
+            blockPrivateConnections: boolean;
+            requireProtocolVersionHeaderOnSessionInit: boolean;
+            shutdownCloseIdleConnections: boolean;
+            shutdownCloseAllConnections: boolean;
+        };
+    };
+    fetcher: {
+        timeout: number;
+        maxRedirects: number;
+        userAgent: string;
+        maxContentLength: number;
+    };
+    transform: {
+        timeoutMs: number;
+        stageWarnRatio: number;
+        metadataFormat: string;
+        maxWorkerScale: number;
+        cancelAckTimeoutMs: number;
+        workerMode: TransformWorkerMode;
+        workerResourceLimits: WorkerResourceLimits | undefined;
+    };
+    tools: {
+        enabled: string[];
+        timeoutMs: number;
+    };
+    tasks: {
+        maxTotal: number;
+        maxPerOwner: number;
+        emitStatusNotifications: boolean;
+    };
+    cache: {
+        enabled: boolean;
+        ttl: number;
+        maxKeys: number;
+        maxSizeBytes: number;
+    };
+    extraction: {
+        maxBlockLength: number;
+        minParagraphLength: number;
+    };
+    noiseRemoval: {
+        extraTokens: string[];
+        extraSelectors: string[];
+        enabledCategories: string[];
+        debug: boolean;
+        aggressiveMode: boolean;
+        preserveSvgCanvas: boolean;
+        weights: {
+            hidden: number;
+            structural: number;
+            promo: number;
+            stickyFixed: number;
+            threshold: number;
+        };
+    };
+    markdownCleanup: {
+        promoteOrphanHeadings: boolean;
+        removeSkipLinks: boolean;
+        removeTocBlocks: boolean;
+        removeTypeDocComments: boolean;
+        headingKeywords: string[];
+    };
+    i18n: {
+        locale: string | undefined;
+    };
+    logging: {
+        level: LogLevel;
+        format: string;
+    };
+    constants: {
+        maxHtmlSize: number;
+        maxUrlLength: number;
+        maxInlineContentChars: number;
+    };
+    security: {
+        blockedHosts: Set<string>;
+        allowedHosts: Set<string>;
+        apiKey: string | undefined;
+        allowRemote: boolean;
+    };
+    auth: AuthConfig;
+    rateLimit: {
+        enabled: boolean;
+        maxRequests: number;
+        windowMs: number;
+        cleanupIntervalMs: number;
+    };
+    runtime: RuntimeState;
+};
+export declare function enableHttpMode(): void;
+declare const CachedPayloadSchema: z.ZodObject<{
+    content: z.ZodOptional<z.ZodString>;
+    markdown: z.ZodOptional<z.ZodString>;
+    title: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+type CachedPayload = z.infer<typeof CachedPayloadSchema>;
+interface CacheEntry {
+    url: string;
+    title?: string;
+    content: string;
+    fetchedAt: string;
+    expiresAt: string;
+}
+interface CacheKeyParts {
+    namespace: string;
+    urlHash: string;
+}
+interface CacheSetOptions {
+    force?: boolean;
+}
+interface CacheGetOptions {
+    force?: boolean;
+}
+interface CacheEntryMetadata {
+    url: string;
+    title?: string;
+}
+interface CacheUpdateEvent {
+    cacheKey: string;
+    namespace: string;
+    urlHash: string;
+    listChanged: boolean;
+}
+type CacheUpdateListener = (event: CacheUpdateEvent) => unknown;
+export declare function parseCachedPayload(raw: string): CachedPayload | null;
+export declare function resolveCachedPayloadContent(payload: CachedPayload): string | null;
+export declare function createCacheKey(namespace: string, url: string, vary?: Record<string, unknown> | string): string | null;
+export declare function parseCacheKey(cacheKey: string): CacheKeyParts | null;
+export declare function onCacheUpdate(listener: CacheUpdateListener): () => void;
+export declare function get(cacheKey: string | null, options?: CacheGetOptions): CacheEntry | undefined;
+export declare function set(cacheKey: string | null, content: string, metadata: CacheEntryMetadata, options?: CacheSetOptions): void;
+export declare function keys(): readonly string[];
+export declare function getEntryMeta(cacheKey: string): {
+    url: string;
+    title?: string;
+} | undefined;
+export declare function isEnabled(): boolean;
+type LogMetadata = Record<string, unknown>;
+interface RequestContext {
+    readonly requestId: string;
+    readonly sessionId?: string;
+    readonly operationId?: string;
+}
+export declare function setMcpServer(server: McpServer): void;
+export declare function registerMcpSessionServer(sessionId: string, server: McpServer): void;
+export declare function unregisterMcpSessionServer(sessionId: string): void;
+export declare function unregisterMcpSessionServerByServer(server: McpServer): void;
+export declare function resolveMcpSessionIdByServer(server: McpServer): string | undefined;
+export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
+export declare function getRequestId(): string | undefined;
+export declare function getOperationId(): string | undefined;
+export declare function logInfo(message: string, meta?: LogMetadata): void;
+export declare function logDebug(message: string, meta?: LogMetadata): void;
+export declare function logWarn(message: string, meta?: LogMetadata): void;
+export declare function logError(message: string, error?: Error | LogMetadata): void;
+export declare function setLogLevel(level: string): void;
+export declare function redactUrl(rawUrl: string): string;
+export interface SessionEntry {
+    readonly server: McpServer;
+    readonly transport: StreamableHTTPServerTransport;
+    createdAt: number;
+    lastSeen: number;
+    protocolInitialized: boolean;
+    negotiatedProtocolVersion: string;
+    authFingerprint: string;
+}
+export interface SessionStore {
+    get: (sessionId: string) => SessionEntry | undefined;
+    touch: (sessionId: string) => void;
+    set: (sessionId: string, entry: SessionEntry) => void;
+    remove: (sessionId: string) => SessionEntry | undefined;
+    size: () => number;
+    inFlight: () => number;
+    incrementInFlight: () => void;
+    decrementInFlight: () => void;
+    clear: () => SessionEntry[];
+    evictExpired: () => SessionEntry[];
+    evictOldest: () => SessionEntry | undefined;
+}
+interface SlotTracker {
+    readonly releaseSlot: () => void;
+    readonly markInitialized: () => void;
+    readonly isInitialized: () => boolean;
+}
+type CloseHandler = (() => void) | undefined;
+export declare function composeCloseHandlers(first: CloseHandler, second: CloseHandler): CloseHandler;
+export declare function startSessionCleanupLoop(store: SessionStore, sessionTtlMs: number, options?: {
+    onEvictSession?: (session: SessionEntry) => Promise<void> | void;
+    cleanupIntervalMs?: number;
+}): AbortController;
+export declare function createSessionStore(sessionTtlMs: number): SessionStore;
+export declare function createSlotTracker(store: SessionStore): SlotTracker;
+export declare function reserveSessionSlot(store: SessionStore, maxSessions: number): boolean;
+export declare function ensureSessionCapacity({ store, maxSessions, evictOldest, }: {
+    store: SessionStore;
+    maxSessions: number;
+    evictOldest: (store: SessionStore) => boolean;
+}): boolean;
+export {};