@j0hanz/fetch-url-mcp 1.11.2 → 1.11.4

package/dist/http/helpers.d.ts

@@ -4,7 +4,7 @@ import type { AuthInfo } from '@modelcontextprotocol/sdk/server/auth/types.js';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
 import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js';
-import type { JsonRpcId } from '../lib/mcp-tools.js';
+import type { JsonRpcId } from '../lib/mcp-interop.js';
 export type NetworkServer = Server | HttpsServer;
 export interface RequestContext {
     req: IncomingMessage;

package/dist/http/helpers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/http/helpers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,WAAW,EAAE,MAAM,YAAY,CAAC;AAKxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gDAAgD,CAAC;AAC/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACzE,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AACxG,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+CAA+C,CAAC;AAQ/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AASrD,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,WAAW,CAAC;AAcjD,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,eAAe,CAAC;IACrB,GAAG,EAAE,cAAc,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,oBAAqB,SAAQ,cAAc;IAC1D,IAAI,EAAE,QAAQ,CAAC;CAChB;AAWD,wBAAgB,QAAQ,CACtB,GAAG,EAAE,cAAc,EACnB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,OAAO,GACZ,IAAI,CAKN;AAED,wBAAgB,SAAS,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAInE;AAED,wBAAgB,SAAS,CACvB,GAAG,EAAE,cAAc,EACnB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,MAAM,SAAM,EACZ,EAAE,GAAE,SAAgB,GACnB,IAAI,CAMN;AAMD,wBAAgB,cAAc,CAC5B,GAAG,EAAE,eAAe,EACpB,IAAI,EAAE,MAAM,GACX,MAAM,GAAG,IAAI,CAIf;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAKnE;AAkBD,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,eAAe,GACnB,MAAM,GAAG,IAAI,CAKf;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CAOvD;AAMD,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,eAAe,GAAG;IAC9D,MAAM,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,MAAM,IAAI,CAAC;CACrB,CAwCA;AAgBD,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,CAoBpE;AAMD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,MAAM,CAAC,EAAE,WAAW,GACnB,cAAc,GAAG,IAAI,CAgBvB;AAMD,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE;IAAE,KAAK,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;CAAE,EAC5C,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,SAAS,EACjB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAgB,sBAAsB,CACpC,aAAa,EAAE,6BAA6B,GAC3C,SAAS,CA4CX;AAMD,KAAK,iBAAiB,GAAG,mBAAmB,GAAG,cAAc,GAAG,aAAa,CAAC;AAE9E,qBAAa,aAAc,SAAQ,KAAK;IACtC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;gBAErB,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM;CAKrD;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAMpD,cAAM,cAAc;IACZ,IAAI,CACR,GAAG,EAAE,eAAe,EACpB,KAAK,SAA2B,EAChC,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,OAAO,CAAC;YA0BL,QAAQ;YAyCR,aAAa;IAkD3B,OAAO,CAAC,cAAc;CAIvB;AAED,eAAO,MAAM,cAAc,gBAAuB,CAAC;AAEnD,UAAU,iBAAiB;IACzB,MAAM,EAAE,SAAS,CAAC;IAClB,SAAS,EAAE,6BAA6B,CAAC;CAC1C;AAED,UAAU,sBAAsB;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAqCD,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,wBAAsB,oCAAoC,CACxD,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,SAAS,EACjB,aAAa,EAAE,MAAM,GACpB,IAAI,CAEN"}
+{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/http/helpers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,WAAW,EAAE,MAAM,YAAY,CAAC;AAKxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gDAAgD,CAAC;AAC/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACzE,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AACxG,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+CAA+C,CAAC;AAQ/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAUvD,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,WAAW,CAAC;AAcjD,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,eAAe,CAAC;IACrB,GAAG,EAAE,cAAc,CAAC;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,oBAAqB,SAAQ,cAAc;IAC1D,IAAI,EAAE,QAAQ,CAAC;CAChB;AAWD,wBAAgB,QAAQ,CACtB,GAAG,EAAE,cAAc,EACnB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,OAAO,GACZ,IAAI,CAKN;AAED,wBAAgB,SAAS,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAInE;AAED,wBAAgB,SAAS,CACvB,GAAG,EAAE,cAAc,EACnB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,MAAM,SAAM,EACZ,EAAE,GAAE,SAAgB,GACnB,IAAI,CAMN;AAMD,wBAAgB,cAAc,CAC5B,GAAG,EAAE,eAAe,EACpB,IAAI,EAAE,MAAM,GACX,MAAM,GAAG,IAAI,CAIf;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAKnE;AAkBD,wBAAgB,8BAA8B,CAC5C,GAAG,EAAE,eAAe,GACnB,MAAM,GAAG,IAAI,CAKf;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CAOvD;AAMD,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,eAAe,GAAG;IAC9D,MAAM,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,MAAM,IAAI,CAAC;CACrB,CAwCA;AAgBD,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,CAoBpE;AAMD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,MAAM,CAAC,EAAE,WAAW,GACnB,cAAc,GAAG,IAAI,CAgBvB;AAMD,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE;IAAE,KAAK,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;CAAE,EAC5C,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,SAAS,EACjB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAgB,sBAAsB,CACpC,aAAa,EAAE,6BAA6B,GAC3C,SAAS,CA4CX;AAMD,KAAK,iBAAiB,GAAG,mBAAmB,GAAG,cAAc,GAAG,aAAa,CAAC;AAE9E,qBAAa,aAAc,SAAQ,KAAK;IACtC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;gBAErB,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM;CAKrD;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAEtE;AAED,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAMpD,cAAM,cAAc;IACZ,IAAI,CACR,GAAG,EAAE,eAAe,EACpB,KAAK,SAA2B,EAChC,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,OAAO,CAAC;YA0BL,QAAQ;YAyCR,aAAa;IAkD3B,OAAO,CAAC,cAAc;CAIvB;AAED,eAAO,MAAM,cAAc,gBAAuB,CAAC;AAEnD,UAAU,iBAAiB;IACzB,MAAM,EAAE,SAAS,CAAC;IAClB,SAAS,EAAE,6BAA6B,CAAC;CAC1C;AAED,UAAU,sBAAsB;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAqCD,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,wBAAsB,oCAAoC,CACxD,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,SAAS,EACjB,aAAa,EAAE,MAAM,GACpB,IAAI,CAEN"}

package/dist/http/helpers.js

@@ -2,9 +2,9 @@ import { Writable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { composeCloseHandlers, config, logWarn } from '../lib/core.js';
 import { resolveMcpSessionIdByServer, unregisterMcpSessionServer, unregisterMcpSessionServerByServer, } from '../lib/core.js';
-import { cancelTasksForOwner } from '../lib/task-handlers.js';
 import { createDefaultBlockList, normalizeIpForBlockList } from '../lib/url.js';
 import { getErrorMessage, toError } from '../lib/utils.js';
+import { cancelTasksForOwner } from '../tasks/handlers.js';
 function abortControllerBestEffort(controller) {
     if (!controller.signal.aborted)
         controller.abort();

package/dist/http/native.js

@@ -11,7 +11,7 @@ import { config, enableHttpMode } from '../lib/core.js';
 import { logError, logInfo, registerMcpSessionServer, runWithRequestContext, } from '../lib/core.js';
 import { composeCloseHandlers, createSessionStore, createSlotTracker, ensureSessionCapacity, reserveSessionSlot, startSessionCleanupLoop, } from '../lib/core.js';
 import { handleDownload } from '../lib/http.js';
-import { acceptsEventStream, acceptsJsonAndEventStream, isJsonRpcBatchRequest, isMcpMessageBody, isMcpRequestBody, } from '../lib/mcp-tools.js';
+import { acceptsEventStream, acceptsJsonAndEventStream, isJsonRpcBatchRequest, isMcpMessageBody, isMcpRequestBody, } from '../lib/mcp-interop.js';
 import { toError } from '../lib/utils.js';
 import { applyHttpServerTuning, drainConnectionsOnShutdown, } from '../lib/utils.js';
 import { isObject } from '../lib/utils.js';

package/dist/lib/config.d.ts

@@ -0,0 +1,152 @@
+export declare const serverVersion: string;
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+type TransformWorkerMode = 'threads' | 'process';
+type AuthMode = 'oauth' | 'static';
+interface WorkerResourceLimits {
+    maxOldGenerationSizeMb?: number;
+    maxYoungGenerationSizeMb?: number;
+    codeRangeSizeMb?: number;
+    stackSizeMb?: number;
+}
+interface AuthConfig {
+    mode: AuthMode;
+    issuerUrl: URL | undefined;
+    authorizationUrl: URL | undefined;
+    tokenUrl: URL | undefined;
+    revocationUrl: URL | undefined;
+    registrationUrl: URL | undefined;
+    introspectionUrl: URL | undefined;
+    resourceUrl: URL;
+    requiredScopes: string[];
+    clientId: string | undefined;
+    clientSecret: string | undefined;
+    introspectionTimeoutMs: number;
+    staticTokens: string[];
+}
+interface HttpsConfig {
+    enabled: boolean;
+    keyFile: string | undefined;
+    certFile: string | undefined;
+    caFile: string | undefined;
+}
+interface RuntimeState {
+    httpMode: boolean;
+}
+interface AppServerHttpConfig {
+    headersTimeoutMs: number | undefined;
+    requestTimeoutMs: number | undefined;
+    keepAliveTimeoutMs: number | undefined;
+    keepAliveTimeoutBufferMs: number | undefined;
+    maxHeadersCount: number | undefined;
+    maxConnections: number;
+    blockPrivateConnections: boolean;
+    shutdownCloseIdleConnections: boolean;
+    shutdownCloseAllConnections: boolean;
+}
+interface AppServerConfig {
+    name: string;
+    version: string;
+    port: number;
+    host: string;
+    https: HttpsConfig;
+    sessionTtlMs: number;
+    sessionInitTimeoutMs: number;
+    maxSessions: number;
+    http: AppServerHttpConfig;
+}
+interface AppFetcherConfig {
+    timeout: number;
+    maxRedirects: number;
+    userAgent: string;
+    maxContentLength: number;
+}
+interface AppTransformConfig {
+    timeoutMs: number;
+    stageWarnRatio: number;
+    metadataFormat: string;
+    maxWorkerScale: number;
+    cancelAckTimeoutMs: number;
+    workerMode: TransformWorkerMode;
+    workerResourceLimits: WorkerResourceLimits | undefined;
+}
+interface AppTasksConfig {
+    maxTotal: number;
+    maxPerOwner: number;
+    emitStatusNotifications: boolean;
+    requireInterception: boolean;
+}
+interface AppCacheConfig {
+    enabled: boolean;
+    ttl: number;
+    maxKeys: number;
+    maxSizeBytes: number;
+}
+interface AppNoiseRemovalConfig {
+    extraTokens: string[];
+    extraSelectors: string[];
+    enabledCategories: string[];
+    debug: boolean;
+    aggressiveMode: boolean;
+    preserveSvgCanvas: boolean;
+    weights: {
+        hidden: number;
+        structural: number;
+        promo: number;
+        stickyFixed: number;
+        threshold: number;
+    };
+}
+interface AppMarkdownCleanupConfig {
+    promoteOrphanHeadings: boolean;
+    removeSkipLinks: boolean;
+    removeTocBlocks: boolean;
+    removeTypeDocComments: boolean;
+    headingKeywords: string[];
+}
+export declare const config: {
+    server: AppServerConfig;
+    fetcher: AppFetcherConfig;
+    transform: AppTransformConfig;
+    tools: {
+        enabled: string[];
+        timeoutMs: number;
+    };
+    tasks: AppTasksConfig;
+    cache: AppCacheConfig;
+    extraction: {
+        maxBlockLength: number;
+        minParagraphLength: number;
+    };
+    noiseRemoval: AppNoiseRemovalConfig;
+    markdownCleanup: AppMarkdownCleanupConfig;
+    i18n: {
+        locale: string | undefined;
+    };
+    logging: {
+        level: LogLevel;
+        format: string;
+    };
+    constants: {
+        maxHtmlBytes: number;
+        maxUrlLength: number;
+        maxInlineContentChars: number;
+    };
+    security: {
+        blockedHosts: Set<string>;
+        allowedHosts: Set<string>;
+        apiKey: string | undefined;
+        allowRemote: boolean;
+        allowLocalFetch: boolean;
+    };
+    auth: AuthConfig;
+    rateLimit: {
+        enabled: boolean;
+        maxRequests: number;
+        windowMs: number;
+        cleanupIntervalMs: number;
+    };
+    runtime: RuntimeState;
+};
+export declare function enableHttpMode(): void;
+export {};
+//# sourceMappingURL=config.d.ts.map

package/dist/lib/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AA4CA,eAAO,MAAM,aAAa,EAAE,MAA2C,CAAC;AAIxE,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AA2B3D,KAAK,mBAAmB,GAAG,SAAS,GAAG,SAAS,CAAC;AACjD,KAAK,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AA4OnC,UAAU,oBAAoB;IAC5B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAyCD,UAAU,UAAU;IAClB,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,GAAG,GAAG,SAAS,CAAC;IAC3B,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,GAAG,GAAG,SAAS,CAAC;IAC1B,aAAa,EAAE,GAAG,GAAG,SAAS,CAAC;IAC/B,eAAe,EAAE,GAAG,GAAG,SAAS,CAAC;IACjC,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,WAAW,EAAE,GAAG,CAAC;IACjB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B;AAoGD,UAAU,YAAY;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAQD,UAAU,mBAAmB;IAC3B,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,EAAE,OAAO,CAAC;IACjC,4BAA4B,EAAE,OAAO,CAAC;IACtC,2BAA2B,EAAE,OAAO,CAAC;CACtC;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,WAAW,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,mBAAmB,CAAC;CAC3B;AAuCD,UAAU,gBAAgB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAWD,UAAU,kBAAkB;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,mBAAmB,CAAC;IAChC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;CACxD;AAkBD,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,OAAO,CAAC;IACjC,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAmBD,UAAU,cAAc;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB;AAWD,UAAU,qBAAqB;IAC7B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,KAAK,EAAE,OAAO,CAAC;IACf,cAAc,EAAE,OAAO,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AA2BD,UAAU,wBAAwB;IAChC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAiBD,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+ClB,CAAC;AAEF,wBAAgB,cAAc,IAAI,IAAI,CAErC"}

package/dist/lib/config.js

@@ -0,0 +1,478 @@
+import { readFileSync } from 'node:fs';
+import { findPackageJSON } from 'node:module';
+import process from 'node:process';
+import { z } from 'zod';
+import { buildIpv4, isIP, normalizeHostname, stripTrailingDots, } from './url.js';
+import { getErrorMessage } from './utils.js';
+// ── Version ─────────────────────────────────────────────────────────
+function hasPackageJsonVersion(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        typeof value.version === 'string');
+}
+function readServerVersion(moduleUrl) {
+    const packageJsonPath = findPackageJSON(moduleUrl);
+    if (!packageJsonPath)
+        throw new Error('package.json not found');
+    let packageJson;
+    try {
+        packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf-8'));
+    }
+    catch (error) {
+        throw new Error(`Failed to parse package.json at ${packageJsonPath}: ${getErrorMessage(error)}`, { cause: error });
+    }
+    if (!hasPackageJsonVersion(packageJson)) {
+        throw new Error(`package.json version is missing at ${packageJsonPath}`);
+    }
+    return packageJson.version;
+}
+export const serverVersion = readServerVersion(import.meta.url);
+const LOG_LEVELS = ['debug', 'info', 'warn', 'error'];
+const ALLOWED_LOG_LEVELS = new Set(LOG_LEVELS);
+const DEFAULT_HEADING_KEYWORDS = [
+    'overview',
+    'introduction',
+    'summary',
+    'conclusion',
+    'prerequisites',
+    'requirements',
+    'installation',
+    'configuration',
+    'usage',
+    'features',
+    'limitations',
+    'troubleshooting',
+    'faq',
+    'resources',
+    'references',
+    'changelog',
+    'license',
+    'acknowledgments',
+    'appendix',
+];
+class ConfigError extends Error {
+    name = 'ConfigError';
+}
+// ── Environment ─────────────────────────────────────────────────────
+function isMissingEnvFileError(error) {
+    if (!error || typeof error !== 'object')
+        return false;
+    const { code } = error;
+    return code === 'ENOENT' || code === 'ERR_ENV_FILE_NOT_FOUND';
+}
+function loadEnvFileIfAvailable() {
+    try {
+        process.loadEnvFile();
+    }
+    catch (error) {
+        if (isMissingEnvFileError(error))
+            return;
+        throw error;
+    }
+}
+loadEnvFileIfAvailable();
+const { env } = process;
+const ENV_BOOLEAN_SCHEMA = z.stringbool({
+    truthy: ['true', '1', 'yes', 'on'],
+    falsy: ['false', '0', 'no', 'off'],
+});
+// ── Host parsing helpers ────────────────────────────────────────────
+function tryParseUrlHost(raw) {
+    if (raw.includes('://')) {
+        const hostname = URL.parse(raw)?.hostname;
+        if (hostname)
+            return normalizeHostname(hostname);
+    }
+    const candidateHostname = URL.parse(`http://${raw}`)?.hostname;
+    if (candidateHostname)
+        return normalizeHostname(candidateHostname);
+    return undefined;
+}
+function tryParseIpv6Bracket(lowered) {
+    if (!lowered.startsWith('['))
+        return undefined;
+    const end = lowered.indexOf(']');
+    return end === -1 ? null : normalizeHostname(lowered.slice(1, end));
+}
+function tryParseHostPort(lowered) {
+    const firstColon = lowered.indexOf(':');
+    if (firstColon === -1)
+        return normalizeHostname(lowered);
+    if (lowered.includes(':', firstColon + 1))
+        return null;
+    const host = lowered.slice(0, firstColon);
+    return host ? normalizeHostname(host) : null;
+}
+// ── EnvParser ───────────────────────────────────────────────────────
+const EnvParser = {
+    integerValue(envValue, opts) {
+        if (!envValue)
+            return null;
+        const parsed = Number.parseInt(envValue, 10);
+        if (Number.isNaN(parsed)) {
+            if (opts?.envName)
+                process.stderr.write(`Warning: ignoring invalid ${opts.envName} value "${envValue}" (not an integer)\n`);
+            return null;
+        }
+        if ((opts?.min !== undefined && parsed < opts.min) ||
+            (opts?.max !== undefined && parsed > opts.max)) {
+            if (opts.envName)
+                process.stderr.write(`Warning: ignoring out-of-range ${opts.envName} value ${parsed} (allowed: ${opts.min ?? '-∞'}..${opts.max ?? '∞'})\n`);
+            return null;
+        }
+        return parsed;
+    },
+    optionalInteger(envValue, opts) {
+        return EnvParser.integerValue(envValue, opts) ?? undefined;
+    },
+    integer(envValue, defaultValue, opts) {
+        return EnvParser.integerValue(envValue, opts) ?? defaultValue;
+    },
+    boolean(envValue, defaultValue, envName) {
+        const trimmed = envValue?.trim();
+        if (!trimmed)
+            return defaultValue;
+        const parsed = ENV_BOOLEAN_SCHEMA.safeParse(trimmed);
+        if (!parsed.success) {
+            if (envName)
+                process.stderr.write(`Warning: ignoring invalid ${envName} value "${envValue ?? ''}" (expected true/false, 1/0, yes/no, or on/off)\n`);
+            return defaultValue;
+        }
+        return parsed.data;
+    },
+    list(envValue, defaultValue) {
+        if (!envValue)
+            return defaultValue ? [...defaultValue] : [];
+        const parsed = envValue
+            .split(/[\s,]+/)
+            .map((e) => e.trim())
+            .filter(Boolean);
+        return parsed.length > 0 || !defaultValue ? parsed : [...defaultValue];
+    },
+    locale(value) {
+        const lowered = value?.trim().toLowerCase();
+        return !lowered || lowered === 'system' || lowered === 'default'
+            ? undefined
+            : lowered;
+    },
+    logLevel(envValue) {
+        const level = envValue?.toLowerCase();
+        if (!level || !ALLOWED_LOG_LEVELS.has(level)) {
+            if (envValue)
+                process.stderr.write(`Warning: ignoring invalid LOG_LEVEL value "${envValue}", using default "info"\n`);
+            return 'info';
+        }
+        return level;
+    },
+    transformWorkerMode(envValue) {
+        const normalized = envValue?.trim().toLowerCase();
+        return normalized === 'process' || normalized === 'fork'
+            ? 'process'
+            : 'threads';
+    },
+    url(value, name) {
+        if (!value)
+            return undefined;
+        const parsed = URL.parse(value);
+        if (!parsed)
+            throw new ConfigError(`Invalid ${name} value: ${value}`);
+        return parsed;
+    },
+    allowedHosts(envValue) {
+        return new Set(EnvParser.list(envValue)
+            .map((h) => EnvParser.normalizeHostValue(h))
+            .filter((h) => h !== null));
+    },
+    optionalFilePath(value) {
+        const trimmed = value?.trim();
+        return trimmed === '' ? undefined : trimmed;
+    },
+    normalizeHostValue(value) {
+        const raw = value.trim();
+        if (!raw)
+            return null;
+        const fromUrl = tryParseUrlHost(raw);
+        if (fromUrl !== undefined)
+            return fromUrl;
+        const lowered = raw.toLowerCase();
+        const fromBracket = tryParseIpv6Bracket(lowered);
+        if (fromBracket !== undefined)
+            return fromBracket;
+        if (isIP(lowered) === 6)
+            return stripTrailingDots(lowered);
+        return tryParseHostPort(lowered);
+    },
+    formatHostForUrl(hostname) {
+        if (hostname.includes(':') && !hostname.startsWith('['))
+            return `[${hostname}]`;
+        return hostname;
+    },
+};
+// ── Constants ───────────────────────────────────────────────────────
+const MAX_HTML_BYTES = 10 * 1024 * 1024;
+const MAX_INLINE_CONTENT_CHARS = EnvParser.integer(env['MAX_INLINE_CONTENT_CHARS'], 0, { min: 0, max: MAX_HTML_BYTES, envName: 'MAX_INLINE_CONTENT_CHARS' });
+const DEFAULT_SESSION_TTL_MS = 30 * 60 * 1000;
+const DEFAULT_SESSION_INIT_TIMEOUT_MS = 10000;
+const DEFAULT_MAX_SESSIONS = 200;
+const DEFAULT_USER_AGENT = `fetch-url-mcp/${serverVersion}`;
+const DEFAULT_TOOL_TIMEOUT_PADDING_MS = 5000;
+const DEFAULT_TRANSFORM_TIMEOUT_MS = 30000;
+const DEFAULT_FETCH_TIMEOUT_MS = EnvParser.integer(env['FETCH_TIMEOUT_MS'], 15000, { min: 1000, max: 60000, envName: 'FETCH_TIMEOUT_MS' });
+const DEFAULT_TOOL_TIMEOUT_MS = DEFAULT_FETCH_TIMEOUT_MS +
+    DEFAULT_TRANSFORM_TIMEOUT_MS +
+    DEFAULT_TOOL_TIMEOUT_PADDING_MS;
+const DEFAULT_TASKS_MAX_TOTAL = EnvParser.integer(env['TASKS_MAX_TOTAL'], 5000, { min: 1, envName: 'TASKS_MAX_TOTAL' });
+const DEFAULT_TASKS_MAX_PER_OWNER = EnvParser.integer(env['TASKS_MAX_PER_OWNER'], 1000, { min: 1, envName: 'TASKS_MAX_PER_OWNER' });
+const RESOLVED_TASKS_MAX_PER_OWNER = Math.min(DEFAULT_TASKS_MAX_PER_OWNER, DEFAULT_TASKS_MAX_TOTAL);
+function resolveWorkerResourceLimits() {
+    const limits = {};
+    const keys = {
+        maxOldGenerationSizeMb: 'TRANSFORM_WORKER_MAX_OLD_GENERATION_MB',
+        maxYoungGenerationSizeMb: 'TRANSFORM_WORKER_MAX_YOUNG_GENERATION_MB',
+        codeRangeSizeMb: 'TRANSFORM_WORKER_CODE_RANGE_MB',
+        stackSizeMb: 'TRANSFORM_WORKER_STACK_MB',
+    };
+    let hasLimits = false;
+    for (const [prop, envKey] of Object.entries(keys)) {
+        const val = EnvParser.optionalInteger(env[envKey], {
+            min: 1,
+            envName: envKey,
+        });
+        if (val !== undefined) {
+            limits[prop] = val;
+            hasLimits = true;
+        }
+    }
+    if (hasLimits)
+        return limits;
+    // If no explicit limits are set, apply a default maxOldGenerationSizeMb based on constrained memory if available.
+    // This helps prevent OOM crashes in memory-constrained environments like AWS Lambda.
+    const constrained = process.constrainedMemory();
+    if (constrained > 0) {
+        const availableMb = Math.floor(constrained / (1024 * 1024));
+        // Set maxOldGenerationSizeMb to 75% of available memory, but no less than 64MB to allow some headroom for the young generation and code.
+        limits.maxOldGenerationSizeMb = Math.max(64, Math.floor(availableMb * 0.75));
+        return limits;
+    }
+    return undefined;
+}
+function buildAuthConfig(baseUrl) {
+    const parseUrl = (key) => EnvParser.url(env[key], key);
+    const issuerUrl = parseUrl('OAUTH_ISSUER_URL');
+    const authorizationUrl = parseUrl('OAUTH_AUTHORIZATION_URL');
+    const tokenUrl = parseUrl('OAUTH_TOKEN_URL');
+    const revocationUrl = parseUrl('OAUTH_REVOCATION_URL');
+    const registrationUrl = parseUrl('OAUTH_REGISTRATION_URL');
+    const introspectionUrl = parseUrl('OAUTH_INTROSPECTION_URL');
+    const resourceUrl = new URL('/mcp', baseUrl);
+    const oauthConfigured = issuerUrl !== undefined ||
+        authorizationUrl !== undefined ||
+        tokenUrl !== undefined ||
+        introspectionUrl !== undefined;
+    const tokens = EnvParser.list(env['ACCESS_TOKENS']);
+    if (env['API_KEY'])
+        tokens.push(env['API_KEY']);
+    return {
+        mode: oauthConfigured ? 'oauth' : 'static',
+        issuerUrl,
+        authorizationUrl,
+        tokenUrl,
+        revocationUrl,
+        registrationUrl,
+        introspectionUrl,
+        resourceUrl,
+        requiredScopes: EnvParser.list(env['OAUTH_REQUIRED_SCOPES']),
+        clientId: env['OAUTH_CLIENT_ID'],
+        clientSecret: env['OAUTH_CLIENT_SECRET'],
+        introspectionTimeoutMs: 5000,
+        staticTokens: [...new Set(tokens)],
+    };
+}
+function buildHttpsConfig() {
+    const keyFile = EnvParser.optionalFilePath(env['SERVER_TLS_KEY_FILE']);
+    const certFile = EnvParser.optionalFilePath(env['SERVER_TLS_CERT_FILE']);
+    const caFile = EnvParser.optionalFilePath(env['SERVER_TLS_CA_FILE']);
+    if ((keyFile && !certFile) || (!keyFile && certFile)) {
+        throw new ConfigError('Both SERVER_TLS_KEY_FILE and SERVER_TLS_CERT_FILE must be set together');
+    }
+    return {
+        enabled: Boolean(keyFile && certFile),
+        keyFile,
+        certFile,
+        caFile,
+    };
+}
+// ── Blocked hosts ───────────────────────────────────────────────────
+const LOOPBACK_V4 = buildIpv4([127, 0, 0, 1]);
+const ANY_V4 = buildIpv4([0, 0, 0, 0]);
+const METADATA_V4_AWS = buildIpv4([169, 254, 169, 254]);
+const METADATA_V4_AZURE = buildIpv4([100, 100, 100, 200]);
+const BLOCKED_HOSTS = new Set([
+    'localhost',
+    LOOPBACK_V4,
+    ANY_V4,
+    '::1',
+    METADATA_V4_AWS,
+    'metadata.google.internal',
+    'metadata.azure.com',
+    METADATA_V4_AZURE,
+    'instance-data',
+]);
+// ── Server binding ──────────────────────────────────────────────────
+const host = (env['HOST'] ?? LOOPBACK_V4).trim();
+const port = env['PORT']?.trim() === '0'
+    ? 0
+    : EnvParser.integer(env['PORT'], 3000, {
+        min: 1024,
+        max: 65535,
+        envName: 'PORT',
+    });
+const httpsConfig = buildHttpsConfig();
+const allowRemote = EnvParser.boolean(env['ALLOW_REMOTE'], false, 'ALLOW_REMOTE');
+const baseUrl = new URL(`${httpsConfig.enabled ? 'https' : 'http'}://${EnvParser.formatHostForUrl(host)}:${port}`);
+const runtimeState = {
+    httpMode: false,
+};
+function buildServerConfig() {
+    const parseOptInt = (key, min = 1) => EnvParser.optionalInteger(env[key], { min, envName: key });
+    return {
+        name: 'fetch-url-mcp',
+        version: serverVersion,
+        port,
+        host,
+        https: httpsConfig,
+        sessionTtlMs: DEFAULT_SESSION_TTL_MS,
+        sessionInitTimeoutMs: DEFAULT_SESSION_INIT_TIMEOUT_MS,
+        maxSessions: DEFAULT_MAX_SESSIONS,
+        http: {
+            headersTimeoutMs: parseOptInt('SERVER_HEADERS_TIMEOUT_MS'),
+            requestTimeoutMs: parseOptInt('SERVER_REQUEST_TIMEOUT_MS', 0),
+            keepAliveTimeoutMs: parseOptInt('SERVER_KEEP_ALIVE_TIMEOUT_MS'),
+            keepAliveTimeoutBufferMs: parseOptInt('SERVER_KEEP_ALIVE_TIMEOUT_BUFFER_MS', 0),
+            maxHeadersCount: parseOptInt('SERVER_MAX_HEADERS_COUNT'),
+            maxConnections: EnvParser.integer(env['SERVER_MAX_CONNECTIONS'], 0, {
+                min: 0,
+                envName: 'SERVER_MAX_CONNECTIONS',
+            }),
+            blockPrivateConnections: EnvParser.boolean(env['SERVER_BLOCK_PRIVATE_CONNECTIONS'], false, 'SERVER_BLOCK_PRIVATE_CONNECTIONS'),
+            shutdownCloseIdleConnections: true,
+            shutdownCloseAllConnections: false,
+        },
+    };
+}
+function buildFetcherConfig() {
+    return {
+        timeout: DEFAULT_FETCH_TIMEOUT_MS,
+        maxRedirects: 5,
+        userAgent: env['USER_AGENT'] ?? DEFAULT_USER_AGENT,
+        maxContentLength: MAX_HTML_BYTES,
+    };
+}
+function buildTransformConfig() {
+    return {
+        timeoutMs: DEFAULT_TRANSFORM_TIMEOUT_MS,
+        stageWarnRatio: 0.5,
+        metadataFormat: 'markdown',
+        maxWorkerScale: 4,
+        cancelAckTimeoutMs: EnvParser.integer(env['TRANSFORM_CANCEL_ACK_TIMEOUT_MS'], 200, { min: 50, max: 5000, envName: 'TRANSFORM_CANCEL_ACK_TIMEOUT_MS' }),
+        workerMode: EnvParser.transformWorkerMode(env['TRANSFORM_WORKER_MODE']),
+        workerResourceLimits: resolveWorkerResourceLimits(),
+    };
+}
+function buildTasksConfig() {
+    return {
+        maxTotal: DEFAULT_TASKS_MAX_TOTAL,
+        maxPerOwner: RESOLVED_TASKS_MAX_PER_OWNER,
+        emitStatusNotifications: EnvParser.boolean(env['TASKS_STATUS_NOTIFICATIONS'], false, 'TASKS_STATUS_NOTIFICATIONS'),
+        requireInterception: EnvParser.boolean(env['TASKS_REQUIRE_INTERCEPTION'], true, 'TASKS_REQUIRE_INTERCEPTION'),
+    };
+}
+function buildCacheConfig() {
+    return {
+        enabled: EnvParser.boolean(env['CACHE_ENABLED'], true, 'CACHE_ENABLED'),
+        ttl: 86400,
+        maxKeys: 100,
+        maxSizeBytes: 50 * 1024 * 1024,
+    };
+}
+function buildNoiseRemovalConfig() {
+    return {
+        extraTokens: EnvParser.list(env['FETCH_URL_MCP_EXTRA_NOISE_TOKENS']),
+        extraSelectors: EnvParser.list(env['FETCH_URL_MCP_EXTRA_NOISE_SELECTORS']),
+        enabledCategories: [
+            'cookie-banners',
+            'newsletters',
+            'social-share',
+            'nav-footer',
+            'author-blocks',
+            'related-content',
+        ],
+        debug: false,
+        aggressiveMode: false,
+        preserveSvgCanvas: false,
+        weights: {
+            hidden: 50,
+            structural: 50,
+            promo: 35,
+            stickyFixed: 30,
+            threshold: 50,
+        },
+    };
+}
+function buildMarkdownCleanupConfig() {
+    return {
+        promoteOrphanHeadings: true,
+        removeSkipLinks: true,
+        removeTocBlocks: true,
+        removeTypeDocComments: true,
+        headingKeywords: EnvParser.list(env['MARKDOWN_HEADING_KEYWORDS'], DEFAULT_HEADING_KEYWORDS),
+    };
+}
+// ── Config object ───────────────────────────────────────────────────
+export const config = {
+    server: buildServerConfig(),
+    fetcher: buildFetcherConfig(),
+    transform: buildTransformConfig(),
+    tools: {
+        enabled: ['fetch-url'],
+        timeoutMs: DEFAULT_TOOL_TIMEOUT_MS,
+    },
+    tasks: buildTasksConfig(),
+    cache: buildCacheConfig(),
+    extraction: {
+        maxBlockLength: 5000,
+        minParagraphLength: 10,
+    },
+    noiseRemoval: buildNoiseRemovalConfig(),
+    markdownCleanup: buildMarkdownCleanupConfig(),
+    i18n: {
+        locale: EnvParser.locale(env['FETCH_URL_MCP_LOCALE']),
+    },
+    logging: {
+        level: EnvParser.logLevel(env['LOG_LEVEL']),
+        format: env['LOG_FORMAT']?.toLowerCase() === 'json' ? 'json' : 'text',
+    },
+    constants: {
+        maxHtmlBytes: MAX_HTML_BYTES,
+        maxUrlLength: 2048,
+        maxInlineContentChars: MAX_INLINE_CONTENT_CHARS,
+    },
+    security: {
+        blockedHosts: BLOCKED_HOSTS,
+        allowedHosts: EnvParser.allowedHosts(env['ALLOWED_HOSTS']),
+        apiKey: env['API_KEY'],
+        allowRemote,
+        allowLocalFetch: EnvParser.boolean(env['ALLOW_LOCAL_FETCH'], false, 'ALLOW_LOCAL_FETCH'),
+    },
+    auth: buildAuthConfig(baseUrl),
+    rateLimit: {
+        enabled: true,
+        maxRequests: 100,
+        windowMs: 60000,
+        cleanupIntervalMs: 60000,
+    },
+    runtime: runtimeState,
+};
+export function enableHttpMode() {
+    runtimeState.httpMode = true;
+}