@j0hanz/superfetch 2.0.1 → 2.1.0

package/dist/config.d.ts

@@ -0,0 +1,77 @@
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+interface AuthConfig {
+    mode: 'oauth' | 'static';
+    issuerUrl: URL | undefined;
+    authorizationUrl: URL | undefined;
+    tokenUrl: URL | undefined;
+    revocationUrl: URL | undefined;
+    registrationUrl: URL | undefined;
+    introspectionUrl: URL | undefined;
+    resourceUrl: URL;
+    requiredScopes: string[];
+    clientId: string | undefined;
+    clientSecret: string | undefined;
+    introspectionTimeoutMs: number;
+    staticTokens: string[];
+}
+interface RuntimeState {
+    httpMode: boolean;
+}
+export declare const config: {
+    server: {
+        name: string;
+        version: string;
+        port: number;
+        host: string;
+        sessionTtlMs: number;
+        sessionInitTimeoutMs: number;
+        maxSessions: number;
+        http: {
+            headersTimeoutMs: number | undefined;
+            requestTimeoutMs: number | undefined;
+            keepAliveTimeoutMs: number | undefined;
+            shutdownCloseIdleConnections: boolean;
+            shutdownCloseAllConnections: boolean;
+        };
+    };
+    fetcher: {
+        timeout: number;
+        maxRedirects: number;
+        userAgent: string;
+        maxContentLength: number;
+    };
+    cache: {
+        enabled: boolean;
+        ttl: number;
+        maxKeys: number;
+    };
+    extraction: {
+        maxBlockLength: number;
+        minParagraphLength: number;
+    };
+    logging: {
+        level: LogLevel;
+    };
+    constants: {
+        maxHtmlSize: number;
+        maxUrlLength: number;
+        maxInlineContentChars: number;
+    };
+    security: {
+        blockedHosts: Set<string>;
+        blockedIpPatterns: RegExp[];
+        allowedHosts: Set<string>;
+        apiKey: string | undefined;
+        allowRemote: boolean;
+    };
+    auth: AuthConfig;
+    rateLimit: {
+        enabled: boolean;
+        maxRequests: number;
+        windowMs: number;
+        cleanupIntervalMs: number;
+    };
+    runtime: RuntimeState;
+};
+export declare function enableHttpMode(): void;
+export {};

package/dist/config.js

@@ -0,0 +1,261 @@
+import packageJson from '../package.json' with { type: 'json' };
+function buildIpv4(parts) {
+    return parts.join('.');
+}
+function formatHostForUrl(hostname) {
+    if (hostname.includes(':') && !hostname.startsWith('[')) {
+        return `[${hostname}]`;
+    }
+    return hostname;
+}
+function normalizeHostValue(value) {
+    const trimmed = value.trim().toLowerCase();
+    if (!trimmed)
+        return null;
+    if (trimmed.startsWith('[')) {
+        const end = trimmed.indexOf(']');
+        if (end === -1)
+            return null;
+        return trimmed.slice(1, end);
+    }
+    const colonIndex = trimmed.indexOf(':');
+    if (colonIndex !== -1) {
+        return trimmed.slice(0, colonIndex);
+    }
+    return trimmed;
+}
+const ALLOWED_LOG_LEVELS = new Set([
+    'debug',
+    'info',
+    'warn',
+    'error',
+]);
+function isLogLevel(value) {
+    return ALLOWED_LOG_LEVELS.has(value);
+}
+function isBelowMin(value, min) {
+    if (min === undefined)
+        return false;
+    return value < min;
+}
+function isAboveMax(value, max) {
+    if (max === undefined)
+        return false;
+    return value > max;
+}
+function parseInteger(envValue, defaultValue, min, max) {
+    if (!envValue)
+        return defaultValue;
+    const parsed = parseInt(envValue, 10);
+    if (Number.isNaN(parsed))
+        return defaultValue;
+    if (isBelowMin(parsed, min))
+        return defaultValue;
+    if (isAboveMax(parsed, max))
+        return defaultValue;
+    return parsed;
+}
+function parseOptionalInteger(envValue, min, max) {
+    if (!envValue)
+        return undefined;
+    const parsed = parseInt(envValue, 10);
+    if (Number.isNaN(parsed))
+        return undefined;
+    if (isBelowMin(parsed, min))
+        return undefined;
+    if (isAboveMax(parsed, max))
+        return undefined;
+    return parsed;
+}
+function parseBoolean(envValue, defaultValue) {
+    if (!envValue)
+        return defaultValue;
+    return envValue !== 'false';
+}
+function parseList(envValue) {
+    if (!envValue)
+        return [];
+    return envValue
+        .split(/[\s,]+/)
+        .map((entry) => entry.trim())
+        .filter((entry) => entry.length > 0);
+}
+function parseUrlEnv(value, name) {
+    if (!value)
+        return undefined;
+    if (!URL.canParse(value)) {
+        throw new Error(`Invalid ${name} value: ${value}`);
+    }
+    return new URL(value);
+}
+function parseAllowedHosts(envValue) {
+    const hosts = new Set();
+    for (const entry of parseList(envValue)) {
+        const normalized = normalizeHostValue(entry);
+        if (normalized) {
+            hosts.add(normalized);
+        }
+    }
+    return hosts;
+}
+function parseLogLevel(envValue) {
+    const level = envValue?.toLowerCase();
+    if (!level)
+        return 'info';
+    return isLogLevel(level) ? level : 'info';
+}
+const SIZE_LIMITS = {
+    TEN_MB: 10 * 1024 * 1024,
+};
+const TIMEOUT = {
+    DEFAULT_FETCH_TIMEOUT_MS: 15000,
+    DEFAULT_SESSION_TTL_MS: 30 * 60 * 1000,
+};
+function readCoreOAuthUrls() {
+    return {
+        issuerUrl: parseUrlEnv(process.env.OAUTH_ISSUER_URL, 'OAUTH_ISSUER_URL'),
+        authorizationUrl: parseUrlEnv(process.env.OAUTH_AUTHORIZATION_URL, 'OAUTH_AUTHORIZATION_URL'),
+        tokenUrl: parseUrlEnv(process.env.OAUTH_TOKEN_URL, 'OAUTH_TOKEN_URL'),
+    };
+}
+function readOptionalOAuthUrls(baseUrl) {
+    return {
+        revocationUrl: parseUrlEnv(process.env.OAUTH_REVOCATION_URL, 'OAUTH_REVOCATION_URL'),
+        registrationUrl: parseUrlEnv(process.env.OAUTH_REGISTRATION_URL, 'OAUTH_REGISTRATION_URL'),
+        introspectionUrl: parseUrlEnv(process.env.OAUTH_INTROSPECTION_URL, 'OAUTH_INTROSPECTION_URL'),
+        resourceUrl: parseUrlEnv(process.env.OAUTH_RESOURCE_URL, 'OAUTH_RESOURCE_URL') ??
+            new URL('/mcp', baseUrl),
+    };
+}
+function readOAuthUrls(baseUrl) {
+    return { ...readCoreOAuthUrls(), ...readOptionalOAuthUrls(baseUrl) };
+}
+function resolveAuthMode(authModeEnv, urls) {
+    if (authModeEnv === 'oauth')
+        return 'oauth';
+    if (authModeEnv === 'static')
+        return 'static';
+    const oauthConfigured = [
+        urls.issuerUrl,
+        urls.authorizationUrl,
+        urls.tokenUrl,
+        urls.introspectionUrl,
+    ].some((value) => value !== undefined);
+    return oauthConfigured ? 'oauth' : 'static';
+}
+function collectStaticTokens() {
+    const staticTokens = new Set(parseList(process.env.ACCESS_TOKENS));
+    if (process.env.API_KEY) {
+        staticTokens.add(process.env.API_KEY);
+    }
+    return Array.from(staticTokens);
+}
+function buildAuthConfig(baseUrl) {
+    const urls = readOAuthUrls(baseUrl);
+    const mode = resolveAuthMode(process.env.AUTH_MODE?.toLowerCase(), urls);
+    return {
+        mode,
+        ...urls,
+        requiredScopes: parseList(process.env.OAUTH_REQUIRED_SCOPES),
+        clientId: process.env.OAUTH_CLIENT_ID,
+        clientSecret: process.env.OAUTH_CLIENT_SECRET,
+        introspectionTimeoutMs: parseInteger(process.env.OAUTH_INTROSPECTION_TIMEOUT_MS, 5000, 1000, 30000),
+        staticTokens: collectStaticTokens(),
+    };
+}
+const LOOPBACK_V4 = buildIpv4([127, 0, 0, 1]);
+const ANY_V4 = buildIpv4([0, 0, 0, 0]);
+const METADATA_V4_AWS = buildIpv4([169, 254, 169, 254]);
+const METADATA_V4_AZURE = buildIpv4([100, 100, 100, 200]);
+const host = process.env.HOST ?? LOOPBACK_V4;
+const port = parseInteger(process.env.PORT, 3000, 1024, 65535);
+const baseUrl = new URL(`http://${formatHostForUrl(host)}:${port}`);
+const allowRemote = parseBoolean(process.env.ALLOW_REMOTE, false);
+const runtimeState = {
+    httpMode: false,
+};
+export const config = {
+    server: {
+        name: 'superFetch',
+        version: packageJson.version,
+        port,
+        host,
+        sessionTtlMs: TIMEOUT.DEFAULT_SESSION_TTL_MS,
+        sessionInitTimeoutMs: 10000,
+        maxSessions: 200,
+        http: {
+            headersTimeoutMs: parseOptionalInteger(process.env.SERVER_HEADERS_TIMEOUT_MS, 1000, 600000),
+            requestTimeoutMs: parseOptionalInteger(process.env.SERVER_REQUEST_TIMEOUT_MS, 1000, 600000),
+            keepAliveTimeoutMs: parseOptionalInteger(process.env.SERVER_KEEP_ALIVE_TIMEOUT_MS, 1000, 600000),
+            shutdownCloseIdleConnections: parseBoolean(process.env.SERVER_SHUTDOWN_CLOSE_IDLE, false),
+            shutdownCloseAllConnections: parseBoolean(process.env.SERVER_SHUTDOWN_CLOSE_ALL, false),
+        },
+    },
+    fetcher: {
+        timeout: TIMEOUT.DEFAULT_FETCH_TIMEOUT_MS,
+        maxRedirects: 5,
+        userAgent: process.env.USER_AGENT ?? 'superFetch-MCP/2.0',
+        maxContentLength: SIZE_LIMITS.TEN_MB,
+    },
+    cache: {
+        enabled: parseBoolean(process.env.CACHE_ENABLED, true),
+        ttl: parseInteger(process.env.CACHE_TTL, 3600, 60, 86400),
+        maxKeys: 100,
+    },
+    extraction: {
+        maxBlockLength: 5000,
+        minParagraphLength: 10,
+    },
+    logging: {
+        level: parseLogLevel(process.env.LOG_LEVEL),
+    },
+    constants: {
+        maxHtmlSize: SIZE_LIMITS.TEN_MB,
+        maxUrlLength: 2048,
+        maxInlineContentChars: 20000,
+    },
+    security: {
+        blockedHosts: new Set([
+            'localhost',
+            LOOPBACK_V4,
+            ANY_V4,
+            '::1',
+            METADATA_V4_AWS,
+            'metadata.google.internal',
+            'metadata.azure.com',
+            METADATA_V4_AZURE,
+            'instance-data',
+        ]),
+        blockedIpPatterns: [
+            /^10\./,
+            /^172\.(1[6-9]|2\d|3[01])\./,
+            /^192\.168\./,
+            /^127\./,
+            /^0\./,
+            /^169\.254\./,
+            /^100\.64\./,
+            /^fc00:/i,
+            /^fd00:/i,
+            /^fe80:/i,
+            /^::ffff:127\./,
+            /^::ffff:10\./,
+            /^::ffff:172\.(1[6-9]|2\d|3[01])\./,
+            /^::ffff:192\.168\./,
+            /^::ffff:169\.254\./,
+        ],
+        allowedHosts: parseAllowedHosts(process.env.ALLOWED_HOSTS),
+        apiKey: process.env.API_KEY,
+        allowRemote,
+    },
+    auth: buildAuthConfig(baseUrl),
+    rateLimit: {
+        enabled: true,
+        maxRequests: 100,
+        windowMs: 60000,
+        cleanupIntervalMs: 60000,
+    },
+    runtime: runtimeState,
+};
+export function enableHttpMode() {
+    runtimeState.httpMode = true;
+}

package/dist/crypto.d.ts

@@ -0,0 +1,2 @@
+export declare function timingSafeEqualUtf8(a: string, b: string): boolean;
+export declare function sha256Hex(input: string | Uint8Array): string;

package/dist/crypto.js

@@ -0,0 +1,32 @@
+import { createHash, hash as oneShotHash, timingSafeEqual } from 'node:crypto';
+const MAX_HASH_INPUT_BYTES = 5 * 1024 * 1024;
+const ALLOWED_HASH_ALGORITHMS = new Set([
+    'sha256',
+    'sha512',
+]);
+function byteLength(input) {
+    return typeof input === 'string'
+        ? Buffer.byteLength(input, 'utf8')
+        : input.byteLength;
+}
+export function timingSafeEqualUtf8(a, b) {
+    const aBuffer = Buffer.from(a, 'utf8');
+    const bBuffer = Buffer.from(b, 'utf8');
+    if (aBuffer.length !== bBuffer.length)
+        return false;
+    return timingSafeEqual(aBuffer, bBuffer);
+}
+function hashHex(algorithm, input) {
+    if (!ALLOWED_HASH_ALGORITHMS.has(algorithm)) {
+        throw new Error(`Hash algorithm not allowed: ${algorithm}`);
+    }
+    if (byteLength(input) <= MAX_HASH_INPUT_BYTES) {
+        return oneShotHash(algorithm, input, 'hex');
+    }
+    const hasher = createHash(algorithm);
+    hasher.update(input);
+    return hasher.digest('hex');
+}
+export function sha256Hex(input) {
+    return hashHex('sha256', input);
+}

package/dist/errors.d.ts

@@ -0,0 +1,10 @@
+export declare class FetchError extends Error {
+    readonly url: string;
+    readonly statusCode: number;
+    readonly code: string;
+    readonly details: Readonly<Record<string, unknown>>;
+    constructor(message: string, url: string, httpStatus?: number, details?: Record<string, unknown>);
+}
+export declare function getErrorMessage(error: unknown): string;
+export declare function createErrorWithCode(message: string, code: string): NodeJS.ErrnoException;
+export declare function isSystemError(error: unknown): error is NodeJS.ErrnoException;

package/dist/errors.js

@@ -0,0 +1,28 @@
+const DEFAULT_HTTP_STATUS = 502;
+export class FetchError extends Error {
+    url;
+    statusCode;
+    code;
+    details;
+    constructor(message, url, httpStatus, details = {}) {
+        super(message);
+        this.url = url;
+        this.name = 'FetchError';
+        this.statusCode = httpStatus ?? DEFAULT_HTTP_STATUS;
+        this.code = httpStatus ? `HTTP_${httpStatus}` : 'FETCH_ERROR';
+        this.details = Object.freeze({ url, httpStatus, ...details });
+        Error.captureStackTrace(this, this.constructor);
+    }
+}
+export function getErrorMessage(error) {
+    return error instanceof Error ? error.message : 'Unknown error';
+}
+export function createErrorWithCode(message, code) {
+    const error = new Error(message);
+    return Object.assign(error, { code });
+}
+export function isSystemError(error) {
+    return (error instanceof Error &&
+        'code' in error &&
+        typeof Reflect.get(error, 'code') === 'string');
+}

package/dist/fetch.d.ts

@@ -0,0 +1,40 @@
+import type { Dispatcher } from 'undici';
+export interface FetchOptions {
+    signal?: AbortSignal;
+}
+export declare function isBlockedIp(ip: string): boolean;
+export declare function normalizeUrl(urlString: string): {
+    normalizedUrl: string;
+    hostname: string;
+};
+export declare function validateAndNormalizeUrl(urlString: string): string;
+export interface TransformResult {
+    readonly url: string;
+    readonly transformed: boolean;
+    readonly platform?: string;
+}
+export declare function transformToRawUrl(url: string): TransformResult;
+export declare function isRawTextContentUrl(url: string): boolean;
+export declare const dispatcher: Dispatcher;
+export declare function destroyAgents(): void;
+interface FetchTelemetryContext {
+    requestId: string;
+    startTime: number;
+    url: string;
+    method: string;
+    contextRequestId?: string;
+    operationId?: string;
+}
+export declare function startFetchTelemetry(url: string, method: string): FetchTelemetryContext;
+export declare function recordFetchResponse(context: FetchTelemetryContext, response: Response, contentSize?: number): void;
+export declare function recordFetchError(context: FetchTelemetryContext, error: unknown, status?: number): void;
+export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
+    response: Response;
+    url: string;
+}>;
+export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal): Promise<{
+    text: string;
+    size: number;
+}>;
+export declare function fetchNormalizedUrl(normalizedUrl: string, options?: FetchOptions): Promise<string>;
+export {};