mocktp 0.0.1-security → 3.15.3

package/src/rules/passthrough-handling-definitions.ts

@@ -0,0 +1,111 @@
+import { ProxyConfig } from "./proxy-config";
+
+export interface ForwardingOptions {
+    targetHost: string,
+    // Should the host (H1) or :authority (H2) header be updated to match?
+    updateHostHeader?: true | false | string // Change automatically/ignore/change to custom value
+}
+
+export interface PassThroughLookupOptions {
+    /**
+     * The maximum time to cache a DNS response. Up to this limit,
+     * responses will be cached according to their own TTL. Defaults
+     * to Infinity.
+     */
+    maxTtl?: number;
+    /**
+     * How long to cache a DNS ENODATA or ENOTFOUND response. Defaults
+     * to 0.15.
+     */
+    errorTtl?: number;
+    /**
+     * The primary servers to use. DNS queries will be resolved against
+     * these servers first. If no data is available, queries will fall
+     * back to dns.lookup, and use the OS's default DNS servers.
+     *
+     * This defaults to dns.getServers().
+     */
+    servers?: string[];
+}
+
+export type CADefinition =
+    | { cert: string | Buffer }
+    | { certPath: string };
+
+/**
+ * This defines the upstream connection parameters. These passthrough parameters
+ * are shared between both WebSocket & Request passthrough rules.
+ */
+export interface PassThroughHandlerConnectionOptions {
+    /**
+     * The forwarding configuration for the passthrough rule.
+     * This generally shouldn't be used explicitly unless you're
+     * building rule data by hand. Instead, call `thenPassThrough`
+     * to send data directly or `thenForwardTo` with options to
+     * configure traffic forwarding.
+     */
+    forwarding?: ForwardingOptions,
+
+    /**
+     * A list of hostnames for which server certificate and TLS version errors
+     * should be ignored (none, by default).
+     *
+     * If set to 'true', HTTPS errors will be ignored for all hosts. WARNING:
+     * Use this at your own risk. Setting this to `true` can open your
+     * application to MITM attacks and should never be used over any network
+     * that is not completed trusted end-to-end.
+     */
+    ignoreHostHttpsErrors?: string[] | boolean;
+
+    /**
+     * An array of additional certificates, which should be trusted as certificate
+     * authorities for upstream hosts, in addition to Node.js's built-in certificate
+     * authorities.
+     *
+     * Each certificate should be an object with either a `cert` key and a string
+     * or buffer value containing the PEM certificate, or a `certPath` key and a
+     * string value containing the local path to the PEM certificate.
+     */
+    additionalTrustedCAs?: Array<CADefinition>;
+
+    /**
+     * Deprecated alias for `additionalTrustedCAs`
+     *
+     * @deprecated
+     */
+    trustAdditionalCAs?: Array<CADefinition>;
+
+    /**
+     * A mapping of hosts to client certificates to use, in the form of
+     * `{ key, cert }` objects (none, by default)
+     */
+    clientCertificateHostMap?: {
+        [host: string]: { pfx: Buffer, passphrase?: string }
+    };
+
+    /**
+     * Upstream proxy configuration: pass through requests via this proxy.
+     *
+     * If this is undefined, no proxy will be used. To configure a proxy
+     * provide either:
+     * - a ProxySettings object
+     * - a callback which will be called with an object containing the
+     *   hostname, and must return a ProxySettings object or undefined.
+     * - an array of ProxySettings or callbacks. The array will be
+     *   processed in order, and the first not-undefined ProxySettings
+     *   found will be used.
+     *
+     * When using a remote client, this parameter or individual array
+     * values may be passed by reference, using the name of a rule
+     * parameter configured in the admin server.
+     */
+    proxyConfig?: ProxyConfig;
+
+    /**
+     * Custom DNS options, to allow configuration of the resolver used
+     * when forwarding requests upstream. Passing any option switches
+     * from using node's default dns.lookup function to using the
+     * cacheable-lookup module, which will cache responses.
+     */
+    lookupOptions?: PassThroughLookupOptions;
+}

package/src/rules/passthrough-handling.ts

@@ -0,0 +1,376 @@
+import * as _ from 'lodash';
+import * as fs from 'fs/promises';
+import * as tls from 'tls';
+import url = require('url');
+import { oneLine } from 'common-tags';
+import CacheableLookup from 'cacheable-lookup';
+
+import { CompletedBody, Headers } from '../types';
+import { byteLength } from '../util/util';
+import { asBuffer } from '../util/buffer-utils';
+import { isLocalhostAddress } from '../util/socket-util';
+import { CachedDns, dnsLookup, DnsLookupFunction } from '../util/dns';
+import { isMockttpBody, encodeBodyBuffer } from '../util/request-utils';
+import { areFFDHECurvesSupported } from '../util/openssl-compat';
+
+import {
+    CallbackRequestResult,
+    CallbackResponseMessageResult
+} from './requests/request-handler-definitions';
+import {
+    CADefinition,
+    PassThroughLookupOptions
+} from './passthrough-handling-definitions';
+
+// TLS settings for proxied connections, intended to avoid TLS fingerprint blocking
+// issues so far as possible, by closely emulating a Firefox Client Hello:
+const NEW_CURVES_SUPPORTED = areFFDHECurvesSupported(process.versions.openssl);
+
+const SSL_OP_LEGACY_SERVER_CONNECT = 1 << 2;
+const SSL_OP_TLSEXT_PADDING = 1 << 4;
+const SSL_OP_NO_ENCRYPT_THEN_MAC = 1 << 19;
+
+// All settings are designed to exactly match Firefox v103, since that's a good baseline
+// that seems to be widely accepted and is easy to emulate from Node.js.
+export const getUpstreamTlsOptions = (strictChecks: boolean): tls.SecureContextOptions => ({
+    ecdhCurve: [
+        'X25519',
+        'prime256v1', // N.B. Equivalent to secp256r1
+        'secp384r1',
+        'secp521r1',
+        ...(NEW_CURVES_SUPPORTED
+            ? [ // Only available with OpenSSL v3+:
+                'ffdhe2048',
+                'ffdhe3072'
+            ] : []
+        )
+    ].join(':'),
+    sigalgs: [
+        'ecdsa_secp256r1_sha256',
+        'ecdsa_secp384r1_sha384',
+        'ecdsa_secp521r1_sha512',
+        'rsa_pss_rsae_sha256',
+        'rsa_pss_rsae_sha384',
+        'rsa_pss_rsae_sha512',
+        'rsa_pkcs1_sha256',
+        'rsa_pkcs1_sha384',
+        'rsa_pkcs1_sha512',
+        'ECDSA+SHA1',
+        'rsa_pkcs1_sha1'
+    ].join(':'),
+    ciphers: [
+        'TLS_AES_128_GCM_SHA256',
+        'TLS_CHACHA20_POLY1305_SHA256',
+        'TLS_AES_256_GCM_SHA384',
+        'ECDHE-ECDSA-AES128-GCM-SHA256',
+        'ECDHE-RSA-AES128-GCM-SHA256',
+        'ECDHE-ECDSA-CHACHA20-POLY1305',
+        'ECDHE-RSA-CHACHA20-POLY1305',
+        'ECDHE-ECDSA-AES256-GCM-SHA384',
+        'ECDHE-RSA-AES256-GCM-SHA384',
+        'ECDHE-ECDSA-AES256-SHA',
+        'ECDHE-ECDSA-AES128-SHA',
+        'ECDHE-RSA-AES128-SHA',
+        'ECDHE-RSA-AES256-SHA',
+        'AES128-GCM-SHA256',
+        'AES256-GCM-SHA384',
+        'AES128-SHA',
+        'AES256-SHA',
+
+        // This magic cipher is the very obtuse way that OpenSSL downgrades the overall
+        // security level to allow various legacy settings, protocols & ciphers:
+        ...(!strictChecks
+            ? ['@SECLEVEL=0']
+            : []
+        )
+    ].join(':'),
+    secureOptions: strictChecks
+        ? SSL_OP_TLSEXT_PADDING | SSL_OP_NO_ENCRYPT_THEN_MAC
+        : SSL_OP_TLSEXT_PADDING | SSL_OP_NO_ENCRYPT_THEN_MAC | SSL_OP_LEGACY_SERVER_CONNECT,
+    ...({
+        // Valid, but not included in Node.js TLS module types:
+        requestOSCP: true
+    } as any),
+
+    // Allow TLSv1, if !strict:
+    minVersion: strictChecks ? tls.DEFAULT_MIN_VERSION : 'TLSv1',
+
+    // Skip certificate validation entirely, if not strict:
+    rejectUnauthorized: strictChecks,
+});
+
+export async function getTrustedCAs(
+    trustedCAs: Array<string | CADefinition> | undefined,
+    additionalTrustedCAs: Array<CADefinition> | undefined
+): Promise<Array<string> | undefined> {
+    if (trustedCAs && additionalTrustedCAs?.length) {
+        throw new Error(`trustedCAs and additionalTrustedCAs options are mutually exclusive`);
+    }
+
+    if (trustedCAs) {
+        return Promise.all(trustedCAs.map((caDefinition) =>  getCA(caDefinition)));
+    }
+
+    if (additionalTrustedCAs) {
+        const CAs = await Promise.all(additionalTrustedCAs.map((caDefinition) =>  getCA(caDefinition)));
+        return tls.rootCertificates.concat(CAs);
+    }
+}
+
+const getCA = async (caDefinition: string | CADefinition) => {
+    return typeof caDefinition === 'string'
+        ? caDefinition
+    : 'certPath' in caDefinition
+        ? await fs.readFile(caDefinition.certPath, 'utf8')
+    // 'cert' in caDefinition
+        : caDefinition.cert.toString('utf8')
+}
+
+
+// --- Various helpers for deriving parts of request/response data given partial overrides: ---
+
+/**
+ * Takes a callback result and some headers, and returns a ready to send body, using the headers
+ * (and potentially modifying them) to match the content type & encoding.
+ */
+export async function buildOverriddenBody(
+    callbackResult: CallbackRequestResult | CallbackResponseMessageResult | void,
+    headers: Headers
+) {
+    // Raw bodies are easy: use them as is.
+    if (callbackResult?.rawBody) return callbackResult?.rawBody!;
+
+    // In the json/body case, we need to get the body and transform it into a buffer
+    // for consistent handling later, and encode it to match the headers.
+
+    let replacementBody: string | Uint8Array | Buffer | CompletedBody | undefined;
+    if (callbackResult?.json) {
+        headers['content-type'] = 'application/json';
+        replacementBody = JSON.stringify(callbackResult?.json);
+    } else {
+        replacementBody = callbackResult?.body;
+    }
+
+    if (replacementBody === undefined) return replacementBody;
+
+    let rawBuffer: Buffer;
+    if (isMockttpBody(replacementBody)) {
+        // It's our own bodyReader instance. That's not supposed to happen, but
+        // it's ok, we just need to use the buffer data instead of the whole object
+        rawBuffer = Buffer.from((replacementBody as CompletedBody).buffer);
+    } else if (replacementBody === '') {
+        // For empty bodies, it's slightly more convenient if they're truthy
+        rawBuffer = Buffer.alloc(0);
+    } else {
+        rawBuffer = asBuffer(replacementBody);
+    }
+
+    return await encodeBodyBuffer(rawBuffer, headers);
+}
+
+/**
+ * If you override some headers, they have implications for the effective URL we send the
+ * request to. If you override that and the URL at the same time, it gets complicated.
+ *
+ * This method calculates the correct header value we should use: prioritising the header
+ * value you provide, printing a warning if it's contradictory, or return the URL-inferred
+ * value to override the header correctly if you didn't specify.
+ */
+function deriveUrlLinkedHeader(
+    originalHeaders: Headers,
+    replacementHeaders: Headers | undefined,
+    headerName: 'host' | ':authority' | ':scheme',
+    expectedValue: string // The inferred 'correct' value from the URL
+) {
+    const replacementValue = replacementHeaders?.[headerName];
+
+    if (replacementValue !== undefined) {
+        if (replacementValue !== expectedValue && replacementValue === originalHeaders[headerName]) {
+            // If you rewrite the URL-based header wrongly, by explicitly setting it to the
+            // existing value, we accept it but print a warning. This would be easy to
+            // do if you mutate the existing headers, for example, and ignore the host.
+            console.warn(oneLine`
+                Passthrough callback overrode the URL and the ${headerName} header
+                with mismatched values, which may be a mistake. The URL implies
+                ${expectedValue}, whilst the header was set to ${replacementValue}.
+            `);
+        }
+        // Whatever happens, if you explicitly set a value, we use it.
+        return replacementValue;
+    }
+
+    // If you didn't override the header at all, then we automatically ensure
+    // the correct value is set automatically.
+    return expectedValue;
+}
+
+/**
+ * Autocorrect the host header only in the case that if you didn't explicitly
+ * override it yourself for some reason (e.g. if you're testing bad behaviour).
+ */
+export function getHostAfterModification(
+    reqUrl: string,
+    originalHeaders: Headers,
+    replacementHeaders: Headers | undefined
+): string {
+    return deriveUrlLinkedHeader(
+        originalHeaders,
+        replacementHeaders,
+        'host',
+        url.parse(reqUrl).host!
+    );
+}
+
+export const OVERRIDABLE_REQUEST_PSEUDOHEADERS = [
+    ':authority',
+    ':scheme'
+] as const;
+
+/**
+ * Automatically update the :scheme and :authority headers to match the updated URL,
+ * as long as they weren't explicitly overriden themselves, in which case let them
+ * be set to any invalid value you like (e.g. to send a request to one server but
+ * pretend it was sent to another).
+ */
+export function getH2HeadersAfterModification(
+    reqUrl: string,
+    originalHeaders: Headers,
+    replacementHeaders: Headers | undefined
+): { [K in typeof OVERRIDABLE_REQUEST_PSEUDOHEADERS[number]]: string } {
+    const parsedUrl = url.parse(reqUrl);
+
+    return {
+        ':scheme': deriveUrlLinkedHeader(
+            originalHeaders,
+            replacementHeaders,
+            ':scheme',
+            parsedUrl.protocol!.slice(0, -1)
+        ),
+        ':authority': deriveUrlLinkedHeader(
+            originalHeaders,
+            replacementHeaders,
+            ':authority',
+            parsedUrl.host!
+        )
+    };
+}
+
+// Helper to handle content-length nicely for you when rewriting requests with callbacks
+export function getContentLengthAfterModification(
+    body: string | Uint8Array | Buffer,
+    originalHeaders: Headers,
+    replacementHeaders: Headers | undefined,
+    mismatchAllowed: boolean = false
+): string | undefined {
+    // If there was a content-length header, it might now be wrong, and it's annoying
+    // to need to set your own content-length override when you just want to change
+    // the body. To help out, if you override the body but don't explicitly override
+    // the (now invalid) content-length, then we fix it for you.
+
+    if (!_.has(originalHeaders, 'content-length')) {
+        // Nothing to override - use the replacement value, or undefined
+        return (replacementHeaders || {})['content-length'];
+    }
+
+    if (!replacementHeaders) {
+        // There was a length set, and you've provided a body but not changed it.
+        // You probably just want to send this body and have it work correctly,
+        // so we should fix the content length for you automatically.
+        return byteLength(body).toString();
+    }
+
+    // There was a content length before, and you're replacing the headers entirely
+    const lengthOverride = replacementHeaders['content-length']?.toString();
+
+    // If you're setting the content-length to the same as the origin headers, even
+    // though that's the wrong value, it *might* be that you're just extending the
+    // existing headers, and you're doing this by accident (we can't tell for sure).
+    // We use invalid content-length as instructed, but print a warning just in case.
+    if (
+        lengthOverride === originalHeaders['content-length'] &&
+        lengthOverride !== byteLength(body).toString() &&
+        !mismatchAllowed // Set for HEAD responses
+    ) {
+        console.warn(oneLine`
+            Passthrough modifications overrode the body and the content-length header
+            with mismatched values, which may be a mistake. The body contains
+            ${byteLength(body)} bytes, whilst the header was set to ${lengthOverride}.
+        `);
+    }
+
+    return lengthOverride;
+}
+
+// Function to check if we should skip https errors for the current hostname and port,
+// based on the given config
+export function shouldUseStrictHttps(
+    hostname: string,
+    port: number,
+    ignoreHostHttpsErrors: string[] | boolean
+) {
+    let skipHttpsErrors = false;
+
+    if (ignoreHostHttpsErrors === true) {
+        // Ignore cert errors if `ignoreHostHttpsErrors` is set to true, or
+        skipHttpsErrors = true;
+    } else if (Array.isArray(ignoreHostHttpsErrors) && (
+        // if the whole hostname or host+port is whitelisted
+        _.includes(ignoreHostHttpsErrors, hostname) ||
+        _.includes(ignoreHostHttpsErrors, `${hostname}:${port}`)
+    )) {
+        skipHttpsErrors = true;
+    }
+    return !skipHttpsErrors;
+}
+
+export const getDnsLookupFunction = _.memoize((lookupOptions: PassThroughLookupOptions | undefined) => {
+    if (!lookupOptions) {
+        // By default, use 10s caching of hostnames, just to reduce the delay from
+        // endlessly 10ms query delay for 'localhost' with every request.
+        return new CachedDns(10000).lookup;
+    } else {
+        // Or if options are provided, use those to configure advanced DNS cases:
+        const cacheableLookup = new CacheableLookup({
+            maxTtl: lookupOptions.maxTtl,
+            errorTtl: lookupOptions.errorTtl,
+            // As little caching of "use the fallback server" as possible:
+            fallbackDuration: 0
+        });
+
+        if (lookupOptions.servers) {
+            cacheableLookup.servers = lookupOptions.servers;
+        }
+
+        return cacheableLookup.lookup;
+    }
+});
+
+export async function getClientRelativeHostname(
+    hostname: string | null,
+    remoteIp: string | undefined,
+    lookupFn: DnsLookupFunction
+) {
+    if (!hostname || !remoteIp || isLocalhostAddress(remoteIp)) return hostname;
+
+    // Otherwise, we have a request from a different machine (or Docker container/VM/etc) and we need
+    // to make sure that 'localhost' means _that_ machine, not ourselves.
+
+    // This check must be run before req modifications. If a modification changes the address to localhost,
+    // then presumably it really does mean *this* localhost.
+
+    if (
+        // If the hostname is a known localhost address, we're done:
+        isLocalhostAddress(hostname) ||
+        // Otherwise, we look up the IP, so we can accurately check for localhost-bound requests. This adds a little
+        // delay, but since it's cached we save the equivalent delay in request lookup later, so it should be
+        // effectively free. We ignore errors to delegate unresolvable etc to request processing later.
+        isLocalhostAddress(await dnsLookup(lookupFn, hostname).catch(() => null))
+    ) {
+        return remoteIp;
+
+        // Note that we just redirect - we don't update the host header. From the POV of the target, it's still
+        // 'localhost' traffic that should appear identical to normal.
+    } else {
+        return hostname;
+    }
+}

package/src/rules/proxy-config.ts

@@ -0,0 +1,136 @@
+import * as _ from 'lodash';
+
+import { MaybePromise } from '../util/type-utils';
+import { RuleParameterReference } from './rule-parameters';
+import { CADefinition } from './passthrough-handling-definitions';
+
+/**
+ * A ProxySetting is a specific proxy setting to use, which is passed to a proxy agent
+ * who will manage creating a socket for the request (directly, or tunnelled, or whatever).
+ */
+export interface ProxySetting {
+    /**
+     * The URL for the proxy to forward traffic through.
+     *
+     * This can be any URL supported by https://www.npmjs.com/package/proxy-agent.
+     * For example: http://..., socks5://..., pac+http://...
+     */
+    proxyUrl: string;
+
+    /**
+     * A list of no-proxy values, matching hosts' traffic should *not* be proxied.
+     *
+     * This is a common proxy feature, but unfortunately isn't standardized. See
+     * https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/ for some
+     * background. This implementation is intended to match Curl's behaviour, and
+     * any differences are a bug.
+     *
+     * The currently supported formats are:
+     * - example.com (matches domain and all subdomains)
+     * - example.com:443 (matches domain and all subdomains, but only on that port)
+     * - 10.0.0.1 (matches IP, but only when used directly - does not resolve domains)
+     *
+     * Some other formats (e.g. leading dots or *.) will work, but the leading
+     * characters are ignored. More formats may be added in future, e.g. CIDR ranges.
+     * To maximize compatibility with values used elsewhere, unrecognized formats
+     * will generally be ignored, but may match in unexpected ways.
+     */
+    noProxy?: string[];
+
+    /**
+     * CAs to trust for HTTPS connections to the proxy. Ignored if the connection to
+     * the proxy is not HTTPS. If not specified, this will default to the Node
+     * defaults, or you can override them here completely.
+     *
+     * This sets the complete list of trusted CAs, and is mutually exclusive with the
+     * `additionalTrustedCAs` option, which adds additional CAs (but also trusts the
+     * Node default CAs too).
+     *
+     * This should be specified as either a { cert: string | Buffer } object or a
+     * { certPath: string } object (to read the cert from disk). The previous
+     * simple string format is supported but deprecated.
+     */
+    trustedCAs?: Array<
+        | string // Deprecated
+        | CADefinition
+    >;
+
+    /**
+     * Extra CAs to trust for HTTPS connections to the proxy. Ignored if the connection
+     * to the proxy is not HTTPS.
+     *
+     * This appends to the list of trusted CAs, and is mutually exclusive with the
+     * `trustedCAs` option, which completely overrides the list of CAs.
+     */
+    additionalTrustedCAs?: Array<CADefinition>;
+}
+
+/**
+ * A ProxySettingSource is a way to calculate the ProxySetting for a given request. It
+ * may be a fixed ProxySetting value, or a callback to get ProxySetting values, or an
+ * array of sources, which should be iterated to get the first usable value
+ */
+export type ProxySettingSource =
+    | ProxySetting
+    | ProxySettingCallback
+    | Array<ProxySettingSource>
+    | undefined;
+
+export type ProxySettingCallbackParams = { hostname: string };
+export type ProxySettingCallback = (params: ProxySettingCallbackParams) => MaybePromise<ProxySetting | undefined>;
+
+/**
+ * A ProxyConfig is externally provided config that specifies a ProxySettingSource.
+ * It might be a ProxySettingSource itself, or it might include references to rule
+ * parameters, which must be dereferenced to make it usable as a ProxySettingSource.
+ */
+export type ProxyConfig =
+ | ProxySettingSource
+ | RuleParameterReference<ProxySettingSource>
+ | Array<ProxySettingSource | RuleParameterReference<ProxySettingSource>>;
+
+export async function getProxySetting(
+    configSource: ProxySettingSource,
+    params: ProxySettingCallbackParams
+) {
+    if (_.isFunction(configSource)) return configSource(params);
+    else if (_.isArray(configSource)) {
+        let result: ProxySetting | undefined;
+        for (let configArrayOption of configSource) {
+            result = await getProxySetting(configArrayOption, params);
+            if (result) break;
+        }
+        return result;
+    }
+    else return configSource;
+}
+
+export const matchesNoProxy = (hostname: string, portNum: number, noProxyValues: string[] | undefined) => {
+    if (!noProxyValues || noProxyValues.length === 0) return false; // Skip everything in the common case.
+
+    const port = portNum.toString();
+    const hostParts = hostname.split('.').reverse();
+
+    return noProxyValues.some((noProxy) => {
+        const [noProxyHost, noProxyPort] = noProxy.split(':') as [string, string | undefined];
+
+        let noProxyParts = noProxyHost.split('.').reverse();
+        const lastPart = noProxyParts[noProxyParts.length - 1];
+        if (lastPart === '' || lastPart === '*') {
+            noProxyParts = noProxyParts.slice(0, -1);
+        }
+
+        if (noProxyPort && port !== noProxyPort) return false;
+
+        for (let i = 0; i < noProxyParts.length; i++) {
+            let noProxyPart = noProxyParts[i];
+            let hostPart = hostParts[i];
+
+            if (hostPart === undefined) return false; // No-proxy is longer than hostname
+            if (noProxyPart !== hostPart) return false; // Mismatch
+        }
+
+        // If we run out of no-proxy parts with no mismatch then we've matched
+        return true;
+    });
+}