mocktp 0.0.1-security → 3.15.3

package/src/types.ts

@@ -0,0 +1,433 @@
+import stream = require('stream');
+import http = require('http');
+import { EventEmitter } from 'events';
+
+export const DEFAULT_ADMIN_SERVER_PORT = 45454;
+
+export enum Method {
+    GET,
+    POST,
+    PUT,
+    DELETE,
+    PATCH,
+    HEAD,
+    OPTIONS
+}
+
+export enum RulePriority {
+    FALLBACK = 0,
+    DEFAULT = 1
+}
+
+export interface Headers {
+    // An arbitrary set of headers that are known to
+    // only ever appear once (for valid requests).
+    host?: string;
+    'content-length'?: string;
+    'content-type'?: string;
+    'user-agent'?: string;
+    cookie?: string;
+    ':method'?: string;
+    ':scheme'?: string;
+    ':authority'?: string;
+    ':path'?: string;
+
+    // In general there may be 0+ of any header
+    [key: string]: undefined | string | string[];
+}
+
+export interface Trailers {
+    // 0+ of any trailer
+    [key: string]: undefined | string | string[];
+}
+
+export type RawHeaders = Array<[key: string, value: string]>;
+export type RawTrailers = RawHeaders; // Just a convenient alias
+
+export interface Request {
+    id: string;
+    matchedRuleId?: string;
+
+    protocol: string;
+    httpVersion?: string; // Like timingEvents - not set remotely with older servers
+    method: string;
+    url: string;
+    path: string;
+
+    remoteIpAddress?: string; // Not set remotely with older servers
+    remotePort?: number; // Not set remotely with older servers
+
+    // Exists only if a host header is sent. A strong candidate for deprecation
+    // in future, since it's not clear that this comes from headers not the URL, and
+    // either way it duplicates existing data.
+    hostname?: string;
+
+    headers: Headers;
+    rawHeaders: RawHeaders;
+
+    timingEvents: TimingEvents;
+    tags: string[];
+}
+
+export interface TlsConnectionEvent {
+    hostname?: string;
+    remoteIpAddress: string;
+    remotePort: number;
+    tags: string[];
+    timingEvents: TlsTimingEvents;
+    tlsMetadata: TlsSocketMetadata;
+}
+
+export interface TlsSocketMetadata {
+    sniHostname?: string;
+    connectHostname?: string;
+    connectPort?: string;
+    clientAlpn?: string[];
+    ja3Fingerprint?: string;
+}
+
+export interface TlsPassthroughEvent extends TlsConnectionEvent {
+    id: string;
+    upstreamPort: number;
+}
+
+export interface TlsHandshakeFailure extends TlsConnectionEvent {
+    failureCause:
+        | 'closed'
+        | 'reset'
+        | 'cert-rejected'
+        | 'no-shared-cipher'
+        | 'handshake-timeout'
+        | 'unknown';
+    timingEvents: TlsFailureTimingEvents;
+}
+
+export interface TlsTimingEvents {
+    /**
+     * When the socket initially connected, in MS since the unix
+     * epoch.
+     */
+    startTime: number;
+
+    /**
+     * When the socket initially connected, equivalent to startTime.
+     *
+     * High-precision floating-point monotonically increasing timestamps.
+     * Comparable and precise, but not related to specific current time.
+     */
+    connectTimestamp: number;
+
+    /**
+     * When Mockttp's handshake for this connection was completed (if there
+     * was one). This is not set for passed through connections.
+     */
+    handshakeTimestamp?: number;
+
+    /**
+     * When the outer tunnel (e.g. a preceeding CONNECT request) was created,
+     * if there was one.
+     */
+    tunnelTimestamp?: number;
+
+    /**
+     * When the connection was closed, if it has been closed.
+     */
+    disconnectTimestamp?: number;
+}
+
+export interface TlsFailureTimingEvents extends TlsTimingEvents {
+    /**
+     * When the TLS connection failed. This may be due to a failed handshake
+     * (in which case `handshakeTimestamp` will be undefined) or due to a
+     * subsequent error which means the TLS connection was not usable (like
+     * an immediate closure due to an async certificate rejection).
+     */
+    failureTimestamp: number;
+}
+
+// Internal representation of an ongoing HTTP request whilst it's being processed
+export interface OngoingRequest extends Request, EventEmitter {
+    body: OngoingBody;
+    rawTrailers?: RawHeaders;
+}
+
+export interface OngoingBody {
+    asStream: () => stream.Readable;
+    asBuffer: () => Promise<Buffer>;
+    asDecodedBuffer: () => Promise<Buffer>;
+    asText: () => Promise<string>;
+    asJson: () => Promise<object>;
+    asFormData: () => Promise<{ [key: string]: string | string[] | undefined }>;
+}
+
+export interface CompletedBody {
+    /**
+     * The raw bytes of the response. If a content encoding was used, this is
+     * the raw encoded data.
+     */
+    buffer: Buffer;
+
+    /**
+     * The decoded bytes of the response. If no encoding was used, this is the
+     * same as `.buffer`. The response is decoded and returned asynchronously
+     * as a Promise.
+     */
+    getDecodedBuffer(): Promise<Buffer | undefined>;
+
+    /**
+     * The contents of the response, decoded and parsed as a UTF-8 string.
+     * The response is decoded and returned asynchronously as a Promise.
+     */
+    getText(): Promise<string | undefined>;
+
+    /**
+     * The contents of the response, decoded, parsed as UTF-8 string, and
+     * then parsed a JSON. The response is decoded and returned asynchronously
+     * as a Promise.
+     */
+    getJson(): Promise<object | undefined>;
+
+    /**
+     * The contents of the response, decoded, and then parsed automatically as
+     * either one of the form encoding types (either URL-encoded or multipart),
+     * determined automatically from the message content-type header.
+     *
+     * This method is convenient and offers a single mechanism to parse both
+     * formats, but you may want to consider parsing on format explicitly with
+     * the `getUrlEncodedFormData()` or `getMultipartFormData()` methods instead.
+     *
+     * After parsing & decoding, the result is returned asynchronously as a
+     * Promise for a key-value(s) object.
+     */
+    getFormData(): Promise<{ [key: string]: string | string[] | undefined } | undefined>;
+
+    /**
+     * The contents of the response, decoded, parsed as UTF-8 string, and then
+     * parsed as URL-encoded form data. After parsing & decoding, the result is
+     * returned asynchronously as a Promise for a key-value(s) object.
+     */
+    getUrlEncodedFormData(): Promise<{ [key: string]: string | string[] | undefined } | undefined>;
+
+    /**
+     * The contents of the response, decoded, and then parsed as multi-part
+     * form data. The response is result is returned asynchronously as a
+     * Promise for an array of parts with their names, data and metadata.
+     */
+    getMultipartFormData(): Promise<Array<{ name?: string, filename?: string, type?: string, data: Buffer }> | undefined>;
+}
+
+// Internal & external representation of an initiated (no body yet received) HTTP request.
+export type InitiatedRequest = Request;
+
+export interface AbortedRequest extends InitiatedRequest {
+    error?: {
+        name?: string;
+        code?: string;
+        message?: string;
+        stack?: string;
+    };
+}
+
+// Internal & external representation of a fully completed HTTP request
+export interface CompletedRequest extends Request {
+    body: CompletedBody;
+    rawTrailers: RawTrailers;
+    trailers: Trailers;
+}
+
+export interface TimingEvents {
+    // Milliseconds since unix epoch
+    startTime: number;
+
+    // High-precision floating-point monotonically increasing timestamps.
+    // Comparable and precise, but not related to specific current time.
+    startTimestamp: number; // When the request was initially received
+    bodyReceivedTimestamp?: number; // When the request body was fully received
+    headersSentTimestamp?: number; // When the response headers were sent
+    responseSentTimestamp?: number; // When the response was fully completed
+
+    wsAcceptedTimestamp?: number; // When the websocket was accepted
+    wsClosedTimestamp?: number; // When the websocket was closed
+
+    abortedTimestamp?: number; // When the connected was aborted
+}
+
+export interface OngoingResponse extends http.ServerResponse {
+    id: string;
+    getHeaders(): Headers;
+    getRawHeaders(): RawHeaders;
+    body: OngoingBody;
+    getRawTrailers(): RawTrailers;
+    timingEvents: TimingEvents;
+    tags: string[];
+}
+
+export interface CompletedResponse {
+    id: string;
+    statusCode: number;
+    statusMessage: string;
+    headers: Headers;
+    rawHeaders: RawHeaders;
+    body: CompletedBody;
+    rawTrailers: RawTrailers;
+    trailers: Trailers;
+    timingEvents: TimingEvents;
+    tags: string[];
+}
+
+export interface WebSocketMessage {
+    /**
+     * The id of this websocket stream. This will match the id of the request,
+     * the initial connection response, and any other WebSocket events for the
+     * same connection stream.
+     */
+    streamId: string;
+
+    /**
+     * Whether the message was sent by Mockttp, or received from a Mockttp client.
+     */
+    direction: 'sent' | 'received';
+
+    /**
+     * The contents of the message as a raw buffer. This is already decompressed,
+     * if the WebSocket uses compression.
+     */
+    content: Uint8Array;
+
+    /**
+     * Whether this is a string message or a raw binary data message.
+     */
+    isBinary: boolean;
+
+    /**
+     * A high-precision floating-point monotonically increasing timestamp.
+     * Comparable and precise, but not related to specific current time.
+     *
+     * To link this to the current time, compare it to `timingEvents.startTime`.
+     */
+    eventTimestamp: number;
+
+    timingEvents: TimingEvents;
+    tags: string[];
+}
+
+export interface WebSocketClose {
+    /**
+     * The id of this websocket stream. This will match the id of the request,
+     * the initial connection response, and any other WebSocket events for the
+     * same connection stream.
+     */
+    streamId: string;
+
+    /**
+     * The close code of the shutdown. This is the close code that was received
+     * from the remote client (either initiated remotely, or echoing our own sent
+     * close frame).
+     *
+     * This may be undefined only if a close frame was received but did not contain
+     * any close code. If no close frame was received before the connection was
+     * lost (i.e. the connection was not cleanly closed) this event will not
+     * fire at all, and an 'abort' event will fire instead.
+     */
+    closeCode: number | undefined;
+
+    /**
+     * The close reason of the shutdown.
+     */
+    closeReason: string;
+
+    timingEvents: TimingEvents;
+    tags: string[];
+}
+
+/**
+ * A client error event describes a request (or our best guess at parsing it),
+ * that wasn't correctly completed, and the error response it received, or
+ * 'aborted' if the connection was disconnected before we could respond.
+ */
+export interface ClientError {
+    errorCode?: string;
+    request: {
+        id: string;
+        timingEvents: TimingEvents;
+        tags: string[];
+
+        // All of these are best guess, depending on what's parseable:
+        protocol?: string;
+        httpVersion?: string;
+        method?: string;
+        url?: string;
+        path?: string;
+
+        headers: Headers;
+        rawHeaders: RawHeaders;
+
+        remoteIpAddress?: string;
+        remotePort?: number;
+    };
+    response: CompletedResponse | 'aborted';
+}
+
+/**
+ * An event fired from an individual rule during request processing.
+ */
+export interface RuleEvent<T = unknown> {
+    requestId: string;
+    ruleId: string;
+    eventType: string;
+    eventData: T;
+}
+
+/**
+ * A mocked endpoint provides methods to see the current state of
+ * a mock rule.
+ */
+export interface MockedEndpoint {
+    id: string;
+
+    /**
+     * Get the requests that this endpoint has seen so far.
+     *
+     * This method returns a promise, which resolves with the requests seen
+     * up until now, once all ongoing requests have terminated. The returned
+     * lists are immutable, so won't change if more requests arrive in future.
+     * Call `getSeenRequests` again later to get an updated list.
+     *
+     * Requests are included here once the response is completed, even if the request
+     * itself failed, the responses failed or exceptions are thrown elsewhere. To
+     * watch for errors or detailed response info, look at the various server.on(event)
+     * methods.
+     */
+    getSeenRequests(): Promise<CompletedRequest[]>;
+
+    /**
+     * Reports whether this endpoint is still pending: if it either hasn't seen the
+     * specified number of requests (if one was specified e.g. with .twice())
+     * or if it hasn't seen at least one request, by default.
+     *
+     * This method returns a promise, which resolves with the result once all
+     * ongoing requests have terminated.
+     */
+    isPending(): Promise<boolean>;
+}
+
+export interface MockedEndpointData {
+    id: string;
+    explanation?: string;
+    seenRequests: CompletedRequest[];
+    isPending: boolean;
+}
+
+export interface Explainable {
+    explain(): string;
+}
+
+export interface ProxyEnvConfig {
+    HTTP_PROXY: string;
+    HTTPS_PROXY: string;
+}
+
+// A slightly weird one: this is necessary because we export types that inherit from EventEmitter,
+// so the docs include EventEmitter's methods, which @link to this type, that's otherwise not
+// defined in this module. Reexporting the values avoids warnings for that.
+export type defaultMaxListeners = typeof EventEmitter.defaultMaxListeners;

package/src/util/buffer-utils.ts

@@ -0,0 +1,164 @@
+import * as _ from 'lodash';
+import { EventEmitter } from 'events';
+import * as stream from 'stream';
+
+import { isNode } from './util';
+
+const MAX_BUFFER_SIZE = isNode
+    ? require('buffer').constants.MAX_LENGTH
+    : Infinity;
+
+export const asBuffer = (input: Buffer | Uint8Array | string) =>
+    Buffer.isBuffer(input)
+        ? input
+    : typeof input === "string"
+        ? Buffer.from(input, 'utf8')
+    // Is Array:
+        : Buffer.from(input);
+
+export type BufferInProgress = Promise<Buffer> & {
+    currentChunks: Buffer[]; // Stores the body chunks as they arrive
+    failedWith?: Error; // Stores the error that killed the stream, if one did
+    events: EventEmitter; // Emits events - notably 'truncate' if data is truncated
+};
+
+// Takes a buffer and a stream, returns a simple stream that outputs the buffer then the stream. The stream
+// is lazy, so doesn't read data in from the buffer or input until something here starts reading.
+export const bufferThenStream = (buffer: BufferInProgress, inputStream: stream.Readable): stream.Readable => {
+    let active = false;
+
+    const outputStream = new stream.PassThrough({
+        // Note we use the default highWaterMark, which means this applies backpressure, pushing buffering
+        // onto the OS & backpressure on network instead of accepting data before we're ready to stream it.
+
+        // Without changing behaviour, we listen for read() events, and don't start streaming until we get one.
+        read(size) {
+            // On the first actual read of this stream, we pull from the buffer
+            // and then hook it up to the input.
+            if (!active) {
+                if (buffer.failedWith) {
+                    outputStream.destroy(buffer.failedWith);
+                } else {
+                    // First stream everything that's been buffered so far:
+                    outputStream.write(Buffer.concat(buffer.currentChunks));
+
+                    // Then start streaming all future incoming data:
+                    inputStream.pipe(outputStream);
+
+                    if (inputStream.readableEnded) outputStream.end();
+                    if (inputStream.readableAborted) outputStream.destroy();
+
+                    // Forward any future errors from the input stream:
+                    inputStream.on('error', (e) => {
+                        outputStream.emit('error', e)
+                    });
+
+                    // Silence 'unhandled rejection' warnings here, since we'll handle
+                    // them on the stream instead
+                    buffer.catch(() => {});
+                }
+                active = true;
+            }
+
+            // Except for the first activation logic (above) do the default transform read() steps just
+            // like a normal PassThrough stream.
+            return stream.Transform.prototype._read.call(this, size);
+        }
+    });
+
+    buffer.events.on('truncate', (chunks) => {
+        // If the stream hasn't started yet, start it now, so it grabs the buffer
+        // data before it gets truncated:
+        if (!active) outputStream.read(0);
+    });
+
+    return outputStream;
+};
+
+export const bufferToStream = (buffer: Buffer): stream.Readable => {
+    const outputStream = new stream.PassThrough();
+    outputStream.end(buffer);
+    return outputStream;
+};
+
+export const streamToBuffer = (input: stream.Readable, maxSize = MAX_BUFFER_SIZE) => {
+    let chunks: Buffer[] = [];
+
+    const bufferPromise = <BufferInProgress> new Promise(
+        (resolve, reject) => {
+            function failWithAbortError() {
+                bufferPromise.failedWith = new Error('Aborted');
+                reject(bufferPromise.failedWith);
+            }
+
+            // If stream has already finished/aborted, resolve accordingly immediately:
+            if (input.readableEnded) return resolve(Buffer.from([]));
+            if (input.readableAborted) return failWithAbortError();
+
+            let currentSize = 0;
+            const onData = (d: Buffer) => {
+                currentSize += d.length;
+                chunks.push(d);
+
+                // If we go over maxSize, drop the whole stream, so the buffer
+                // resolves empty. MaxSize should be large, so this is rare,
+                // and only happens as an alternative to crashing the process.
+                if (currentSize > maxSize) {
+                    // Announce truncation, so that other mechanisms (asStream) can
+                    // capture this data if they're interested in it.
+                    bufferPromise.events.emit('truncate', chunks);
+
+                    // Drop all the data so far & stop reading
+                    bufferPromise.currentChunks = chunks = [];
+                    input.removeListener('data', onData);
+
+                    // We then resolve immediately - the buffer is done, even if the body
+                    // might still be streaming in we're not listening to it. This means
+                    // that requests can 'complete' for event/callback purposes while
+                    // they're actually still streaming, but only in this scenario where
+                    // the data is too large to really be used by the events/callbacks.
+
+                    // If we don't resolve, then cases which intentionally don't consume
+                    // the raw stream but do consume the buffer (beforeRequest) would
+                    // deadlock: beforeRequest must complete to begin streaming the
+                    // full body to the target clients.
+
+                    resolve(Buffer.from([]));
+                }
+            };
+            input.on('data', onData);
+
+            input.once('end', () => {
+                resolve(Buffer.concat(chunks));
+            });
+            input.once('aborted', failWithAbortError);
+            input.on('error', (e) => {
+                bufferPromise.failedWith = bufferPromise.failedWith || e;
+                reject(e);
+            });
+        }
+    );
+    bufferPromise.currentChunks = chunks;
+    bufferPromise.events = new EventEmitter();
+    return bufferPromise;
+};
+
+export function splitBuffer(input: Buffer, splitter: string, maxParts = Infinity) {
+    const parts: Buffer[] = [];
+
+    let remainingBuffer = input;
+    while (remainingBuffer.length) {
+        let endOfPart = remainingBuffer.indexOf(splitter);
+        if (endOfPart === -1) endOfPart = remainingBuffer.length;
+
+        parts.push(remainingBuffer.slice(0, endOfPart));
+        remainingBuffer = remainingBuffer.slice(endOfPart + splitter.length);
+
+        if (parts.length === maxParts - 1) {
+            parts.push(remainingBuffer);
+            break;
+        }
+    }
+
+    return parts;
+}

package/src/util/dns.ts

@@ -0,0 +1,52 @@
+import * as dns from 'dns';
+
+// We exclude the __promisify__ type hack from the official DNS type, to give us a type
+// that otherwise enforces correctness. We still have to cast past the missing 'field'
+// at the end of the day, but this allows us to avoid that until the last minute without
+// sacrificing any strictness the rest of the time.
+export type DnsLookupFunction = Omit<typeof dns.lookup, '__promisify__'>;
+
+// A drop-in alternative to dns.lookup, but where results are briefly cached to avoid completely
+// unnecessary lookups, while remaining fairly reactive to actual host file changes etc.
+export class CachedDns {
+
+    private cache = new Map<string, [address: string | dns.LookupAddress[], family: number]>();
+
+    constructor(
+        private cacheDurationMs: number
+    ) {}
+
+    private cacheKey(hostname: Parameters<typeof dns.lookup>[0], options?: dns.LookupAllOptions | dns.LookupOneOptions) {
+        return `${hostname}-${options?.all}-${options?.family}-${options?.hints}-${options?.verbatim}`;
+    }
+
+    lookup: DnsLookupFunction = (...args: Parameters<typeof dns.lookup>) => {
+        const [hostname, options] = args.slice(0, -1) as [string, dns.LookupOptions | undefined];
+        const cb = args[args.length - 1] as (err: NodeJS.ErrnoException | null, address: string | dns.LookupAddress[], family: number) => void;
+
+        const key = this.cacheKey(hostname, options);
+        const cachedResult = this.cache.get(key);
+        if (cachedResult) {
+            setImmediate(() => cb(null, ...cachedResult));
+        } else {
+            dns.lookup(hostname, options ?? {}, (err, ...cbArgs) => {
+                if (!err) {
+                    this.cache.set(key, cbArgs);
+                    // Always refresh Xms after initially inserted - no LRU or similar
+                    setTimeout(() => this.cache.delete(key), this.cacheDurationMs).unref();
+                }
+                return cb(err, ...cbArgs);
+            });
+        }
+    }
+
+}
+
+export function dnsLookup(lookupFn: typeof dns.lookup | DnsLookupFunction, hostname: string) {
+    return new Promise<string>((resolve, reject) => {
+        (lookupFn as typeof dns.lookup)(hostname!, (err, address) => {
+            if (err) reject(err);
+            else resolve(address);
+        });
+    })
+}

package/src/util/error.ts

@@ -0,0 +1,18 @@
+export type ErrorLike = Partial<Error> & {
+    // Various properties we might want to look for on errors:
+    code?: string;
+    cmd?: string;
+    signal?: string;
+    statusCode?: number;
+    statusMessage?: string;
+};
+
+// Useful to easily cast and then examine errors that are otherwise 'unknown':
+export function isErrorLike(error: any): error is ErrorLike {
+    return typeof error === 'object' && (
+        error instanceof Error ||
+        error.message ||
+        error.code ||
+        error.stack
+    )
+}