@npy/fetch 0.1.1 → 0.1.3

package/src/agent.ts

@@ -0,0 +1,408 @@
+import { Deferred } from "@fuman/utils";
+import { toConnectError, toSendError } from "./_internal/error-mapping";
+import { raceSignal } from "./_internal/promises";
+import { bodyErrorMapperSymbol } from "./_internal/symbols";
+import {
+    AgentBusyError,
+    AgentClosedError,
+    OriginMismatchError,
+    RequestAbortedError,
+    UnsupportedAlpnProtocolError,
+    UnsupportedMethodError,
+    UnsupportedProtocolError,
+} from "./errors";
+import { readResponse, writeRequest } from "./io/io";
+import type { Agent } from "./types/agent";
+import type { Dialer } from "./types/dialer";
+
+const PORT_MAP = {
+    "http:": 80,
+    "https:": 443,
+} as const;
+
+const DEFAULT_ALPN_PROTOCOLS = ["http/1.1"] as const;
+
+function resolvedDeferred(): Deferred<void> {
+    const deferred = new Deferred<void>();
+    deferred.resolve();
+    return deferred;
+}
+
+function withSignal<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
+    return signal ? raceSignal(promise, signal) : promise;
+}
+
+function isTlsConnection(
+    conn: Dialer.ConnectionLike,
+): conn is Dialer.ConnectionLike & { getAlpnProtocol(): string | null } {
+    return (
+        "getAlpnProtocol" in conn && typeof conn.getAlpnProtocol === "function"
+    );
+}
+
+export function createAgent(
+    dialer: Dialer,
+    baseUrl: string,
+    options: Agent.Options = {},
+): Agent {
+    const base = new URL(baseUrl);
+
+    if (base.protocol !== "http:" && base.protocol !== "https:") {
+        throw new UnsupportedProtocolError(base.protocol, {
+            origin: base.origin,
+            scheme: base.protocol,
+            host: base.hostname,
+            port: base.port ? Number.parseInt(base.port, 10) : undefined,
+            url: base.toString(),
+        });
+    }
+
+    const secure = base.protocol === "https:";
+    const hostname = base.hostname;
+    const port = base.port
+        ? Number.parseInt(base.port, 10)
+        : PORT_MAP[base.protocol];
+
+    if (!Number.isFinite(port) || port <= 0 || port > 65535) {
+        throw new TypeError(`Invalid port in base URL: ${baseUrl}`);
+    }
+
+    const target: Dialer.Target = {
+        address: hostname,
+        port,
+        secure,
+        sni: secure ? hostname : undefined,
+        alpnProtocols: secure ? [...DEFAULT_ALPN_PROTOCOLS] : undefined,
+    };
+
+    const connectOptions = options.connect ?? {};
+    const readerOptions = options.io?.reader ?? {};
+    const writerOptions = options.io?.writer ?? {};
+
+    let conn: Dialer.ConnectionLike | undefined;
+    let connectPromise: Promise<Dialer.ConnectionLike> | undefined;
+
+    let closed = false;
+    let isBusy = false;
+    let lastUsedTime = Date.now();
+    let idleDeferred = resolvedDeferred();
+
+    function createBaseErrorContext() {
+        return {
+            origin: base.origin,
+            scheme: base.protocol,
+            host: hostname,
+            port,
+        } as const;
+    }
+
+    function createRequestErrorContext(url: URL, method?: string) {
+        return {
+            ...createBaseErrorContext(),
+            url: url.toString(),
+            method,
+        } as const;
+    }
+
+    function markIdle(): void {
+        isBusy = false;
+        lastUsedTime = Date.now();
+        idleDeferred.resolve();
+    }
+
+    function disposeConn(): void {
+        const current = conn;
+        conn = undefined;
+        if (!current) return;
+        try {
+            current.close();
+        } catch {}
+    }
+
+    function forceClose(): void {
+        if (closed) return;
+        closed = true;
+        disposeConn();
+        if (!isBusy) markIdle();
+    }
+
+    function assertUsable(): void {
+        if (closed) throw new AgentClosedError(createBaseErrorContext());
+    }
+
+    function assertSameOrigin(url: URL): void {
+        if (url.origin !== base.origin) {
+            throw new OriginMismatchError(base.origin, url.origin, {
+                ...createBaseErrorContext(),
+                url: url.toString(),
+            });
+        }
+    }
+
+    function configureConnection(nextConn: Dialer.ConnectionLike): void {
+        nextConn.setNoDelay(connectOptions.noDelay ?? true);
+        if (connectOptions.keepAlive !== null) {
+            nextConn.setKeepAlive(connectOptions.keepAlive ?? true);
+        }
+    }
+
+    async function connect(
+        signal?: AbortSignal,
+    ): Promise<Dialer.ConnectionLike> {
+        assertUsable();
+        if (conn) return conn;
+        if (connectPromise) return withSignal(connectPromise, signal);
+
+        let timedOut = false;
+        let timeoutId: ReturnType<typeof setTimeout> | undefined;
+        const abortController = new AbortController();
+
+        const onAbort = () => abortController.abort(signal?.reason);
+        const cleanup = () => {
+            if (timeoutId !== undefined) {
+                clearTimeout(timeoutId);
+                timeoutId = undefined;
+            }
+            if (signal) signal.removeEventListener("abort", onAbort);
+        };
+
+        if (signal) {
+            if (signal.aborted) abortController.abort(signal.reason);
+            else signal.addEventListener("abort", onAbort, { once: true });
+        }
+
+        if (
+            connectOptions.timeout != null &&
+            Number.isFinite(connectOptions.timeout) &&
+            connectOptions.timeout > 0
+        ) {
+            timeoutId = setTimeout(() => {
+                timedOut = true;
+                abortController.abort(
+                    new DOMException("Connection timed out", "TimeoutError"),
+                );
+            }, connectOptions.timeout);
+        }
+
+        connectPromise = (async () => {
+            try {
+                const nextConn = await dialer.dial(target, {
+                    signal: abortController.signal,
+                });
+
+                if (closed) {
+                    try {
+                        nextConn.close();
+                    } catch {}
+                    throw new AgentClosedError(createBaseErrorContext());
+                }
+
+                configureConnection(nextConn);
+
+                if (secure && isTlsConnection(nextConn)) {
+                    const alpn = nextConn.getAlpnProtocol();
+                    if (alpn != null && alpn !== "" && alpn !== "http/1.1") {
+                        try {
+                            nextConn.close();
+                        } catch {}
+                        throw new UnsupportedAlpnProtocolError(
+                            alpn,
+                            createBaseErrorContext(),
+                        );
+                    }
+                }
+
+                conn = nextConn;
+                return nextConn;
+            } catch (error) {
+                throw toConnectError(error, {
+                    signal,
+                    timedOut,
+                    context: createBaseErrorContext(),
+                });
+            } finally {
+                cleanup();
+                connectPromise = undefined;
+            }
+        })();
+
+        return withSignal(connectPromise, signal);
+    }
+
+    async function executeRequest(
+        sendOptions: Agent.SendOptions,
+        mapBodyError?: (err: unknown) => unknown,
+    ): Promise<Response> {
+        assertUsable();
+
+        const url =
+            typeof sendOptions.url === "string"
+                ? new URL(sendOptions.url)
+                : sendOptions.url;
+
+        const method = sendOptions.method.toUpperCase();
+        const errorContext = createRequestErrorContext(url, method);
+
+        if (sendOptions.signal?.aborted) {
+            throw new RequestAbortedError(
+                sendOptions.signal.reason,
+                errorContext,
+            );
+        }
+        if (isBusy) throw new AgentBusyError(errorContext);
+        assertSameOrigin(url);
+        if (method === "CONNECT")
+            throw new UnsupportedMethodError("CONNECT", errorContext);
+
+        isBusy = true;
+        idleDeferred = new Deferred<void>();
+
+        let finalized = false;
+        let activeConn: Dialer.ConnectionLike | undefined;
+
+        const finalize = (reusable: boolean) => {
+            if (finalized) return;
+            finalized = true;
+            if (!reusable || closed) {
+                if (conn === activeConn) disposeConn();
+                else if (activeConn) {
+                    try {
+                        activeConn.close();
+                    } catch {}
+                }
+            }
+            markIdle();
+        };
+
+        const abortListener = () => {
+            if (activeConn) {
+                if (conn === activeConn) conn = undefined;
+                try {
+                    activeConn.close();
+                } catch {}
+            }
+        };
+
+        try {
+            activeConn = await connect(sendOptions.signal);
+
+            sendOptions.signal?.addEventListener("abort", abortListener, {
+                once: true,
+            });
+
+            try {
+                await withSignal(
+                    writeRequest(
+                        activeConn,
+                        {
+                            url,
+                            method,
+                            headers: sendOptions.headers,
+                            body: sendOptions.body ?? null,
+                            signal: sendOptions.signal,
+                        },
+                        writerOptions,
+                    ),
+                    sendOptions.signal,
+                );
+            } catch (error) {
+                throw toSendError(error, {
+                    signal: sendOptions.signal,
+                    context: errorContext,
+                    phase: "request",
+                });
+            }
+
+            const isHeadRequest = method === "HEAD";
+            const shouldIgnoreBody = (status: number) =>
+                isHeadRequest ||
+                (status >= 100 && status < 200) ||
+                status === 204 ||
+                status === 304;
+
+            let response: Response;
+
+            try {
+                response = await withSignal(
+                    readResponse(
+                        activeConn,
+                        readerOptions,
+                        shouldIgnoreBody,
+                        (reusable) => {
+                            sendOptions.signal?.removeEventListener(
+                                "abort",
+                                abortListener,
+                            );
+                            finalize(reusable);
+                        },
+                        mapBodyError,
+                    ),
+                    sendOptions.signal,
+                );
+            } catch (error) {
+                throw toSendError(error, {
+                    signal: sendOptions.signal,
+                    context: errorContext,
+                    phase: "response",
+                });
+            }
+
+            return response;
+        } catch (error) {
+            sendOptions.signal?.removeEventListener("abort", abortListener);
+
+            if (activeConn) {
+                if (conn === activeConn) conn = undefined;
+                try {
+                    activeConn.close();
+                } catch {}
+            }
+
+            finalize(false);
+            throw error;
+        }
+    }
+
+    async function send(sendOptions: Agent.SendOptions): Promise<Response> {
+        const url =
+            typeof sendOptions.url === "string"
+                ? new URL(sendOptions.url)
+                : sendOptions.url;
+        const errorContext = createRequestErrorContext(
+            url,
+            sendOptions.method.toUpperCase(),
+        );
+
+        const mapBodyError =
+            (
+                sendOptions as {
+                    [bodyErrorMapperSymbol]?: (err: unknown) => unknown;
+                }
+            )[bodyErrorMapperSymbol] ??
+            ((error: unknown) =>
+                toSendError(error, {
+                    signal: sendOptions.signal,
+                    context: errorContext,
+                    phase: "body",
+                }));
+
+        return executeRequest(sendOptions, mapBodyError);
+    }
+
+    return {
+        [Symbol.dispose]: forceClose,
+        close: forceClose,
+        hostname,
+        port,
+        send,
+        whenIdle(): Promise<void> {
+            return idleDeferred.promise;
+        },
+        get isIdle(): boolean {
+            return !isBusy;
+        },
+        get lastUsed(): number {
+            return lastUsedTime;
+        },
+    };
+}

package/src/body.ts

@@ -0,0 +1,179 @@
+import { randomBytes } from "node:crypto";
+import { Readable } from "node:stream";
+import { isAnyArrayBuffer } from "node:util/types";
+import type { IClosable, IReadable } from "@fuman/io";
+import { webReadableToFuman } from "@fuman/io";
+import { utf8 } from "@fuman/utils";
+import { CRLF_LENGTH, CRLF_STR } from "./_internal/consts";
+import {
+    type FormDataPolyfill,
+    isBlob,
+    isFormData,
+    isFumanReadable,
+    isIterable,
+    isMultipartFormDataStream,
+    isReadable,
+    isReadableStream,
+    isURLSearchParameters,
+} from "./_internal/guards";
+
+export type BodyInit =
+    | Exclude<RequestInit["body"], undefined | null>
+    | FormDataPolyfill
+    | Readable
+    | (IReadable & IClosable);
+
+export interface BodyState {
+    contentLength: number | null;
+    contentType: string | null;
+
+    body:
+        | Readable
+        | ReadableStream
+        | Uint8Array
+        | (IReadable & IClosable)
+        | null;
+}
+
+type Bytes = Uint8Array<ArrayBufferLike>;
+
+const BOUNDARY = "-".repeat(2);
+
+const makeFormBoundary = (): string =>
+    `formdata-${randomBytes(8).toString("hex")}`;
+
+const getFormHeader = (
+    boundary: string,
+    name: string,
+    field: File | Blob | string,
+): string => {
+    let header = `${BOUNDARY}${boundary}${CRLF_STR}`;
+    header += `Content-Disposition: form-data; name="${name}"`;
+    if (isBlob(field)) {
+        header += `; filename="${(field as File).name ?? "blob"}"${CRLF_STR}`;
+        header += `Content-Type: ${field.type || "application/octet-stream"}`;
+    }
+    return `${header}${CRLF_STR}${CRLF_STR}`;
+};
+
+const getFormFooter = (boundary: string) =>
+    `${BOUNDARY}${boundary}${BOUNDARY}${CRLF_STR}${CRLF_STR}`;
+
+export const getFormDataLength = (form: FormData, boundary: string) => {
+    let length = Buffer.byteLength(getFormFooter(boundary));
+    for (const [name, value] of form)
+        length +=
+            Buffer.byteLength(getFormHeader(boundary, name, value)) +
+            (isBlob(value) ? value.size : Buffer.byteLength(`${value}`)) +
+            CRLF_LENGTH;
+    return length;
+};
+
+async function* generatorOfFormData(
+    form: FormData,
+    boundary: string,
+): AsyncGenerator<Bytes> {
+    for (const [name, value] of form) {
+        if (isBlob(value)) {
+            yield utf8.encoder.encode(
+                getFormHeader(boundary, name, value),
+            ) as Bytes;
+
+            for await (const chunk of value.stream() as any as AsyncIterable<Bytes>) {
+                yield chunk;
+            }
+
+            yield utf8.encoder.encode(CRLF_STR) as Bytes;
+        } else {
+            yield utf8.encoder.encode(
+                getFormHeader(boundary, name, value) + value + CRLF_STR,
+            ) as Bytes;
+        }
+    }
+    yield utf8.encoder.encode(getFormFooter(boundary)) as Bytes;
+}
+
+export const extractBody = (object: BodyInit | null): BodyState => {
+    let type: string | null = null;
+    let body:
+        | Readable
+        | ReadableStream
+        | Uint8Array
+        | (IReadable & IClosable)
+        | null;
+    let size: number | null = null;
+
+    if (object == null) {
+        body = null;
+        size = 0;
+    } else if (typeof object === "string") {
+        const bytes = utf8.encoder.encode(`${object}`);
+        type = "text/plain;charset=UTF-8";
+        size = bytes.byteLength;
+        body = bytes;
+    } else if (isURLSearchParameters(object)) {
+        const bytes = utf8.encoder.encode(object.toString());
+        body = bytes;
+        size = bytes.byteLength;
+        type = "application/x-www-form-urlencoded;charset=UTF-8";
+    } else if (isBlob(object)) {
+        size = object.size;
+        type = object.type || null;
+        body = object.stream();
+    } else if (object instanceof Uint8Array) {
+        body = object;
+        size = object.byteLength;
+    } else if (isAnyArrayBuffer(object)) {
+        const bytes = new Uint8Array(object);
+        body = bytes;
+        size = bytes.byteLength;
+    } else if (ArrayBuffer.isView(object)) {
+        const bytes = new Uint8Array(
+            object.buffer,
+            object.byteOffset,
+            object.byteLength,
+        );
+        body = bytes;
+        size = bytes.byteLength;
+    } else if (isReadableStream(object)) {
+        body = object;
+    } else if (isFumanReadable(object)) {
+        body = object;
+    } else if (isFormData(object)) {
+        const boundary = makeFormBoundary();
+        type = `multipart/form-data; boundary=${boundary}`;
+        size = getFormDataLength(object, boundary);
+        body = Readable.from(generatorOfFormData(object, boundary));
+    } else if (isMultipartFormDataStream(object)) {
+        type = `multipart/form-data; boundary=${object.getBoundary()}`;
+        size = object.hasKnownLength() ? object.getLengthSync() : null;
+        body = object as Readable;
+    } else if (isReadable(object)) {
+        body = object as Readable;
+    } else if (isIterable(object)) {
+        body = Readable.from(object);
+    } else {
+        const bytes = utf8.encoder.encode(`${object}`);
+        type = "text/plain;charset=UTF-8";
+        body = bytes;
+        size = bytes.byteLength;
+    }
+
+    return {
+        contentLength: size,
+        contentType: type,
+        body,
+    };
+};
+
+export function fromRequestBody(request: Request): BodyInit | null {
+    if (request.bodyUsed) {
+        throw new TypeError("Request body has already been used");
+    }
+
+    if (request.body == null) {
+        return null;
+    }
+
+    return webReadableToFuman(request.body);
+}

package/src/dialers/index.ts

@@ -0,0 +1,3 @@
+export type { ConnectionLike, Dialer, DialTarget } from "../types/dialer";
+export * from "./proxy";
+export * from "./tcp";

package/src/dialers/proxy.ts

@@ -0,0 +1,102 @@
+import type { ITcpConnection } from "@fuman/net";
+import type { NodeTlsUpgradeOptions } from "@fuman/node";
+import {
+    createProxyConnection,
+    type ProxyConnectionFn,
+    type ProxyInfo,
+    parse as parseProxy,
+} from "@npy/proxy-kit";
+import { connectTcp, upgradeTls } from "../_internal/net";
+import type { Dialer } from "../types/dialer";
+
+const DEFAULT_HTTP_ALPN_PROTOCOLS = ["http/1.1"] as const;
+
+type ProxyConnectOptions = Parameters<typeof connectTcp>[0];
+type UpgradableTcpConnection = Parameters<typeof upgradeTls>[0];
+type ResolvedProxy = Parameters<typeof createProxyConnection>[0]["proxy"];
+
+function normalizeProxy(proxy: ProxyDialer.Input): ResolvedProxy {
+    if (typeof proxy !== "string") {
+        return proxy;
+    }
+
+    const parsed = parseProxy(proxy, { strict: true });
+    if (parsed == null) {
+        throw new TypeError(`Invalid proxy string: ${proxy}`);
+    }
+
+    return parsed;
+}
+
+/**
+ * Dialer that routes connections through an HTTP, HTTPS or SOCKS proxy.
+ *
+ * @remarks
+ * Secure targets are tunneled and then upgraded to TLS after the proxy connection
+ * has been established.
+ */
+export class ProxyDialer implements Dialer {
+    readonly proxy: ResolvedProxy;
+    readonly #options: Readonly<ProxyDialer.Options>;
+    readonly #connectThroughProxy: ProxyConnectionFn<ProxyConnectOptions>;
+
+    constructor(proxy: ProxyDialer.Input, options: ProxyDialer.Options = {}) {
+        this.proxy = normalizeProxy(proxy);
+        this.#options = { ...options };
+        this.#connectThroughProxy = createProxyConnection({
+            proxy: this.proxy,
+            connectionFn: connectTcp,
+        });
+    }
+
+    async dial(
+        target: Dialer.Target,
+        options: Dialer.Options = {},
+    ): Promise<Dialer.ConnectionLike> {
+        const tunneled = await this.#connectThroughProxy({
+            address: target.address,
+            port: target.port,
+            signal: options.signal,
+        });
+
+        if (!target.secure) {
+            return tunneled;
+        }
+
+        return this.#upgradeSecureTarget(tunneled, target, options.signal);
+    }
+
+    async #upgradeSecureTarget(
+        conn: ITcpConnection,
+        target: Dialer.Target,
+        signal?: AbortSignal,
+    ): Promise<Dialer.ConnectionLike> {
+        const sni = target.sni ?? this.#options.sni ?? target.address;
+        const extraOptions =
+            this.#options.extraOptions || target.extraOptions
+                ? { ...this.#options.extraOptions, ...target.extraOptions }
+                : undefined;
+
+        const tlsOptions: NodeTlsUpgradeOptions & { signal?: AbortSignal } = {
+            signal,
+            caCerts: this.#options.caCerts,
+            sni,
+            alpnProtocols: target.alpnProtocols ??
+                this.#options.alpnProtocols ?? [...DEFAULT_HTTP_ALPN_PROTOCOLS],
+            extraOptions,
+        };
+
+        return upgradeTls(conn as UpgradableTcpConnection, tlsOptions);
+    }
+}
+
+export namespace ProxyDialer {
+    export type Input = string | ProxyInfo;
+
+    export interface Options {
+        caCerts?: string[];
+        sni?: string;
+        alpnProtocols?: string[];
+        extraOptions?: NodeTlsUpgradeOptions["extraOptions"];
+    }
+}