apify-client 3.0.0-beta.9 → 3.0.0-rc.0

package/dist/http_clients/axios.d.ts

@@ -0,0 +1,60 @@
+import type http from 'node:http';
+import type https from 'node:https';
+import type { AxiosInstance, AxiosInterceptorManager, InternalAxiosRequestConfig } from 'axios';
+import type { HttpClientOptions, HttpRequest, HttpResponse } from './base.js';
+import { HttpClient } from './base.js';
+/**
+ * An [axios request interceptor](https://axios-http.com/docs/interceptors). It runs on every attempt, right before
+ * the request is handed to axios, and sees the request as the shared pipeline prepared it: headers merged and the
+ * body already serialized and compressed.
+ */
+export type RequestInterceptorFunction = Parameters<AxiosInterceptorManager<InternalAxiosRequestConfig>['use']>[0];
+/**
+ * Configuration of {@link AxiosHttpClient}.
+ */
+export interface AxiosHttpClientOptions extends HttpClientOptions {
+    /** @default [] */
+    requestInterceptors?: RequestInterceptorFunction[];
+}
+/**
+ * The default HTTP client of {@link ApifyClient}, built on [axios](https://axios-http.com).
+ *
+ * It inherits the request pipeline from {@link HttpClient} and adds the transport: in Node.js, keep-alive agents
+ * that honor the `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` environment variables; in the browser, the XHR
+ * adapter of axios. Network errors are retried, timeouts included, and a stream body is sent without following
+ * redirects, since the part of it already sent could not be replayed.
+ */
+export declare class AxiosHttpClient extends HttpClient {
+    #private;
+    /** The axios instance the requests go through. */
+    axios: AxiosInstance;
+    /**
+     * Keep-alive agent for plain HTTP, created with the first request in Node.js. It is the same `ProxyAgent` as
+     * {@link httpsAgent}, so the socket pool is shared across both schemes.
+     */
+    httpAgent?: http.Agent;
+    /** Keep-alive agent for HTTPS. The same instance as {@link httpAgent}. */
+    httpsAgent?: https.Agent;
+    constructor(options?: AxiosHttpClientOptions);
+    /**
+     * A timeout is an axios error coded `ECONNABORTED` or `ETIMEDOUT`, besides the `TimeoutError` the base class
+     * recognizes.
+     */
+    isTimeoutError(error: unknown): boolean;
+    /**
+     * A network failure is an axios error that carries the request it was sending: a connection that could not be
+     * made, dropped mid-way or timed out. Errors raised before a request exists, such as an invalid URL, are not
+     * retried.
+     */
+    isRetryableTransportError(error: unknown): boolean;
+    /**
+     * Destroys the keep-alive agents, closing every pooled socket. A client that never sent a request in Node.js
+     * has nothing to close.
+     */
+    close(): Promise<void>;
+    /**
+     * Sends the request through axios and hands the raw response back. A stream body is sent without following
+     * redirects, since the part of it already sent could not be replayed.
+     */
+    sendRequest(request: HttpRequest): Promise<HttpResponse>;
+}

package/dist/http_clients/axios.js

@@ -0,0 +1,117 @@
+import axios, { AxiosError, AxiosHeaders } from 'axios';
+import { runtime } from '#runtime';
+import { isStream } from '../utils.js';
+import { HttpClient } from './base.js';
+/**
+ * The default HTTP client of {@link ApifyClient}, built on [axios](https://axios-http.com).
+ *
+ * It inherits the request pipeline from {@link HttpClient} and adds the transport: in Node.js, keep-alive agents
+ * that honor the `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` environment variables; in the browser, the XHR
+ * adapter of axios. Network errors are retried, timeouts included, and a stream body is sent without following
+ * redirects, since the part of it already sent could not be replayed.
+ */
+export class AxiosHttpClient extends HttpClient {
+    /** The axios instance the requests go through. */
+    axios;
+    /**
+     * Keep-alive agent for plain HTTP, created with the first request in Node.js. It is the same `ProxyAgent` as
+     * {@link httpsAgent}, so the socket pool is shared across both schemes.
+     */
+    httpAgent;
+    /** Keep-alive agent for HTTPS. The same instance as {@link httpAgent}. */
+    httpsAgent;
+    #httpAgentsPromise;
+    constructor(options = {}) {
+        super(options);
+        this.axios = axios.create({
+            // The agents handle the proxy environment variables.
+            proxy: false,
+            validateStatus: null,
+            // The shared pipeline serializes and parses the bodies, so axios only moves bytes.
+            transformRequest: [],
+            transformResponse: [],
+            responseType: 'arraybuffer',
+            // maxBodyLength needs to be Infinity, because -1 falls back to a 10 MB default
+            // from an axios subdependency - 'follow-redirects'
+            maxBodyLength: Infinity,
+            // maxContentLength must be -1, because Infinity will cause axios to run super slow
+            // thanks to a bug that's now fixed, but not released yet https://github.com/axios/axios/pull/3738
+            maxContentLength: -1,
+        });
+        // The pipeline merges every header the client sends, so the axios defaults would only get in the way.
+        this.axios.defaults.headers = new AxiosHeaders();
+        for (const interceptor of options.requestInterceptors ?? []) {
+            this.axios.interceptors.request.use(interceptor);
+        }
+    }
+    /**
+     * A timeout is an axios error coded `ECONNABORTED` or `ETIMEDOUT`, besides the `TimeoutError` the base class
+     * recognizes.
+     */
+    isTimeoutError(error) {
+        if (super.isTimeoutError(error))
+            return true;
+        // Axios aborts the request when its own timeout fires, which gives ECONNABORTED, or ETIMEDOUT under
+        // `clarifyTimeoutError`. ETIMEDOUT also arrives straight from the socket, when the connection itself
+        // times out.
+        return (axios.isAxiosError(error) && (error.code === AxiosError.ECONNABORTED || error.code === AxiosError.ETIMEDOUT));
+    }
+    /**
+     * A network failure is an axios error that carries the request it was sending: a connection that could not be
+     * made, dropped mid-way or timed out. Errors raised before a request exists, such as an invalid URL, are not
+     * retried.
+     */
+    isRetryableTransportError(error) {
+        if (!axios.isAxiosError(error))
+            return false;
+        return typeof error.request === 'object' && error.request !== null && error.config != null;
+    }
+    /**
+     * Destroys the keep-alive agents, closing every pooled socket. A client that never sent a request in Node.js
+     * has nothing to close.
+     */
+    async close() {
+        // A failed initialization left nothing to release, and its error belongs to the request that hit it.
+        await this.#httpAgentsPromise?.catch(() => { });
+        this.httpAgent?.destroy();
+        this.httpsAgent?.destroy();
+    }
+    /**
+     * Sends the request through axios and hands the raw response back. A stream body is sent without following
+     * redirects, since the part of it already sent could not be replayed.
+     */
+    async sendRequest(request) {
+        await this.#ensureHttpAgents();
+        const { method, url, headers, body, timeoutMillis, stream, signal } = request;
+        const response = await this.axios.request({
+            method,
+            url,
+            headers,
+            data: body,
+            // Axios reads 0 as no timeout.
+            timeout: timeoutMillis ?? 0,
+            signal,
+            responseType: stream ? 'stream' : 'arraybuffer',
+            // See axios/axios#1045 for why axios must not buffer a stream body to follow redirects.
+            ...(isStream(body) ? { maxRedirects: 0 } : {}),
+        });
+        return {
+            status: response.status,
+            headers: AxiosHeaders.from(response.headers).toJSON(),
+            body: response.data,
+        };
+    }
+    async #ensureHttpAgents() {
+        this.#httpAgentsPromise ??= this.#initHttpAgents();
+        return this.#httpAgentsPromise;
+    }
+    async #initHttpAgents() {
+        const agents = await runtime.createHttpAgents({ timeoutMillis: this.timeoutMaxMillis });
+        if (!agents)
+            return;
+        this.httpAgent = agents.httpAgent;
+        this.httpsAgent = agents.httpsAgent;
+        this.axios.defaults.httpAgent = this.httpAgent;
+        this.axios.defaults.httpsAgent = this.httpsAgent;
+    }
+}

package/dist/http_clients/base.d.ts

@@ -0,0 +1,331 @@
+import type { Readable } from 'node:stream';
+import { z } from 'zod';
+import type { Log } from '@apify/log';
+import type { HttpCompressor } from '../http_compressors/base.js';
+import type { HttpCompressionAlgorithm } from '../http_compressors/resolve.js';
+import type { HttpMethod } from '../models.js';
+import { Statistics } from '../statistics.js';
+import type { Timeout, TimeoutTier } from '../timeouts.js';
+export type { HttpMethod } from '../models.js';
+export declare const DEFAULT_MAX_RETRIES = 8;
+export declare const DEFAULT_MIN_DELAY_BETWEEN_RETRIES_MILLIS = 500;
+/**
+ * Bounds of the numeric {@link HttpClientOptions}. `ApifyClientOptions` checks the same bounds and adds the defaults.
+ * @internal
+ */
+export declare const httpClientOptionsShape: {
+    maxRetries: z.ZodNumber;
+    minDelayBetweenRetriesMillis: z.ZodNumber;
+    timeoutShortSecs: z.ZodNumber;
+    timeoutMediumSecs: z.ZodNumber;
+    timeoutLongSecs: z.ZodNumber;
+    timeoutMaxSecs: z.ZodNumber;
+};
+/**
+ * Response headers keyed by lowercase header name. A header the server sent more than once may arrive as an array.
+ */
+export type HttpResponseHeaders = Record<string, string | string[] | undefined>;
+/**
+ * A request body as the transport receives it: already serialized, and compressed when that paid off. A `Readable`
+ * is passed through untouched and is only available in Node.js.
+ */
+export type HttpRequestBody = string | Buffer | ArrayBuffer | ArrayBufferView | Readable;
+/**
+ * A response body as the transport hands it back: the raw bytes, a `Readable` when the request asked for a streamed
+ * response, or `undefined` when the response carried no body.
+ */
+export type HttpResponseBody = Buffer | ArrayBuffer | Readable | undefined;
+/**
+ * How {@link HttpClient.call} hands back the response body.
+ *
+ * - `parsed` (the default) decodes it by content type: JSON becomes an object, text and XML become a string, anything
+ *   else stays a `Buffer` (an `ArrayBuffer` in the browser).
+ * - `buffer` skips the decoding and hands back the raw bytes.
+ * - `stream` hands back a `Readable` with the body unread. Node.js only.
+ */
+export type ApifyResponseType = 'parsed' | 'buffer' | 'stream';
+/**
+ * A request as the resource clients hand it to {@link HttpClient.call}.
+ */
+export interface ApifyRequestConfig {
+    /** Full URL of the endpoint, without the query string. */
+    url: string;
+    /** HTTP method. */
+    method: HttpMethod;
+    /**
+     * Query parameters. `undefined` values are dropped, booleans are sent as `1` and `0`, `Date` values as ISO 8601
+     * strings and arrays comma-separated.
+     */
+    params?: Record<string, unknown>;
+    /** Per-request headers. They win over the client's default headers, header names compared case-insensitively. */
+    headers?: Record<string, string>;
+    /**
+     * Request body. A plain object or array is serialized to JSON and sent as `application/json`, or form-encoded
+     * when a `Content-Type: application/x-www-form-urlencoded` header asks for it. A string, binary value
+     * (`Buffer`, `ArrayBuffer`, typed array) or `Readable` is sent as it is, and `URLSearchParams` is sent
+     * form-encoded. A `Blob`, `File`, `FormData` or web `ReadableStream` is rejected with a `TypeError`, since
+     * serializing one would send an empty body.
+     */
+    data?: unknown;
+    /**
+     * Timeout of the request: a tier name, a number of seconds, or `'noTimeout'`. The client resolves it to the
+     * timeout of each attempt in milliseconds, doubling it with every retry up to `timeoutMaxSecs`.
+     * @default 'medium'
+     */
+    timeoutSecs?: Timeout;
+    /** @default 'parsed' */
+    responseType?: ApifyResponseType;
+    /**
+     * Serialize function-valued fields of a JSON body to their source instead of dropping them, which is how
+     * Actor input carrying page functions reaches the API.
+     */
+    stringifyFunctions?: boolean;
+    /** Give up on the first transport timeout instead of retrying it. */
+    doNotRetryTimeouts?: boolean;
+    /**
+     * Aborts the call. Once the signal aborts, the attempt in flight is ended, no retry follows, and the call
+     * rejects with the signal's `reason`.
+     */
+    signal?: AbortSignal;
+}
+/**
+ * A successful response as {@link HttpClient.call} resolves to it.
+ *
+ * @template T - Type of the parsed body.
+ */
+export interface ApifyResponse<T = any> {
+    /** HTTP status code. */
+    status: number;
+    /** Response headers keyed by lowercase header name. */
+    headers: HttpResponseHeaders;
+    /** The response body, decoded according to the request's {@link ApifyRequestConfig.responseType}. */
+    data: T;
+    /** The request this response answers, as it was passed to {@link HttpClient.call}. */
+    config: ApifyRequestConfig;
+}
+/**
+ * One prepared request attempt, as {@link HttpClient.sendRequest} receives it from the shared pipeline.
+ */
+export interface HttpRequest {
+    /** HTTP method. */
+    method: HttpMethod;
+    /** Full request URL with the query string already encoded into it. */
+    url: string;
+    /** Final request headers, with the client's default headers already merged in. */
+    headers: Record<string, string>;
+    /** Request body, already serialized and compressed, or `undefined` for a request without one. */
+    body?: HttpRequestBody;
+    /**
+     * Timeout for this attempt in milliseconds, or `undefined` for a request that runs without one, which is what
+     * `timeoutSecs: 'noTimeout'` asks for.
+     */
+    timeoutMillis?: number;
+    /** Whether to hand the body back unread as a `Readable`, so the caller can stream it. */
+    stream: boolean;
+    /**
+     * The caller's abort signal, to hand to the HTTP library alongside the timeout. The pipeline stops retrying
+     * once it aborts, so the transport only has to end the request in flight.
+     */
+    signal?: AbortSignal;
+}
+/**
+ * A response as the transport returns it from {@link HttpClient.sendRequest}, error statuses included.
+ */
+export interface HttpResponse {
+    /** HTTP status code. */
+    status: number;
+    /** Response headers keyed by lowercase header name. */
+    headers: HttpResponseHeaders;
+    /** The body as raw bytes, or unread as a `Readable` when the request asked for a stream. */
+    body: HttpResponseBody;
+}
+/**
+ * Configuration shared by every {@link HttpClient}.
+ */
+export interface HttpClientOptions {
+    /** Apify API token, sent as a `Bearer` token in the `Authorization` header. */
+    token?: string;
+    /** How many times a failed request is retried at most. @default 8 */
+    maxRetries?: number;
+    /**
+     * Lower bound for the delay before the first retry in milliseconds. It doubles with every further retry.
+     * @default 500
+     */
+    minDelayBetweenRetriesMillis?: number;
+    /** Duration of the `short` timeout tier, in seconds. @default 5 */
+    timeoutShortSecs?: number;
+    /** Duration of the `medium` timeout tier, in seconds. @default 30 */
+    timeoutMediumSecs?: number;
+    /** Duration of the `long` timeout tier, in seconds. @default 360 */
+    timeoutLongSecs?: number;
+    /** Upper bound for the timeout of a single attempt, in seconds. @default 360 */
+    timeoutMaxSecs?: number;
+    /**
+     * Compression of request bodies: the name of a built-in algorithm, or an {@link HttpCompressor} for a custom
+     * quality or algorithm.
+     * @default 'brotli'
+     */
+    compression?: HttpCompressionAlgorithm | HttpCompressor;
+    /** Additional headers sent with every request. They win over the built-in defaults. */
+    headers?: Record<string, string>;
+    /** Statistics the client records its calls into. Created when omitted. */
+    stats?: Statistics;
+    /** Logger for the retry warnings. Defaults to a child of the `@apify/log` default with an `ApifyClient` prefix. */
+    logger?: Log;
+    /** Value of the `X-Apify-Workflow-Key` header. Defaults to the `APIFY_WORKFLOW_KEY` environment variable. */
+    workflowKey?: string;
+    /** @internal */
+    userAgentSuffix?: string | string[];
+}
+/**
+ * Base class for the HTTP clients used by {@link ApifyClient}.
+ *
+ * It holds the shared request pipeline: {@link call} merges the default headers in, serializes and compresses the
+ * body, encodes the query parameters, retries transient failures with exponential backoff, grows the timeout with
+ * every attempt, decodes the response body by content type, records statistics and converts error statuses to
+ * {@link ApifyApiError}. A concrete client only has to implement {@link sendRequest}, the transport that moves one
+ * prepared request over the wire, and can override the classification and lifecycle hooks {@link isTimeoutError},
+ * {@link isRetryableTransportError} and {@link close}.
+ *
+ * Overriding {@link call} itself is also supported and bypasses the transport hooks entirely. Such a client has
+ * to send the default headers from `defaultHeaders` with every request, otherwise the `Authorization` header never
+ * reaches the API. The protected helpers `prepareRequest()`, `buildUrl()` and `computeTimeoutMillis()` stay
+ * available to it.
+ *
+ * @example
+ * ```javascript
+ * import { ApifyClient, HttpClient } from 'apify-client';
+ *
+ * class FetchHttpClient extends HttpClient {
+ *     async sendRequest({ method, url, headers, body, timeoutMillis, signal }) {
+ *         const signals = signal ? [signal] : [];
+ *         if (timeoutMillis !== undefined) signals.push(AbortSignal.timeout(timeoutMillis));
+ *         const response = await fetch(url, { method, headers, body, signal: AbortSignal.any(signals) });
+ *         return {
+ *             status: response.status,
+ *             headers: Object.fromEntries(response.headers),
+ *             body: Buffer.from(await response.arrayBuffer()),
+ *         };
+ *     }
+ *
+ *     isRetryableTransportError(error) {
+ *         // Without this the client gives up on the first connection failure.
+ *         return this.isTimeoutError(error) || error instanceof TypeError;
+ *     }
+ * }
+ *
+ * const client = ApifyClient.withCustomHttpClient({ token: 'my-token', httpClient: new FetchHttpClient() });
+ * ```
+ */
+export declare abstract class HttpClient {
+    #private;
+    /** Statistics of the API calls made through this client. */
+    stats: Statistics;
+    /** Logger for the retry warnings. */
+    logger: Log;
+    /** How many times a failed request is retried at most. */
+    maxRetries: number;
+    /** Lower bound for the delay before the first retry in milliseconds. It doubles with every further retry. */
+    minDelayBetweenRetriesMillis: number;
+    /** Duration of each timeout tier, in milliseconds. */
+    timeoutMillis: Record<TimeoutTier, number>;
+    /** Upper bound for the timeout of a single attempt, in milliseconds. */
+    timeoutMaxMillis: number;
+    /** Compressor the request pipeline runs the bodies worth compressing through. */
+    readonly httpCompressor: HttpCompressor;
+    /**
+     * Headers sent with every request: the `Authorization` header built from the token, the `User-Agent` where the
+     * runtime lets a client set one, the `X-Apify-Workflow-Key` when a workflow key is configured, and whatever
+     * {@link HttpClientOptions.headers} added.
+     */
+    protected readonly defaultHeaders: Record<string, string>;
+    /**
+     * @param options - Configuration of the pipeline. The retry and timeout options are validated the same way the
+     * `ApifyClient` constructor validates them.
+     * @throws {ArgumentValidationError} When a retry or timeout option is out of bounds.
+     */
+    constructor(options?: HttpClientOptions);
+    /**
+     * Sets the `Authorization` header from the token, unless an authorization header is already configured.
+     *
+     * @param token - The Apify API token to send as the `Bearer` token.
+     */
+    setDefaultAuthorization(token: string): void;
+    /**
+     * Whether an error thrown by the transport is a timeout.
+     *
+     * Recognizes errors named `TimeoutError`, which is what `AbortSignal.timeout()` produces. Transports extend it
+     * with the timeout errors their HTTP library throws. The classification is independent of retryability: a
+     * timeout the retry loop should retry has to be covered by {@link isRetryableTransportError} too.
+     */
+    isTimeoutError(error: unknown): boolean;
+    /**
+     * Whether an error thrown by the transport is worth retrying.
+     *
+     * The default classifies nothing as retryable, so a transport that does not override it gives up on the first
+     * connection failure. Every transport should map its own transient errors here: connection resets, refused
+     * connections, timeouts and the like. Error responses are not the transport's concern, the pipeline decides on
+     * them from the status code.
+     */
+    isRetryableTransportError(_error: unknown): boolean;
+    /**
+     * Releases resources owned by the transport, such as a connection pool. The default does nothing.
+     */
+    close(): Promise<void>;
+    /**
+     * Sends one prepared request through the underlying HTTP library.
+     *
+     * The inherited {@link call} needs it, so every transport has to implement it. Let the library's errors
+     * propagate unwrapped: {@link call} classifies them through {@link isRetryableTransportError} and
+     * {@link isTimeoutError}. Return error responses as they are too, the pipeline turns them into
+     * {@link ApifyApiError} and decides whether to retry.
+     *
+     * @param request - The request to send, with the headers merged, the body serialized and the query encoded.
+     * @returns The response, with the body unread when `request.stream` is set and as raw bytes otherwise.
+     */
+    sendRequest(request: HttpRequest): Promise<HttpResponse>;
+    /**
+     * Makes an API request with automatic retries and exponential backoff.
+     *
+     * Network errors the transport classifies as retryable, rate limits (HTTP 429) and server errors (HTTP 5xx)
+     * are retried up to {@link maxRetries} times. Any other error status is thrown as {@link ApifyApiError} right
+     * away. A request whose body is a `Readable` is never retried, since part of the stream has already been
+     * consumed by the time the failure shows. Aborting `config.signal` ends the attempt in flight, skips the
+     * remaining retries and rejects the call with the signal's `reason`.
+     *
+     * @template T - Type of the parsed response body.
+     * @param config - The request to make.
+     * @returns The successful response.
+     * @throws {ApifyApiError} When the API responds with an error status the retries could not fix.
+     */
+    call<T = any>(config: ApifyRequestConfig): Promise<ApifyResponse<T>>;
+    /**
+     * Prepares the headers and the body of a request.
+     *
+     * Merges the client's default headers with the per-request ones, header names compared case-insensitively and
+     * the per-request values winning. Serializes an object body to JSON, setting `Content-Type: application/json`
+     * unless the caller supplied a content type. Runs the body through {@link httpCompressor} unless a
+     * `Content-Encoding` header is already set, the body is too small, or its content type says the payload is
+     * already compressed. A caller-supplied `Content-Encoding` is forwarded as it is, which is how a pre-encoded
+     * body is uploaded, and `Content-Encoding: identity` opts a single request out of compression.
+     */
+    protected prepareRequest(config: ApifyRequestConfig): Promise<{
+        headers: Record<string, string>;
+        body: HttpRequestBody | undefined;
+    }>;
+    /**
+     * Appends the query parameters to the URL. `undefined` values are dropped, booleans are sent as `1` and `0`,
+     * `Date` values as ISO 8601 strings and arrays comma-separated.
+     */
+    protected buildUrl(url: string, params?: Record<string, unknown>): string;
+    /**
+     * Resolves `timeoutSecs` to the number of milliseconds the given attempt gets. A tier name resolves to its
+     * configured duration, a number is taken as seconds, and `'noTimeout'` becomes `undefined`. The result doubles
+     * with each attempt and is capped at `timeoutMaxMillis`. A requested value above the cap is capped too, which
+     * warns once, since it does not take effect in full.
+     *
+     * @param attempt - Current attempt number, starting at 1.
+     * @param timeoutSecs - The request's timeout. Defaults to the `medium` tier.
+     */
+    protected computeTimeoutMillis(attempt: number, timeoutSecs?: Timeout): number | undefined;
+}