apify-client 3.0.0-beta.22 → 3.0.0-beta.24

package/dist/http_client.js

@@ -1,260 +0,0 @@
-import retry from 'async-retry';
-import axios, { AxiosHeaders } from 'axios';
-import { APIFY_ENV_VARS } from '@apify/consts';
-import { concatStreamToBuffer } from '@apify/utilities';
-import { ApifyApiError } from './apify_api_error.js';
-import { InvalidResponseBodyError, requestInterceptors, responseInterceptors } from './interceptors.js';
-import { runtime } from '#runtime';
-import { asArray, cast, getEnv, isStream, version } from './utils.js';
-const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
-export class HttpClient {
-    stats;
-    maxRetries;
-    minDelayBetweenRetriesMillis;
-    userProvidedRequestInterceptors;
-    logger;
-    /** Duration of each timeout tier, in milliseconds. */
-    timeoutMillis;
-    /** Cap on the timeout of a single request attempt, in milliseconds. */
-    timeoutMaxMillis;
-    httpAgent;
-    httpsAgent;
-    axios;
-    workflowKey;
-    #httpAgentsPromise;
-    constructor(options) {
-        const { token } = options;
-        this.stats = options.apifyClientStats;
-        this.maxRetries = options.maxRetries;
-        this.minDelayBetweenRetriesMillis = options.minDelayBetweenRetriesMillis;
-        this.userProvidedRequestInterceptors = options.requestInterceptors;
-        this.timeoutMillis = {
-            short: options.timeoutShortSecs * 1000,
-            medium: options.timeoutMediumSecs * 1000,
-            long: options.timeoutLongSecs * 1000,
-        };
-        this.timeoutMaxMillis = options.timeoutMaxSecs * 1000;
-        this.logger = options.logger;
-        this.workflowKey = options.workflowKey || getEnv(APIFY_ENV_VARS.WORKFLOW_KEY);
-        this.axios = axios.create({
-            // Disable axios's built-in proxy handling since we're using custom agents
-            proxy: false,
-            paramsSerializer: (params) => {
-                const formattedParams = Object.entries(params)
-                    .filter(([, value]) => value !== undefined)
-                    .map(([key, value]) => {
-                    if (value instanceof Date) {
-                        return [key, value.toISOString()];
-                    }
-                    const updatedValue = typeof value === 'boolean' ? Number(value) : value;
-                    return [key, String(updatedValue)];
-                });
-                return new URLSearchParams(formattedParams).toString();
-            },
-            validateStatus: null,
-            // Interceptors serialize requests and parse responses instead. Empty arrays rather than `undefined`,
-            // which axios fills in with its default transforms.
-            transformRequest: [],
-            transformResponse: [],
-            responseType: 'arraybuffer',
-            // Every request sets its own timeout in `createRequestHandler`, so the default only backs a raw
-            // `axios.request()` call.
-            timeout: this.timeoutMaxMillis,
-            // maxBodyLength needs to be Infinity, because -1 falls back to a 10 MB default
-            // from an axios subdependency - 'follow-redirects'
-            maxBodyLength: Infinity,
-            // maxContentLength must be -1, because Infinity will cause axios to run super slow
-            // thanks to a bug that's now fixed, but not released yet https://github.com/axios/axios/pull/3738
-            maxContentLength: -1,
-        });
-        // Clean all default headers because they only make a mess and their merging is difficult to understand and buggy.
-        this.axios.defaults.headers = new AxiosHeaders();
-        // If workflow key is available, pass it as a header
-        if (this.workflowKey) {
-            this.axios.defaults.headers['X-Apify-Workflow-Key'] = this.workflowKey;
-        }
-        // Attach Authorization header for all requests if token was provided
-        if (token) {
-            this.axios.defaults.headers.Authorization = `Bearer ${token}`;
-        }
-        // Browsers do not let a page set the header, so it is only sent where the runtime describes its platform.
-        if (runtime.platform) {
-            const isAtHome = !!getEnv(APIFY_ENV_VARS.IS_AT_HOME);
-            let userAgent = `ApifyClient/${version} (${runtime.platform}); isAtHome/${isAtHome}`;
-            if (options.userAgentSuffix) {
-                userAgent += `; ${asArray(options.userAgentSuffix).join('; ')}`;
-            }
-            this.axios.defaults.headers['User-Agent'] = userAgent;
-        }
-        requestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
-        this.userProvidedRequestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
-        responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
-    }
-    async #ensureHttpAgents() {
-        this.#httpAgentsPromise ??= this.#initHttpAgents();
-        return this.#httpAgentsPromise;
-    }
-    async #initHttpAgents() {
-        const agents = await runtime.createHttpAgents({ timeoutMillis: this.timeoutMaxMillis });
-        if (!agents)
-            return;
-        this.httpAgent = agents.httpAgent;
-        this.httpsAgent = agents.httpsAgent;
-        this.axios.defaults.httpAgent = this.httpAgent;
-        this.axios.defaults.httpsAgent = this.httpsAgent;
-    }
-    async call(config) {
-        await this.#ensureHttpAgents();
-        this.stats.calls++;
-        const makeRequest = this.#createRequestHandler(config);
-        return retry(makeRequest, {
-            retries: this.maxRetries,
-            minTimeout: this.minDelayBetweenRetriesMillis,
-            onRetry: (error, attempt) => this.#onRequestRetry(error, attempt),
-        });
-    }
-    #informAboutStreamNoRetry() {
-        this.logger.warningOnce('Request body was a stream - retrying will not work, as part of it was already consumed.');
-        this.logger.warningOnce('If you want Apify client to handle retries for you, collect the stream into a buffer before sending it.');
-    }
-    /**
-     * Successful responses are returned, errors and unsuccessful
-     * status codes are retried. See the following functions for the
-     * retrying logic.
-     */
-    #createRequestHandler(config) {
-        const { timeoutSecs = 'medium', ...axiosConfig } = config;
-        const makeRequest = async (stopTrying, attempt) => {
-            this.stats.requests++;
-            let response;
-            const requestIsStream = isStream(config.data);
-            try {
-                if (requestIsStream) {
-                    // Handling redirects is not possible without buffering - part of the stream has already been sent and can't be recovered
-                    // when server sends the redirect. Therefore we need to override this in Axios config to prevent it from buffering the body.
-                    // see also axios/axios#1045
-                    axiosConfig.maxRedirects = 0;
-                }
-                response = await this.axios.request({
-                    ...axiosConfig,
-                    timeout: this.#computeTimeoutMillis(timeoutSecs, attempt),
-                });
-                if (this.#isStatusOk(response.status))
-                    return response;
-                // A failed request with `responseType: 'stream'` carries the API error body in the stream. Read
-                // it so that `ApifyApiError` can parse it like any other error body. A body that cannot be read
-                // leaves the error without a message, which beats losing the status code to a stream error.
-                if (isStream(response.data)) {
-                    response.data = await concatStreamToBuffer(response.data).catch(() => undefined);
-                }
-            }
-            catch (err) {
-                return cast(this.#handleRequestError(err, config, stopTrying));
-            }
-            if (response.status === RATE_LIMIT_EXCEEDED_STATUS_CODE) {
-                this.stats.addRateLimitError(attempt);
-            }
-            const apiError = ApifyApiError.fromResponse(response, attempt);
-            if (this.#isStatusCodeRetryable(response.status)) {
-                if (requestIsStream) {
-                    this.#informAboutStreamNoRetry();
-                }
-                else {
-                    // allow a retry
-                    throw apiError;
-                }
-            }
-            stopTrying(apiError);
-            return response;
-        };
-        return makeRequest;
-    }
-    #isStatusOk(statusCode) {
-        return statusCode < 300;
-    }
-    /**
-     * Resolves `timeoutSecs` to the number of milliseconds the given attempt gets. A tier name resolves to
-     * its configured duration, a number is taken as seconds, and `'noTimeout'` becomes `0`, which axios reads
-     * as no timeout. The result doubles with each attempt and is capped at `timeoutMaxMillis`. A requested
-     * value above the cap is capped too, which warns once, since it does not take effect in full.
-     */
-    #computeTimeoutMillis(timeoutSecs, attempt) {
-        if (timeoutSecs === 'noTimeout')
-            return 0;
-        const requestedMillis = typeof timeoutSecs === 'number' ? timeoutSecs * 1000 : this.timeoutMillis[timeoutSecs];
-        if (requestedMillis > this.timeoutMaxMillis) {
-            // `warningOnce` keys by message, so each requested value warns once.
-            this.logger.warningOnce(`The requested timeout of ${requestedMillis / 1000}s exceeds timeoutMaxSecs ` +
-                `(${this.timeoutMaxMillis / 1000}s) and is capped at it. ` +
-                'Raise timeoutMaxSecs on the client to allow longer request timeouts.');
-        }
-        return Math.min(requestedMillis * 2 ** (attempt - 1), this.timeoutMaxMillis);
-    }
-    /**
-     * Handles all unexpected errors that can happen, but are not
-     * Apify API typed errors. E.g. network errors, timeouts and so on.
-     */
-    #handleRequestError(err, config, stopTrying) {
-        if (this.#isTimeoutError(err) && config.doNotRetryTimeouts) {
-            return stopTrying(err);
-        }
-        if (this.#isRetryableError(err)) {
-            if (isStream(config.data)) {
-                this.#informAboutStreamNoRetry();
-            }
-            else {
-                throw err;
-            }
-        }
-        return stopTrying(err);
-    }
-    /**
-     * Axios calls req.abort() on timeouts so timeout errors will
-     * have a code ECONNABORTED.
-     */
-    #isTimeoutError(err) {
-        return err.code === 'ECONNABORTED';
-    }
-    /**
-     * We don't want to retry every exception thrown from Axios.
-     * The common denominator for retryable errors are network issues.
-     * @param {Error} err
-     * @private
-     */
-    #isRetryableError(err) {
-        return this.#isNetworkError(err) || this.#isResponseBodyInvalid(err);
-    }
-    /**
-     * When a network connection to our API is interrupted in the middle of streaming
-     * a response, the request often does not fail, but simply contains
-     * an incomplete response. This can often be fixed by retrying.
-     */
-    #isResponseBodyInvalid(err) {
-        return err instanceof InvalidResponseBodyError;
-    }
-    /**
-     * When a network request is attempted by axios and fails,
-     * it throws an AxiosError, which will have the request
-     * and config (and other) properties.
-     */
-    #isNetworkError(err) {
-        const hasRequest = err.request && typeof err.request === 'object';
-        const hasConfig = err.config && typeof err.config === 'object';
-        return hasRequest && hasConfig;
-    }
-    /**
-     * We retry 429 (rate limit) and 500+.
-     * For status codes 300-499 (except 429) we do not retry the request,
-     * because it's probably caused by invalid url (redirect 3xx) or invalid user input (4xx).
-     */
-    #isStatusCodeRetryable(statusCode) {
-        const isRateLimitError = statusCode === RATE_LIMIT_EXCEEDED_STATUS_CODE;
-        const isInternalError = statusCode >= 500;
-        return isRateLimitError || isInternalError;
-    }
-    #onRequestRetry(error, attempt) {
-        if (attempt === Math.round(this.maxRetries / 2)) {
-            this.logger.warning(`API request failed ${attempt} times. Max attempts: ${this.maxRetries + 1}.\nCause:${error.stack}`);
-        }
-    }
-}

package/dist/interceptors.d.ts

@@ -1,19 +0,0 @@
-import type { AxiosInterceptorManager, AxiosResponse } from 'axios';
-import type { ApifyRequestConfig, ApifyResponse } from './http_client.js';
-/**
- * This error exists for the quite common situation, where only a partial JSON response is received and
- * an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
- * request. We do that by identifying this error in HttpClient.
- *
- * The properties mimic AxiosError for easier integration in HttpClient error handling.
- */
-export declare class InvalidResponseBodyError extends Error {
-    code: string;
-    response: AxiosResponse;
-    cause: Error;
-    constructor(response: AxiosResponse, cause: Error);
-}
-export type RequestInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyRequestConfig>['use']>[0];
-export type ResponseInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyResponse>['use']>[0];
-export declare const requestInterceptors: RequestInterceptorFunction[];
-export declare const responseInterceptors: ResponseInterceptorFunction[];

package/dist/interceptors.js

@@ -1,123 +0,0 @@
-import axios, { AxiosHeaders } from 'axios';
-import contentTypeParser from 'content-type';
-import { maybeParseBody } from './body_parser.js';
-import { isCompressibleContentType, maybeCompressValue } from './utils.js';
-/**
- * This error exists for the quite common situation, where only a partial JSON response is received and
- * an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
- * request. We do that by identifying this error in HttpClient.
- *
- * The properties mimic AxiosError for easier integration in HttpClient error handling.
- */
-export class InvalidResponseBodyError extends Error {
-    code;
-    response;
-    constructor(response, cause) {
-        super(`Response body could not be parsed.\nCause:${cause.message}`);
-        this.name = this.constructor.name;
-        this.code = 'invalid-response-body';
-        this.response = response;
-        this.cause = cause;
-    }
-}
-/**
- * Reads a request header regardless of the casing it was set with, since HTTP header names are case-insensitive
- * while the config keeps whatever casing the caller used.
- */
-function getHeader(config, name) {
-    const wanted = name.toLowerCase();
-    const key = Object.keys(config.headers ?? {}).find((candidate) => candidate.toLowerCase() === wanted);
-    const value = key === undefined ? undefined : config.headers?.[key];
-    return typeof value === 'string' ? value : undefined;
-}
-function serializeRequest(config) {
-    // A string body with an explicit content type is already serialized and goes out as it is. The axios default
-    // transform would otherwise parse a JSON one in full just to check that it is valid, which for a body assembled
-    // from thousands of pre-serialized requests costs about as much as serializing them did.
-    if (typeof config.data === 'string' && getHeader(config, 'content-type'))
-        return config;
-    const [defaultTransform] = axios.defaults.transformRequest;
-    // The function not only serializes data, but it also adds correct headers.
-    const data = defaultTransform(config.data, config.headers);
-    // Actor inputs can include functions and we don't want to omit those,
-    // because it's convenient for users. JSON.stringify removes them.
-    // It's a bit inefficient that we serialize the JSON twice, but I feel
-    // it's a small price to pay. The axios default transform does a lot
-    // of body type checks and we would have to copy all of them to the resource clients.
-    if (config.stringifyFunctions) {
-        const contentTypeHeader = getHeader(config, 'content-type');
-        try {
-            const type = contentTypeHeader ? contentTypeParser.parse(contentTypeHeader).type : undefined;
-            if (type === 'application/json' && typeof config.data === 'object') {
-                config.data = stringifyWithFunctions(config.data);
-            }
-            else {
-                config.data = data;
-            }
-        }
-        catch {
-            config.data = data;
-        }
-    }
-    else {
-        config.data = data;
-    }
-    return config;
-}
-function ensureHeadersPrototype(config) {
-    if (config.headers && !(config.headers instanceof AxiosHeaders)) {
-        Object.setPrototypeOf(config.headers, AxiosHeaders.prototype);
-    }
-    return config;
-}
-/**
- * JSON.stringify() that serializes functions to string instead
- * of replacing them with null or removing them.
- */
-function stringifyWithFunctions(obj) {
-    return JSON.stringify(obj, (_key, value) => {
-        return typeof value === 'function' ? value.toString() : value;
-    });
-}
-async function maybeCompressRequest(config) {
-    // A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone.
-    if (getHeader(config, 'content-encoding'))
-        return config;
-    if (!isCompressibleContentType(getHeader(config, 'content-type')))
-        return config;
-    const maybeCompressed = await maybeCompressValue(config.data);
-    if (maybeCompressed) {
-        config.headers ??= {};
-        config.headers['content-encoding'] = maybeCompressed.encoding;
-        config.data = maybeCompressed.data;
-    }
-    return config;
-}
-function parseResponseData(response) {
-    if (!response.data || // Nothing to do here.
-        response.config.responseType !== 'arraybuffer' || // We don't want to parse custom response types.
-        response.config.forceBuffer // Apify custom property to prevent parsing of buffer.
-    ) {
-        return response;
-    }
-    // A `Buffer` from the Node.js adapter and an `ArrayBuffer` from the browser ones both carry `byteLength`.
-    if (!response.data.byteLength) {
-        // undefined is better than an empty buffer
-        response.data = undefined;
-        return response;
-    }
-    const contentTypeHeader = response.headers['content-type'];
-    try {
-        response.data = maybeParseBody(response.data, contentTypeHeader);
-    }
-    catch (err) {
-        throw new InvalidResponseBodyError(response, err);
-    }
-    return response;
-}
-export const requestInterceptors = [
-    maybeCompressRequest,
-    serializeRequest,
-    ensureHeadersPrototype,
-];
-export const responseInterceptors = [parseResponseData];