apify-client 3.0.0-beta.6 → 3.0.0-beta.60

package/dist/http_client.d.ts

@@ -1,88 +0,0 @@
-import type http from 'node:http';
-import type https from 'node:https';
-import type { AxiosInstance, AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig } from 'axios';
-import type { Log } from '@apify/log';
-import type { RequestInterceptorFunction } from './interceptors.js';
-import type { Statistics } from './statistics.js';
-export declare class HttpClient {
-    stats: Statistics;
-    maxRetries: number;
-    minDelayBetweenRetriesMillis: number;
-    userProvidedRequestInterceptors: RequestInterceptorFunction[];
-    logger: Log;
-    timeoutMillis: number;
-    httpAgent?: http.Agent;
-    httpsAgent?: https.Agent;
-    axios: AxiosInstance;
-    workflowKey?: string;
-    private nodeInitPromise?;
-    private userAgentSuffix?;
-    constructor(options: HttpClientOptions);
-    private ensureNodeInit;
-    private initNode;
-    call<T = any>(config: ApifyRequestConfig): Promise<ApifyResponse<T>>;
-    private _informAboutStreamNoRetry;
-    /**
-     * Successful responses are returned, errors and unsuccessful
-     * status codes are retried. See the following functions for the
-     * retrying logic.
-     */
-    private _createRequestHandler;
-    private _isStatusOk;
-    /**
-     * Handles all unexpected errors that can happen, but are not
-     * Apify API typed errors. E.g. network errors, timeouts and so on.
-     */
-    private _handleRequestError;
-    /**
-     * Axios calls req.abort() on timeouts so timeout errors will
-     * have a code ECONNABORTED.
-     */
-    private _isTimeoutError;
-    /**
-     * We don't want to retry every exception thrown from Axios.
-     * The common denominator for retryable errors are network issues.
-     * @param {Error} err
-     * @private
-     */
-    private _isRetryableError;
-    /**
-     * When a network connection to our API is interrupted in the middle of streaming
-     * a response, the request often does not fail, but simply contains
-     * an incomplete response. This can often be fixed by retrying.
-     */
-    private _isResponseBodyInvalid;
-    /**
-     * When a network request is attempted by axios and fails,
-     * it throws an AxiosError, which will have the request
-     * and config (and other) properties.
-     */
-    private _isNetworkError;
-    /**
-     * We retry 429 (rate limit) and 500+.
-     * For status codes 300-499 (except 429) we do not retry the request,
-     * because it's probably caused by invalid url (redirect 3xx) or invalid user input (4xx).
-     */
-    private _isStatusCodeRetryable;
-    private _onRequestRetry;
-}
-export interface ApifyRequestConfig extends AxiosRequestConfig {
-    stringifyFunctions?: boolean;
-    forceBuffer?: boolean;
-    doNotRetryTimeouts?: boolean;
-}
-export interface ApifyResponse<T = any> extends AxiosResponse<T> {
-    config: ApifyRequestConfig & InternalAxiosRequestConfig;
-}
-export interface HttpClientOptions {
-    apifyClientStats: Statistics;
-    maxRetries: number;
-    minDelayBetweenRetriesMillis: number;
-    requestInterceptors: RequestInterceptorFunction[];
-    timeoutSecs: number;
-    logger: Log;
-    token?: string;
-    workflowKey?: string;
-    /** @internal */
-    userAgentSuffix?: string | string[];
-}

package/dist/http_client.js

@@ -1,255 +0,0 @@
-import retry from 'async-retry';
-import axios, { AxiosHeaders } from 'axios';
-import { APIFY_ENV_VARS } from '@apify/consts';
-import { ApifyApiError } from './apify_api_error.js';
-import { InvalidResponseBodyError, requestInterceptors, responseInterceptors } from './interceptors.js';
-import { asArray, cast, getVersionData, isNode, isStream } from './utils.js';
-const { version } = getVersionData();
-const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
-export class HttpClient {
-    stats;
-    maxRetries;
-    minDelayBetweenRetriesMillis;
-    userProvidedRequestInterceptors;
-    logger;
-    timeoutMillis;
-    httpAgent;
-    httpsAgent;
-    axios;
-    workflowKey;
-    nodeInitPromise;
-    userAgentSuffix;
-    constructor(options) {
-        const { token } = options;
-        this.stats = options.apifyClientStats;
-        this.maxRetries = options.maxRetries;
-        this.minDelayBetweenRetriesMillis = options.minDelayBetweenRetriesMillis;
-        this.userProvidedRequestInterceptors = options.requestInterceptors;
-        this.timeoutMillis = options.timeoutSecs * 1000;
-        this.logger = options.logger;
-        this.workflowKey = options.workflowKey || process.env[APIFY_ENV_VARS.WORKFLOW_KEY];
-        this.userAgentSuffix = options.userAgentSuffix;
-        this._onRequestRetry = this._onRequestRetry.bind(this);
-        this.axios = axios.create({
-            // Disable axios's built-in proxy handling since we're using custom agents
-            proxy: false,
-            paramsSerializer: (params) => {
-                const formattedParams = Object.entries(params)
-                    .filter(([, value]) => value !== undefined)
-                    .map(([key, value]) => {
-                    if (value instanceof Date) {
-                        return [key, value.toISOString()];
-                    }
-                    const updatedValue = typeof value === 'boolean' ? Number(value) : value;
-                    return [key, String(updatedValue)];
-                });
-                return new URLSearchParams(formattedParams).toString();
-            },
-            validateStatus: null,
-            // Using interceptors for this functionality.
-            transformRequest: undefined,
-            transformResponse: undefined,
-            responseType: 'arraybuffer',
-            timeout: this.timeoutMillis,
-            // maxBodyLength needs to be Infinity, because -1 falls back to a 10 MB default
-            // from an axios subdependency - 'follow-redirects'
-            maxBodyLength: Infinity,
-            // maxContentLength must be -1, because Infinity will cause axios to run super slow
-            // thanks to a bug that's now fixed, but not released yet https://github.com/axios/axios/pull/3738
-            maxContentLength: -1,
-        });
-        // Clean all default headers because they only make a mess and their merging is difficult to understand and buggy.
-        this.axios.defaults.headers = new AxiosHeaders();
-        // If workflow key is available, pass it as a header
-        if (this.workflowKey) {
-            this.axios.defaults.headers['X-Apify-Workflow-Key'] = this.workflowKey;
-        }
-        // Attach Authorization header for all requests if token was provided
-        if (token) {
-            this.axios.defaults.headers.Authorization = `Bearer ${token}`;
-        }
-        requestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
-        this.userProvidedRequestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
-        responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
-    }
-    async ensureNodeInit() {
-        if (!isNode())
-            return;
-        this.nodeInitPromise ??= this.initNode();
-        return this.nodeInitPromise;
-    }
-    async initNode() {
-        if (!isNode())
-            return;
-        const [{ ProxyAgent }, os] = await Promise.all([import('proxy-agent'), import('node:os')]);
-        // We want to keep sockets alive for better performance.
-        // Enhanced agent configuration based on agentkeepalive best practices:
-        // - Nagle's algorithm disabled for lower latency
-        // - Free socket timeout to prevent socket leaks
-        // - LIFO scheduling to reuse recent sockets
-        // - Socket TTL for connection freshness
-        const agentOptions = {
-            keepAlive: true,
-            // Timeout for inactive sockets
-            // Prevents socket leaks from idle connections
-            timeout: this.timeoutMillis,
-            // Keep alive timeout for free sockets (15 seconds)
-            // Node.js will close unused sockets after this period
-            keepAliveMsecs: 15_000,
-            // Maximum number of sockets per host
-            maxSockets: 256,
-            maxFreeSockets: 256,
-            // LIFO scheduling - reuse most recently used sockets for better performance
-            scheduling: 'lifo',
-        };
-        // Use ProxyAgent which automatically detects proxy from environment variables
-        // and supports CONNECT tunneling
-        const proxyAgent = new ProxyAgent(agentOptions);
-        this.httpAgent = proxyAgent;
-        this.httpsAgent = proxyAgent;
-        // Disable Nagle's algorithm for lower latency
-        // This sends data immediately instead of buffering small packets
-        const setNoDelay = (socket) => {
-            socket.setNoDelay(true);
-        };
-        this.httpAgent.on('socket', setNoDelay);
-        this.httpsAgent.on('socket', setNoDelay);
-        this.axios.defaults.httpAgent = this.httpAgent;
-        this.axios.defaults.httpsAgent = this.httpsAgent;
-        // Works only in Node. Cannot be set in browser
-        const isAtHome = !!process.env[APIFY_ENV_VARS.IS_AT_HOME];
-        let userAgent = `ApifyClient/${version} (${os.platform()}; Node/${process.version}); isAtHome/${isAtHome}`;
-        if (this.userAgentSuffix) {
-            userAgent += `; ${asArray(this.userAgentSuffix).join('; ')}`;
-        }
-        this.axios.defaults.headers['User-Agent'] = userAgent;
-    }
-    async call(config) {
-        await this.ensureNodeInit();
-        this.stats.calls++;
-        const makeRequest = this._createRequestHandler(config);
-        return retry(makeRequest, {
-            retries: this.maxRetries,
-            minTimeout: this.minDelayBetweenRetriesMillis,
-            onRetry: this._onRequestRetry,
-        });
-    }
-    _informAboutStreamNoRetry() {
-        this.logger.warningOnce('Request body was a stream - retrying will not work, as part of it was already consumed.');
-        this.logger.warningOnce('If you want Apify client to handle retries for you, collect the stream into a buffer before sending it.');
-    }
-    /**
-     * Successful responses are returned, errors and unsuccessful
-     * status codes are retried. See the following functions for the
-     * retrying logic.
-     */
-    _createRequestHandler(config) {
-        const makeRequest = async (stopTrying, attempt) => {
-            this.stats.requests++;
-            let response;
-            const requestIsStream = isStream(config.data);
-            try {
-                if (requestIsStream) {
-                    // Handling redirects is not possible without buffering - part of the stream has already been sent and can't be recovered
-                    // when server sends the redirect. Therefore we need to override this in Axios config to prevent it from buffering the body.
-                    // see also axios/axios#1045
-                    config = { ...config, maxRedirects: 0 };
-                }
-                // Increase timeout with each attempt. Max timeout is bounded by the client timeout.
-                config.timeout = Math.min(this.timeoutMillis, (config.timeout ?? this.timeoutMillis) * 2 ** (attempt - 1));
-                response = await this.axios.request(config);
-                if (this._isStatusOk(response.status))
-                    return response;
-            }
-            catch (err) {
-                return cast(this._handleRequestError(err, config, stopTrying));
-            }
-            if (response.status === RATE_LIMIT_EXCEEDED_STATUS_CODE) {
-                this.stats.addRateLimitError(attempt);
-            }
-            const apiError = ApifyApiError.fromResponse(response, attempt);
-            if (this._isStatusCodeRetryable(response.status)) {
-                if (requestIsStream) {
-                    this._informAboutStreamNoRetry();
-                }
-                else {
-                    // allow a retry
-                    throw apiError;
-                }
-            }
-            stopTrying(apiError);
-            return response;
-        };
-        return makeRequest;
-    }
-    _isStatusOk(statusCode) {
-        return statusCode < 300;
-    }
-    /**
-     * Handles all unexpected errors that can happen, but are not
-     * Apify API typed errors. E.g. network errors, timeouts and so on.
-     */
-    _handleRequestError(err, config, stopTrying) {
-        if (this._isTimeoutError(err) && config.doNotRetryTimeouts) {
-            return stopTrying(err);
-        }
-        if (this._isRetryableError(err)) {
-            if (isStream(config.data)) {
-                this._informAboutStreamNoRetry();
-            }
-            else {
-                throw err;
-            }
-        }
-        return stopTrying(err);
-    }
-    /**
-     * Axios calls req.abort() on timeouts so timeout errors will
-     * have a code ECONNABORTED.
-     */
-    _isTimeoutError(err) {
-        return err.code === 'ECONNABORTED';
-    }
-    /**
-     * We don't want to retry every exception thrown from Axios.
-     * The common denominator for retryable errors are network issues.
-     * @param {Error} err
-     * @private
-     */
-    _isRetryableError(err) {
-        return this._isNetworkError(err) || this._isResponseBodyInvalid(err);
-    }
-    /**
-     * When a network connection to our API is interrupted in the middle of streaming
-     * a response, the request often does not fail, but simply contains
-     * an incomplete response. This can often be fixed by retrying.
-     */
-    _isResponseBodyInvalid(err) {
-        return err instanceof InvalidResponseBodyError;
-    }
-    /**
-     * When a network request is attempted by axios and fails,
-     * it throws an AxiosError, which will have the request
-     * and config (and other) properties.
-     */
-    _isNetworkError(err) {
-        const hasRequest = err.request && typeof err.request === 'object';
-        const hasConfig = err.config && typeof err.config === 'object';
-        return hasRequest && hasConfig;
-    }
-    /**
-     * We retry 429 (rate limit) and 500+.
-     * For status codes 300-499 (except 429) we do not retry the request,
-     * because it's probably caused by invalid url (redirect 3xx) or invalid user input (4xx).
-     */
-    _isStatusCodeRetryable(statusCode) {
-        const isRateLimitError = statusCode === RATE_LIMIT_EXCEEDED_STATUS_CODE;
-        const isInternalError = statusCode >= 500;
-        return isRateLimitError || isInternalError;
-    }
-    _onRequestRetry(error, attempt) {
-        if (attempt === Math.round(this.maxRetries / 2)) {
-            this.logger.warning(`API request failed ${attempt} times. Max attempts: ${this.maxRetries + 1}.\nCause:${error.stack}`);
-        }
-    }
-}

package/dist/interceptors.d.ts

@@ -1,19 +0,0 @@
-import type { AxiosInterceptorManager, AxiosResponse } from 'axios';
-import type { ApifyRequestConfig, ApifyResponse } from './http_client.js';
-/**
- * This error exists for the quite common situation, where only a partial JSON response is received and
- * an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
- * request. We do that by identifying this error in HttpClient.
- *
- * The properties mimic AxiosError for easier integration in HttpClient error handling.
- */
-export declare class InvalidResponseBodyError extends Error {
-    code: string;
-    response: AxiosResponse;
-    cause: Error;
-    constructor(response: AxiosResponse, cause: Error);
-}
-export type RequestInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyRequestConfig>['use']>[0];
-export type ResponseInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyResponse>['use']>[0];
-export declare const requestInterceptors: RequestInterceptorFunction[];
-export declare const responseInterceptors: ResponseInterceptorFunction[];

package/dist/interceptors.js

@@ -1,105 +0,0 @@
-import axios, { AxiosHeaders } from 'axios';
-import contentTypeParser from 'content-type';
-import { maybeParseBody } from './body_parser.js';
-import { isNode, maybeCompressValue } from './utils.js';
-/**
- * This error exists for the quite common situation, where only a partial JSON response is received and
- * an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
- * request. We do that by identifying this error in HttpClient.
- *
- * The properties mimic AxiosError for easier integration in HttpClient error handling.
- */
-export class InvalidResponseBodyError extends Error {
-    code;
-    response;
-    constructor(response, cause) {
-        super(`Response body could not be parsed.\nCause:${cause.message}`);
-        this.name = this.constructor.name;
-        this.code = 'invalid-response-body';
-        this.response = response;
-        this.cause = cause;
-    }
-}
-function serializeRequest(config) {
-    const [defaultTransform] = axios.defaults.transformRequest;
-    // The function not only serializes data, but it also adds correct headers.
-    const data = defaultTransform(config.data, config.headers);
-    // Actor inputs can include functions and we don't want to omit those,
-    // because it's convenient for users. JSON.stringify removes them.
-    // It's a bit inefficient that we serialize the JSON twice, but I feel
-    // it's a small price to pay. The axios default transform does a lot
-    // of body type checks and we would have to copy all of them to the resource clients.
-    if (config.stringifyFunctions) {
-        const contentTypeHeader = config.headers?.['Content-Type'] || config.headers?.['content-type'];
-        try {
-            const { type } = contentTypeParser.parse(contentTypeHeader);
-            if (type === 'application/json' && typeof config.data === 'object') {
-                config.data = stringifyWithFunctions(config.data);
-            }
-            else {
-                config.data = data;
-            }
-        }
-        catch {
-            config.data = data;
-        }
-    }
-    else {
-        config.data = data;
-    }
-    return config;
-}
-function ensureHeadersPrototype(config) {
-    if (config.headers && !(config.headers instanceof AxiosHeaders)) {
-        Object.setPrototypeOf(config.headers, AxiosHeaders.prototype);
-    }
-    return config;
-}
-/**
- * JSON.stringify() that serializes functions to string instead
- * of replacing them with null or removing them.
- */
-function stringifyWithFunctions(obj) {
-    return JSON.stringify(obj, (_key, value) => {
-        return typeof value === 'function' ? value.toString() : value;
-    });
-}
-async function maybeCompressRequest(config) {
-    if (config.headers?.['content-encoding'])
-        return config;
-    const maybeCompressed = await maybeCompressValue(config.data);
-    if (maybeCompressed) {
-        config.headers ??= {};
-        config.headers['content-encoding'] = maybeCompressed.encoding;
-        config.data = maybeCompressed.data;
-    }
-    return config;
-}
-function parseResponseData(response) {
-    if (!response.data || // Nothing to do here.
-        response.config.responseType !== 'arraybuffer' || // We don't want to parse custom response types.
-        response.config.forceBuffer // Apify custom property to prevent parsing of buffer.
-    ) {
-        return response;
-    }
-    const isBufferEmpty = isNode() ? !response.data.length : !response.data.byteLength;
-    if (isBufferEmpty) {
-        // undefined is better than an empty buffer
-        response.data = undefined;
-        return response;
-    }
-    const contentTypeHeader = response.headers['content-type'];
-    try {
-        response.data = maybeParseBody(response.data, contentTypeHeader);
-    }
-    catch (err) {
-        throw new InvalidResponseBodyError(response, err);
-    }
-    return response;
-}
-export const requestInterceptors = [
-    maybeCompressRequest,
-    serializeRequest,
-    ensureHeadersPrototype,
-];
-export const responseInterceptors = [parseResponseData];