apify-client 3.0.0-beta.2 → 3.0.0-beta.21

package/dist/http_client.d.ts

@@ -4,72 +4,36 @@ import type { AxiosInstance, AxiosRequestConfig, AxiosResponse, InternalAxiosReq
 import type { Log } from '@apify/log';
 import type { RequestInterceptorFunction } from './interceptors.js';
 import type { Statistics } from './statistics.js';
+import type { Timeout, TimeoutTier } from './timeouts.js';
 export declare class HttpClient {
+    #private;
     stats: Statistics;
     maxRetries: number;
     minDelayBetweenRetriesMillis: number;
     userProvidedRequestInterceptors: RequestInterceptorFunction[];
     logger: Log;
-    timeoutMillis: number;
+    /** Duration of each timeout tier, in milliseconds. */
+    timeoutMillis: Record<TimeoutTier, number>;
+    /** Cap on the timeout of a single request attempt, in milliseconds. */
+    timeoutMaxMillis: number;
     httpAgent?: http.Agent;
     httpsAgent?: https.Agent;
     axios: AxiosInstance;
     workflowKey?: string;
-    private nodeInitPromise?;
-    private userAgentSuffix?;
     constructor(options: HttpClientOptions);
-    private ensureNodeInit;
-    private initNode;
     call<T = any>(config: ApifyRequestConfig): Promise<ApifyResponse<T>>;
-    private _informAboutStreamNoRetry;
-    /**
-     * Successful responses are returned, errors and unsuccessful
-     * status codes are retried. See the following functions for the
-     * retrying logic.
-     */
-    private _createRequestHandler;
-    private _isStatusOk;
-    /**
-     * Handles all unexpected errors that can happen, but are not
-     * Apify API typed errors. E.g. network errors, timeouts and so on.
-     */
-    private _handleRequestError;
-    /**
-     * Axios calls req.abort() on timeouts so timeout errors will
-     * have a code ECONNABORTED.
-     */
-    private _isTimeoutError;
-    /**
-     * We don't want to retry every exception thrown from Axios.
-     * The common denominator for retryable errors are network issues.
-     * @param {Error} err
-     * @private
-     */
-    private _isRetryableError;
-    /**
-     * When a network connection to our API is interrupted in the middle of streaming
-     * a response, the request often does not fail, but simply contains
-     * an incomplete response. This can often be fixed by retrying.
-     */
-    private _isResponseBodyInvalid;
-    /**
-     * When a network request is attempted by axios and fails,
-     * it throws an AxiosError, which will have the request
-     * and config (and other) properties.
-     */
-    private _isNetworkError;
-    /**
-     * We retry 429 (rate limit) and 500+.
-     * For status codes 300-499 (except 429) we do not retry the request,
-     * because it's probably caused by invalid url (redirect 3xx) or invalid user input (4xx).
-     */
-    private _isStatusCodeRetryable;
-    private _onRequestRetry;
 }
-export interface ApifyRequestConfig extends AxiosRequestConfig {
+export interface ApifyRequestConfig extends Omit<AxiosRequestConfig, 'timeout'> {
     stringifyFunctions?: boolean;
     forceBuffer?: boolean;
     doNotRetryTimeouts?: boolean;
+    /**
+     * Timeout of the request: a tier name, a number of seconds, or `'noTimeout'`. The client resolves it to
+     * the axios `timeout` in milliseconds for each attempt, before axios runs its interceptors, so a request
+     * interceptor already sees a number of milliseconds.
+     * @default 'medium'
+     */
+    timeoutSecs?: Timeout;
 }
 export interface ApifyResponse<T = any> extends AxiosResponse<T> {
     config: ApifyRequestConfig & InternalAxiosRequestConfig;
@@ -79,7 +43,10 @@ export interface HttpClientOptions {
     maxRetries: number;
     minDelayBetweenRetriesMillis: number;
     requestInterceptors: RequestInterceptorFunction[];
-    timeoutSecs: number;
+    timeoutShortSecs: number;
+    timeoutMediumSecs: number;
+    timeoutLongSecs: number;
+    timeoutMaxSecs: number;
     logger: Log;
     token?: string;
     workflowKey?: string;

package/dist/http_client.js

@@ -1,10 +1,11 @@
 import retry from 'async-retry';
 import axios, { AxiosHeaders } from 'axios';
 import { APIFY_ENV_VARS } from '@apify/consts';
+import { concatStreamToBuffer } from '@apify/utilities';
 import { ApifyApiError } from './apify_api_error.js';
 import { InvalidResponseBodyError, requestInterceptors, responseInterceptors } from './interceptors.js';
-import { asArray, cast, getVersionData, isNode, isStream } from './utils.js';
-const { version } = getVersionData();
+import { runtime } from '#runtime';
+import { asArray, cast, getEnv, isStream, version } from './utils.js';
 const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
 export class HttpClient {
     stats;
@@ -12,24 +13,29 @@ export class HttpClient {
     minDelayBetweenRetriesMillis;
     userProvidedRequestInterceptors;
     logger;
+    /** Duration of each timeout tier, in milliseconds. */
     timeoutMillis;
+    /** Cap on the timeout of a single request attempt, in milliseconds. */
+    timeoutMaxMillis;
     httpAgent;
     httpsAgent;
     axios;
     workflowKey;
-    nodeInitPromise;
-    userAgentSuffix;
+    #httpAgentsPromise;
     constructor(options) {
         const { token } = options;
         this.stats = options.apifyClientStats;
         this.maxRetries = options.maxRetries;
         this.minDelayBetweenRetriesMillis = options.minDelayBetweenRetriesMillis;
         this.userProvidedRequestInterceptors = options.requestInterceptors;
-        this.timeoutMillis = options.timeoutSecs * 1000;
+        this.timeoutMillis = {
+            short: options.timeoutShortSecs * 1000,
+            medium: options.timeoutMediumSecs * 1000,
+            long: options.timeoutLongSecs * 1000,
+        };
+        this.timeoutMaxMillis = options.timeoutMaxSecs * 1000;
         this.logger = options.logger;
-        this.workflowKey = options.workflowKey || process.env[APIFY_ENV_VARS.WORKFLOW_KEY];
-        this.userAgentSuffix = options.userAgentSuffix;
-        this._onRequestRetry = this._onRequestRetry.bind(this);
+        this.workflowKey = options.workflowKey || getEnv(APIFY_ENV_VARS.WORKFLOW_KEY);
         this.axios = axios.create({
             // Disable axios's built-in proxy handling since we're using custom agents
             proxy: false,
@@ -46,11 +52,14 @@ export class HttpClient {
                 return new URLSearchParams(formattedParams).toString();
             },
             validateStatus: null,
-            // Using interceptors for this functionality.
-            transformRequest: undefined,
-            transformResponse: undefined,
+            // Interceptors serialize requests and parse responses instead. Empty arrays rather than `undefined`,
+            // which axios fills in with its default transforms.
+            transformRequest: [],
+            transformResponse: [],
             responseType: 'arraybuffer',
-            timeout: this.timeoutMillis,
+            // Every request sets its own timeout in `createRequestHandler`, so the default only backs a raw
+            // `axios.request()` call.
+            timeout: this.timeoutMaxMillis,
             // maxBodyLength needs to be Infinity, because -1 falls back to a 10 MB default
             // from an axios subdependency - 'follow-redirects'
             maxBodyLength: Infinity,
@@ -68,73 +77,43 @@ export class HttpClient {
         if (token) {
             this.axios.defaults.headers.Authorization = `Bearer ${token}`;
         }
+        // Browsers do not let a page set the header, so it is only sent where the runtime describes its platform.
+        if (runtime.platform) {
+            const isAtHome = !!getEnv(APIFY_ENV_VARS.IS_AT_HOME);
+            let userAgent = `ApifyClient/${version} (${runtime.platform}); isAtHome/${isAtHome}`;
+            if (options.userAgentSuffix) {
+                userAgent += `; ${asArray(options.userAgentSuffix).join('; ')}`;
+            }
+            this.axios.defaults.headers['User-Agent'] = userAgent;
+        }
         requestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
         this.userProvidedRequestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
         responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
     }
-    async ensureNodeInit() {
-        if (!isNode())
-            return;
-        this.nodeInitPromise ??= this.initNode();
-        return this.nodeInitPromise;
+    async #ensureHttpAgents() {
+        this.#httpAgentsPromise ??= this.#initHttpAgents();
+        return this.#httpAgentsPromise;
     }
-    async initNode() {
-        if (!isNode())
+    async #initHttpAgents() {
+        const agents = await runtime.createHttpAgents({ timeoutMillis: this.timeoutMaxMillis });
+        if (!agents)
             return;
-        const [{ ProxyAgent }, os] = await Promise.all([import('proxy-agent'), import('node:os')]);
-        // We want to keep sockets alive for better performance.
-        // Enhanced agent configuration based on agentkeepalive best practices:
-        // - Nagle's algorithm disabled for lower latency
-        // - Free socket timeout to prevent socket leaks
-        // - LIFO scheduling to reuse recent sockets
-        // - Socket TTL for connection freshness
-        const agentOptions = {
-            keepAlive: true,
-            // Timeout for inactive sockets
-            // Prevents socket leaks from idle connections
-            timeout: this.timeoutMillis,
-            // Keep alive timeout for free sockets (15 seconds)
-            // Node.js will close unused sockets after this period
-            keepAliveMsecs: 15_000,
-            // Maximum number of sockets per host
-            maxSockets: 256,
-            maxFreeSockets: 256,
-            // LIFO scheduling - reuse most recently used sockets for better performance
-            scheduling: 'lifo',
-        };
-        // Use ProxyAgent which automatically detects proxy from environment variables
-        // and supports CONNECT tunneling
-        const proxyAgent = new ProxyAgent(agentOptions);
-        this.httpAgent = proxyAgent;
-        this.httpsAgent = proxyAgent;
-        // Disable Nagle's algorithm for lower latency
-        // This sends data immediately instead of buffering small packets
-        const setNoDelay = (socket) => {
-            socket.setNoDelay(true);
-        };
-        this.httpAgent.on('socket', setNoDelay);
-        this.httpsAgent.on('socket', setNoDelay);
+        this.httpAgent = agents.httpAgent;
+        this.httpsAgent = agents.httpsAgent;
         this.axios.defaults.httpAgent = this.httpAgent;
         this.axios.defaults.httpsAgent = this.httpsAgent;
-        // Works only in Node. Cannot be set in browser
-        const isAtHome = !!process.env[APIFY_ENV_VARS.IS_AT_HOME];
-        let userAgent = `ApifyClient/${version} (${os.platform()}; Node/${process.version}); isAtHome/${isAtHome}`;
-        if (this.userAgentSuffix) {
-            userAgent += `; ${asArray(this.userAgentSuffix).join('; ')}`;
-        }
-        this.axios.defaults.headers['User-Agent'] = userAgent;
     }
     async call(config) {
-        await this.ensureNodeInit();
+        await this.#ensureHttpAgents();
         this.stats.calls++;
-        const makeRequest = this._createRequestHandler(config);
+        const makeRequest = this.#createRequestHandler(config);
         return retry(makeRequest, {
             retries: this.maxRetries,
             minTimeout: this.minDelayBetweenRetriesMillis,
-            onRetry: this._onRequestRetry,
+            onRetry: (error, attempt) => this.#onRequestRetry(error, attempt),
         });
     }
-    _informAboutStreamNoRetry() {
+    #informAboutStreamNoRetry() {
         this.logger.warningOnce('Request body was a stream - retrying will not work, as part of it was already consumed.');
         this.logger.warningOnce('If you want Apify client to handle retries for you, collect the stream into a buffer before sending it.');
     }
@@ -143,7 +122,8 @@ export class HttpClient {
      * status codes are retried. See the following functions for the
      * retrying logic.
      */
-    _createRequestHandler(config) {
+    #createRequestHandler(config) {
+        const { timeoutSecs = 'medium', ...axiosConfig } = config;
         const makeRequest = async (stopTrying, attempt) => {
             this.stats.requests++;
             let response;
@@ -153,24 +133,31 @@ export class HttpClient {
                     // Handling redirects is not possible without buffering - part of the stream has already been sent and can't be recovered
                     // when server sends the redirect. Therefore we need to override this in Axios config to prevent it from buffering the body.
                     // see also axios/axios#1045
-                    config = { ...config, maxRedirects: 0 };
+                    axiosConfig.maxRedirects = 0;
                 }
-                // Increase timeout with each attempt. Max timeout is bounded by the client timeout.
-                config.timeout = Math.min(this.timeoutMillis, (config.timeout ?? this.timeoutMillis) * 2 ** (attempt - 1));
-                response = await this.axios.request(config);
-                if (this._isStatusOk(response.status))
+                response = await this.axios.request({
+                    ...axiosConfig,
+                    timeout: this.#computeTimeoutMillis(timeoutSecs, attempt),
+                });
+                if (this.#isStatusOk(response.status))
                     return response;
+                // A failed request with `responseType: 'stream'` carries the API error body in the stream. Read
+                // it so that `ApifyApiError` can parse it like any other error body. A body that cannot be read
+                // leaves the error without a message, which beats losing the status code to a stream error.
+                if (isStream(response.data)) {
+                    response.data = await concatStreamToBuffer(response.data).catch(() => undefined);
+                }
             }
             catch (err) {
-                return cast(this._handleRequestError(err, config, stopTrying));
+                return cast(this.#handleRequestError(err, config, stopTrying));
             }
             if (response.status === RATE_LIMIT_EXCEEDED_STATUS_CODE) {
                 this.stats.addRateLimitError(attempt);
             }
-            const apiError = new ApifyApiError(response, attempt);
-            if (this._isStatusCodeRetryable(response.status)) {
+            const apiError = ApifyApiError.fromResponse(response, attempt);
+            if (this.#isStatusCodeRetryable(response.status)) {
                 if (requestIsStream) {
-                    this._informAboutStreamNoRetry();
+                    this.#informAboutStreamNoRetry();
                 }
                 else {
                     // allow a retry
@@ -182,20 +169,38 @@ export class HttpClient {
         };
         return makeRequest;
     }
-    _isStatusOk(statusCode) {
+    #isStatusOk(statusCode) {
         return statusCode < 300;
     }
+    /**
+     * Resolves `timeoutSecs` to the number of milliseconds the given attempt gets. A tier name resolves to
+     * its configured duration, a number is taken as seconds, and `'noTimeout'` becomes `0`, which axios reads
+     * as no timeout. The result doubles with each attempt and is capped at `timeoutMaxMillis`. A requested
+     * value above the cap is capped too, which warns once, since it does not take effect in full.
+     */
+    #computeTimeoutMillis(timeoutSecs, attempt) {
+        if (timeoutSecs === 'noTimeout')
+            return 0;
+        const requestedMillis = typeof timeoutSecs === 'number' ? timeoutSecs * 1000 : this.timeoutMillis[timeoutSecs];
+        if (requestedMillis > this.timeoutMaxMillis) {
+            // `warningOnce` keys by message, so each requested value warns once.
+            this.logger.warningOnce(`The requested timeout of ${requestedMillis / 1000}s exceeds timeoutMaxSecs ` +
+                `(${this.timeoutMaxMillis / 1000}s) and is capped at it. ` +
+                'Raise timeoutMaxSecs on the client to allow longer request timeouts.');
+        }
+        return Math.min(requestedMillis * 2 ** (attempt - 1), this.timeoutMaxMillis);
+    }
     /**
      * Handles all unexpected errors that can happen, but are not
      * Apify API typed errors. E.g. network errors, timeouts and so on.
      */
-    _handleRequestError(err, config, stopTrying) {
-        if (this._isTimeoutError(err) && config.doNotRetryTimeouts) {
+    #handleRequestError(err, config, stopTrying) {
+        if (this.#isTimeoutError(err) && config.doNotRetryTimeouts) {
             return stopTrying(err);
         }
-        if (this._isRetryableError(err)) {
+        if (this.#isRetryableError(err)) {
             if (isStream(config.data)) {
-                this._informAboutStreamNoRetry();
+                this.#informAboutStreamNoRetry();
             }
             else {
                 throw err;
@@ -207,7 +212,7 @@ export class HttpClient {
      * Axios calls req.abort() on timeouts so timeout errors will
      * have a code ECONNABORTED.
      */
-    _isTimeoutError(err) {
+    #isTimeoutError(err) {
         return err.code === 'ECONNABORTED';
     }
     /**
@@ -216,15 +221,15 @@ export class HttpClient {
      * @param {Error} err
      * @private
      */
-    _isRetryableError(err) {
-        return this._isNetworkError(err) || this._isResponseBodyInvalid(err);
+    #isRetryableError(err) {
+        return this.#isNetworkError(err) || this.#isResponseBodyInvalid(err);
     }
     /**
      * When a network connection to our API is interrupted in the middle of streaming
      * a response, the request often does not fail, but simply contains
      * an incomplete response. This can often be fixed by retrying.
      */
-    _isResponseBodyInvalid(err) {
+    #isResponseBodyInvalid(err) {
         return err instanceof InvalidResponseBodyError;
     }
     /**
@@ -232,7 +237,7 @@ export class HttpClient {
      * it throws an AxiosError, which will have the request
      * and config (and other) properties.
      */
-    _isNetworkError(err) {
+    #isNetworkError(err) {
         const hasRequest = err.request && typeof err.request === 'object';
         const hasConfig = err.config && typeof err.config === 'object';
         return hasRequest && hasConfig;
@@ -242,12 +247,12 @@ export class HttpClient {
      * For status codes 300-499 (except 429) we do not retry the request,
      * because it's probably caused by invalid url (redirect 3xx) or invalid user input (4xx).
      */
-    _isStatusCodeRetryable(statusCode) {
+    #isStatusCodeRetryable(statusCode) {
         const isRateLimitError = statusCode === RATE_LIMIT_EXCEEDED_STATUS_CODE;
         const isInternalError = statusCode >= 500;
         return isRateLimitError || isInternalError;
     }
-    _onRequestRetry(error, attempt) {
+    #onRequestRetry(error, attempt) {
         if (attempt === Math.round(this.maxRetries / 2)) {
             this.logger.warning(`API request failed ${attempt} times. Max attempts: ${this.maxRetries + 1}.\nCause:${error.stack}`);
         }

package/dist/index.d.ts

@@ -27,6 +27,8 @@ export * from './resource_clients/webhook_collection.js';
 export * from './resource_clients/webhook_dispatch.js';
 export * from './resource_clients/webhook_dispatch_collection.js';
 export * from './apify_api_error.js';
-export * from './argument_validation_error.js';
+export { ArgumentValidationError } from '@apify/validations';
+export * from './response_validation_error.js';
 export { InvalidResponseBodyError } from './interceptors.js';
 export type { PaginatedList, Dictionary } from './utils.js';
+export type { Timeout, TimeoutOptions, TimeoutTier } from './timeouts.js';

package/dist/index.js

@@ -27,5 +27,6 @@ export * from './resource_clients/webhook_collection.js';
 export * from './resource_clients/webhook_dispatch.js';
 export * from './resource_clients/webhook_dispatch_collection.js';
 export * from './apify_api_error.js';
-export * from './argument_validation_error.js';
+export { ArgumentValidationError } from '@apify/validations';
+export * from './response_validation_error.js';
 export { InvalidResponseBodyError } from './interceptors.js';

package/dist/interceptors.js

@@ -1,7 +1,7 @@
 import axios, { AxiosHeaders } from 'axios';
 import contentTypeParser from 'content-type';
 import { maybeParseBody } from './body_parser.js';
-import { isNode, maybeCompressValue } from './utils.js';
+import { isCompressibleContentType, maybeCompressValue } from './utils.js';
 /**
  * This error exists for the quite common situation, where only a partial JSON response is received and
  * an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
@@ -20,7 +20,22 @@ export class InvalidResponseBodyError extends Error {
         this.cause = cause;
     }
 }
+/**
+ * Reads a request header regardless of the casing it was set with, since HTTP header names are case-insensitive
+ * while the config keeps whatever casing the caller used.
+ */
+function getHeader(config, name) {
+    const wanted = name.toLowerCase();
+    const key = Object.keys(config.headers ?? {}).find((candidate) => candidate.toLowerCase() === wanted);
+    const value = key === undefined ? undefined : config.headers?.[key];
+    return typeof value === 'string' ? value : undefined;
+}
 function serializeRequest(config) {
+    // A string body with an explicit content type is already serialized and goes out as it is. The axios default
+    // transform would otherwise parse a JSON one in full just to check that it is valid, which for a body assembled
+    // from thousands of pre-serialized requests costs about as much as serializing them did.
+    if (typeof config.data === 'string' && getHeader(config, 'content-type'))
+        return config;
     const [defaultTransform] = axios.defaults.transformRequest;
     // The function not only serializes data, but it also adds correct headers.
     const data = defaultTransform(config.data, config.headers);
@@ -30,9 +45,9 @@ function serializeRequest(config) {
     // it's a small price to pay. The axios default transform does a lot
     // of body type checks and we would have to copy all of them to the resource clients.
     if (config.stringifyFunctions) {
-        const contentTypeHeader = config.headers?.['Content-Type'] || config.headers?.['content-type'];
+        const contentTypeHeader = getHeader(config, 'content-type');
         try {
-            const { type } = contentTypeParser.parse(contentTypeHeader);
+            const type = contentTypeHeader ? contentTypeParser.parse(contentTypeHeader).type : undefined;
             if (type === 'application/json' && typeof config.data === 'object') {
                 config.data = stringifyWithFunctions(config.data);
             }
@@ -65,7 +80,10 @@ function stringifyWithFunctions(obj) {
     });
 }
 async function maybeCompressRequest(config) {
-    if (config.headers?.['content-encoding'])
+    // A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone.
+    if (getHeader(config, 'content-encoding'))
+        return config;
+    if (!isCompressibleContentType(getHeader(config, 'content-type')))
         return config;
     const maybeCompressed = await maybeCompressValue(config.data);
     if (maybeCompressed) {
@@ -82,8 +100,8 @@ function parseResponseData(response) {
     ) {
         return response;
     }
-    const isBufferEmpty = isNode() ? !response.data.length : !response.data.byteLength;
-    if (isBufferEmpty) {
+    // A `Buffer` from the Node.js adapter and an `ArrayBuffer` from the browser ones both carry `byteLength`.
+    if (!response.data.byteLength) {
         // undefined is better than an empty buffer
         response.data = undefined;
         return response;

package/dist/lazy_schema.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Wraps the construction of a zod schema so it runs on the first call and only once.
+ *
+ * The generated response schemas are exported this way: a couple of hundred `z.looseObject()` calls at import time
+ * would cost more than importing zod itself, so each schema is built by the first response validated against it and
+ * kept for the rest of the process.
+ */
+export declare function lazySchema<T>(build: () => T): () => T;

package/dist/lazy_schema.js

@@ -0,0 +1,11 @@
+/**
+ * Wraps the construction of a zod schema so it runs on the first call and only once.
+ *
+ * The generated response schemas are exported this way: a couple of hundred `z.looseObject()` calls at import time
+ * would cost more than importing zod itself, so each schema is built by the first response validated against it and
+ * kept for the rest of the process.
+ */
+export function lazySchema(build) {
+    let schema;
+    return () => (schema ??= build());
+}