apify-client 3.0.0-beta.2 → 3.0.0-beta.20

package/dist/http_client.d.ts

@@ -4,72 +4,36 @@ import type { AxiosInstance, AxiosRequestConfig, AxiosResponse, InternalAxiosReq
 import type { Log } from '@apify/log';
 import type { RequestInterceptorFunction } from './interceptors.js';
 import type { Statistics } from './statistics.js';
+import type { Timeout, TimeoutTier } from './timeouts.js';
 export declare class HttpClient {
+    #private;
     stats: Statistics;
     maxRetries: number;
     minDelayBetweenRetriesMillis: number;
     userProvidedRequestInterceptors: RequestInterceptorFunction[];
     logger: Log;
-    timeoutMillis: number;
+    /** Duration of each timeout tier, in milliseconds. */
+    timeoutMillis: Record<TimeoutTier, number>;
+    /** Cap on the timeout of a single request attempt, in milliseconds. */
+    timeoutMaxMillis: number;
     httpAgent?: http.Agent;
     httpsAgent?: https.Agent;
     axios: AxiosInstance;
     workflowKey?: string;
-    private nodeInitPromise?;
-    private userAgentSuffix?;
     constructor(options: HttpClientOptions);
-    private ensureNodeInit;
-    private initNode;
     call<T = any>(config: ApifyRequestConfig): Promise<ApifyResponse<T>>;
-    private _informAboutStreamNoRetry;
-    /**
-     * Successful responses are returned, errors and unsuccessful
-     * status codes are retried. See the following functions for the
-     * retrying logic.
-     */
-    private _createRequestHandler;
-    private _isStatusOk;
-    /**
-     * Handles all unexpected errors that can happen, but are not
-     * Apify API typed errors. E.g. network errors, timeouts and so on.
-     */
-    private _handleRequestError;
-    /**
-     * Axios calls req.abort() on timeouts so timeout errors will
-     * have a code ECONNABORTED.
-     */
-    private _isTimeoutError;
-    /**
-     * We don't want to retry every exception thrown from Axios.
-     * The common denominator for retryable errors are network issues.
-     * @param {Error} err
-     * @private
-     */
-    private _isRetryableError;
-    /**
-     * When a network connection to our API is interrupted in the middle of streaming
-     * a response, the request often does not fail, but simply contains
-     * an incomplete response. This can often be fixed by retrying.
-     */
-    private _isResponseBodyInvalid;
-    /**
-     * When a network request is attempted by axios and fails,
-     * it throws an AxiosError, which will have the request
-     * and config (and other) properties.
-     */
-    private _isNetworkError;
-    /**
-     * We retry 429 (rate limit) and 500+.
-     * For status codes 300-499 (except 429) we do not retry the request,
-     * because it's probably caused by invalid url (redirect 3xx) or invalid user input (4xx).
-     */
-    private _isStatusCodeRetryable;
-    private _onRequestRetry;
 }
-export interface ApifyRequestConfig extends AxiosRequestConfig {
+export interface ApifyRequestConfig extends Omit<AxiosRequestConfig, 'timeout'> {
     stringifyFunctions?: boolean;
     forceBuffer?: boolean;
     doNotRetryTimeouts?: boolean;
+    /**
+     * Timeout of the request: a tier name, a number of seconds, or `'noTimeout'`. The client resolves it to
+     * the axios `timeout` in milliseconds for each attempt, before axios runs its interceptors, so a request
+     * interceptor already sees a number of milliseconds.
+     * @default 'medium'
+     */
+    timeoutSecs?: Timeout;
 }
 export interface ApifyResponse<T = any> extends AxiosResponse<T> {
     config: ApifyRequestConfig & InternalAxiosRequestConfig;
@@ -79,7 +43,10 @@ export interface HttpClientOptions {
     maxRetries: number;
     minDelayBetweenRetriesMillis: number;
     requestInterceptors: RequestInterceptorFunction[];
-    timeoutSecs: number;
+    timeoutShortSecs: number;
+    timeoutMediumSecs: number;
+    timeoutLongSecs: number;
+    timeoutMaxSecs: number;
     logger: Log;
     token?: string;
     workflowKey?: string;

package/dist/http_client.js

@@ -1,6 +1,7 @@
 import retry from 'async-retry';
 import axios, { AxiosHeaders } from 'axios';
 import { APIFY_ENV_VARS } from '@apify/consts';
+import { concatStreamToBuffer } from '@apify/utilities';
 import { ApifyApiError } from './apify_api_error.js';
 import { InvalidResponseBodyError, requestInterceptors, responseInterceptors } from './interceptors.js';
 import { asArray, cast, getVersionData, isNode, isStream } from './utils.js';
@@ -12,24 +13,31 @@ export class HttpClient {
     minDelayBetweenRetriesMillis;
     userProvidedRequestInterceptors;
     logger;
+    /** Duration of each timeout tier, in milliseconds. */
     timeoutMillis;
+    /** Cap on the timeout of a single request attempt, in milliseconds. */
+    timeoutMaxMillis;
     httpAgent;
     httpsAgent;
     axios;
     workflowKey;
-    nodeInitPromise;
-    userAgentSuffix;
+    #nodeInitPromise;
+    #userAgentSuffix;
     constructor(options) {
         const { token } = options;
         this.stats = options.apifyClientStats;
         this.maxRetries = options.maxRetries;
         this.minDelayBetweenRetriesMillis = options.minDelayBetweenRetriesMillis;
         this.userProvidedRequestInterceptors = options.requestInterceptors;
-        this.timeoutMillis = options.timeoutSecs * 1000;
+        this.timeoutMillis = {
+            short: options.timeoutShortSecs * 1000,
+            medium: options.timeoutMediumSecs * 1000,
+            long: options.timeoutLongSecs * 1000,
+        };
+        this.timeoutMaxMillis = options.timeoutMaxSecs * 1000;
         this.logger = options.logger;
         this.workflowKey = options.workflowKey || process.env[APIFY_ENV_VARS.WORKFLOW_KEY];
-        this.userAgentSuffix = options.userAgentSuffix;
-        this._onRequestRetry = this._onRequestRetry.bind(this);
+        this.#userAgentSuffix = options.userAgentSuffix;
         this.axios = axios.create({
             // Disable axios's built-in proxy handling since we're using custom agents
             proxy: false,
@@ -46,11 +54,14 @@ export class HttpClient {
                 return new URLSearchParams(formattedParams).toString();
             },
             validateStatus: null,
-            // Using interceptors for this functionality.
-            transformRequest: undefined,
-            transformResponse: undefined,
+            // Interceptors serialize requests and parse responses instead. Empty arrays rather than `undefined`,
+            // which axios fills in with its default transforms.
+            transformRequest: [],
+            transformResponse: [],
             responseType: 'arraybuffer',
-            timeout: this.timeoutMillis,
+            // Every request sets its own timeout in `createRequestHandler`, so the default only backs a raw
+            // `axios.request()` call.
+            timeout: this.timeoutMaxMillis,
             // maxBodyLength needs to be Infinity, because -1 falls back to a 10 MB default
             // from an axios subdependency - 'follow-redirects'
             maxBodyLength: Infinity,
@@ -72,13 +83,13 @@ export class HttpClient {
         this.userProvidedRequestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
         responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
     }
-    async ensureNodeInit() {
+    async #ensureNodeInit() {
         if (!isNode())
             return;
-        this.nodeInitPromise ??= this.initNode();
-        return this.nodeInitPromise;
+        this.#nodeInitPromise ??= this.#initNode();
+        return this.#nodeInitPromise;
     }
-    async initNode() {
+    async #initNode() {
         if (!isNode())
             return;
         const [{ ProxyAgent }, os] = await Promise.all([import('proxy-agent'), import('node:os')]);
@@ -92,7 +103,7 @@ export class HttpClient {
             keepAlive: true,
             // Timeout for inactive sockets
             // Prevents socket leaks from idle connections
-            timeout: this.timeoutMillis,
+            timeout: this.timeoutMaxMillis,
             // Keep alive timeout for free sockets (15 seconds)
             // Node.js will close unused sockets after this period
             keepAliveMsecs: 15_000,
@@ -119,22 +130,22 @@ export class HttpClient {
         // Works only in Node. Cannot be set in browser
         const isAtHome = !!process.env[APIFY_ENV_VARS.IS_AT_HOME];
         let userAgent = `ApifyClient/${version} (${os.platform()}; Node/${process.version}); isAtHome/${isAtHome}`;
-        if (this.userAgentSuffix) {
-            userAgent += `; ${asArray(this.userAgentSuffix).join('; ')}`;
+        if (this.#userAgentSuffix) {
+            userAgent += `; ${asArray(this.#userAgentSuffix).join('; ')}`;
         }
         this.axios.defaults.headers['User-Agent'] = userAgent;
     }
     async call(config) {
-        await this.ensureNodeInit();
+        await this.#ensureNodeInit();
         this.stats.calls++;
-        const makeRequest = this._createRequestHandler(config);
+        const makeRequest = this.#createRequestHandler(config);
         return retry(makeRequest, {
             retries: this.maxRetries,
             minTimeout: this.minDelayBetweenRetriesMillis,
-            onRetry: this._onRequestRetry,
+            onRetry: (error, attempt) => this.#onRequestRetry(error, attempt),
         });
     }
-    _informAboutStreamNoRetry() {
+    #informAboutStreamNoRetry() {
         this.logger.warningOnce('Request body was a stream - retrying will not work, as part of it was already consumed.');
         this.logger.warningOnce('If you want Apify client to handle retries for you, collect the stream into a buffer before sending it.');
     }
@@ -143,7 +154,8 @@ export class HttpClient {
      * status codes are retried. See the following functions for the
      * retrying logic.
      */
-    _createRequestHandler(config) {
+    #createRequestHandler(config) {
+        const { timeoutSecs = 'medium', ...axiosConfig } = config;
         const makeRequest = async (stopTrying, attempt) => {
             this.stats.requests++;
             let response;
@@ -153,24 +165,31 @@ export class HttpClient {
                     // Handling redirects is not possible without buffering - part of the stream has already been sent and can't be recovered
                     // when server sends the redirect. Therefore we need to override this in Axios config to prevent it from buffering the body.
                     // see also axios/axios#1045
-                    config = { ...config, maxRedirects: 0 };
+                    axiosConfig.maxRedirects = 0;
                 }
-                // Increase timeout with each attempt. Max timeout is bounded by the client timeout.
-                config.timeout = Math.min(this.timeoutMillis, (config.timeout ?? this.timeoutMillis) * 2 ** (attempt - 1));
-                response = await this.axios.request(config);
-                if (this._isStatusOk(response.status))
+                response = await this.axios.request({
+                    ...axiosConfig,
+                    timeout: this.#computeTimeoutMillis(timeoutSecs, attempt),
+                });
+                if (this.#isStatusOk(response.status))
                     return response;
+                // A failed request with `responseType: 'stream'` carries the API error body in the stream. Read
+                // it so that `ApifyApiError` can parse it like any other error body. A body that cannot be read
+                // leaves the error without a message, which beats losing the status code to a stream error.
+                if (isStream(response.data)) {
+                    response.data = await concatStreamToBuffer(response.data).catch(() => undefined);
+                }
             }
             catch (err) {
-                return cast(this._handleRequestError(err, config, stopTrying));
+                return cast(this.#handleRequestError(err, config, stopTrying));
             }
             if (response.status === RATE_LIMIT_EXCEEDED_STATUS_CODE) {
                 this.stats.addRateLimitError(attempt);
             }
-            const apiError = new ApifyApiError(response, attempt);
-            if (this._isStatusCodeRetryable(response.status)) {
+            const apiError = ApifyApiError.fromResponse(response, attempt);
+            if (this.#isStatusCodeRetryable(response.status)) {
                 if (requestIsStream) {
-                    this._informAboutStreamNoRetry();
+                    this.#informAboutStreamNoRetry();
                 }
                 else {
                     // allow a retry
@@ -182,20 +201,38 @@ export class HttpClient {
         };
         return makeRequest;
     }
-    _isStatusOk(statusCode) {
+    #isStatusOk(statusCode) {
         return statusCode < 300;
     }
+    /**
+     * Resolves `timeoutSecs` to the number of milliseconds the given attempt gets. A tier name resolves to
+     * its configured duration, a number is taken as seconds, and `'noTimeout'` becomes `0`, which axios reads
+     * as no timeout. The result doubles with each attempt and is capped at `timeoutMaxMillis`. A requested
+     * value above the cap is capped too, which warns once, since it does not take effect in full.
+     */
+    #computeTimeoutMillis(timeoutSecs, attempt) {
+        if (timeoutSecs === 'noTimeout')
+            return 0;
+        const requestedMillis = typeof timeoutSecs === 'number' ? timeoutSecs * 1000 : this.timeoutMillis[timeoutSecs];
+        if (requestedMillis > this.timeoutMaxMillis) {
+            // `warningOnce` keys by message, so each requested value warns once.
+            this.logger.warningOnce(`The requested timeout of ${requestedMillis / 1000}s exceeds timeoutMaxSecs ` +
+                `(${this.timeoutMaxMillis / 1000}s) and is capped at it. ` +
+                'Raise timeoutMaxSecs on the client to allow longer request timeouts.');
+        }
+        return Math.min(requestedMillis * 2 ** (attempt - 1), this.timeoutMaxMillis);
+    }
     /**
      * Handles all unexpected errors that can happen, but are not
      * Apify API typed errors. E.g. network errors, timeouts and so on.
      */
-    _handleRequestError(err, config, stopTrying) {
-        if (this._isTimeoutError(err) && config.doNotRetryTimeouts) {
+    #handleRequestError(err, config, stopTrying) {
+        if (this.#isTimeoutError(err) && config.doNotRetryTimeouts) {
             return stopTrying(err);
         }
-        if (this._isRetryableError(err)) {
+        if (this.#isRetryableError(err)) {
             if (isStream(config.data)) {
-                this._informAboutStreamNoRetry();
+                this.#informAboutStreamNoRetry();
             }
             else {
                 throw err;
@@ -207,7 +244,7 @@ export class HttpClient {
      * Axios calls req.abort() on timeouts so timeout errors will
      * have a code ECONNABORTED.
      */
-    _isTimeoutError(err) {
+    #isTimeoutError(err) {
         return err.code === 'ECONNABORTED';
     }
     /**
@@ -216,15 +253,15 @@ export class HttpClient {
      * @param {Error} err
      * @private
      */
-    _isRetryableError(err) {
-        return this._isNetworkError(err) || this._isResponseBodyInvalid(err);
+    #isRetryableError(err) {
+        return this.#isNetworkError(err) || this.#isResponseBodyInvalid(err);
     }
     /**
      * When a network connection to our API is interrupted in the middle of streaming
      * a response, the request often does not fail, but simply contains
      * an incomplete response. This can often be fixed by retrying.
      */
-    _isResponseBodyInvalid(err) {
+    #isResponseBodyInvalid(err) {
         return err instanceof InvalidResponseBodyError;
     }
     /**
@@ -232,7 +269,7 @@ export class HttpClient {
      * it throws an AxiosError, which will have the request
      * and config (and other) properties.
      */
-    _isNetworkError(err) {
+    #isNetworkError(err) {
         const hasRequest = err.request && typeof err.request === 'object';
         const hasConfig = err.config && typeof err.config === 'object';
         return hasRequest && hasConfig;
@@ -242,12 +279,12 @@ export class HttpClient {
      * For status codes 300-499 (except 429) we do not retry the request,
      * because it's probably caused by invalid url (redirect 3xx) or invalid user input (4xx).
      */
-    _isStatusCodeRetryable(statusCode) {
+    #isStatusCodeRetryable(statusCode) {
         const isRateLimitError = statusCode === RATE_LIMIT_EXCEEDED_STATUS_CODE;
         const isInternalError = statusCode >= 500;
         return isRateLimitError || isInternalError;
     }
-    _onRequestRetry(error, attempt) {
+    #onRequestRetry(error, attempt) {
         if (attempt === Math.round(this.maxRetries / 2)) {
             this.logger.warning(`API request failed ${attempt} times. Max attempts: ${this.maxRetries + 1}.\nCause:${error.stack}`);
         }

package/dist/index.d.ts

@@ -27,6 +27,8 @@ export * from './resource_clients/webhook_collection.js';
 export * from './resource_clients/webhook_dispatch.js';
 export * from './resource_clients/webhook_dispatch_collection.js';
 export * from './apify_api_error.js';
-export * from './argument_validation_error.js';
+export { ArgumentValidationError } from '@apify/validations';
+export * from './response_validation_error.js';
 export { InvalidResponseBodyError } from './interceptors.js';
 export type { PaginatedList, Dictionary } from './utils.js';
+export type { Timeout, TimeoutOptions, TimeoutTier } from './timeouts.js';

package/dist/index.js

@@ -27,5 +27,6 @@ export * from './resource_clients/webhook_collection.js';
 export * from './resource_clients/webhook_dispatch.js';
 export * from './resource_clients/webhook_dispatch_collection.js';
 export * from './apify_api_error.js';
-export * from './argument_validation_error.js';
+export { ArgumentValidationError } from '@apify/validations';
+export * from './response_validation_error.js';
 export { InvalidResponseBodyError } from './interceptors.js';

package/dist/interceptors.js

@@ -1,7 +1,7 @@
 import axios, { AxiosHeaders } from 'axios';
 import contentTypeParser from 'content-type';
 import { maybeParseBody } from './body_parser.js';
-import { isNode, maybeCompressValue } from './utils.js';
+import { isCompressibleContentType, isNode, maybeCompressValue } from './utils.js';
 /**
  * This error exists for the quite common situation, where only a partial JSON response is received and
  * an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
@@ -20,7 +20,22 @@ export class InvalidResponseBodyError extends Error {
         this.cause = cause;
     }
 }
+/**
+ * Reads a request header regardless of the casing it was set with, since HTTP header names are case-insensitive
+ * while the config keeps whatever casing the caller used.
+ */
+function getHeader(config, name) {
+    const wanted = name.toLowerCase();
+    const key = Object.keys(config.headers ?? {}).find((candidate) => candidate.toLowerCase() === wanted);
+    const value = key === undefined ? undefined : config.headers?.[key];
+    return typeof value === 'string' ? value : undefined;
+}
 function serializeRequest(config) {
+    // A string body with an explicit content type is already serialized and goes out as it is. The axios default
+    // transform would otherwise parse a JSON one in full just to check that it is valid, which for a body assembled
+    // from thousands of pre-serialized requests costs about as much as serializing them did.
+    if (typeof config.data === 'string' && getHeader(config, 'content-type'))
+        return config;
     const [defaultTransform] = axios.defaults.transformRequest;
     // The function not only serializes data, but it also adds correct headers.
     const data = defaultTransform(config.data, config.headers);
@@ -30,9 +45,9 @@ function serializeRequest(config) {
     // it's a small price to pay. The axios default transform does a lot
     // of body type checks and we would have to copy all of them to the resource clients.
     if (config.stringifyFunctions) {
-        const contentTypeHeader = config.headers?.['Content-Type'] || config.headers?.['content-type'];
+        const contentTypeHeader = getHeader(config, 'content-type');
         try {
-            const { type } = contentTypeParser.parse(contentTypeHeader);
+            const type = contentTypeHeader ? contentTypeParser.parse(contentTypeHeader).type : undefined;
             if (type === 'application/json' && typeof config.data === 'object') {
                 config.data = stringifyWithFunctions(config.data);
             }
@@ -65,7 +80,10 @@ function stringifyWithFunctions(obj) {
     });
 }
 async function maybeCompressRequest(config) {
-    if (config.headers?.['content-encoding'])
+    // A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone.
+    if (getHeader(config, 'content-encoding'))
+        return config;
+    if (!isCompressibleContentType(getHeader(config, 'content-type')))
         return config;
     const maybeCompressed = await maybeCompressValue(config.data);
     if (maybeCompressed) {

package/dist/lazy_schema.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Wraps the construction of a zod schema so it runs on the first call and only once.
+ *
+ * The generated response schemas are exported this way: a couple of hundred `z.looseObject()` calls at import time
+ * would cost more than importing zod itself, so each schema is built by the first response validated against it and
+ * kept for the rest of the process.
+ */
+export declare function lazySchema<T>(build: () => T): () => T;

package/dist/lazy_schema.js

@@ -0,0 +1,11 @@
+/**
+ * Wraps the construction of a zod schema so it runs on the first call and only once.
+ *
+ * The generated response schemas are exported this way: a couple of hundred `z.looseObject()` calls at import time
+ * would cost more than importing zod itself, so each schema is built by the first response validated against it and
+ * kept for the rest of the process.
+ */
+export function lazySchema(build) {
+    let schema;
+    return () => (schema ??= build());
+}