apify-client 3.0.0-beta.22 → 3.0.0-beta.24

package/README.md

@@ -59,6 +59,12 @@ By default, it will retry up to 8 times. First retry will be attempted after ~50
 and so on. You can configure those parameters using the `maxRetries` and `minDelayBetweenRetriesMillis`
 options of the `ApifyClient` constructor.
 
+### Pluggable HTTP client
+
+The client ships with an axios-based HTTP client and accepts a custom one. Extend `HttpClient`, implement
+`sendRequest()` over the HTTP library of your choice and plug it in with `ApifyClient.withCustomHttpClient()`. The
+custom client inherits the retries, timeouts, compression and error handling of the built-in one.
+
 ### Convenience functions and options
 
 Some actions can't be performed by the API itself, such as indefinite waiting for an actor run to finish

package/dist/apify_api_error.d.ts

@@ -1,5 +1,5 @@
-import type { AxiosResponse } from 'axios';
 import type { LiteralUnion } from 'type-fest';
+import type { ApifyResponse } from './http_clients/base.js';
 import type { ApifyApiErrorType } from './models.js';
 export type { ApifyApiErrorType } from './models.js';
 /**
@@ -58,12 +58,12 @@ export declare class ApifyApiError extends Error {
     /**
      * @hidden
      */
-    constructor(response: AxiosResponse, attempt: number);
+    constructor(response: ApifyResponse, attempt: number);
     /**
      * Creates the error for a failed response as an instance of the subclass matching its HTTP status code.
      * @hidden
      */
-    static fromResponse(response: AxiosResponse, attempt: number): ApifyApiError;
+    static fromResponse(response: ApifyResponse, attempt: number): ApifyApiError;
 }
 /**
  * Thrown when the Apify API responds with HTTP 400 Bad Request, typically because the request

package/dist/apify_api_error.js

@@ -6,9 +6,16 @@ import { isBuffer } from './utils.js';
  * 2: "ActorCollectionClient"
  * 3: undefined
  * 4: "listResources"
+ *
+ * The error is created inside the HTTP client's pipeline, whose frames sit above the resource client's. The
+ * pipeline's own methods are private, so V8 prints them with a `#` that no method name can match. Its public
+ * `call()` prints under the name of the transport's class, which can be anything, so the lookaheads skip a class
+ * named after `HttpClient` and the method `call`. A transport named `SomethingClient` whose `call()` override
+ * awaits a helper of its own is reported under that helper. `ActorClient.call()` and `TaskClient.call()` reach the
+ * pipeline through `start()`, whose frame sits closer to the error and matches first.
  * @private
  */
-const CLIENT_METHOD_REGEX = /at( async)? ([A-Za-z]+(Collection)?Client)\.([A-Za-z]+) \(/;
+const CLIENT_METHOD_REGEX = /at( async)? (?![A-Za-z]*HttpClient\.)([A-Za-z]+(Collection)?Client)\.(?!call \()([A-Za-z]+) \(/;
 /**
  * An `ApifyApiError` is thrown for successful HTTP requests that reach the API,
  * but the API responds with an error response. Typically, those are rate limit
@@ -69,8 +76,8 @@ export class ApifyApiError extends Error {
         let type;
         let responseData = response.data;
         let errorData;
-        // A `forceBuffer` request (e.g. `downloadItems()`) and a failed streaming request, whose body `HttpClient`
-        // has read into a buffer, both arrive unparsed. Parse the body here to get at the error.
+        // A `responseType: 'buffer'` request (e.g. `downloadItems()`) and a failed streaming request, whose body
+        // `HttpClient` has read into a buffer, both arrive unparsed. Parse the body here to get at the error.
         if (isBuffer(responseData)) {
             const body = new TextDecoder().decode(response.data);
             try {
@@ -146,7 +153,7 @@ export class ApifyApiError extends Error {
      *   statusCode: 404
      *   type: record-not-found
      *   attempt: 1
-     *   httpMethod: post
+     *   httpMethod: POST
      *   path: /v2/actor-tasks/user~my-task/runs
      */
     #createApiStack() {

package/dist/apify_client.d.ts

@@ -1,7 +1,8 @@
 import type { SetStatusMessageOptions } from '@crawlee/types';
 import type { Log } from '@apify/log';
-import { HttpClient } from './http_client.js';
-import type { RequestInterceptorFunction } from './interceptors.js';
+import { HttpClient } from './http_clients/base.js';
+import type { HttpCompressor } from './http_compressors/base.js';
+import type { HttpCompressionAlgorithm } from './http_compressors/resolve.js';
 import { ActorClient } from './resource_clients/actor.js';
 import { ActorCollectionClient } from './resource_clients/actor_collection.js';
 import { BuildClient } from './resource_clients/build.js';
@@ -56,10 +57,59 @@ export declare class ApifyClient {
      */
     publicBaseUrl: string;
     token?: string;
+    /**
+     * Statistics of the API calls made through the client. With a custom HTTP client, this is the client's own
+     * `stats` object, so it only reflects what that client records.
+     */
     stats: Statistics;
     logger: Log;
-    httpClient: HttpClient;
+    /**
+     * Creates a client that sends its requests through the built-in {@link AxiosHttpClient}, configured from
+     * `options`: the API token, retries, the timeout tiers, request compression and default headers. The HTTP
+     * client itself is created when {@link httpClient} is first read. To send the requests through a custom HTTP
+     * client, use {@link ApifyClient.withCustomHttpClient} instead.
+     *
+     * @param options - Configuration of the client and of its HTTP client.
+     * @throws {ArgumentValidationError} When an option is unknown or has an invalid value.
+     */
     constructor(options?: ApifyClientOptions);
+    /**
+     * Creates an `ApifyClient` that sends its requests through a custom HTTP client.
+     *
+     * The custom client owns its transport configuration: retries, timeouts, default headers and whatever its
+     * transport needs are set on the client itself. This is also how the built-in {@link AxiosHttpClient} is
+     * configured beyond what the constructor options offer, for example with axios request interceptors. The
+     * token is set as the client's `Authorization` header, unless the client already has one configured.
+     *
+     * @param options - The token, the API URLs and the HTTP client to use.
+     * @returns A client whose {@link httpClient} is the given one.
+     *
+     * @example
+     * ```javascript
+     * import { ApifyClient, HttpClient } from 'apify-client';
+     *
+     * class MyHttpClient extends HttpClient {
+     *     async sendRequest({ method, url, headers, body, timeoutMillis, stream, signal }) {
+     *         // Send the request with the HTTP library of your choice and return { status, headers, body }.
+     *     }
+     * }
+     *
+     * const client = ApifyClient.withCustomHttpClient({ token: 'my-token', httpClient: new MyHttpClient() });
+     * ```
+     */
+    static withCustomHttpClient(options: ApifyClientCustomHttpClientOptions): ApifyClient;
+    /**
+     * The HTTP client the requests go through: the custom one when set through
+     * {@link ApifyClient.withCustomHttpClient}, and otherwise an {@link AxiosHttpClient} configured from the
+     * constructor options, created on first access. Assigning a client applies {@link token} to it through
+     * {@link HttpClient.setDefaultAuthorization} and points {@link stats} at its statistics, so the counters keep
+     * tracking the calls the client actually makes. Resource clients hold on to the HTTP client they were created
+     * with, so assign before reaching for them. Assigning does not close the client it replaces. A default client
+     * that already sent a request keeps its keep-alive agents open until {@link HttpClient.close} is called on it,
+     * so close it yourself, or assign before the first request.
+     */
+    get httpClient(): HttpClient;
+    set httpClient(httpClient: HttpClient);
     /**
      * Returns a client for managing Actors in your account.
      *
@@ -353,6 +403,18 @@ export declare class ApifyClient {
 export interface ApifyClientOptions {
     /** @default https://api.apify.com */
     baseUrl?: string;
+    /**
+     * Compression of request bodies: the name of a built-in algorithm, or an {@link HttpCompressor} for a custom
+     * quality or algorithm. Bodies are compressed only in Node.js, and only when they are large enough to benefit
+     * and their content type does not already carry its own compression.
+     * @default 'brotli'
+     */
+    compression?: HttpCompressionAlgorithm | HttpCompressor;
+    /**
+     * Additional headers sent with every request, such as an integration platform marker. They win over the
+     * built-in defaults, and a header set on a single call wins over them.
+     */
+    headers?: Record<string, string>;
     /**
      * @default https://api.apify.com
      * @since Added in 2.17.0
@@ -362,8 +424,6 @@ export interface ApifyClientOptions {
     maxRetries?: number;
     /** @default 500 */
     minDelayBetweenRetriesMillis?: number;
-    /** @default [] */
-    requestInterceptors?: RequestInterceptorFunction[];
     /**
      * Duration of the `short` timeout tier, in seconds: simple metadata reads and writes.
      * @default 5
@@ -391,3 +451,17 @@ export interface ApifyClientOptions {
      */
     userAgentSuffix?: string | string[];
 }
+/**
+ * Configuration options for {@link ApifyClient.withCustomHttpClient}. Retries, timeouts and headers are configured
+ * on the HTTP client itself.
+ */
+export interface ApifyClientCustomHttpClientOptions {
+    /** @default https://api.apify.com */
+    baseUrl?: string;
+    /** @default https://api.apify.com */
+    publicBaseUrl?: string;
+    /** Set as the HTTP client's `Authorization` header, unless it already has one. */
+    token?: string;
+    /** The HTTP client to send the requests through. */
+    httpClient: HttpClient;
+}

package/dist/apify_client.js

@@ -1,7 +1,9 @@
 import { z } from 'zod';
 import { ACTOR_ENV_VARS, ME_USER_NAME_PLACEHOLDER } from '@apify/consts';
 import logger from '@apify/log';
-import { HttpClient } from './http_client.js';
+import { AxiosHttpClient } from './http_clients/axios.js';
+import { DEFAULT_MAX_RETRIES, DEFAULT_MIN_DELAY_BETWEEN_RETRIES_MILLIS, HttpClient, httpClientOptionsShape, } from './http_clients/base.js';
+import { compressionSchema } from './http_compressors/resolve.js';
 import { ActorClient } from './resource_clients/actor.js';
 import { ActorCollectionClient } from './resource_clients/actor_collection.js';
 import { BuildClient } from './resource_clients/build.js';
@@ -28,19 +30,27 @@ import { WebhookDispatchCollectionClient } from './resource_clients/webhook_disp
 import { Statistics } from './statistics.js';
 import { DEFAULT_TIMEOUT_LONG_SECS, DEFAULT_TIMEOUT_MAX_SECS, DEFAULT_TIMEOUT_MEDIUM_SECS, DEFAULT_TIMEOUT_SHORT_SECS, } from './timeouts.js';
 import { getEnv, parseArgument } from './utils.js';
+const DEFAULT_API_URL = 'https://api.apify.com';
 const clientOptionsSchema = z.strictObject({
-    baseUrl: z.string().default('https://api.apify.com'),
-    publicBaseUrl: z.string().default('https://api.apify.com'),
-    maxRetries: z.number().default(8),
-    minDelayBetweenRetriesMillis: z.number().default(500),
-    requestInterceptors: z.array(z.unknown()).default([]),
-    timeoutShortSecs: z.number().positive().default(DEFAULT_TIMEOUT_SHORT_SECS),
-    timeoutMediumSecs: z.number().positive().default(DEFAULT_TIMEOUT_MEDIUM_SECS),
-    timeoutLongSecs: z.number().positive().default(DEFAULT_TIMEOUT_LONG_SECS),
-    timeoutMaxSecs: z.number().positive().default(DEFAULT_TIMEOUT_MAX_SECS),
+    baseUrl: z.string().default(DEFAULT_API_URL),
+    compression: compressionSchema.default('brotli'),
+    headers: z.record(z.string(), z.string()).optional(),
+    publicBaseUrl: z.string().default(DEFAULT_API_URL),
+    maxRetries: httpClientOptionsShape.maxRetries.default(DEFAULT_MAX_RETRIES),
+    minDelayBetweenRetriesMillis: httpClientOptionsShape.minDelayBetweenRetriesMillis.default(DEFAULT_MIN_DELAY_BETWEEN_RETRIES_MILLIS),
+    timeoutShortSecs: httpClientOptionsShape.timeoutShortSecs.default(DEFAULT_TIMEOUT_SHORT_SECS),
+    timeoutMediumSecs: httpClientOptionsShape.timeoutMediumSecs.default(DEFAULT_TIMEOUT_MEDIUM_SECS),
+    timeoutLongSecs: httpClientOptionsShape.timeoutLongSecs.default(DEFAULT_TIMEOUT_LONG_SECS),
+    timeoutMaxSecs: httpClientOptionsShape.timeoutMaxSecs.default(DEFAULT_TIMEOUT_MAX_SECS),
     token: z.string().optional(),
     userAgentSuffix: z.union([z.string(), z.array(z.string())]).optional(),
 });
+const customHttpClientOptionsSchema = z.strictObject({
+    baseUrl: z.string().default(DEFAULT_API_URL),
+    publicBaseUrl: z.string().default(DEFAULT_API_URL),
+    token: z.string().optional(),
+    httpClient: z.instanceof(HttpClient),
+});
 const resourceIdSchema = z.string().min(1);
 const requestQueueOptionsSchema = z.strictObject({
     clientKey: z.string().min(1).optional(),
@@ -74,12 +84,27 @@ export class ApifyClient {
      */
     publicBaseUrl;
     token;
+    /**
+     * Statistics of the API calls made through the client. With a custom HTTP client, this is the client's own
+     * `stats` object, so it only reflects what that client records.
+     */
     stats;
     logger;
-    httpClient;
+    #httpClient;
+    /** Configuration of the default HTTP client, applied when no custom one is set. */
+    #httpClientOptions;
+    /**
+     * Creates a client that sends its requests through the built-in {@link AxiosHttpClient}, configured from
+     * `options`: the API token, retries, the timeout tiers, request compression and default headers. The HTTP
+     * client itself is created when {@link httpClient} is first read. To send the requests through a custom HTTP
+     * client, use {@link ApifyClient.withCustomHttpClient} instead.
+     *
+     * @param options - Configuration of the client and of its HTTP client.
+     * @throws {ArgumentValidationError} When an option is unknown or has an invalid value.
+     */
     constructor(options = {}) {
         const parsed = parseArgument(options, clientOptionsSchema, 'ApifyClientOptions');
-        const { baseUrl, publicBaseUrl, maxRetries, minDelayBetweenRetriesMillis, requestInterceptors, timeoutShortSecs, timeoutMediumSecs, timeoutLongSecs, timeoutMaxSecs, token, } = parsed;
+        const { baseUrl, compression, headers, publicBaseUrl, maxRetries, minDelayBetweenRetriesMillis, timeoutShortSecs, timeoutMediumSecs, timeoutLongSecs, timeoutMaxSecs, token, } = parsed;
         const tempBaseUrl = baseUrl.endsWith('/') ? baseUrl.slice(0, baseUrl.length - 1) : baseUrl;
         this.baseUrl = `${tempBaseUrl}/v2`;
         const tempPublicBaseUrl = publicBaseUrl.endsWith('/')
@@ -89,11 +114,12 @@ export class ApifyClient {
         this.token = token;
         this.stats = new Statistics();
         this.logger = logger.child({ prefix: 'ApifyClient' });
-        this.httpClient = new HttpClient({
-            apifyClientStats: this.stats,
+        this.#httpClientOptions = {
+            stats: this.stats,
             maxRetries,
             minDelayBetweenRetriesMillis,
-            requestInterceptors,
+            compression,
+            headers,
             timeoutShortSecs,
             timeoutMediumSecs,
             timeoutLongSecs,
@@ -101,7 +127,57 @@ export class ApifyClient {
             logger: this.logger,
             token: this.token,
             userAgentSuffix: parsed.userAgentSuffix,
-        });
+        };
+    }
+    /**
+     * Creates an `ApifyClient` that sends its requests through a custom HTTP client.
+     *
+     * The custom client owns its transport configuration: retries, timeouts, default headers and whatever its
+     * transport needs are set on the client itself. This is also how the built-in {@link AxiosHttpClient} is
+     * configured beyond what the constructor options offer, for example with axios request interceptors. The
+     * token is set as the client's `Authorization` header, unless the client already has one configured.
+     *
+     * @param options - The token, the API URLs and the HTTP client to use.
+     * @returns A client whose {@link httpClient} is the given one.
+     *
+     * @example
+     * ```javascript
+     * import { ApifyClient, HttpClient } from 'apify-client';
+     *
+     * class MyHttpClient extends HttpClient {
+     *     async sendRequest({ method, url, headers, body, timeoutMillis, stream, signal }) {
+     *         // Send the request with the HTTP library of your choice and return { status, headers, body }.
+     *     }
+     * }
+     *
+     * const client = ApifyClient.withCustomHttpClient({ token: 'my-token', httpClient: new MyHttpClient() });
+     * ```
+     */
+    static withCustomHttpClient(options) {
+        const { httpClient, ...rest } = parseArgument(options, customHttpClientOptionsSchema, 'ApifyClientCustomHttpClientOptions');
+        const client = new ApifyClient(rest);
+        client.httpClient = httpClient;
+        return client;
+    }
+    /**
+     * The HTTP client the requests go through: the custom one when set through
+     * {@link ApifyClient.withCustomHttpClient}, and otherwise an {@link AxiosHttpClient} configured from the
+     * constructor options, created on first access. Assigning a client applies {@link token} to it through
+     * {@link HttpClient.setDefaultAuthorization} and points {@link stats} at its statistics, so the counters keep
+     * tracking the calls the client actually makes. Resource clients hold on to the HTTP client they were created
+     * with, so assign before reaching for them. Assigning does not close the client it replaces. A default client
+     * that already sent a request keeps its keep-alive agents open until {@link HttpClient.close} is called on it,
+     * so close it yourself, or assign before the first request.
+     */
+    get httpClient() {
+        this.#httpClient ??= new AxiosHttpClient(this.#httpClientOptions);
+        return this.#httpClient;
+    }
+    set httpClient(httpClient) {
+        if (this.token)
+            httpClient.setDefaultAuthorization(this.token);
+        this.#httpClient = httpClient;
+        this.stats = httpClient.stats;
     }
     #subClientOptions() {
         return {

package/dist/base/api_client.d.ts

@@ -1,5 +1,5 @@
 import type { ApifyClient } from '../apify_client.js';
-import type { HttpClient } from '../http_client.js';
+import type { HttpClient } from '../http_clients/index.js';
 import type { PaginatedResponse, PaginationOptions } from '../utils.js';
 /** @private */
 export interface ApiClientOptions {