apify-client 3.0.0-beta.8 → 3.0.0-rc.0

package/README.md

@@ -59,6 +59,12 @@ By default, it will retry up to 8 times. First retry will be attempted after ~50
 and so on. You can configure those parameters using the `maxRetries` and `minDelayBetweenRetriesMillis`
 options of the `ApifyClient` constructor.
 
+### Pluggable HTTP client
+
+The client ships with an axios-based HTTP client and accepts a custom one. Extend `HttpClient`, implement
+`sendRequest()` over the HTTP library of your choice and plug it in with `ApifyClient.withCustomHttpClient()`. The
+custom client inherits the retries, timeouts, compression and error handling of the built-in one.
+
 ### Convenience functions and options
 
 Some actions can't be performed by the API itself, such as indefinite waiting for an actor run to finish
@@ -145,13 +151,15 @@ property to get only a subset of results. Other props are also available, depend
 
 ## Bundled environments
 
-The package includes a pre-built browser bundle that is automatically resolved by bundlers targeting browser environments. You can also import it explicitly via
+The package includes a pre-built browser bundle that bundlers targeting browsers resolve automatically. You can also import it explicitly:
 
 ```typescript
 import { ApifyClient } from 'apify-client/browser';
 ```
 
-For edge runtimes like Cloudflare Workers, you may need to enable Node compatibility (e.g. `node_compat = true` in `wrangler.toml`). Note that some Node-specific features (streaming, proxy support) are not available in the bundle.
+Only two parts of the client need Node.js built-ins: the HTTP agents and request compression. The `node` condition selects the Node.js implementation of those, and every other target gets one built on Web APIs. Log streaming, proxy support, and request compression are only available in Node.js.
+
+For details, see [Bundled environments](https://docs.apify.com/api/client/js/docs/concepts/bundled-environments).
 
 ## API Reference
 

package/dist/apify_api_error.d.ts

@@ -1,7 +1,7 @@
-import type { AxiosResponse } from 'axios';
 import type { LiteralUnion } from 'type-fest';
-import type { ApifyApiErrorType } from './models.js';
-export type { ApifyApiErrorType } from './models.js';
+import type { ApifyResponse } from './http_clients/base.js';
+import type { ErrorType } from './models.js';
+export type { ErrorType } from './models.js';
 /**
  * An `ApifyApiError` is thrown for successful HTTP requests that reach the API,
  * but the API responds with an error response. Typically, those are rate limit
@@ -17,10 +17,12 @@ export type { ApifyApiErrorType } from './models.js';
  * share a status code are told apart by their `type`.
  */
 export declare class ApifyApiError extends Error {
+    #private;
     name: string;
     /**
-     * The invoked resource client and the method. Known issue: Sometimes it displays
-     * as `unknown` because it can't be parsed from a stack trace.
+     * The resource client and the method parsed from the stack trace, such as `ActorClient.getResource`. A public
+     * method that delegates to a shared helper of its class is reported under the helper. The value is `unknown`
+     * when the stack trace cannot be parsed.
      */
     clientMethod: string;
     /**
@@ -28,10 +30,10 @@ export declare class ApifyApiError extends Error {
      */
     statusCode: number;
     /**
-     * The type of the error, as returned by the API. Typed as the known {@link ApifyApiErrorType}
+     * The type of the error, as returned by the API. Typed as the known {@link ErrorType}
      * values for autocompletion, while still accepting any string the API may return.
      */
-    type?: LiteralUnion<ApifyApiErrorType, string>;
+    type?: LiteralUnion<ErrorType, string>;
     /**
      * Number of the API call attempt.
      */
@@ -56,29 +58,12 @@ export declare class ApifyApiError extends Error {
     /**
      * @hidden
      */
-    constructor(response: AxiosResponse, attempt: number);
+    constructor(response: ApifyResponse, attempt: number);
     /**
      * Creates the error for a failed response as an instance of the subclass matching its HTTP status code.
      * @hidden
      */
-    static fromResponse(response: AxiosResponse, attempt: number): ApifyApiError;
-    private _safelyParsePathFromResponse;
-    private _extractClientAndMethodFromStack;
-    /**
-     * Creates a better looking and more informative stack that will be printed
-     * out when API errors are thrown.
-     *
-     * Example:
-     *
-     * NotFoundError: Actor task was not found
-     *   clientMethod: TaskClient.start
-     *   statusCode: 404
-     *   type: record-not-found
-     *   attempt: 1
-     *   httpMethod: post
-     *   path: /v2/actor-tasks/user~my-task/runs
-     */
-    private _createApiStack;
+    static fromResponse(response: ApifyResponse, attempt: number): ApifyApiError;
 }
 /**
  * Thrown when the Apify API responds with HTTP 400 Bad Request, typically because the request

package/dist/apify_api_error.js

@@ -1,15 +1,21 @@
-import { isomorphicBufferToString } from './body_parser.js';
 import { isBuffer } from './utils.js';
 /**
- * Examples of capturing groups for "...at ActorCollectionClient._list (/Users/..."
- * 0: "at ActorCollectionClient._list ("
+ * Examples of capturing groups for "...at ActorCollectionClient.listResources (/Users/..."
+ * 0: "at ActorCollectionClient.listResources ("
  * 1: undefined
  * 2: "ActorCollectionClient"
  * 3: undefined
- * 4: "list"
+ * 4: "listResources"
+ *
+ * The error is created inside the HTTP client's pipeline, whose frames sit above the resource client's. The
+ * pipeline's own methods are private, so V8 prints them with a `#` that no method name can match. Its public
+ * `call()` prints under the name of the transport's class, which can be anything, so the lookaheads skip a class
+ * named after `HttpClient` and the method `call`. A transport named `SomethingClient` whose `call()` override
+ * awaits a helper of its own is reported under that helper. `ActorClient.call()` and `TaskClient.call()` reach the
+ * pipeline through `start()`, whose frame sits closer to the error and matches first.
  * @private
  */
-const CLIENT_METHOD_REGEX = /at( async)? ([A-Za-z]+(Collection)?Client)\._?([A-Za-z]+) \(/;
+const CLIENT_METHOD_REGEX = /at( async)? (?![A-Za-z]*HttpClient\.)([A-Za-z]+(Collection)?Client)\.(?!call \()([A-Za-z]+) \(/;
 /**
  * An `ApifyApiError` is thrown for successful HTTP requests that reach the API,
  * but the API responds with an error response. Typically, those are rate limit
@@ -27,8 +33,9 @@ const CLIENT_METHOD_REGEX = /at( async)? ([A-Za-z]+(Collection)?Client)\._?([A-Z
 export class ApifyApiError extends Error {
     name;
     /**
-     * The invoked resource client and the method. Known issue: Sometimes it displays
-     * as `unknown` because it can't be parsed from a stack trace.
+     * The resource client and the method parsed from the stack trace, such as `ActorClient.getResource`. A public
+     * method that delegates to a shared helper of its class is reported under the helper. The value is `unknown`
+     * when the stack trace cannot be parsed.
      */
     clientMethod;
     /**
@@ -36,7 +43,7 @@ export class ApifyApiError extends Error {
      */
     statusCode;
     /**
-     * The type of the error, as returned by the API. Typed as the known {@link ApifyApiErrorType}
+     * The type of the error, as returned by the API. Typed as the known {@link ErrorType}
      * values for autocompletion, while still accepting any string the API may return.
      */
     type;
@@ -69,14 +76,16 @@ export class ApifyApiError extends Error {
         let type;
         let responseData = response.data;
         let errorData;
-        // Some methods (e.g. downloadItems) set up forceBuffer on request response. If this request failed
-        // the body buffer needs to parse to get the correct error.
+        // A `responseType: 'buffer'` request (e.g. `downloadItems()`) and a failed streaming request, whose body
+        // `HttpClient` has read into a buffer, both arrive unparsed. Parse the body here to get at the error.
         if (isBuffer(responseData)) {
+            const body = new TextDecoder().decode(response.data);
             try {
-                responseData = JSON.parse(isomorphicBufferToString(response.data, 'utf-8'));
+                responseData = JSON.parse(body);
             }
             catch {
-                // This can happen. The data in the response body are malformed.
+                // A body that is not JSON at all, such as an HTML error page from a proxy, is kept as text.
+                responseData = body;
             }
         }
         if (responseData && responseData.error) {
@@ -97,15 +106,15 @@ export class ApifyApiError extends Error {
         }
         super(message);
         this.name = this.constructor.name;
-        this.clientMethod = this._extractClientAndMethodFromStack();
+        this.clientMethod = this.#extractClientAndMethodFromStack();
         this.statusCode = response.status;
         this.type = type;
         this.attempt = attempt;
         this.httpMethod = response.config?.method;
-        this.path = this._safelyParsePathFromResponse(response);
+        this.path = this.#safelyParsePathFromResponse(response);
         const stack = this.stack;
         this.originalStack = stack.slice(stack.indexOf('\n'));
-        this.stack = this._createApiStack();
+        this.stack = this.#createApiStack();
         this.data = errorData;
     }
     /**
@@ -116,7 +125,7 @@ export class ApifyApiError extends Error {
         const ErrorClass = ERROR_CLASS_BY_STATUS[response.status] ?? (response.status >= 500 ? ServerError : ApifyApiError);
         return new ErrorClass(response, attempt);
     }
-    _safelyParsePathFromResponse(response) {
+    #safelyParsePathFromResponse(response) {
         const urlString = response.config?.url;
         let url;
         try {
@@ -127,11 +136,11 @@ export class ApifyApiError extends Error {
         }
         return url.pathname + url.search;
     }
-    _extractClientAndMethodFromStack() {
+    #extractClientAndMethodFromStack() {
         const match = this.stack.match(CLIENT_METHOD_REGEX);
-        if (match)
-            return `${match[2]}.${match[4]}`;
-        return 'unknown';
+        if (!match)
+            return 'unknown';
+        return `${match[2]}.${match[4]}`;
     }
     /**
      * Creates a better looking and more informative stack that will be printed
@@ -144,10 +153,10 @@ export class ApifyApiError extends Error {
      *   statusCode: 404
      *   type: record-not-found
      *   attempt: 1
-     *   httpMethod: post
+     *   httpMethod: POST
      *   path: /v2/actor-tasks/user~my-task/runs
      */
-    _createApiStack() {
+    #createApiStack() {
         const { name, ...props } = this;
         const stack = Object.entries(props)
             .map(([k, v]) => {

package/dist/apify_client.d.ts

@@ -1,7 +1,7 @@
-import type { SetStatusMessageOptions } from '@crawlee/types';
 import type { Log } from '@apify/log';
-import { HttpClient } from './http_client.js';
-import type { RequestInterceptorFunction } from './interceptors.js';
+import { HttpClient } from './http_clients/base.js';
+import type { HttpCompressor } from './http_compressors/base.js';
+import type { HttpCompressionAlgorithm } from './http_compressors/resolve.js';
 import { ActorClient } from './resource_clients/actor.js';
 import { ActorCollectionClient } from './resource_clients/actor_collection.js';
 import { BuildClient } from './resource_clients/build.js';
@@ -49,17 +49,66 @@ import { Statistics } from './statistics.js';
  * @see https://docs.apify.com/api/v2
  */
 export declare class ApifyClient {
+    #private;
     baseUrl: string;
     /**
      * @since Added in 2.17.0
      */
     publicBaseUrl: string;
     token?: string;
+    /**
+     * Statistics of the API calls made through the client. With a custom HTTP client, this is the client's own
+     * `stats` object, so it only reflects what that client records.
+     */
     stats: Statistics;
     logger: Log;
-    httpClient: HttpClient;
+    /**
+     * Creates a client that sends its requests through the built-in {@link AxiosHttpClient}, configured from
+     * `options`: the API token, retries, the timeout tiers, request compression and default headers. The HTTP
+     * client itself is created when {@link httpClient} is first read. To send the requests through a custom HTTP
+     * client, use {@link ApifyClient.withCustomHttpClient} instead.
+     *
+     * @param options - Configuration of the client and of its HTTP client.
+     * @throws {ArgumentValidationError} When an option is unknown or has an invalid value.
+     */
     constructor(options?: ApifyClientOptions);
-    private _options;
+    /**
+     * Creates an `ApifyClient` that sends its requests through a custom HTTP client.
+     *
+     * The custom client owns its transport configuration: retries, timeouts, default headers and whatever its
+     * transport needs are set on the client itself. This is also how the built-in {@link AxiosHttpClient} is
+     * configured beyond what the constructor options offer, for example with axios request interceptors. The
+     * token is set as the client's `Authorization` header, unless the client already has one configured.
+     *
+     * @param options - The token, the API URLs and the HTTP client to use.
+     * @returns A client whose {@link httpClient} is the given one.
+     *
+     * @example
+     * ```javascript
+     * import { ApifyClient, HttpClient } from 'apify-client';
+     *
+     * class MyHttpClient extends HttpClient {
+     *     async sendRequest({ method, url, headers, body, timeoutMillis, stream, signal }) {
+     *         // Send the request with the HTTP library of your choice and return { status, headers, body }.
+     *     }
+     * }
+     *
+     * const client = ApifyClient.withCustomHttpClient({ token: 'my-token', httpClient: new MyHttpClient() });
+     * ```
+     */
+    static withCustomHttpClient(options: ApifyClientCustomHttpClientOptions): ApifyClient;
+    /**
+     * The HTTP client the requests go through: the custom one when set through
+     * {@link ApifyClient.withCustomHttpClient}, and otherwise an {@link AxiosHttpClient} configured from the
+     * constructor options, created on first access. Assigning a client applies {@link token} to it through
+     * {@link HttpClient.setDefaultAuthorization} and points {@link stats} at its statistics, so the counters keep
+     * tracking the calls the client actually makes. Resource clients hold on to the HTTP client they were created
+     * with, so assign before reaching for them. Assigning does not close the client it replaces. A default client
+     * that already sent a request keeps its keep-alive agents open until {@link HttpClient.close} is called on it,
+     * so close it yourself, or assign before the first request.
+     */
+    get httpClient(): HttpClient;
+    set httpClient(httpClient: HttpClient);
     /**
      * Returns a client for managing Actors in your account.
      *
@@ -347,13 +396,41 @@ export declare class ApifyClient {
      */
     setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<void>;
 }
+/**
+ * Options for {@apilink ApifyClient.setStatusMessage}. Structurally identical to crawlee's
+ * `SetStatusMessageOptions`, so the client stays a drop-in `StorageClient` implementation
+ * without a dependency on the crawlee packages.
+ */
+export interface SetStatusMessageOptions {
+    /** If true, the status message is marked as the final one for the run. */
+    isStatusMessageTerminal?: boolean;
+    /** The log level for the status message. */
+    level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
+}
 /**
  * Configuration options for ApifyClient.
  */
 export interface ApifyClientOptions {
-    /** @default https://api.apify.com */
+    /**
+     * URL of the Apify API, with or without the `/v2` version path, which is appended when missing.
+     * @default https://api.apify.com
+     */
     baseUrl?: string;
     /**
+     * Compression of request bodies: the name of a built-in algorithm, or an {@link HttpCompressor} for a custom
+     * quality or algorithm. Bodies are compressed only in Node.js, and only when they are large enough to benefit
+     * and their content type does not already carry its own compression.
+     * @default 'brotli'
+     */
+    compression?: HttpCompressionAlgorithm | HttpCompressor;
+    /**
+     * Additional headers sent with every request, such as an integration platform marker. They win over the
+     * built-in defaults, and a header set on a single call wins over them.
+     */
+    headers?: Record<string, string>;
+    /**
+     * Publicly accessible URL of the Apify API, used in the URLs the client generates. Like `baseUrl`, with or
+     * without the `/v2` version path.
      * @default https://api.apify.com
      * @since Added in 2.17.0
      */
@@ -362,13 +439,51 @@ export interface ApifyClientOptions {
     maxRetries?: number;
     /** @default 500 */
     minDelayBetweenRetriesMillis?: number;
-    /** @default [] */
-    requestInterceptors?: RequestInterceptorFunction[];
-    /** @default 360 */
-    timeoutSecs?: number;
+    /**
+     * Duration of the `short` timeout tier, in seconds: simple metadata reads and writes.
+     * @default 5
+     */
+    timeoutShortSecs?: number;
+    /**
+     * Duration of the `medium` timeout tier, in seconds: listing, batch and trigger operations.
+     * @default 30
+     */
+    timeoutMediumSecs?: number;
+    /**
+     * Duration of the `long` timeout tier, in seconds: downloads, uploads and streaming.
+     * @default 360
+     */
+    timeoutLongSecs?: number;
+    /**
+     * Cap on the timeout of a single request attempt, in seconds. It bounds the doubling of the timeout across
+     * retries, and caps tier and per-call timeouts alike, so raise it to allow a request timeout above 360 s.
+     * @default 360
+     */
+    timeoutMaxSecs?: number;
     token?: string;
     /**
      * @since Added in 2.10.0
      */
     userAgentSuffix?: string | string[];
 }
+/**
+ * Configuration options for {@link ApifyClient.withCustomHttpClient}. Retries, timeouts and headers are configured
+ * on the HTTP client itself.
+ */
+export interface ApifyClientCustomHttpClientOptions {
+    /**
+     * URL of the Apify API, with or without the `/v2` version path, which is appended when missing.
+     * @default https://api.apify.com
+     */
+    baseUrl?: string;
+    /**
+     * Publicly accessible URL of the Apify API, used in the URLs the client generates. Like `baseUrl`, with or
+     * without the `/v2` version path.
+     * @default https://api.apify.com
+     */
+    publicBaseUrl?: string;
+    /** Set as the HTTP client's `Authorization` header, unless it already has one. */
+    token?: string;
+    /** The HTTP client to send the requests through. */
+    httpClient: HttpClient;
+}