apify-client 3.0.0-beta.5 → 3.0.0-beta.51

package/README.md

@@ -47,7 +47,9 @@ Besides greatly simplifying the process of querying the Apify API, the client pr
 Based on the endpoint, the client automatically extracts the relevant data and returns it in the
 expected format. Date strings are automatically converted to `Date` objects. For exceptions,
 we throw an `ApifyApiError`, which wraps the plain JSON errors returned by API and enriches
-them with other context for easier debugging.
+them with other context for easier debugging. The error is an instance of the subclass matching the
+HTTP status code, such as `NotFoundError` or `RateLimitError`, so a `catch` block can tell them apart
+with `instanceof`.
 
 ### Retries with exponential backoff
 
@@ -57,6 +59,12 @@ By default, it will retry up to 8 times. First retry will be attempted after ~50
 and so on. You can configure those parameters using the `maxRetries` and `minDelayBetweenRetriesMillis`
 options of the `ApifyClient` constructor.
 
+### Pluggable HTTP client
+
+The client ships with an axios-based HTTP client and accepts a custom one. Extend `HttpClient`, implement
+`sendRequest()` over the HTTP library of your choice and plug it in with `ApifyClient.withCustomHttpClient()`. The
+custom client inherits the retries, timeouts, compression and error handling of the built-in one.
+
 ### Convenience functions and options
 
 Some actions can't be performed by the API itself, such as indefinite waiting for an actor run to finish
@@ -143,13 +151,15 @@ property to get only a subset of results. Other props are also available, depend
 
 ## Bundled environments
 
-The package includes a pre-built browser bundle that is automatically resolved by bundlers targeting browser environments. You can also import it explicitly via
+The package includes a pre-built browser bundle that bundlers targeting browsers resolve automatically. You can also import it explicitly:
 
 ```typescript
 import { ApifyClient } from 'apify-client/browser';
 ```
 
-For edge runtimes like Cloudflare Workers, you may need to enable Node compatibility (e.g. `node_compat = true` in `wrangler.toml`). Note that some Node-specific features (streaming, proxy support) are not available in the bundle.
+Only two parts of the client need Node.js built-ins: the HTTP agents and request compression. The `node` condition selects the Node.js implementation of those, and every other target gets one built on Web APIs. Log streaming, proxy support, and request compression are only available in Node.js.
+
+For details, see [Bundled environments](https://docs.apify.com/api/client/js/docs/concepts/bundled-environments).
 
 ## API Reference
 

package/dist/apify_api_error.d.ts

@@ -1,16 +1,28 @@
-import type { AxiosResponse } from 'axios';
+import type { LiteralUnion } from 'type-fest';
+import type { ApifyResponse } from './http_clients/base.js';
+import type { ApifyApiErrorType } from './models.js';
+export type { ApifyApiErrorType } from './models.js';
 /**
  * An `ApifyApiError` is thrown for successful HTTP requests that reach the API,
  * but the API responds with an error response. Typically, those are rate limit
  * errors and internal errors, which are automatically retried, or validation
  * errors, which are thrown immediately, because a correction by the user is
  * needed.
+ *
+ * The thrown error is an instance of the subclass matching the HTTP status code of the response:
+ * {@link InvalidRequestError} (400), {@link UnauthorizedError} (401), {@link ForbiddenError} (403),
+ * {@link NotFoundError} (404), {@link ConflictError} (409), {@link RateLimitError} (429) or
+ * {@link ServerError} (5xx). Any other status code is thrown as a plain `ApifyApiError`. Every
+ * subclass extends `ApifyApiError`, so `instanceof ApifyApiError` matches all of them. Errors that
+ * share a status code are told apart by their `type`.
  */
 export declare class ApifyApiError extends Error {
+    #private;
     name: string;
     /**
-     * The invoked resource client and the method. Known issue: Sometimes it displays
-     * as `unknown` because it can't be parsed from a stack trace.
+     * The resource client and the method parsed from the stack trace, such as `ActorClient.getResource`. A public
+     * method that delegates to a shared helper of its class is reported under the helper. The value is `unknown`
+     * when the stack trace cannot be parsed.
      */
     clientMethod: string;
     /**
@@ -18,9 +30,10 @@ export declare class ApifyApiError extends Error {
      */
     statusCode: number;
     /**
-     * The type of the error, as returned by the API.
+     * The type of the error, as returned by the API. Typed as the known {@link ApifyApiErrorType}
+     * values for autocompletion, while still accepting any string the API may return.
      */
-    type?: string;
+    type?: LiteralUnion<ApifyApiErrorType, string>;
     /**
      * Number of the API call attempt.
      */
@@ -45,22 +58,50 @@ export declare class ApifyApiError extends Error {
     /**
      * @hidden
      */
-    constructor(response: AxiosResponse, attempt: number);
-    private _safelyParsePathFromResponse;
-    private _extractClientAndMethodFromStack;
+    constructor(response: ApifyResponse, attempt: number);
     /**
-     * Creates a better looking and more informative stack that will be printed
-     * out when API errors are thrown.
-     *
-     * Example:
-     *
-     * ApifyApiError: Actor task was not found
-     *   clientMethod: TaskClient.start
-     *   statusCode: 404
-     *   type: record-not-found
-     *   attempt: 1
-     *   httpMethod: post
-     *   path: /v2/actor-tasks/user~my-task/runs
+     * Creates the error for a failed response as an instance of the subclass matching its HTTP status code.
+     * @hidden
      */
-    private _createApiStack;
+    static fromResponse(response: ApifyResponse, attempt: number): ApifyApiError;
+}
+/**
+ * Thrown when the Apify API responds with HTTP 400 Bad Request, typically because the request
+ * failed validation.
+ */
+export declare class InvalidRequestError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with HTTP 401 Unauthorized, because the token is missing
+ * or invalid.
+ */
+export declare class UnauthorizedError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with HTTP 403 Forbidden, because the token lacks the
+ * permission for the operation.
+ */
+export declare class ForbiddenError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with HTTP 404 Not Found.
+ */
+export declare class NotFoundError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with HTTP 409 Conflict.
+ */
+export declare class ConflictError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with HTTP 429 Too Many Requests. The client retries such
+ * requests, so the error surfaces once the retries are exhausted.
+ */
+export declare class RateLimitError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with an HTTP 5xx status. The client retries such requests,
+ * so the error surfaces once the retries are exhausted.
+ */
+export declare class ServerError extends ApifyApiError {
 }

package/dist/apify_api_error.js

@@ -1,27 +1,41 @@
-import { isomorphicBufferToString } from './body_parser.js';
 import { isBuffer } from './utils.js';
 /**
- * Examples of capturing groups for "...at ActorCollectionClient._list (/Users/..."
- * 0: "at ActorCollectionClient._list ("
+ * Examples of capturing groups for "...at ActorCollectionClient.listResources (/Users/..."
+ * 0: "at ActorCollectionClient.listResources ("
  * 1: undefined
  * 2: "ActorCollectionClient"
  * 3: undefined
- * 4: "list"
+ * 4: "listResources"
+ *
+ * The error is created inside the HTTP client's pipeline, whose frames sit above the resource client's. The
+ * pipeline's own methods are private, so V8 prints them with a `#` that no method name can match. Its public
+ * `call()` prints under the name of the transport's class, which can be anything, so the lookaheads skip a class
+ * named after `HttpClient` and the method `call`. A transport named `SomethingClient` whose `call()` override
+ * awaits a helper of its own is reported under that helper. `ActorClient.call()` and `TaskClient.call()` reach the
+ * pipeline through `start()`, whose frame sits closer to the error and matches first.
  * @private
  */
-const CLIENT_METHOD_REGEX = /at( async)? ([A-Za-z]+(Collection)?Client)\._?([A-Za-z]+) \(/;
+const CLIENT_METHOD_REGEX = /at( async)? (?![A-Za-z]*HttpClient\.)([A-Za-z]+(Collection)?Client)\.(?!call \()([A-Za-z]+) \(/;
 /**
  * An `ApifyApiError` is thrown for successful HTTP requests that reach the API,
  * but the API responds with an error response. Typically, those are rate limit
  * errors and internal errors, which are automatically retried, or validation
  * errors, which are thrown immediately, because a correction by the user is
  * needed.
+ *
+ * The thrown error is an instance of the subclass matching the HTTP status code of the response:
+ * {@link InvalidRequestError} (400), {@link UnauthorizedError} (401), {@link ForbiddenError} (403),
+ * {@link NotFoundError} (404), {@link ConflictError} (409), {@link RateLimitError} (429) or
+ * {@link ServerError} (5xx). Any other status code is thrown as a plain `ApifyApiError`. Every
+ * subclass extends `ApifyApiError`, so `instanceof ApifyApiError` matches all of them. Errors that
+ * share a status code are told apart by their `type`.
  */
 export class ApifyApiError extends Error {
     name;
     /**
-     * The invoked resource client and the method. Known issue: Sometimes it displays
-     * as `unknown` because it can't be parsed from a stack trace.
+     * The resource client and the method parsed from the stack trace, such as `ActorClient.getResource`. A public
+     * method that delegates to a shared helper of its class is reported under the helper. The value is `unknown`
+     * when the stack trace cannot be parsed.
      */
     clientMethod;
     /**
@@ -29,7 +43,8 @@ export class ApifyApiError extends Error {
      */
     statusCode;
     /**
-     * The type of the error, as returned by the API.
+     * The type of the error, as returned by the API. Typed as the known {@link ApifyApiErrorType}
+     * values for autocompletion, while still accepting any string the API may return.
      */
     type;
     /**
@@ -61,14 +76,16 @@ export class ApifyApiError extends Error {
         let type;
         let responseData = response.data;
         let errorData;
-        // Some methods (e.g. downloadItems) set up forceBuffer on request response. If this request failed
-        // the body buffer needs to parse to get the correct error.
+        // A `responseType: 'buffer'` request (e.g. `downloadItems()`) and a failed streaming request, whose body
+        // `HttpClient` has read into a buffer, both arrive unparsed. Parse the body here to get at the error.
         if (isBuffer(responseData)) {
+            const body = new TextDecoder().decode(response.data);
             try {
-                responseData = JSON.parse(isomorphicBufferToString(response.data, 'utf-8'));
+                responseData = JSON.parse(body);
             }
             catch {
-                // This can happen. The data in the response body are malformed.
+                // A body that is not JSON at all, such as an HTML error page from a proxy, is kept as text.
+                responseData = body;
             }
         }
         if (responseData && responseData.error) {
@@ -89,18 +106,26 @@ export class ApifyApiError extends Error {
         }
         super(message);
         this.name = this.constructor.name;
-        this.clientMethod = this._extractClientAndMethodFromStack();
+        this.clientMethod = this.#extractClientAndMethodFromStack();
         this.statusCode = response.status;
         this.type = type;
         this.attempt = attempt;
         this.httpMethod = response.config?.method;
-        this.path = this._safelyParsePathFromResponse(response);
+        this.path = this.#safelyParsePathFromResponse(response);
         const stack = this.stack;
         this.originalStack = stack.slice(stack.indexOf('\n'));
-        this.stack = this._createApiStack();
+        this.stack = this.#createApiStack();
         this.data = errorData;
     }
-    _safelyParsePathFromResponse(response) {
+    /**
+     * Creates the error for a failed response as an instance of the subclass matching its HTTP status code.
+     * @hidden
+     */
+    static fromResponse(response, attempt) {
+        const ErrorClass = ERROR_CLASS_BY_STATUS[response.status] ?? (response.status >= 500 ? ServerError : ApifyApiError);
+        return new ErrorClass(response, attempt);
+    }
+    #safelyParsePathFromResponse(response) {
         const urlString = response.config?.url;
         let url;
         try {
@@ -111,11 +136,11 @@ export class ApifyApiError extends Error {
         }
         return url.pathname + url.search;
     }
-    _extractClientAndMethodFromStack() {
+    #extractClientAndMethodFromStack() {
         const match = this.stack.match(CLIENT_METHOD_REGEX);
-        if (match)
-            return `${match[2]}.${match[4]}`;
-        return 'unknown';
+        if (!match)
+            return 'unknown';
+        return `${match[2]}.${match[4]}`;
     }
     /**
      * Creates a better looking and more informative stack that will be printed
@@ -123,15 +148,15 @@ export class ApifyApiError extends Error {
      *
      * Example:
      *
-     * ApifyApiError: Actor task was not found
+     * NotFoundError: Actor task was not found
      *   clientMethod: TaskClient.start
      *   statusCode: 404
      *   type: record-not-found
      *   attempt: 1
-     *   httpMethod: post
+     *   httpMethod: POST
      *   path: /v2/actor-tasks/user~my-task/runs
      */
-    _createApiStack() {
+    #createApiStack() {
         const { name, ...props } = this;
         const stack = Object.entries(props)
             .map(([k, v]) => {
@@ -145,3 +170,51 @@ export class ApifyApiError extends Error {
         return `${name}: ${this.message}\n${stack}`;
     }
 }
+/**
+ * Thrown when the Apify API responds with HTTP 400 Bad Request, typically because the request
+ * failed validation.
+ */
+export class InvalidRequestError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with HTTP 401 Unauthorized, because the token is missing
+ * or invalid.
+ */
+export class UnauthorizedError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with HTTP 403 Forbidden, because the token lacks the
+ * permission for the operation.
+ */
+export class ForbiddenError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with HTTP 404 Not Found.
+ */
+export class NotFoundError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with HTTP 409 Conflict.
+ */
+export class ConflictError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with HTTP 429 Too Many Requests. The client retries such
+ * requests, so the error surfaces once the retries are exhausted.
+ */
+export class RateLimitError extends ApifyApiError {
+}
+/**
+ * Thrown when the Apify API responds with an HTTP 5xx status. The client retries such requests,
+ * so the error surfaces once the retries are exhausted.
+ */
+export class ServerError extends ApifyApiError {
+}
+const ERROR_CLASS_BY_STATUS = {
+    400: InvalidRequestError,
+    401: UnauthorizedError,
+    403: ForbiddenError,
+    404: NotFoundError,
+    409: ConflictError,
+    429: RateLimitError,
+};

package/dist/apify_client.d.ts

@@ -1,7 +1,7 @@
-import type { SetStatusMessageOptions } from '@crawlee/types';
 import type { Log } from '@apify/log';
-import { HttpClient } from './http_client.js';
-import type { RequestInterceptorFunction } from './interceptors.js';
+import { HttpClient } from './http_clients/base.js';
+import type { HttpCompressor } from './http_compressors/base.js';
+import type { HttpCompressionAlgorithm } from './http_compressors/resolve.js';
 import { ActorClient } from './resource_clients/actor.js';
 import { ActorCollectionClient } from './resource_clients/actor_collection.js';
 import { BuildClient } from './resource_clients/build.js';
@@ -49,17 +49,66 @@ import { Statistics } from './statistics.js';
  * @see https://docs.apify.com/api/v2
  */
 export declare class ApifyClient {
+    #private;
     baseUrl: string;
     /**
      * @since Added in 2.17.0
      */
     publicBaseUrl: string;
     token?: string;
+    /**
+     * Statistics of the API calls made through the client. With a custom HTTP client, this is the client's own
+     * `stats` object, so it only reflects what that client records.
+     */
     stats: Statistics;
     logger: Log;
-    httpClient: HttpClient;
+    /**
+     * Creates a client that sends its requests through the built-in {@link AxiosHttpClient}, configured from
+     * `options`: the API token, retries, the timeout tiers, request compression and default headers. The HTTP
+     * client itself is created when {@link httpClient} is first read. To send the requests through a custom HTTP
+     * client, use {@link ApifyClient.withCustomHttpClient} instead.
+     *
+     * @param options - Configuration of the client and of its HTTP client.
+     * @throws {ArgumentValidationError} When an option is unknown or has an invalid value.
+     */
     constructor(options?: ApifyClientOptions);
-    private _options;
+    /**
+     * Creates an `ApifyClient` that sends its requests through a custom HTTP client.
+     *
+     * The custom client owns its transport configuration: retries, timeouts, default headers and whatever its
+     * transport needs are set on the client itself. This is also how the built-in {@link AxiosHttpClient} is
+     * configured beyond what the constructor options offer, for example with axios request interceptors. The
+     * token is set as the client's `Authorization` header, unless the client already has one configured.
+     *
+     * @param options - The token, the API URLs and the HTTP client to use.
+     * @returns A client whose {@link httpClient} is the given one.
+     *
+     * @example
+     * ```javascript
+     * import { ApifyClient, HttpClient } from 'apify-client';
+     *
+     * class MyHttpClient extends HttpClient {
+     *     async sendRequest({ method, url, headers, body, timeoutMillis, stream, signal }) {
+     *         // Send the request with the HTTP library of your choice and return { status, headers, body }.
+     *     }
+     * }
+     *
+     * const client = ApifyClient.withCustomHttpClient({ token: 'my-token', httpClient: new MyHttpClient() });
+     * ```
+     */
+    static withCustomHttpClient(options: ApifyClientCustomHttpClientOptions): ApifyClient;
+    /**
+     * The HTTP client the requests go through: the custom one when set through
+     * {@link ApifyClient.withCustomHttpClient}, and otherwise an {@link AxiosHttpClient} configured from the
+     * constructor options, created on first access. Assigning a client applies {@link token} to it through
+     * {@link HttpClient.setDefaultAuthorization} and points {@link stats} at its statistics, so the counters keep
+     * tracking the calls the client actually makes. Resource clients hold on to the HTTP client they were created
+     * with, so assign before reaching for them. Assigning does not close the client it replaces. A default client
+     * that already sent a request keeps its keep-alive agents open until {@link HttpClient.close} is called on it,
+     * so close it yourself, or assign before the first request.
+     */
+    get httpClient(): HttpClient;
+    set httpClient(httpClient: HttpClient);
     /**
      * Returns a client for managing Actors in your account.
      *
@@ -347,13 +396,41 @@ export declare class ApifyClient {
      */
     setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<void>;
 }
+/**
+ * Options for {@apilink ApifyClient.setStatusMessage}. Structurally identical to crawlee's
+ * `SetStatusMessageOptions`, so the client stays a drop-in `StorageClient` implementation
+ * without a dependency on the crawlee packages.
+ */
+export interface SetStatusMessageOptions {
+    /** If true, the status message is marked as the final one for the run. */
+    isStatusMessageTerminal?: boolean;
+    /** The log level for the status message. */
+    level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
+}
 /**
  * Configuration options for ApifyClient.
  */
 export interface ApifyClientOptions {
-    /** @default https://api.apify.com */
+    /**
+     * URL of the Apify API, with or without the `/v2` version path, which is appended when missing.
+     * @default https://api.apify.com
+     */
     baseUrl?: string;
     /**
+     * Compression of request bodies: the name of a built-in algorithm, or an {@link HttpCompressor} for a custom
+     * quality or algorithm. Bodies are compressed only in Node.js, and only when they are large enough to benefit
+     * and their content type does not already carry its own compression.
+     * @default 'brotli'
+     */
+    compression?: HttpCompressionAlgorithm | HttpCompressor;
+    /**
+     * Additional headers sent with every request, such as an integration platform marker. They win over the
+     * built-in defaults, and a header set on a single call wins over them.
+     */
+    headers?: Record<string, string>;
+    /**
+     * Publicly accessible URL of the Apify API, used in the URLs the client generates. Like `baseUrl`, with or
+     * without the `/v2` version path.
      * @default https://api.apify.com
      * @since Added in 2.17.0
      */
@@ -362,13 +439,51 @@ export interface ApifyClientOptions {
     maxRetries?: number;
     /** @default 500 */
     minDelayBetweenRetriesMillis?: number;
-    /** @default [] */
-    requestInterceptors?: RequestInterceptorFunction[];
-    /** @default 360 */
-    timeoutSecs?: number;
+    /**
+     * Duration of the `short` timeout tier, in seconds: simple metadata reads and writes.
+     * @default 5
+     */
+    timeoutShortSecs?: number;
+    /**
+     * Duration of the `medium` timeout tier, in seconds: listing, batch and trigger operations.
+     * @default 30
+     */
+    timeoutMediumSecs?: number;
+    /**
+     * Duration of the `long` timeout tier, in seconds: downloads, uploads and streaming.
+     * @default 360
+     */
+    timeoutLongSecs?: number;
+    /**
+     * Cap on the timeout of a single request attempt, in seconds. It bounds the doubling of the timeout across
+     * retries, and caps tier and per-call timeouts alike, so raise it to allow a request timeout above 360 s.
+     * @default 360
+     */
+    timeoutMaxSecs?: number;
     token?: string;
     /**
      * @since Added in 2.10.0
      */
     userAgentSuffix?: string | string[];
 }
+/**
+ * Configuration options for {@link ApifyClient.withCustomHttpClient}. Retries, timeouts and headers are configured
+ * on the HTTP client itself.
+ */
+export interface ApifyClientCustomHttpClientOptions {
+    /**
+     * URL of the Apify API, with or without the `/v2` version path, which is appended when missing.
+     * @default https://api.apify.com
+     */
+    baseUrl?: string;
+    /**
+     * Publicly accessible URL of the Apify API, used in the URLs the client generates. Like `baseUrl`, with or
+     * without the `/v2` version path.
+     * @default https://api.apify.com
+     */
+    publicBaseUrl?: string;
+    /** Set as the HTTP client's `Authorization` header, unless it already has one. */
+    token?: string;
+    /** The HTTP client to send the requests through. */
+    httpClient: HttpClient;
+}