apify-client 3.0.0-beta.71 → 3.0.0-beta.73

package/dist/http_clients/base.d.ts

@@ -27,9 +27,10 @@ export declare const httpClientOptionsShape: {
 export type HttpResponseHeaders = Record<string, string | string[] | undefined>;
 /**
  * A request body as the transport receives it: already serialized, and compressed when that paid off. A `Readable`
- * is passed through untouched and is only available in Node.js.
+ * is passed through untouched and is only available in Node.js. A `Blob` body reaches a Node.js transport as a
+ * fresh `Readable` over its bytes for every attempt, and a transport in any other runtime as the `Blob` itself.
  */
-export type HttpRequestBody = string | Buffer | ArrayBuffer | ArrayBufferView | Readable;
+export type HttpRequestBody = string | Buffer | ArrayBuffer | ArrayBufferView | Readable | Blob;
 /**
  * A response body as the transport hands it back: the raw bytes, a `Readable` when the request asked for a streamed
  * response, or `undefined` when the response carried no body.
@@ -63,8 +64,9 @@ export interface ApifyRequestConfig {
      * Request body. A plain object or array is serialized to JSON and sent as `application/json`, or form-encoded
      * when a `Content-Type: application/x-www-form-urlencoded` header asks for it. A string, binary value
      * (`Buffer`, `ArrayBuffer`, typed array) or `Readable` is sent as it is, and `URLSearchParams` is sent
-     * form-encoded. A `Blob`, `File`, `FormData` or web `ReadableStream` is rejected with a `TypeError`, since
-     * serializing one would send an empty body.
+     * form-encoded. A `Blob` or `File`, such as one from `fs.openAsBlob()`, is streamed without being held in
+     * memory, under its own `type` when no `Content-Type` header is set. A `FormData` or web `ReadableStream` is
+     * rejected with a `TypeError`, since serializing one would send an empty body.
      */
     data?: unknown;
     /**
@@ -278,7 +280,8 @@ export declare abstract class HttpClient {
      * The inherited {@link call} needs it, so every transport has to implement it. Let the library's errors
      * propagate unwrapped: {@link call} classifies them through {@link isRetryableTransportError} and
      * {@link isTimeoutError}. Return error responses as they are too, the pipeline turns them into
-     * {@link ApifyApiError} and decides whether to retry.
+     * {@link ApifyApiError} and decides whether to retry. Reject when the connection drops before the body is
+     * complete: the pipeline takes a returned body as whole and does not retry one that fails to parse.
      *
      * @param request - The request to send, with the headers merged, the body serialized and the query encoded.
      * @returns The response, with the body unread when `request.stream` is set and as raw bytes otherwise.
@@ -290,8 +293,10 @@ export declare abstract class HttpClient {
      * Network errors the transport classifies as retryable, rate limits (HTTP 429) and server errors (HTTP 5xx)
      * are retried up to {@link maxRetries} times. Any other error status is thrown as {@link ApifyApiError} right
      * away. A request whose body is a `Readable` is never retried, since part of the stream has already been
-     * consumed by the time the failure shows. Aborting `config.signal` ends the attempt in flight, skips the
-     * remaining retries and rejects the call with the signal's `reason`.
+     * consumed by the time the failure shows. A `Blob` body is read again for every attempt, so it is retried like
+     * any other, unless reading the blob itself fails: that error is thrown right away, with the transport error
+     * as its `cause` when that error does not already wrap it. Aborting `config.signal` ends the attempt in flight,
+     * skips the remaining retries and rejects the call with the signal's `reason`.
      *
      * @template T - Type of the parsed response body.
      * @param config - The request to make.

package/dist/http_clients/base.js

@@ -9,7 +9,7 @@ import { InvalidResponseBodyError } from '../invalid_response_body_error.js';
 import { Statistics } from '../statistics.js';
 import { DEFAULT_TIMEOUT_LONG_SECS, DEFAULT_TIMEOUT_MAX_SECS, DEFAULT_TIMEOUT_MEDIUM_SECS, DEFAULT_TIMEOUT_SHORT_SECS, } from '../timeouts.js';
 import { runtime } from '#runtime';
-import { asArray, getEnv, isBuffer, isCompressibleContentType, isStream, MIN_COMPRESS_BYTES, parseArgument, sleep, toBytes, version, } from '../utils.js';
+import { asArray, getEnv, isBlob, isBuffer, isCompressibleContentType, isStream, MIN_COMPRESS_BYTES, parseArgument, sleep, toBytes, version, } from '../utils.js';
 export const DEFAULT_MAX_RETRIES = 8;
 export const DEFAULT_MIN_DELAY_BETWEEN_RETRIES_MILLIS = 500;
 const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
@@ -29,7 +29,7 @@ export const httpClientOptionsShape = {
 };
 const httpClientOptionsSchema = z.looseObject(httpClientOptionsShape).partial();
 /** Objects whose payload does not live in their own enumerable keys, so JSON serialization loses it. */
-const UNSERIALIZABLE_OBJECT_TAGS = new Set(['Blob', 'File', 'FormData', 'ReadableStream']);
+const UNSERIALIZABLE_OBJECT_TAGS = new Set(['FormData', 'ReadableStream']);
 /**
  * Base class for the HTTP clients used by {@link ApifyClient}.
  *
@@ -169,7 +169,8 @@ export class HttpClient {
      * The inherited {@link call} needs it, so every transport has to implement it. Let the library's errors
      * propagate unwrapped: {@link call} classifies them through {@link isRetryableTransportError} and
      * {@link isTimeoutError}. Return error responses as they are too, the pipeline turns them into
-     * {@link ApifyApiError} and decides whether to retry.
+     * {@link ApifyApiError} and decides whether to retry. Reject when the connection drops before the body is
+     * complete: the pipeline takes a returned body as whole and does not retry one that fails to parse.
      *
      * @param request - The request to send, with the headers merged, the body serialized and the query encoded.
      * @returns The response, with the body unread when `request.stream` is set and as raw bytes otherwise.
@@ -183,8 +184,10 @@ export class HttpClient {
      * Network errors the transport classifies as retryable, rate limits (HTTP 429) and server errors (HTTP 5xx)
      * are retried up to {@link maxRetries} times. Any other error status is thrown as {@link ApifyApiError} right
      * away. A request whose body is a `Readable` is never retried, since part of the stream has already been
-     * consumed by the time the failure shows. Aborting `config.signal` ends the attempt in flight, skips the
-     * remaining retries and rejects the call with the signal's `reason`.
+     * consumed by the time the failure shows. A `Blob` body is read again for every attempt, so it is retried like
+     * any other, unless reading the blob itself fails: that error is thrown right away, with the transport error
+     * as its `cause` when that error does not already wrap it. Aborting `config.signal` ends the attempt in flight,
+     * skips the remaining retries and rejects the call with the signal's `reason`.
      *
      * @template T - Type of the parsed response body.
      * @param config - The request to make.
@@ -219,6 +222,11 @@ export class HttpClient {
             headers = mergeHeaders(headers, { 'Content-Encoding': this.httpCompressor.contentEncoding });
             deleteHeader(headers, 'content-length');
         }
+        // A blob reaches a Node.js transport as a stream, which would go out chunked without a known length.
+        // Browsers set the header themselves and refuse to let a page set it.
+        if (isBlob(body) && runtime.isNode) {
+            headers = mergeHeaders(headers, { 'Content-Length': String(body.size) });
+        }
         return { headers, body };
     }
     /**
@@ -303,13 +311,20 @@ export class HttpClient {
         const { config, url, headers, body, attempt, stopRetrying } = options;
         this.stats.requests++;
         const requestIsStream = isStream(config.data);
+        // The transport reports a failure of the blob as an error of its own, which may pass as transient.
+        let sourceFailure;
+        const attemptBody = isBlob(body)
+            ? runtime.openBlobBody(body, (error) => {
+                sourceFailure = { error };
+            })
+            : body;
         let response;
         try {
             response = await this.sendRequest({
                 method: config.method,
                 url,
                 headers,
-                body,
+                body: attemptBody,
                 timeoutMillis: this.computeTimeoutMillis(attempt, config.timeoutSecs),
                 stream: config.responseType === 'stream',
                 signal: config.signal,
@@ -318,9 +333,20 @@ export class HttpClient {
         catch (err) {
             // Every HTTP library reports an abort with an error of its own, so the caller gets the reason instead.
             config.signal?.throwIfAborted();
+            if (sourceFailure) {
+                // Sending the body again cannot fix its source.
+                stopRetrying();
+                throw withCause(sourceFailure.error, err);
+            }
             this.#handleRequestError(err, config, stopRetrying);
             throw err;
         }
+        finally {
+            // A transport may stop reading the body early, for example on an error response sent before the whole
+            // body arrived, which would leave the blob's file open.
+            if (attemptBody !== body && isStream(attemptBody))
+                attemptBody.destroy();
+        }
         // A failed streaming request carries the API error body in the stream, so read it and let
         // `ApifyApiError` parse it like any other error body. A body that cannot be read leaves the error
         // without a message, which beats losing the status code to a stream error.
@@ -332,12 +358,14 @@ export class HttpClient {
             data = this.#parseResponseBody(response, config);
         }
         catch (err) {
-            // A body that does not parse is usually a connection dropped mid-response, which a retry fixes.
-            if (requestIsStream) {
-                this.#informAboutStreamNoRetry();
+            // The body of a success arrived in full, since a transfer cut short fails in `sendRequest()`, so a
+            // retry would get the same body back.
+            if (response.status < 300) {
                 stopRetrying();
+                throw err;
             }
-            throw err;
+            // An error status is reported by its code, and `ApifyApiError` makes what it can of the raw body.
+            data = response.body;
         }
         const apifyResponse = {
             status: response.status,
@@ -451,7 +479,7 @@ function deleteHeader(headers, name) {
 /**
  * Views a serialized body as the bytes to compress, or `undefined` when compressing it would not pay off: the
  * caller already labeled the body with a `Content-Encoding`, its content type carries its own compression, it is
- * a stream or smaller than {@link MIN_COMPRESS_BYTES}, or the runtime has no compression to offer.
+ * a stream or a blob, it is smaller than {@link MIN_COMPRESS_BYTES}, or the runtime has no compression to offer.
  */
 function compressibleBytes(body, headers) {
     if (body === undefined || !runtime.isNode)
@@ -463,6 +491,22 @@ function compressibleBytes(body, headers) {
     const bytes = toBytes(body);
     return bytes && bytes.byteLength >= MIN_COMPRESS_BYTES ? bytes : undefined;
 }
+/**
+ * Sets `cause` on an error that has none, and returns the error. A `cause` that already wraps the error, as an axios
+ * error wraps the error of its request body, is left out, since linking it would make the chain circular.
+ */
+function withCause(error, cause) {
+    if (!(error instanceof Error) || error.cause !== undefined)
+        return error;
+    const seen = new Set();
+    for (let link = cause; link instanceof Error && !seen.has(link); link = link.cause) {
+        if (link === error)
+            return error;
+        seen.add(link);
+    }
+    error.cause = cause;
+    return error;
+}
 function headerValue(value) {
     return Array.isArray(value) ? value[0] : value;
 }
@@ -481,6 +525,11 @@ function serializeBody(data, headers, stringifyFunctions) {
         return undefined;
     if (isStream(data))
         return data;
+    if (isBlob(data)) {
+        if (getHeader(headers, 'content-type') === undefined && data.type)
+            headers['Content-Type'] = data.type;
+        return data;
+    }
     if (isBuffer(data)) {
         // The axios Node.js adapter refuses a body that is neither a `Buffer`, an `ArrayBuffer` nor a string.
         if (!runtime.isNode || Buffer.isBuffer(data))

package/dist/invalid_response_body_error.d.ts

@@ -1,9 +1,9 @@
 import { ApifyClientError } from './apify_client_error.js';
 import type { HttpResponse } from './http_clients/base.js';
 /**
- * Thrown when a response body does not parse as its content type claims, typically a JSON document cut short by a
- * connection dropped mid-response. The client retries such requests, so the error surfaces once the retries are
- * exhausted.
+ * Thrown when the body of a successful response does not parse as its content type claims, for example a non-JSON
+ * value stored under `application/json`. The response arrived in full, so the request is not retried. An error
+ * status with an unparsable body throws {@link ApifyApiError} for its status code.
  */
 export declare class InvalidResponseBodyError extends ApifyClientError {
     code: string;

package/dist/invalid_response_body_error.js

@@ -1,8 +1,8 @@
 import { ApifyClientError } from './apify_client_error.js';
 /**
- * Thrown when a response body does not parse as its content type claims, typically a JSON document cut short by a
- * connection dropped mid-response. The client retries such requests, so the error surfaces once the retries are
- * exhausted.
+ * Thrown when the body of a successful response does not parse as its content type claims, for example a non-JSON
+ * value stored under `application/json`. The response arrived in full, so the request is not retried. An error
+ * status with an unparsable body throws {@link ApifyApiError} for its status code.
  */
 export class InvalidResponseBodyError extends ApifyClientError {
     code;

package/dist/resource_clients/key_value_store.d.ts

@@ -210,15 +210,18 @@ export declare class KeyValueStoreClient extends ResourceClient {
      * to be valid JSON already, or {@link getRecord} fails to parse it back.
      *
      * **Note about streams:** If the value is a stream object (has `.pipe` and `.on` methods),
-     * the upload cannot be retried on failure or follow redirects. For reliable uploads,
-     * buffer the entire stream into memory first.
+     * the upload cannot be retried on failure or follow redirects. For reliable uploads of a file,
+     * pass a `Blob` from `fs.openAsBlob()` instead: it is read again for every attempt and never held
+     * in memory whole.
      *
      * @param record - The record to store
      * @param record.key - Record key (unique identifier)
-     * @param record.value - Record value (a JSON-serializable value, Buffer, ArrayBuffer, typed array, or Readable)
+     * @param record.value - Record value (a JSON-serializable value, Buffer, ArrayBuffer, typed array, Blob, or
+     *                       Readable)
      * @param record.contentType - Optional MIME type. Auto-detected if not provided:
      *                             - Objects: `'application/json; charset=utf-8'`
      *                             - Strings: `'text/plain; charset=utf-8'`
+     *                             - Blobs: their `type`, or `'application/octet-stream'` when it is empty
      *                             - Binary values and streams: `'application/octet-stream'`
      *
      *                             Worth setting for media and archives: the client skips compressing a body
@@ -349,10 +352,10 @@ export interface KeyValueClientGetRecordOptions extends TimeoutOptions {
  * A value that `setRecord` accepts.
  *
  * Anything JSON-serializable, or binary content the client uploads as bytes instead of JSON-encoding
- * it: a `Buffer`, an `ArrayBuffer`, a typed array, or a readable stream. `Buffer` is a `Uint8Array`, so
- * `TypedArray` covers it.
+ * it: a `Buffer`, an `ArrayBuffer`, a typed array, a `Blob`, or a readable stream. `Buffer` is a
+ * `Uint8Array`, so `TypedArray` covers it.
  */
-export type KeyValueStoreRecordValue = JsonValue | ArrayBuffer | TypedArray | Readable;
+export type KeyValueStoreRecordValue = JsonValue | ArrayBuffer | TypedArray | Blob | Readable;
 /**
  * Represents a record (key-value pair) in a Key-Value Store.
  *

package/dist/resource_clients/key_value_store.js

@@ -5,7 +5,7 @@ import { ResourceClient } from '../base/resource_client.js';
 import * as schemas from '../schemas.js';
 import { timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
 import { runtime } from '#runtime';
-import { anyObjectSchema, applyQueryParamsToUrl, catchNotFoundOrThrow, isBuffer, isStream, minForLimitParam, parseArgument, parseResponse, } from '../utils.js';
+import { anyObjectSchema, applyQueryParamsToUrl, catchNotFoundOrThrow, isBlob, isBuffer, isStream, minForLimitParam, parseArgument, parseResponse, } from '../utils.js';
 const listKeysOptionsSchema = z.strictObject({
     limit: z.number().min(0).optional(),
     chunkSize: z.number().positive().optional(),
@@ -364,15 +364,18 @@ export class KeyValueStoreClient extends ResourceClient {
      * to be valid JSON already, or {@link getRecord} fails to parse it back.
      *
      * **Note about streams:** If the value is a stream object (has `.pipe` and `.on` methods),
-     * the upload cannot be retried on failure or follow redirects. For reliable uploads,
-     * buffer the entire stream into memory first.
+     * the upload cannot be retried on failure or follow redirects. For reliable uploads of a file,
+     * pass a `Blob` from `fs.openAsBlob()` instead: it is read again for every attempt and never held
+     * in memory whole.
      *
      * @param record - The record to store
      * @param record.key - Record key (unique identifier)
-     * @param record.value - Record value (a JSON-serializable value, Buffer, ArrayBuffer, typed array, or Readable)
+     * @param record.value - Record value (a JSON-serializable value, Buffer, ArrayBuffer, typed array, Blob, or
+     *                       Readable)
      * @param record.contentType - Optional MIME type. Auto-detected if not provided:
      *                             - Objects: `'application/json; charset=utf-8'`
      *                             - Strings: `'text/plain; charset=utf-8'`
+     *                             - Blobs: their `type`, or `'application/octet-stream'` when it is empty
      *                             - Binary values and streams: `'application/octet-stream'`
      *
      *                             Worth setting for media and archives: the client skips compressing a body
@@ -435,18 +438,20 @@ export class KeyValueStoreClient extends ResourceClient {
         if (contentEncoding !== undefined && !runtime.isNode) {
             throw new Error('The contentEncoding option can only be used in Node.js environment.');
         }
-        const isValueStreamOrBuffer = isStream(value) || isBuffer(value);
+        const isBinaryValue = isStream(value) || isBuffer(value) || isBlob(value);
         // The client forwards the header without inspecting the body, so a string or a value serialized to JSON, which
         // cannot hold compressed bytes, would be stored under an encoding that misdescribes it.
         const declaresCompression = contentEncoding !== undefined && contentEncoding.trim().toLowerCase() !== 'identity';
-        if (declaresCompression && !isValueStreamOrBuffer) {
+        if (declaresCompression && !isBinaryValue) {
             throw new TypeError(`Cannot upload a value of type ${typeof value} with Content-Encoding: ${contentEncoding}. ` +
                 'An encoding other than identity declares the value is already compressed, so pass the ' +
-                'compressed bytes as a Buffer, an ArrayBuffer, a typed array, or a readable stream.');
+                'compressed bytes as a Buffer, an ArrayBuffer, a typed array, a Blob, or a readable stream.');
         }
         // To allow saving Objects to JSON without providing content type
         if (!contentType) {
-            if (isValueStreamOrBuffer)
+            if (isBlob(value))
+                contentType = value.type || 'application/octet-stream';
+            else if (isBinaryValue)
                 contentType = 'application/octet-stream';
             else if (typeof value === 'string')
                 contentType = 'text/plain; charset=utf-8';
@@ -454,7 +459,7 @@ export class KeyValueStoreClient extends ResourceClient {
                 contentType = 'application/json; charset=utf-8';
         }
         const isContentTypeJson = contentType.startsWith('application/json');
-        if (isContentTypeJson && !isValueStreamOrBuffer && typeof value !== 'string') {
+        if (isContentTypeJson && !isBinaryValue && typeof value !== 'string') {
             try {
                 value = JSON.stringify(value, null, 2);
             }

package/dist/runtime/node.js

@@ -1,4 +1,5 @@
 import os from 'node:os';
+import { Readable } from 'node:stream';
 import { brotliCompress, constants, gzip } from 'node:zlib';
 /**
  * The Node.js implementation of {@link Runtime}. Deno and Bun resolve the `node` condition too and
@@ -46,4 +47,34 @@ export const runtime = {
         });
         return { httpAgent: agent, httpsAgent: agent };
     },
+    openBlobBody(blob, onSourceError) {
+        return Readable.from(readBlob(blob, onSourceError), { objectMode: false });
+    },
 };
+/**
+ * Yields the bytes of a blob. Ending the iteration early, which destroying the `Readable` around it does, cancels
+ * the read and releases a file the blob reads from. Only a failing read is reported: an error the `Readable` is
+ * destroyed with is thrown in at the `yield` and belongs to the transport. A `for await` loop would surface both at
+ * the same place, so the loop calls `next()` itself to catch only the read.
+ */
+async function* readBlob(blob, onSourceError) {
+    const chunks = blob.stream()[Symbol.asyncIterator]();
+    try {
+        while (true) {
+            let chunk;
+            try {
+                chunk = await chunks.next();
+            }
+            catch (error) {
+                onSourceError(error);
+                throw error;
+            }
+            if (chunk.done)
+                return;
+            yield chunk.value;
+        }
+    }
+    finally {
+        await chunks.return?.();
+    }
+}

package/dist/runtime/types.d.ts

@@ -1,5 +1,6 @@
 import type http from 'node:http';
 import type https from 'node:https';
+import type { Readable } from 'node:stream';
 /**
  * What to compress a request body with, as the built-in `HttpCompressor` implementations ask for it.
  * @internal
@@ -49,4 +50,11 @@ export interface Runtime {
     createHttpAgents(options: {
         timeoutMillis: number;
     }): Promise<HttpAgents | undefined>;
+    /**
+     * Opens a `Blob` request body for one attempt. The Node.js implementation reads the blob into a `Readable` and
+     * reports an error the blob raises while it is read to `onSourceError` before the stream fails with it, so the
+     * pipeline can tell a failing source apart from a failing connection. The Web API one returns the blob itself,
+     * which `fetch` and `XMLHttpRequest` read on their own, and never calls `onSourceError`.
+     */
+    openBlobBody(blob: Blob, onSourceError: (error: unknown) => void): Readable | Blob;
 }

package/dist/runtime/web.js

@@ -15,4 +15,7 @@ export const runtime = {
     async createHttpAgents() {
         return undefined;
     },
+    openBlobBody(blob) {
+        return blob;
+    },
 };

package/dist/utils.d.ts

@@ -101,6 +101,11 @@ export declare function splitIntoJsonArrayBatches<T extends {
     maxByteLength: number;
 }): T[][];
 export declare function isBuffer(value: unknown): value is Buffer | ArrayBuffer | TypedArray;
+/**
+ * Whether a value is a `Blob`, a `File` included. Unlike a stream, a blob can be read again for every attempt of a
+ * request, so a request with a blob body can be retried.
+ */
+export declare function isBlob(value: unknown): value is Blob;
 export declare function isStream(value: unknown): value is Readable;
 /**
  * Helper class to create async iterators from paginated list endpoints.

package/dist/utils.js

@@ -236,6 +236,13 @@ export function isBuffer(value) {
         return !isTagged(value, 'DataView');
     return isTagged(value, 'ArrayBuffer');
 }
+/**
+ * Whether a value is a `Blob`, a `File` included. Unlike a stream, a blob can be read again for every attempt of a
+ * request, so a request with a blob body can be retried.
+ */
+export function isBlob(value) {
+    return isTagged(value, 'Blob') || isTagged(value, 'File');
+}
 function isTagged(value, tag) {
     return Object.prototype.toString.call(value) === `[object ${tag}]`;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify-client",
-  "version": "3.0.0-beta.71",
+  "version": "3.0.0-beta.73",
   "description": "Apify API client for JavaScript",
   "engines": {
     "node": ">=22.13.0"