apify-client 3.0.0-beta.19 → 3.0.0-beta.20

package/dist/http_client.js

@@ -54,9 +54,10 @@ export class HttpClient {
                 return new URLSearchParams(formattedParams).toString();
             },
             validateStatus: null,
-            // Using interceptors for this functionality.
-            transformRequest: undefined,
-            transformResponse: undefined,
+            // Interceptors serialize requests and parse responses instead. Empty arrays rather than `undefined`,
+            // which axios fills in with its default transforms.
+            transformRequest: [],
+            transformResponse: [],
             responseType: 'arraybuffer',
             // Every request sets its own timeout in `createRequestHandler`, so the default only backs a raw
             // `axios.request()` call.

package/dist/interceptors.js

@@ -31,6 +31,11 @@ function getHeader(config, name) {
     return typeof value === 'string' ? value : undefined;
 }
 function serializeRequest(config) {
+    // A string body with an explicit content type is already serialized and goes out as it is. The axios default
+    // transform would otherwise parse a JSON one in full just to check that it is valid, which for a body assembled
+    // from thousands of pre-serialized requests costs about as much as serializing them did.
+    if (typeof config.data === 'string' && getHeader(config, 'content-type'))
+        return config;
     const [defaultTransform] = axios.defaults.transformRequest;
     // The function not only serializes data, but it also adds correct headers.
     const data = defaultTransform(config.data, config.headers);

package/dist/resource_clients/request_queue.d.ts

@@ -3,6 +3,15 @@ import type { ApiClientSubResourceOptions } from '../base/api_client.js';
 import { ResourceClient } from '../base/resource_client.js';
 import type { RequestQueue, RequestQueueClientAddRequestResult, RequestQueueClientBatchDeleteRequestsResult, RequestQueueClientBatchRequestsOperationResult, RequestQueueClientListAndLockHeadResult, RequestQueueClientListHeadResult, RequestQueueClientListRequestsResult, RequestQueueClientProlongRequestLockResult, RequestQueueClientRequestSchema, RequestQueueClientRequestToAdd, RequestQueueClientRequestToUpdate, RequestQueueClientUnlockRequestsResult } from '../models.js';
 import type { TimeoutOptions } from '../timeouts.js';
+/**
+ * A request to add, serialized once up front: `byteLength` decides which batch it goes into, `json` is what the body of
+ * that batch is assembled from, and `request` is what the result bookkeeping needs.
+ */
+interface SerializedRequestToAdd {
+    request: RequestQueueClientRequestToAdd;
+    json: string;
+    byteLength: number;
+}
 export type { AllowedHttpMethods, RequestQueue, RequestQueueClientAddRequestResult, RequestQueueClientBatchDeleteRequestsResult, RequestQueueClientBatchRequestsOperationResult, RequestQueueClientListAndLockHeadResult, RequestQueueClientListHeadResult, RequestQueueClientListItem, RequestQueueClientListRequestsResult, RequestQueueClientLockedListItem, RequestQueueClientProlongRequestLockResult, RequestQueueClientRequestSchema, RequestQueueClientRequestToAdd, RequestQueueClientRequestToUpdate, RequestQueueClientUnlockRequestsResult, RequestQueueStats, } from '../models.js';
 /**
  * Client for managing a specific Request queue.
@@ -160,8 +169,8 @@ export declare class RequestQueueClient extends ResourceClient {
      *
      * @private
      */
-    protected addRequestBatch(requests: RequestQueueClientRequestToAdd[], options?: RequestQueueClientAddRequestOptions): Promise<RequestQueueClientBatchRequestsOperationResult>;
-    protected addRequestBatchWithRetries(requests: RequestQueueClientRequestToAdd[], options?: RequestQueueClientBatchAddRequestWithRetriesOptions): Promise<RequestQueueClientBatchRequestsOperationResult>;
+    protected addRequestBatch(requests: SerializedRequestToAdd[], options?: RequestQueueClientAddRequestOptions): Promise<RequestQueueClientBatchRequestsOperationResult>;
+    protected addRequestBatchWithRetries(requests: SerializedRequestToAdd[], options?: RequestQueueClientBatchAddRequestWithRetriesOptions): Promise<RequestQueueClientBatchRequestsOperationResult>;
     /**
      * Adds multiple requests to the queue in a single operation.
      *

package/dist/resource_clients/request_queue.js

@@ -5,7 +5,7 @@ import { ResourceClient } from '../base/resource_client.js';
 import { ResponseValidationError } from '../response_validation_error.js';
 import * as schemas from '../schemas.js';
 import { optionalTimeoutSchema, timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
-import { anyObjectSchema, catchNotFoundOrThrow, isNonArrayObject, parseArgument, parseResponse, RequestQueuePaginationIterator, sliceArrayByByteLength, } from '../utils.js';
+import { anyObjectSchema, catchNotFoundOrThrow, isNonArrayObject, parseArgument, parseResponse, RequestQueuePaginationIterator, splitIntoJsonArrayBatches, utf8ByteLength, } from '../utils.js';
 const DEFAULT_PARALLEL_BATCH_ADD_REQUESTS = 5;
 const DEFAULT_UNPROCESSED_RETRIES_BATCH_ADD_REQUESTS = 3;
 const DEFAULT_MIN_DELAY_BETWEEN_UNPROCESSED_REQUESTS_RETRIES_MILLIS = 500;
@@ -21,7 +21,6 @@ const listAndLockHeadOptionsSchema = z.strictObject({
 // every key of every request. `id` is assigned by the API, so a new request must not carry one.
 const newRequestSchema = z.custom((value) => isNonArrayObject(value) && value.id === undefined, 'Expected a request object without an `id`');
 const forefrontOptionsSchema = z.strictObject({ forefront: z.boolean().optional(), ...timeoutOptionsShape });
-const batchAddRequestsSchema = z.array(newRequestSchema).min(1).max(REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
 const batchAddRequestsWithRetriesSchema = z.array(newRequestSchema).min(1);
 const optionalBooleanSchema = z.boolean().optional();
 const optionalNumberSchema = z.number().optional();
@@ -277,13 +276,15 @@ export class RequestQueueClient extends ResourceClient {
      * @private
      */
     async addRequestBatch(requests, options = {}) {
-        parseArgument(requests, batchAddRequestsSchema);
         const parsed = parseArgument(options, forefrontOptionsSchema, 'RequestQueueClientAddRequestOptions');
         const response = await this.httpClient.call({
             url: this.buildUrl('requests/batch'),
             method: 'POST',
             timeoutSecs: this.#resolveTimeout(parsed.timeoutSecs, 'medium'),
-            data: requests,
+            // The body is assembled from the requests as `batchAddRequests` serialized them; the explicit content type
+            // makes the request interceptor send the string as it is instead of serializing the requests again.
+            headers: { 'content-type': 'application/json' },
+            data: `[${requests.map(({ json }) => json).join(',')}]`,
             params: this.buildParams({
                 forefront: parsed.forefront,
                 clientKey: this.#clientKey,
@@ -316,7 +317,7 @@ export class RequestQueueClient extends ResourceClient {
                 // Get unique keys of all requests processed so far
                 const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
                 // Requests remaining to be processed are the all that remain
-                remainingRequests = requests.filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey));
+                remainingRequests = requests.filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey));
                 // Stop if all requests have been processed
                 if (remainingRequests.length === 0) {
                     break;
@@ -332,8 +333,8 @@ export class RequestQueueClient extends ResourceClient {
                 // This ensures that this method does not throw and keeps the signature.
                 const processedRequestsUniqueKeys = processedRequests.map(({ uniqueKey }) => uniqueKey);
                 unprocessedRequests = requests
-                    .filter(({ uniqueKey }) => !processedRequestsUniqueKeys.includes(uniqueKey))
-                    .map(({ method, uniqueKey, url }) => ({ method, uniqueKey, url }));
+                    .filter(({ request }) => !processedRequestsUniqueKeys.includes(request.uniqueKey))
+                    .map(({ request: { method, uniqueKey, url } }) => ({ method, uniqueKey, url }));
                 break;
             }
             // Exponential backoff
@@ -394,12 +395,25 @@ export class RequestQueueClient extends ResourceClient {
         const executingRequests = new Set();
         const individualResults = [];
         const payloadSizeLimitBytes = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
+        // Serialize every request once: the byte lengths decide the batch boundaries, and the same strings are joined
+        // into the batch bodies, so nothing is stringified a second time when it is sent.
+        const serializedRequests = requests.map((request, index) => {
+            const json = JSON.stringify(request);
+            const byteLength = utf8ByteLength(json);
+            // Two more bytes for the brackets, which even a batch of one request carries.
+            if (byteLength + 2 > payloadSizeLimitBytes) {
+                throw new Error(`RequestQueueClient.batchAddRequests: The size of the request with index: ${index} ` +
+                    `exceeds the maximum allowed size (${payloadSizeLimitBytes} bytes).`);
+            }
+            return { request, json, byteLength };
+        });
+        const batches = splitIntoJsonArrayBatches(serializedRequests, {
+            maxCount: REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION,
+            maxByteLength: payloadSizeLimitBytes,
+        });
         // Keep a pool of up to `maxParallel` requests running at once
-        let i = 0;
-        while (i < requests.length) {
-            const slicedRequests = requests.slice(i, i + REQUEST_QUEUE_MAX_REQUESTS_PER_BATCH_OPERATION);
-            const requestsInBatch = sliceArrayByByteLength(slicedRequests, payloadSizeLimitBytes, i);
-            const requestPromise = this.addRequestBatchWithRetries(requestsInBatch, options);
+        for (const batch of batches) {
+            const requestPromise = this.addRequestBatchWithRetries(batch, options);
             executingRequests.add(requestPromise);
             // A rejection reaches the caller through the awaits below; this bookkeeping chain only has to avoid
             // turning it into an unhandled one of its own.
@@ -410,7 +424,6 @@ export class RequestQueueClient extends ResourceClient {
             if (executingRequests.size >= maxParallel) {
                 await Promise.race(executingRequests);
             }
-            i += requestsInBatch.length;
         }
         // Get results from remaining operations
         await Promise.all(executingRequests);

package/dist/utils.d.ts

@@ -79,9 +79,21 @@ export declare function isCompressibleContentType(contentType?: string): boolean
  */
 export declare function maybeCompressValue(value: unknown): Promise<CompressedValue | undefined>;
 /**
- * Helper function slice the items from array to fit the max byte length.
+ * Returns the UTF-8 byte length of a string.
  */
-export declare function sliceArrayByByteLength<T>(array: T[], maxByteLength: number, startIndex: number): T[];
+export declare function utf8ByteLength(value: string): number;
+/**
+ * Splits JSON-serialized items into consecutive batches of at most `maxCount` items, each of which fits into a JSON
+ * array body - the items joined by commas between brackets - of at most `maxByteLength` bytes. The `byteLength` of an
+ * item is the UTF-8 byte length of its serialization. An item too large for a body of its own still gets one, so a
+ * caller that cannot send such an item has to reject it beforehand.
+ */
+export declare function splitIntoJsonArrayBatches<T extends {
+    byteLength: number;
+}>(items: readonly T[], { maxCount, maxByteLength }: {
+    maxCount: number;
+    maxByteLength: number;
+}): T[][];
 export declare function isNode(): boolean;
 export declare function isBuffer(value: unknown): value is Buffer | ArrayBuffer | TypedArray;
 export declare function isStream(value: unknown): value is Readable;

package/dist/utils.js

@@ -213,28 +213,35 @@ export async function maybeCompressValue(value) {
     }
 }
 /**
- * Helper function slice the items from array to fit the max byte length.
+ * Returns the UTF-8 byte length of a string.
  */
-export function sliceArrayByByteLength(array, maxByteLength, startIndex) {
-    const stringByteLength = (str) => (isNode() ? Buffer.byteLength(str) : new Blob([str]).size);
-    const arrayByteLength = stringByteLength(JSON.stringify(array));
-    if (arrayByteLength < maxByteLength)
-        return array;
-    const slicedArray = [];
-    let byteLength = 2; // 2 bytes for the empty array []
-    for (let i = 0; i < array.length; i++) {
-        const item = array[i];
-        const itemByteSize = stringByteLength(JSON.stringify(item));
-        if (itemByteSize > maxByteLength) {
-            throw new Error(`RequestQueueClient.batchAddRequests: The size of the request with index: ${startIndex + i} ` +
-                `exceeds the maximum allowed size (${maxByteLength} bytes).`);
+export function utf8ByteLength(value) {
+    return isNode() ? Buffer.byteLength(value) : new Blob([value]).size;
+}
+/**
+ * Splits JSON-serialized items into consecutive batches of at most `maxCount` items, each of which fits into a JSON
+ * array body - the items joined by commas between brackets - of at most `maxByteLength` bytes. The `byteLength` of an
+ * item is the UTF-8 byte length of its serialization. An item too large for a body of its own still gets one, so a
+ * caller that cannot send such an item has to reject it beforehand.
+ */
+export function splitIntoJsonArrayBatches(items, { maxCount, maxByteLength }) {
+    const batches = [];
+    let batch = [];
+    // One byte for the opening bracket; each item then adds its own bytes plus one for the comma or the closing
+    // bracket that follows it.
+    let byteLength = 1;
+    for (const item of items) {
+        if (batch.length > 0 && (batch.length >= maxCount || byteLength + item.byteLength + 1 > maxByteLength)) {
+            batches.push(batch);
+            batch = [];
+            byteLength = 1;
         }
-        if (byteLength + itemByteSize >= maxByteLength)
-            break;
-        byteLength += itemByteSize;
-        slicedArray.push(item);
+        batch.push(item);
+        byteLength += item.byteLength + 1;
     }
-    return slicedArray;
+    if (batch.length > 0)
+        batches.push(batch);
+    return batches;
 }
 export function isNode() {
     if (typeof BROWSER_BUILD !== 'undefined')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify-client",
-  "version": "3.0.0-beta.19",
+  "version": "3.0.0-beta.20",
   "description": "Apify API client for JavaScript",
   "engines": {
     "node": ">=22.0.0"