apify 4.0.0-beta.21 → 4.0.0-beta.23

package/dist/actor.js

@@ -633,7 +633,7 @@ export class Actor {
         }
         const dataset = await this.openDataset();
         // Two code paths for charging:
-        // 1. Intercepted client: PatchedDatasetClient intercepts pushItems() calls, handling charging
+        // 1. Intercepted client: PpeAwareDatasetClient intercepts pushItems() calls, handling charging
         //    internally. This is needed because Crawlee's Dataset may call pushItems() directly,
         //    bypassing Actor.pushData(). We propagate eventName via AsyncLocalStorage context.
         // 2. Direct charging: When using a non-patched client (e.g., forceCloud option or custom client),
@@ -1578,7 +1578,7 @@ export class Actor {
         return Boolean(dataset.backend[USES_PUSH_DATA_INTERCEPTION]);
     }
     async pushDataViaInterceptedClient(dataset, item, eventName) {
-        // PatchedDatasetClient will handle charging and item limiting.
+        // PpeAwareDatasetClient will handle charging and item limiting.
         // We only need to propagate `eventName` and (optionally) return aggregated charge info.
         const context = {
             eventName,

package/dist/apify_dataset_backend.d.ts

@@ -2,8 +2,9 @@ import type { DatasetBackend, DatasetBackendListOptions, DatasetInfo, Dictionary
 import type { DatasetClient } from 'apify-client';
 /**
  * Implements crawlee v4's {@link DatasetBackend} interface on top of `apify-client`'s
- * dataset API. A thin method-mapping wrapper — the interfaces differ only in naming
- * (`getMetadata`/`get`, `drop`/`delete`, `pushData`/`pushItems`, `getData`/`listItems`).
+ * dataset API. Mostly a thin method-mapping wrapper (`getMetadata`/`get`, `drop`/`delete`,
+ * `getData`/`listItems`), except `pushData`, which also splits large pushes into chunks
+ * fitting the API's payload size limit.
  *
  * @internal
  */

package/dist/apify_dataset_backend.js

@@ -1,7 +1,14 @@
+import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
+/** Slight reduction of the API's 9MB payload limit, to stay safely below it. */
+const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
+const EFFECTIVE_LIMIT_BYTES = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
+/** Per-item ceiling — 2 bytes under the chunk limit, so even a lone item fits its `[]` wrapper. */
+const MAX_ITEM_BYTES = EFFECTIVE_LIMIT_BYTES - 2;
 /**
  * Implements crawlee v4's {@link DatasetBackend} interface on top of `apify-client`'s
- * dataset API. A thin method-mapping wrapper — the interfaces differ only in naming
- * (`getMetadata`/`get`, `drop`/`delete`, `pushData`/`pushItems`, `getData`/`listItems`).
+ * dataset API. Mostly a thin method-mapping wrapper (`getMetadata`/`get`, `drop`/`delete`,
+ * `getData`/`listItems`), except `pushData`, which also splits large pushes into chunks
+ * fitting the API's payload size limit.
  *
  * @internal
  */
@@ -25,9 +32,46 @@ export class ApifyDatasetBackend {
             'Use `drop()` to delete the dataset entirely, or open a new dataset instead.');
     }
     async pushData(items) {
-        await this.client.pushItems(items);
+        // The platform API rejects payloads over 9MB — split the items into chunks
+        // that fit, pushed sequentially to preserve item order.
+        const payloads = items.map((item, index) => serializeToSizeLimit(item, index));
+        for (const chunk of chunkBySize(payloads, EFFECTIVE_LIMIT_BYTES)) {
+            await this.client.pushItems(chunk);
+        }
     }
     async getData(options) {
         return await this.client.listItems(options);
     }
 }
+/** Serializes a dataset item, throwing if it alone exceeds the payload size limit. */
+function serializeToSizeLimit(item, index) {
+    const payload = JSON.stringify(item);
+    const bytes = Buffer.byteLength(payload);
+    if (bytes > MAX_ITEM_BYTES) {
+        throw new Error(`Data item at index ${index} is too large (size: ${bytes} bytes, limit: ${MAX_ITEM_BYTES} bytes)`);
+    }
+    return payload;
+}
+/**
+ * Takes an array of JSON-serialized items and groups them into JSON array strings
+ * of at most `limitBytes` each, preserving item order. Assumes (and does not
+ * validate) that no single item exceeds the limit.
+ */
+function chunkBySize(payloads, limitBytes) {
+    const chunks = [];
+    let chunkBytes = Infinity; // Forces the first item to open a new chunk.
+    for (const payload of payloads) {
+        const bytes = Buffer.byteLength(payload);
+        if (chunkBytes + bytes + 1 <= limitBytes) {
+            // Fits into the current chunk — add 1 byte for the ',' separator.
+            chunks[chunks.length - 1].push(payload);
+            chunkBytes += bytes + 1;
+        }
+        else {
+            // Open a new chunk — add 2 bytes for the '[]' wrapper.
+            chunks.push([payload]);
+            chunkBytes = bytes + 2;
+        }
+    }
+    return chunks.map((chunk) => `[${chunk.join(',')}]`);
+}

package/dist/apify_storage_backend.d.ts

@@ -11,7 +11,8 @@ export declare const USES_PUSH_DATA_INTERCEPTION: unique symbol;
  * Context of a single `Actor.pushData()` call, shared with the intercepted
  * `pushItems()` calls so they can (1) know which event to charge and
  * (2) aggregate the {@link ChargeResult} across the multiple `pushItems()`
- * calls a single `pushData()` may trigger (Crawlee batches large pushes).
+ * calls a single `pushData()` may trigger (the backend splits pushes exceeding
+ * the API's payload size limit).
  */
 export interface PpeAwarePushDataContext {
     eventName: string | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify",
-  "version": "4.0.0-beta.21",
+  "version": "4.0.0-beta.23",
   "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
   "engines": {
     "node": ">=22.0.0"
@@ -59,9 +59,9 @@
     "@apify/log": "^2.4.3",
     "@apify/timeout": "^0.3.0",
     "@apify/utilities": "^2.13.0",
-    "@crawlee/core": "^4.0.0-beta.105",
-    "@crawlee/types": "^4.0.0-beta.105",
-    "@crawlee/utils": "^4.0.0-beta.105",
+    "@crawlee/core": "^4.0.0-beta.115",
+    "@crawlee/types": "^4.0.0-beta.115",
+    "@crawlee/utils": "^4.0.0-beta.115",
     "apify-client": "^2.23.4",
     "semver": "^7.5.4",
     "tslib": "^2.6.2",
@@ -79,7 +79,7 @@
     "@types/tough-cookie": "^4.0.5",
     "@types/ws": "^8.5.12",
     "commitlint": "^21.0.0",
-    "crawlee": "^4.0.0-beta.105",
+    "crawlee": "^4.0.0-beta.115",
     "globby": "^16.0.0",
     "husky": "^9.1.7",
     "lint-staged": "^17.0.0",