apify 4.0.0-beta.20 → 4.0.0-beta.22

package/dist/actor.d.ts

@@ -1,5 +1,5 @@
 import type { EventManager, EventTypeName, RecordOptions, UseStateOptions } from '@crawlee/core';
-import { Dataset, RequestQueue } from '@crawlee/core';
+import { Dataset, KeyValueStore, RequestQueue } from '@crawlee/core';
 import type { Awaitable, Dictionary, StorageBackend } from '@crawlee/types';
 import type { ActorCallOptions, ActorStartOptions, ApifyClientOptions, RunAbortOptions, TaskCallOptions, Webhook, WebhookEventType } from 'apify-client';
 import { ActorRun as ClientActorRun, ApifyClient } from 'apify-client';
@@ -9,7 +9,6 @@ import type { ChargeOptions, ChargeResult } from './charging.js';
 import { ChargingManager } from './charging.js';
 import type { ConfigurationOptions } from './configuration.js';
 import { Configuration } from './configuration.js';
-import { KeyValueStore } from './key_value_store.js';
 import type { ProxyConfigurationOptions } from './proxy_configuration.js';
 import { ProxyConfiguration } from './proxy_configuration.js';
 import type { OpenStorageOptions, StorageIdentifier, StorageIdentifierWithoutAlias } from './storage.js';

package/dist/actor.js

@@ -1,5 +1,5 @@
 import { createPrivateKey } from 'node:crypto';
-import { Dataset, purgeDefaultStorages, RequestQueue, serviceLocator } from '@crawlee/core';
+import { Dataset, KeyValueStore, purgeDefaultStorages, RequestQueue, serviceLocator } from '@crawlee/core';
 import { sleep } from '@crawlee/utils';
 import { ApifyClient } from 'apify-client';
 import { z } from 'zod';
@@ -11,7 +11,6 @@ import { ApifyStorageBackend, pushDataChargingContext, USES_PUSH_DATA_INTERCEPTI
 import { ChargingManager, pushDataAndCharge } from './charging.js';
 import { Configuration } from './configuration.js';
 import { getDefaultsFromInputSchema, noActorInputSchemaDefinedMarker, readInputSchema } from './input-schemas.js';
-import { KeyValueStore } from './key_value_store.js';
 import { PlatformEventManager } from './platform_event_manager.js';
 import { ProxyConfiguration } from './proxy_configuration.js';
 import { openStorage } from './storage.js';
@@ -634,7 +633,7 @@ export class Actor {
         }
         const dataset = await this.openDataset();
         // Two code paths for charging:
-        // 1. Intercepted client: PatchedDatasetClient intercepts pushItems() calls, handling charging
+        // 1. Intercepted client: PpeAwareDatasetClient intercepts pushItems() calls, handling charging
         //    internally. This is needed because Crawlee's Dataset may call pushItems() directly,
         //    bypassing Actor.pushData(). We propagate eventName via AsyncLocalStorage context.
         // 2. Direct charging: When using a non-patched client (e.g., forceCloud option or custom client),
@@ -1579,7 +1578,7 @@ export class Actor {
         return Boolean(dataset.backend[USES_PUSH_DATA_INTERCEPTION]);
     }
     async pushDataViaInterceptedClient(dataset, item, eventName) {
-        // PatchedDatasetClient will handle charging and item limiting.
+        // PpeAwareDatasetClient will handle charging and item limiting.
         // We only need to propagate `eventName` and (optionally) return aggregated charge info.
         const context = {
             eventName,

package/dist/apify_dataset_backend.d.ts

@@ -2,8 +2,9 @@ import type { DatasetBackend, DatasetBackendListOptions, DatasetInfo, Dictionary
 import type { DatasetClient } from 'apify-client';
 /**
  * Implements crawlee v4's {@link DatasetBackend} interface on top of `apify-client`'s
- * dataset API. A thin method-mapping wrapper — the interfaces differ only in naming
- * (`getMetadata`/`get`, `drop`/`delete`, `pushData`/`pushItems`, `getData`/`listItems`).
+ * dataset API. Mostly a thin method-mapping wrapper (`getMetadata`/`get`, `drop`/`delete`,
+ * `getData`/`listItems`), except `pushData`, which also splits large pushes into chunks
+ * fitting the API's payload size limit.
  *
  * @internal
  */

package/dist/apify_dataset_backend.js

@@ -1,7 +1,14 @@
+import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
+/** Slight reduction of the API's 9MB payload limit, to stay safely below it. */
+const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
+const EFFECTIVE_LIMIT_BYTES = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
+/** Per-item ceiling — 2 bytes under the chunk limit, so even a lone item fits its `[]` wrapper. */
+const MAX_ITEM_BYTES = EFFECTIVE_LIMIT_BYTES - 2;
 /**
  * Implements crawlee v4's {@link DatasetBackend} interface on top of `apify-client`'s
- * dataset API. A thin method-mapping wrapper — the interfaces differ only in naming
- * (`getMetadata`/`get`, `drop`/`delete`, `pushData`/`pushItems`, `getData`/`listItems`).
+ * dataset API. Mostly a thin method-mapping wrapper (`getMetadata`/`get`, `drop`/`delete`,
+ * `getData`/`listItems`), except `pushData`, which also splits large pushes into chunks
+ * fitting the API's payload size limit.
  *
  * @internal
  */
@@ -25,9 +32,46 @@ export class ApifyDatasetBackend {
             'Use `drop()` to delete the dataset entirely, or open a new dataset instead.');
     }
     async pushData(items) {
-        await this.client.pushItems(items);
+        // The platform API rejects payloads over 9MB — split the items into chunks
+        // that fit, pushed sequentially to preserve item order.
+        const payloads = items.map((item, index) => serializeToSizeLimit(item, index));
+        for (const chunk of chunkBySize(payloads, EFFECTIVE_LIMIT_BYTES)) {
+            await this.client.pushItems(chunk);
+        }
     }
     async getData(options) {
         return await this.client.listItems(options);
     }
 }
+/** Serializes a dataset item, throwing if it alone exceeds the payload size limit. */
+function serializeToSizeLimit(item, index) {
+    const payload = JSON.stringify(item);
+    const bytes = Buffer.byteLength(payload);
+    if (bytes > MAX_ITEM_BYTES) {
+        throw new Error(`Data item at index ${index} is too large (size: ${bytes} bytes, limit: ${MAX_ITEM_BYTES} bytes)`);
+    }
+    return payload;
+}
+/**
+ * Takes an array of JSON-serialized items and groups them into JSON array strings
+ * of at most `limitBytes` each, preserving item order. Assumes (and does not
+ * validate) that no single item exceeds the limit.
+ */
+function chunkBySize(payloads, limitBytes) {
+    const chunks = [];
+    let chunkBytes = Infinity; // Forces the first item to open a new chunk.
+    for (const payload of payloads) {
+        const bytes = Buffer.byteLength(payload);
+        if (chunkBytes + bytes + 1 <= limitBytes) {
+            // Fits into the current chunk — add 1 byte for the ',' separator.
+            chunks[chunks.length - 1].push(payload);
+            chunkBytes += bytes + 1;
+        }
+        else {
+            // Open a new chunk — add 2 bytes for the '[]' wrapper.
+            chunks.push([payload]);
+            chunkBytes = bytes + 2;
+        }
+    }
+    return chunks.map((chunk) => `[${chunk.join(',')}]`);
+}

package/dist/apify_storage_backend.d.ts

@@ -11,7 +11,8 @@ export declare const USES_PUSH_DATA_INTERCEPTION: unique symbol;
  * Context of a single `Actor.pushData()` call, shared with the intercepted
  * `pushItems()` calls so they can (1) know which event to charge and
  * (2) aggregate the {@link ChargeResult} across the multiple `pushItems()`
- * calls a single `pushData()` may trigger (Crawlee batches large pushes).
+ * calls a single `pushData()` may trigger (the backend splits pushes exceeding
+ * the API's payload size limit).
  */
 export interface PpeAwarePushDataContext {
     eventName: string | undefined;

package/dist/index.d.ts

@@ -7,7 +7,6 @@ export { ChargeOptions, ChargeResult, ActorPricingInfo, ChargingManager } from '
 export * from './configuration.js';
 export * from './proxy_configuration.js';
 export * from './platform_event_manager.js';
-export * from './key_value_store.js';
-export { Dataset, DatasetDataOptions, DatasetIteratorOptions, DatasetConsumer, DatasetMapper, DatasetReducer, DatasetOptions, DatasetContent, RequestQueue, RequestQueueOperationOptions, RequestQueueOptions, KeyConsumer, KeyValueStoreOptions, RecordOptions, KeyValueStoreIteratorOptions, log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
+export { Dataset, DatasetDataOptions, DatasetIteratorOptions, DatasetConsumer, DatasetMapper, DatasetReducer, DatasetOptions, DatasetContent, RequestQueue, RequestQueueOperationOptions, RequestQueueOptions, KeyValueStore, KeyConsumer, KeyValueStoreOptions, RecordOptions, KeyValueStoreIteratorOptions, log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
 export type { QueueOperationInfo } from '@crawlee/types';
 export { ApifyClient, ApifyClientOptions } from 'apify-client';

package/dist/index.js

@@ -5,6 +5,5 @@ export { ChargingManager } from './charging.js';
 export * from './configuration.js';
 export * from './proxy_configuration.js';
 export * from './platform_event_manager.js';
-export * from './key_value_store.js';
-export { Dataset, RequestQueue, log, Log, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
+export { Dataset, RequestQueue, KeyValueStore, log, Log, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
 export { ApifyClient } from 'apify-client';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify",
-  "version": "4.0.0-beta.20",
+  "version": "4.0.0-beta.22",
   "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
   "engines": {
     "node": ">=22.0.0"

package/dist/key_value_store.d.ts

@@ -1,21 +0,0 @@
-import type { StorageOpenOptions } from '@crawlee/core';
-import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core';
-/**
- * @inheritDoc
- */
-export declare class KeyValueStore extends CoreKeyValueStore {
-    /**
-     * Returns a URL for the given key that may be used to publicly
-     * access the value in the remote key-value store.
-     *
-     * On the Apify platform the URL is signed with the store's
-     * `urlSigningSecretKey` so that anyone with the URL can read the record
-     * without authentication. Locally we delegate to crawlee's default
-     * implementation (which produces a `file://` URL or returns `undefined`).
-     */
-    getPublicUrl(key: string): Promise<string | undefined>;
-    /**
-     * @inheritDoc
-     */
-    static open(storeIdOrName?: string | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
-}

package/dist/key_value_store.js

@@ -1,40 +0,0 @@
-import { KeyValueStore as CoreKeyValueStore, serviceLocator } from '@crawlee/core';
-import { createHmacSignature } from '@apify/utilities';
-import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js';
-/**
- * @inheritDoc
- */
-export class KeyValueStore extends CoreKeyValueStore {
-    /**
-     * Returns a URL for the given key that may be used to publicly
-     * access the value in the remote key-value store.
-     *
-     * On the Apify platform the URL is signed with the store's
-     * `urlSigningSecretKey` so that anyone with the URL can read the record
-     * without authentication. Locally we delegate to crawlee's default
-     * implementation (which produces a `file://` URL or returns `undefined`).
-     */
-    async getPublicUrl(key) {
-        const config = serviceLocator.getConfiguration();
-        // Detect a remote (Apify) store by its backend type rather than by
-        // `isAtHome`, so that a `forceCloud` store opened locally still gets a
-        // signed Apify URL (matching the platform behaviour). `backend` is
-        // `private` on `CoreKeyValueStore`, so bypass the visibility check.
-        const { backend } = this;
-        if (!(backend instanceof ApifyKeyValueStoreBackend)) {
-            return super.getPublicUrl(key);
-        }
-        const publicUrl = new URL(`${config.apiPublicBaseUrl}/v2/key-value-stores/${this.id}/records/${key}`);
-        const metadata = (await backend.getMetadata());
-        if (metadata?.urlSigningSecretKey) {
-            publicUrl.searchParams.append('signature', createHmacSignature(metadata.urlSigningSecretKey, key));
-        }
-        return publicUrl.toString();
-    }
-    /**
-     * @inheritDoc
-     */
-    static async open(storeIdOrName, options = {}) {
-        return super.open(storeIdOrName, options);
-    }
-}