@crawlee/core 4.0.0-beta.76 → 4.0.0-beta.78

package/memory-storage/memory-storage.d.ts

@@ -26,9 +26,9 @@ export declare class MemoryStorageBackend implements storage.StorageBackend {
      */
     getStorageBackendCacheKey(): string;
     private static resolveStorageKey;
-    createDatasetBackend(options?: storage.CreateDatasetBackendOptions): Promise<storage.DatasetBackend>;
-    createKeyValueStoreBackend(options?: storage.CreateKeyValueStoreBackendOptions): Promise<storage.KeyValueStoreBackend>;
-    createRequestQueueBackend(options?: storage.CreateRequestQueueBackendOptions): Promise<RequestQueueBackend>;
+    createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
+    createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
+    createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<RequestQueueBackend>;
     storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
     /**
      * Cleans up the default storages before the run starts. For the in-memory storage this simply

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.76",
+    "version": "4.0.0-beta.78",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,9 +53,9 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.76",
-        "@crawlee/types": "4.0.0-beta.76",
-        "@crawlee/utils": "4.0.0-beta.76",
+        "@crawlee/fs-storage": "4.0.0-beta.78",
+        "@crawlee/types": "4.0.0-beta.78",
+        "@crawlee/utils": "4.0.0-beta.78",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -79,5 +79,5 @@
             }
         }
     },
-    "gitHead": "0f76e7dbac40567e81475a6a77c7dedd8a829d12"
+    "gitHead": "e6c53f94d11cf6e0885855255c0d9ac8071459fa"
 }

package/storages/dataset.d.ts

@@ -460,8 +460,8 @@ export interface DatasetReducer<T, Data> {
     (memo: T, item: Data, index: number): Awaitable<T>;
 }
 export interface DatasetOptions {
-    id: string;
-    name?: string;
+    /** Resolved metadata for the dataset, as returned by the backend's `getMetadata()`. */
+    metadata: DatasetInfo;
     backend: DatasetBackend;
 }
 export interface DatasetContent<Data> {

package/storages/dataset.js

@@ -97,8 +97,8 @@ export class Dataset {
      */
     constructor(options, config = Configuration.getGlobalConfig()) {
         this.config = config;
-        this.id = options.id;
-        this.name = options.name;
+        this.id = options.metadata.id;
+        this.name = options.metadata.name;
         this.backend = options.backend;
         this.log = serviceLocator.getLogger().child({ prefix: 'Dataset' });
     }

package/storages/key_value_store.d.ts

@@ -1,4 +1,4 @@
-import type { Dictionary, KeyValueStoreBackend } from '@crawlee/types';
+import type { Dictionary, KeyValueStoreBackend, KeyValueStoreInfo } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
 import type { Awaitable } from '../typedefs.js';
 import type { KeyValueStoreStats } from './storage_stats.js';
@@ -509,8 +509,8 @@ export interface KeyConsumer {
     }): Awaitable<void>;
 }
 export interface KeyValueStoreOptions {
-    id: string;
-    name?: string;
+    /** Resolved metadata for the key-value store, as returned by the backend's `getMetadata()`. */
+    metadata: KeyValueStoreInfo;
     backend: KeyValueStoreBackend;
 }
 /**

package/storages/key_value_store.js

@@ -85,8 +85,8 @@ export class KeyValueStore {
      */
     constructor(options, config = Configuration.getGlobalConfig()) {
         this.config = config;
-        this.id = options.id;
-        this.name = options.name;
+        this.id = options.metadata.id;
+        this.name = options.metadata.name;
         this.backend = options.backend;
     }
     /**

package/storages/request_queue.d.ts

@@ -50,12 +50,9 @@ export declare class RequestQueue implements IStorage, IRequestManager {
     protected readonly config: Configuration;
     id: string;
     name?: string;
-    timeoutSecs: number;
-    clientKey: string;
     backend: RequestQueueBackend;
     protected proxyConfiguration?: ProxyConfiguration;
     log: CrawleeLogger;
-    private isInitialized;
     protected requestCache: LruCache<RequestLruItem>;
     /**
      * Remembers the `requestId` of every request already submitted to the client — including background
@@ -294,8 +291,8 @@ interface RequestLruItem {
     forefront: boolean;
 }
 export interface RequestQueueOptions {
-    id: string;
-    name?: string;
+    /** Resolved metadata for the request queue, as returned by the backend's `getMetadata()`. */
+    metadata: RequestQueueInfo;
     backend: RequestQueueBackend;
     /**
      * Used to pass the proxy configuration for the `requestsFromUrl` objects.

package/storages/request_queue.js

@@ -2,7 +2,6 @@ import { inspect } from 'node:util';
 import { chunkedAsyncIterable, downloadListOfUrls, getObjectType, isAsyncIterable, isIterable, peekableAsyncIterable, sleep, } from '@crawlee/utils';
 import ow from 'ow';
 import { LruCache } from '@apify/datastructures';
-import { cryptoRandomObjectId } from '@apify/utilities';
 import { Configuration } from '../configuration.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
@@ -59,12 +58,9 @@ export class RequestQueue {
     config;
     id;
     name;
-    timeoutSecs = 30;
-    clientKey = cryptoRandomObjectId();
     backend;
     proxyConfiguration;
     log;
-    isInitialized = false;
     requestCache;
     /**
      * Remembers the `requestId` of every request already submitted to the client — including background
@@ -98,8 +94,8 @@ export class RequestQueue {
      */
     constructor(options, config = Configuration.getGlobalConfig()) {
         this.config = config;
-        this.id = options.id;
-        this.name = options.name;
+        this.id = options.metadata.id;
+        this.name = options.metadata.name;
         this.events = serviceLocator.getEventManager();
         this.backend = options.backend;
         this.proxyConfiguration = options.proxyConfiguration;
@@ -734,19 +730,6 @@ export class RequestQueue {
         });
         queue.proxyConfiguration = options.proxyConfiguration;
         queue.httpClient = options.httpClient;
-        if (!queue.isInitialized) {
-            // Re-create the request queue backend with clientKey and timeoutSecs so that
-            // request locking works correctly for API-backed implementations.
-            // TODO: clientKey/timeoutSecs are Apify-platform concerns and should eventually be pushed
-            // down into the Apify SDK's client implementation, aligning with crawlee-python's approach
-            // where locking is handled internally by the backend (see crawlee-python PR #1194).
-            queue.backend = await storageBackend.createRequestQueueBackend({
-                id: queue.id,
-                clientKey: queue.clientKey,
-                timeoutSecs: queue.timeoutSecs,
-            });
-            queue.isInitialized = true;
-        }
         return queue;
     }
 }

package/storages/storage_instance_manager.js

@@ -187,11 +187,9 @@ export class StorageInstanceManager {
             // Cache miss — create the sub-backend and storage instance.
             const subBackend = await backendOpener();
             const storageInfo = await subBackend.getMetadata();
-            const instance = new cls({
-                id: storageInfo.id,
-                name: storageInfo.name,
-                backend: subBackend,
-            });
+            // Storage frontends are thin wrappers over the backend. We hand them the resolved metadata
+            // we just fetched (so `id`/`name` etc. are available synchronously) along with the backend.
+            const instance = new cls({ metadata: storageInfo, backend: subBackend });
             // Atomic cache writes (no awaits between these).
             this.cache.set(cls, instance, backendCacheKey, alias);
             return instance;