@crawlee/core 3.17.1-beta.45 → 3.17.1-beta.47

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "3.17.1-beta.45",
+    "version": "3.17.1-beta.47",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -59,9 +59,9 @@
         "@apify/pseudo_url": "^2.0.30",
         "@apify/timeout": "^0.3.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/memory-storage": "3.17.1-beta.45",
-        "@crawlee/types": "3.17.1-beta.45",
-        "@crawlee/utils": "3.17.1-beta.45",
+        "@crawlee/memory-storage": "3.17.1-beta.47",
+        "@crawlee/types": "3.17.1-beta.47",
+        "@crawlee/utils": "3.17.1-beta.47",
         "@sapphire/async-queue": "^1.5.1",
         "@vladfrangu/async_event_emitter": "^2.2.2",
         "csv-stringify": "^6.2.0",
@@ -83,5 +83,5 @@
             }
         }
     },
-    "gitHead": "c0676ff5f444967c1a323e02dd9e34c00f7469f9"
+    "gitHead": "93d7b6fccbd6e4f4b474adbba55e5f05051bbaf2"
 }

package/storages/request_dedup_cache.d.ts

@@ -0,0 +1,23 @@
+/**
+ * A fixed-size, direct-mapped cache for `uniqueKey`-based request deduplication.
+ *
+ * `RequestProvider.requestCache` only remembers the first batch of requests, so repeated
+ * `addRequestsBatched()` calls with overlapping URLs re-submit already-enqueued requests
+ * (https://github.com/apify/crawlee/issues/3120). This is a separate, cheaper cache we can populate on
+ * every batch: a fixed number of slots indexed by a hash of the request's cache key, storing the
+ * server-assigned `requestId`. Memory is capped by the slot count regardless of the working set size;
+ * a hash collision just overwrites a slot, causing an occasional cache miss (a harmless re-submission)
+ * but never a false hit — so a genuinely new request is never dropped.
+ *
+ * @internal
+ */
+export declare class RequestDeduplicationCache {
+    private readonly size;
+    private keys;
+    private ids;
+    constructor(size?: number);
+    get(cacheKey: string): string | null;
+    add(cacheKey: string, requestId: string): void;
+    clear(): void;
+    private indexOf;
+}

package/storages/request_dedup_cache.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RequestDeduplicationCache = void 0;
+/**
+ * A fixed-size, direct-mapped cache for `uniqueKey`-based request deduplication.
+ *
+ * `RequestProvider.requestCache` only remembers the first batch of requests, so repeated
+ * `addRequestsBatched()` calls with overlapping URLs re-submit already-enqueued requests
+ * (https://github.com/apify/crawlee/issues/3120). This is a separate, cheaper cache we can populate on
+ * every batch: a fixed number of slots indexed by a hash of the request's cache key, storing the
+ * server-assigned `requestId`. Memory is capped by the slot count regardless of the working set size;
+ * a hash collision just overwrites a slot, causing an occasional cache miss (a harmless re-submission)
+ * but never a false hit — so a genuinely new request is never dropped.
+ *
+ * @internal
+ */
+class RequestDeduplicationCache {
+    // The slot count is the same for every queue, so it's a fixed default rather than a per-consumer option.
+    constructor(size = 1000000) {
+        Object.defineProperty(this, "size", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: size
+        });
+        Object.defineProperty(this, "keys", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "ids", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.keys = new Array(size);
+        this.ids = new Array(size);
+    }
+    get(cacheKey) {
+        const index = this.indexOf(cacheKey);
+        return this.keys[index] === cacheKey ? this.ids[index] : null;
+    }
+    add(cacheKey, requestId) {
+        const index = this.indexOf(cacheKey);
+        this.keys[index] = cacheKey;
+        this.ids[index] = requestId;
+    }
+    clear() {
+        this.keys = new Array(this.size);
+        this.ids = new Array(this.size);
+    }
+    // A cheap FNV-1a hash of the cache key — avoids pulling in a dedicated hashing dependency.
+    indexOf(cacheKey) {
+        /* eslint-disable no-bitwise */
+        let hash = 0x811c9dc5;
+        for (let i = 0; i < cacheKey.length; i++) {
+            hash ^= cacheKey.charCodeAt(i);
+            hash = Math.imul(hash, 0x01000193);
+        }
+        return (hash >>> 0) % this.size;
+        /* eslint-enable no-bitwise */
+    }
+}
+exports.RequestDeduplicationCache = RequestDeduplicationCache;

package/storages/request_provider.d.ts

@@ -5,6 +5,7 @@ import { Configuration } from '../configuration';
 import type { ProxyConfiguration } from '../proxy_configuration';
 import type { InternalSource, RequestOptions, Source } from '../request';
 import { Request } from '../request';
+import { RequestDeduplicationCache } from './request_dedup_cache';
 import type { IStorage, StorageManagerOptions } from './storage_manager';
 export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
 /**
@@ -74,6 +75,12 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
     private initialHandledCount;
     protected queueHeadIds: ListDictionary<string>;
     protected requestCache: LruCache<RequestLruItem>;
+    /**
+     * Remembers the `requestId` of every request already submitted to the client — including background
+     * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
+     * See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.
+     */
+    protected requestSeenCache: RequestDeduplicationCache;
     protected recentlyHandledRequestsCache: LruCache<boolean>;
     protected queuePausedForMigration: boolean;
     protected lastActivity: Date;

package/storages/request_provider.js

@@ -11,6 +11,7 @@ const configuration_1 = require("../configuration");
 const log_1 = require("../log");
 const request_1 = require("../request");
 const access_checking_1 = require("./access_checking");
+const request_dedup_cache_1 = require("./request_dedup_cache");
 const storage_manager_1 = require("./storage_manager");
 const utils_2 = require("./utils");
 /**
@@ -118,6 +119,17 @@ class RequestProvider {
             writable: true,
             value: void 0
         });
+        /**
+         * Remembers the `requestId` of every request already submitted to the client — including background
+         * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
+         * See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.
+         */
+        Object.defineProperty(this, "requestSeenCache", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "recentlyHandledRequestsCache", {
             enumerable: true,
             configurable: true,
@@ -156,6 +168,7 @@ class RequestProvider {
         });
         this.proxyConfiguration = options.proxyConfiguration;
         this.requestCache = new datastructures_1.LruCache({ maxLength: options.requestCacheMaxSize });
+        this.requestSeenCache = new request_dedup_cache_1.RequestDeduplicationCache();
         this.recentlyHandledRequestsCache = new datastructures_1.LruCache({ maxLength: options.recentlyHandledRequestsMaxSize });
         this.log = log_1.log.child({ prefix: `${options.logPrefix}(${this.id}, ${this.name ?? 'no-name'})` });
         const eventManager = config.getEventManager();
@@ -232,6 +245,7 @@ class RequestProvider {
         };
         const { requestId, wasAlreadyPresent } = queueOperationInfo;
         this._cacheRequest(cacheKey, queueOperationInfo);
+        this.requestSeenCache.add(cacheKey, requestId);
         if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {
             this.assumedTotalCount++;
             // Performance optimization: add request straight to head if possible
@@ -293,16 +307,17 @@ class RequestProvider {
         const requestsToAdd = new Map();
         for (const request of requests) {
             const cacheKey = getCachedRequestId(request.uniqueKey);
+            // Prefer the full `requestCache` record; fall back to the dedup cache for background batches it skips.
             const cachedInfo = this.requestCache.get(cacheKey);
-            if (cachedInfo) {
-                request.id = cachedInfo.id;
+            const knownRequestId = cachedInfo?.id ?? this.requestSeenCache.get(cacheKey);
+            if (knownRequestId) {
+                request.id = knownRequestId;
                 results.processedRequests.push({
                     wasAlreadyPresent: true,
-                    // We may assume that if request is in local cache then also the information if the
-                    // request was already handled is there because just one client should be using one queue.
-                    wasAlreadyHandled: cachedInfo.isHandled,
-                    requestId: cachedInfo.id,
-                    uniqueKey: cachedInfo.uniqueKey,
+                    // The dedup cache doesn't track the handled state; only the full record does.
+                    wasAlreadyHandled: cachedInfo?.isHandled ?? false,
+                    requestId: knownRequestId,
+                    uniqueKey: request.uniqueKey,
                 });
             }
             else if (!requestsToAdd.has(request.uniqueKey)) {
@@ -325,6 +340,8 @@ class RequestProvider {
             if (cache) {
                 this._cacheRequest(cacheKey, { ...newRequest, forefront });
             }
+            // Unlike `requestCache`, populate this on every batch (including background ones).
+            this.requestSeenCache.add(cacheKey, requestId);
             if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {
                 this.assumedTotalCount++;
                 // Performance optimization: add request straight to head if possible
@@ -558,6 +575,7 @@ class RequestProvider {
         this.assumedTotalCount = 0;
         this.assumedHandledCount = 0;
         this.requestCache.clear();
+        this.requestSeenCache.clear();
     }
     /**
      * Caches information about request to beware of unneeded addRequest() calls.