@crawlee/core 3.17.1-beta.9 → 3.18.1-beta.0

package/storages/request_provider.js

@@ -11,8 +11,14 @@ const configuration_1 = require("../configuration");
 const log_1 = require("../log");
 const request_1 = require("../request");
 const access_checking_1 = require("./access_checking");
+const request_dedup_cache_1 = require("./request_dedup_cache");
 const storage_manager_1 = require("./storage_manager");
 const utils_2 = require("./utils");
+/**
+ * Maximum number of consecutive batch-add attempts that make no progress before the remaining
+ * unprocessed requests are skipped, so permanently rejected requests don't retry forever.
+ */
+const MAX_UNPROCESSED_REQUESTS_RETRIES = 3;
 class RequestProvider {
     constructor(options, config = configuration_1.Configuration.getGlobalConfig()) {
         Object.defineProperty(this, "config", {
@@ -113,6 +119,17 @@ class RequestProvider {
             writable: true,
             value: void 0
         });
+        /**
+         * Remembers the `requestId` of every request already submitted to the client — including background
+         * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
+         * See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.
+         */
+        Object.defineProperty(this, "requestSeenCache", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "recentlyHandledRequestsCache", {
             enumerable: true,
             configurable: true,
@@ -151,6 +168,7 @@ class RequestProvider {
         });
         this.proxyConfiguration = options.proxyConfiguration;
         this.requestCache = new datastructures_1.LruCache({ maxLength: options.requestCacheMaxSize });
+        this.requestSeenCache = new request_dedup_cache_1.RequestDeduplicationCache();
         this.recentlyHandledRequestsCache = new datastructures_1.LruCache({ maxLength: options.recentlyHandledRequestsMaxSize });
         this.log = log_1.log.child({ prefix: `${options.logPrefix}(${this.id}, ${this.name ?? 'no-name'})` });
         const eventManager = config.getEventManager();
@@ -227,6 +245,7 @@ class RequestProvider {
         };
         const { requestId, wasAlreadyPresent } = queueOperationInfo;
         this._cacheRequest(cacheKey, queueOperationInfo);
+        this.requestSeenCache.add(cacheKey, requestId);
         if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {
             this.assumedTotalCount++;
             // Performance optimization: add request straight to head if possible
@@ -288,16 +307,17 @@ class RequestProvider {
         const requestsToAdd = new Map();
         for (const request of requests) {
             const cacheKey = getCachedRequestId(request.uniqueKey);
+            // Prefer the full `requestCache` record; fall back to the dedup cache for background batches it skips.
             const cachedInfo = this.requestCache.get(cacheKey);
-            if (cachedInfo) {
-                request.id = cachedInfo.id;
+            const knownRequestId = cachedInfo?.id ?? this.requestSeenCache.get(cacheKey);
+            if (knownRequestId) {
+                request.id = knownRequestId;
                 results.processedRequests.push({
                     wasAlreadyPresent: true,
-                    // We may assume that if request is in local cache then also the information if the
-                    // request was already handled is there because just one client should be using one queue.
-                    wasAlreadyHandled: cachedInfo.isHandled,
-                    requestId: cachedInfo.id,
-                    uniqueKey: cachedInfo.uniqueKey,
+                    // The dedup cache doesn't track the handled state; only the full record does.
+                    wasAlreadyHandled: cachedInfo?.isHandled ?? false,
+                    requestId: knownRequestId,
+                    uniqueKey: request.uniqueKey,
                 });
             }
             else if (!requestsToAdd.has(request.uniqueKey)) {
@@ -320,6 +340,8 @@ class RequestProvider {
             if (cache) {
                 this._cacheRequest(cacheKey, { ...newRequest, forefront });
             }
+            // Unlike `requestCache`, populate this on every batch (including background ones).
+            this.requestSeenCache.add(cacheKey, requestId);
             if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {
                 this.assumedTotalCount++;
                 // Performance optimization: add request straight to head if possible
@@ -385,13 +407,20 @@ class RequestProvider {
         const requestIterator = generateRequests();
         const chunks = (0, utils_1.peekableAsyncIterable)((0, utils_1.chunkedAsyncIterable)(requestIterator, effectiveChunkSize));
         const chunksIterator = chunks[Symbol.asyncIterator]();
-        const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true) => {
+        const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true, unsuccessfulAttempts = 0) => {
             const resultsToReturn = [];
             const apiResult = await this.addRequests(providedRequests, { forefront: options.forefront, cache });
             resultsToReturn.push(...apiResult.processedRequests);
             if (apiResult.unprocessedRequests.length) {
+                // Count attempts that make no progress, so permanently rejected requests (e.g. a malformed
+                // `userData` shape causing a 400) don't loop forever. Any progress resets the counter.
+                const attempts = apiResult.processedRequests.length ? 0 : unsuccessfulAttempts + 1;
+                if (attempts >= MAX_UNPROCESSED_REQUESTS_RETRIES) {
+                    this.log.warning(`Some requests were consistently rejected by the request queue and will be skipped after ${MAX_UNPROCESSED_REQUESTS_RETRIES} attempts. This usually means the request data is malformed (e.g. an invalid 'userData' shape).`, { unprocessedRequests: apiResult.unprocessedRequests });
+                    return resultsToReturn;
+                }
                 await (0, utils_1.sleep)(waitBetweenBatchesMillis);
-                resultsToReturn.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(providedRequests.filter((r) => !apiResult.processedRequests.some((pr) => pr.uniqueKey === r.uniqueKey)), false)));
+                resultsToReturn.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(providedRequests.filter((r) => !apiResult.processedRequests.some((pr) => pr.uniqueKey === r.uniqueKey)), false, attempts)));
             }
             return resultsToReturn;
         };
@@ -546,6 +575,7 @@ class RequestProvider {
         this.assumedTotalCount = 0;
         this.assumedHandledCount = 0;
         this.requestCache.clear();
+        this.requestSeenCache.clear();
     }
     /**
      * Caches information about request to beware of unneeded addRequest() calls.

package/storages/sitemap_request_list.d.ts

@@ -1,6 +1,7 @@
 import { type ParseSitemapOptions } from '@crawlee/utils';
 import { Configuration } from '../configuration';
 import type { GlobInput, RegExpInput } from '../enqueue_links';
+import { EnqueueStrategy } from '../enqueue_links';
 import { Request } from '../request';
 import type { IRequestList } from './request_list';
 interface UrlConstraints {
@@ -76,6 +77,13 @@ export interface SitemapRequestListOptions extends UrlConstraints {
      * @default 200
      */
     maxBufferSize?: number;
+    /**
+     * Keep only sitemap-derived URLs matching this strategy relative to the parent sitemap URL; non-`http(s)`
+     * schemes are always dropped. The filtering stays enforced after navigation (e.g. across redirects).
+     * Pass `'all'` to disable host filtering.
+     * @default EnqueueStrategy.SameHostname
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
     /**
      * Advanced options for the underlying `parseSitemap` call.
      */
@@ -133,6 +141,10 @@ export declare class SitemapRequestList implements IRequestList {
      * Proxy URL to be used for sitemap loading.
      */
     private proxyUrl;
+    /**
+     * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
+     */
+    private enqueueStrategy;
     /**
      * Logger instance.
      */

package/storages/sitemap_request_list.js

@@ -132,6 +132,15 @@ class SitemapRequestList {
             writable: true,
             value: void 0
         });
+        /**
+         * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
+         */
+        Object.defineProperty(this, "enqueueStrategy", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         /**
          * Logger instance.
          */
@@ -173,6 +182,7 @@ class SitemapRequestList {
             signal: ow_1.default.optional.any(),
             timeoutMillis: ow_1.default.optional.number,
             maxBufferSize: ow_1.default.optional.number,
+            enqueueStrategy: ow_1.default.optional.string.oneOf(Object.values(enqueue_links_1.EnqueueStrategy)),
             parseSitemapOptions: ow_1.default.optional.object,
             globs: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('glob'))),
             exclude: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.regExp, ow_1.default.object.hasKeys('glob'), ow_1.default.object.hasKeys('regexp'))),
@@ -200,6 +210,7 @@ class SitemapRequestList {
         this.persistStateKey = options.persistStateKey;
         this.persistenceOptions = { enable: true, ...options.persistenceOptions };
         this.proxyUrl = options.proxyUrl;
+        this.enqueueStrategy = options.enqueueStrategy ?? enqueue_links_1.EnqueueStrategy.SameHostname;
         this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
         this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
         this.events = config.getEventManager();
@@ -308,6 +319,7 @@ class SitemapRequestList {
                     ...parseSitemapOptions,
                     maxDepth: 0,
                     emitNestedSitemaps: true,
+                    enqueueStrategy: this.enqueueStrategy,
                 })) {
                     if (!item.originSitemapUrl) {
                         // This is a nested sitemap
@@ -398,14 +410,21 @@ class SitemapRequestList {
         }
         // Create a new stream, as we have read all the URLs from the current one.
         // Pushing the urls back to the original stream might not be possible if it has been ended.
-        const newStream = this.createNewStream(this.urlQueueStream.readableHighWaterMark);
+        const previousStream = this.urlQueueStream;
+        const newStream = this.createNewStream(previousStream.readableHighWaterMark);
         for (const url of urlQueue) {
             newStream.push(url);
         }
-        if (this.urlQueueStream.writableEnded) {
+        if (previousStream.writableEnded) {
             newStream.end();
         }
         this.urlQueueStream = newStream;
+        // A `pushNextUrl()` call may be blocked on backpressure, waiting for a `readdata` event on the
+        // previous stream. That event is only ever emitted by `readNextUrl()` on the current stream, so
+        // after the swap the waiter would never be notified and the background sitemap loading would hang.
+        // Re-emit `readdata` on the previous stream to release any such pending waiter (its URL has already
+        // been transferred to the new stream above).
+        previousStream.emit('readdata');
         await this.store.setValue(this.persistStateKey, {
             sitemapParsingProgress: {
                 pendingSitemapUrls: Array.from(this.sitemapParsingProgress.pendingSitemapUrls),
@@ -457,7 +476,7 @@ class SitemapRequestList {
             if (!nextUrl) {
                 return null;
             }
-            this.requestData.set(nextUrl, new request_1.Request({ url: nextUrl }));
+            this.requestData.set(nextUrl, new request_1.Request({ url: nextUrl, enqueueStrategy: this.enqueueStrategy }));
         }
         this.inProgress.add(nextUrl);
         return this.requestData.get(nextUrl);