@crawlee/core 4.0.0-beta.160 → 4.0.0-beta.161

package/memory-storage/resource-clients/request-queue.d.ts

@@ -53,7 +53,7 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
      * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
      * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
      * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
-     * soon as the page is filled, keeping those calls O(head) instead of O(N).
+     * soon as the page is filled, keeping those calls O(head) instead of O(pending).
      */
     private listPendingHead;
     fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;

package/memory-storage/resource-clients/request-queue.js

@@ -19,11 +19,11 @@ export class RequestQueueBackend extends BaseClient {
     pendingRequestCount = 0;
     /**
      * Serializes every operation that reads-then-writes this backend's shared queue state — the
-     * `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request
-     * counts. Those mutations span `await` points, so without this mutex a concurrent operation could
-     * interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while
-     * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
-     * `isFinished`, whose head scan also prunes `forefrontRequestIds`.
+     * `requests` map, the `pendingRequestIds` set, the `forefrontRequestIds` array, the
+     * `inProgressRequestIds` set and the request counts. Those mutations span `await` points, so
+     * without this mutex a concurrent operation could interleave and corrupt them (e.g. a head scan
+     * pruning `forefrontRequestIds` while `addBatchOfRequests` pushes to it). Held by every mutating
+     * method as well as by `isEmpty`/`isFinished`, whose head scan also prunes `forefrontRequestIds`.
      */
     #queueStateMutex = new AsyncQueue();
     #forefrontRequestIds = [];
@@ -37,6 +37,12 @@ export class RequestQueueBackend extends BaseClient {
      */
     #inProgressRequestIds = new Set();
     #requests = new Map();
+    /**
+     * IDs of requests that are not yet handled (pending or in progress), in insertion order. Handled
+     * requests stay in `requests` for deduplication but are removed from here, so head scans only
+     * ever walk the unhandled tail instead of every request the queue has ever seen.
+     */
+    #pendingRequestIds = new Set();
     // kept as TS-private: storage-backend tests read this field at runtime
     storageBackend;
     constructor(options) {
@@ -61,6 +67,7 @@ export class RequestQueueBackend extends BaseClient {
                 // leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head
                 // scan would resolve to a missing request and dereference.
                 this.#requests.clear();
+                this.#pendingRequestIds.clear();
                 this.#forefrontRequestIds = [];
                 this.#inProgressRequestIds.clear();
             }
@@ -76,6 +83,7 @@ export class RequestQueueBackend extends BaseClient {
         try {
             // Clear all in-memory state
             this.#requests.clear();
+            this.#pendingRequestIds.clear();
             this.#forefrontRequestIds = [];
             this.#inProgressRequestIds.clear();
             this.handledRequestCount = 0;
@@ -90,9 +98,7 @@ export class RequestQueueBackend extends BaseClient {
         for (let i = this.#forefrontRequestIds.length - 1; i >= 0; i--) {
             yield this.#forefrontRequestIds[i];
         }
-        for (const key of this.#requests.keys()) {
-            yield key;
-        }
+        yield* this.#pendingRequestIds;
     }
     /**
      * Scans the queue and returns the pending head — requests that are neither handled nor currently
@@ -107,7 +113,7 @@ export class RequestQueueBackend extends BaseClient {
      * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
      * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
      * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
-     * soon as the page is filled, keeping those calls O(head) instead of O(N).
+     * soon as the page is filled, keeping those calls O(head) instead of O(pending).
      */
     async listPendingHead(limit, detectInProgressRequests = false) {
         const items = [];
@@ -127,11 +133,10 @@ export class RequestQueueBackend extends BaseClient {
             }
             seenRequestIds.add(requestId);
             const request = this.#requests.get(requestId);
-            // Permanently-handled requests (`orderNo === null`) are in a terminal state and can be skipped.
+            // Only `forefrontRequestIds` can still reference a handled request (`orderNo === null`);
+            // `pendingRequestIds` drops them on handling. Remember the id so the list gets pruned below.
             if (request.orderNo === null) {
-                if (this.#forefrontRequestIds.includes(requestId)) {
-                    handledForefrontIds.add(requestId);
-                }
+                handledForefrontIds.add(requestId);
                 continue;
             }
             // In progress (fetched but not yet handled or reclaimed) — skip it, but remember that the
@@ -193,6 +198,7 @@ export class RequestQueueBackend extends BaseClient {
                 }
                 this.#requests.set(requestModel.id, requestModel);
                 if (requestModel.orderNo) {
+                    this.#pendingRequestIds.add(requestModel.id);
                     this.pendingRequestCount += 1;
                 }
                 else {
@@ -245,6 +251,7 @@ export class RequestQueueBackend extends BaseClient {
             const handledAt = request.handledAt ?? new Date().toISOString();
             const requestModel = this.createInternalRequest({ ...request, handledAt }, false);
             this.#requests.set(id, requestModel);
+            this.#pendingRequestIds.delete(id);
             // The request is no longer in progress for this client.
             this.#inProgressRequestIds.delete(id);
             if (!wasAlreadyHandled) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.160",
+    "version": "4.0.0-beta.161",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.160",
-        "@crawlee/http-client": "4.0.0-beta.160",
-        "@crawlee/types": "4.0.0-beta.160",
-        "@crawlee/utils": "4.0.0-beta.160",
+        "@crawlee/fs-storage": "4.0.0-beta.161",
+        "@crawlee/http-client": "4.0.0-beta.161",
+        "@crawlee/types": "4.0.0-beta.161",
+        "@crawlee/utils": "4.0.0-beta.161",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "09fbce5ade9b84a2b191c635463e172388e52678"
+    "gitHead": "dc353e294d5f4d2ee468325ef11dd7af44443be2"
 }