@crawlee/core 4.0.0-beta.150 → 4.0.0-beta.151

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.150",
+    "version": "4.0.0-beta.151",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.150",
-        "@crawlee/http-client": "4.0.0-beta.150",
-        "@crawlee/types": "4.0.0-beta.150",
-        "@crawlee/utils": "4.0.0-beta.150",
+        "@crawlee/fs-storage": "4.0.0-beta.151",
+        "@crawlee/http-client": "4.0.0-beta.151",
+        "@crawlee/types": "4.0.0-beta.151",
+        "@crawlee/utils": "4.0.0-beta.151",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "ef7e9fb2e2fcaa0f74e60a30fa8ae61c60c64626"
+    "gitHead": "1865bf2b241da81d8ed6f1215c59f9d0b88ec553"
 }

package/storages/throttling_request_manager.d.ts

@@ -1,6 +1,7 @@
 import type { Dictionary } from '@crawlee/types';
 import type { Configuration } from '../configuration.js';
-import type { Request, Source } from '../request.js';
+import type { Source } from '../request.js';
+import { Request } from '../request.js';
 import type { RequestSourceStatus } from './request_loader.js';
 import type { IRequestManager, PacingSignal, RequestsLike } from './request_manager.js';
 import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';

package/storages/throttling_request_manager.js

@@ -2,6 +2,7 @@ import { URL } from 'node:url';
 import { getDomain } from 'tldts';
 import { z } from 'zod';
 import { asyncifyIterable } from '../iterables.js';
+import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
 import { normalizeHostname } from '../url.js';
 import { parseArgument, schemas } from '../validators.js';
@@ -44,6 +45,11 @@ function newDomainState(domain) {
     };
 }
 const DEFAULT_PERSIST_STATE_KEY = 'CRAWLEE_THROTTLED_DOMAINS';
+/**
+ * How many requests one fetch may move out of the wrapped manager before giving up on finding a dispatchable
+ * one. The next fetch resumes where it left off, and a migrated request is never walked again.
+ */
+const MAX_INNER_MIGRATIONS = 1000;
 /**
  * A request manager that wraps another one and paces requests per domain.
  *
@@ -133,6 +139,8 @@ export class ThrottlingRequestManager {
     #subManagersReady;
     /** Batches still being added in the background; keeps {@link ThrottlingRequestManager.checkReadiness} honest. */
     #inProgressBatchCount = 0;
+    /** Requests moved into a sub-queue, which both managers count. Subtracted below; not persisted. */
+    #migratedFromInner = 0;
     /** The latest {@link setExpectedRequestProcessingTimeSecs} hint, kept for a wrapped manager resolved later. */
     #expectedRequestProcessingSecs;
     #warnedAbout = new Set();
@@ -588,13 +596,13 @@ export class ThrottlingRequestManager {
         return this.#selectManagerOrThrow(request.url);
     }
     async getTotalCount() {
-        return this.#sumOverManagers((manager) => manager.getTotalCount());
+        return (await this.#sumOverManagers((manager) => manager.getTotalCount())) - this.#migratedFromInner;
     }
     async getPendingCount() {
         return this.#sumOverManagers((manager) => manager.getPendingCount());
     }
     async getHandledCount() {
-        return this.#sumOverManagers((manager) => manager.getHandledCount());
+        return (await this.#sumOverManagers((manager) => manager.getHandledCount())) - this.#migratedFromInner;
     }
     /**
      * Reports whether anything can be dispatched right now, and if not, when — or why never.
@@ -638,8 +646,9 @@ export class ThrottlingRequestManager {
                 readyAt = until;
             }
         }
+        const inner = await this.#getInner();
         const probed = (await Promise.all([
-            (await this.#getInner()).checkReadiness(),
+            inner.checkReadiness(),
             ...dispatchable.map(async (subManager) => (await subManager).checkReadiness()),
         ])).reduce(joinRequestSourceStatuses);
         // Anything dispatchable outranks a stalled or throttled domain, so we stop here without touching them.
@@ -687,6 +696,7 @@ export class ThrottlingRequestManager {
     async #purgeDomainQueues() {
         const subManagers = await this.#getSubManagers();
         await Promise.all(subManagers.map(async (manager) => manager.purge?.()));
+        this.#migratedFromInner = 0;
         for (const state of this.domainStates.values()) {
             state.consecutive429Count = 0;
             state.backoffUntil = 0;
@@ -746,7 +756,7 @@ export class ThrottlingRequestManager {
             // No dispatch to pace, so the domain keeps its slot.
             state.crawlDelayUntil = crawlDelayUntilBefore;
         }
-        const request = await (await this.#getInner()).fetchNextRequest();
+        const request = await this.#fetchFromInner();
         if (request !== null) {
             this.#inFlightFromInner.add(request.id ?? request.uniqueKey);
         }
@@ -761,6 +771,51 @@ export class ThrottlingRequestManager {
         }
         return request;
     }
+    /**
+     * The next request the wrapped manager can offer whose domain is not being held back.
+     *
+     * Its requests are not stored per domain, so they cannot be skipped the way a sub-queue can - the only
+     * way to find out which domain one belongs to is to take it out. One that turns out to be backing off is
+     * moved into its domain's sub-queue, where that domain's clocks hold it back like any other: dispatching
+     * it would have the crawler hammer the domain that just told us to wait, and putting it back would have
+     * every later fetch walk past it again.
+     */
+    async #fetchFromInner() {
+        const inner = await this.#getInner();
+        for (let migrated = 0; migrated < MAX_INNER_MIGRATIONS; migrated++) {
+            const request = await inner.fetchNextRequest();
+            if (request === null) {
+                return null;
+            }
+            const state = this.#getDomainState(request.url);
+            if (state === null || Date.now() >= throttledUntil(state)) {
+                return request;
+            }
+            await this.#migrateToSubQueue(request, inner);
+        }
+        return null;
+    }
+    /**
+     * Moves a request from the wrapped manager into its domain's sub-queue.
+     *
+     * Added there before it is marked handled here, so that a crash in between duplicates the request rather
+     * than dropping it, and `uniqueKey` absorbs the duplicate. A copy is what lands in the sub-queue: adding
+     * the request itself would overwrite the `id` the wrapped manager knows it by.
+     */
+    async #migrateToSubQueue(request, inner) {
+        try {
+            const subManager = await this.#selectManagerOrThrow(request.url);
+            await subManager.addRequest(new Request({ ...request, id: undefined }));
+        }
+        catch (error) {
+            // Whatever the reason, the request is out of the wrapped manager and in nobody's hands, so it
+            // goes back before the failure is raised.
+            await inner.reclaimRequest(request, { forefront: true });
+            throw error;
+        }
+        await inner.markRequestAsHandled(request);
+        this.#migratedFromInner += 1;
+    }
     async *[Symbol.asyncIterator]() {
         while (true) {
             const req = await this.fetchNextRequest();