@crawlee/basic 4.0.0-beta.182 → 4.0.0-beta.183

package/internals/autoscaling/autoscaled_pool.d.ts

@@ -46,8 +46,10 @@ export interface AutoscaledPoolOptions extends TaskLoopOptions {
     /**
      * A function that performs an asynchronous resource-intensive task.
      * The function must either be labeled `async` or return a promise.
+     * @param taskId The id the task is {@link IConcurrencySystem.tryRegisterTaskStart|booked} under, so work done
+     * inside can be correlated with what the governor sees.
      */
-    runTaskFunction?: () => Promise<unknown>;
+    runTaskFunction?: (taskId: string) => Promise<unknown>;
     /**
      * Timeout in which the `runTaskFunction` needs to finish, given in seconds.
      * @default 0

package/internals/autoscaling/autoscaled_pool.js

@@ -94,6 +94,8 @@ export class AutoscaledPool {
      * (possibly shared) governor's total. `pause()` and `maybeFinish()` care only about this pool draining.
      */
     #ownConcurrency = 0;
+    /** Source of the ids that pair a booking with the release of the same task. */
+    #nextTaskId = 0;
     constructor(options) {
         const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maybeRunIntervalSecs, taskTimeoutSecs, log, concurrencySystem, consumer, } = parseArgument(options, autoscaledPoolOptionsSchema, 'AutoscaledPoolOptions');
         this.#log = log.child({ prefix: 'AutoscaledPool' });
@@ -299,9 +301,10 @@ export class AutoscaledPool {
             // No tasks could mean that we're finished with all tasks.
             return this.maybeFinish();
         }
+        const taskId = String(this.#nextTaskId++);
         // - the budget still has room. Re-checked atomically, because another pool sharing the governor may have taken
         // the last free slot while we awaited `isTaskReadyFunction` above.
-        if (!this.#concurrencySystem.tryRegisterTaskStart(this.#consumer)) {
+        if (!this.#concurrencySystem.tryRegisterTaskStart(this.#consumer, taskId)) {
             return done();
         }
         this.#ownConcurrency++;
@@ -315,10 +318,10 @@ export class AutoscaledPool {
             // Execute the current task.
             this.#log.perf('Running a task.');
             if (this.#taskTimeoutMillis > 0) {
-                await addTimeoutToPromise(async () => this.#runTaskFunction(), this.#taskTimeoutMillis, `runTaskFunction timed out after ${this.#taskTimeoutMillis / 1000} seconds.`);
+                await addTimeoutToPromise(async () => this.#runTaskFunction(taskId), this.#taskTimeoutMillis, `runTaskFunction timed out after ${this.#taskTimeoutMillis / 1000} seconds.`);
             }
             else {
-                await this.#runTaskFunction();
+                await this.#runTaskFunction(taskId);
             }
             this.#log.perf('Task finished.');
             // Run task after the previous one finished. Only on success: a failed task rejects the pool, and
@@ -340,7 +343,7 @@ export class AutoscaledPool {
             }
         }
         finally {
-            this.#concurrencySystem.registerTaskEnd(this.#consumer);
+            this.#concurrencySystem.registerTaskEnd(this.#consumer, taskId);
             this.#ownConcurrency--;
         }
         return undefined;

package/internals/autoscaling/concurrency_system.d.ts

@@ -131,10 +131,17 @@ export interface IConcurrencySystem {
      *
      * Must be an *atomic* (synchronous) check-and-book: several pools may share one governor, and a check separated
      * from the booking by an `await` lets two of them claim the last free slot at once.
+     * @param taskId Identifies the booked task among the ones `consumer` has in flight, and is repeated by the
+     * {@link IConcurrencySystem.registerTaskEnd|release} — pairing the two is what lets an implementation time a
+     * task or tell which ones are still running. Nothing beyond that is promised: another consumer may be using the
+     * same id at the same time, and an id may come back around once its task has ended.
      */
-    tryRegisterTaskStart(consumer: ConcurrencyConsumer): boolean;
-    /** Returns a task's slot to `consumer`'s budget. Called once the task settles (resolve or reject). */
-    registerTaskEnd(consumer: ConcurrencyConsumer): void;
+    tryRegisterTaskStart(consumer: ConcurrencyConsumer, taskId: string): boolean;
+    /**
+     * Returns a task's slot to `consumer`'s budget. Called once the task settles (resolve or reject).
+     * @param taskId The id the task was {@link IConcurrencySystem.tryRegisterTaskStart|booked} under.
+     */
+    registerTaskEnd(consumer: ConcurrencyConsumer, taskId: string): void;
 }
 /**
  * The shareable "governor" behind an {@link AutoscaledPool}: it decides whether there is free compute for one more

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.182",
+    "version": "4.0.0-beta.183",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.182",
-        "@crawlee/http-client": "4.0.0-beta.182",
-        "@crawlee/types": "4.0.0-beta.182",
-        "@crawlee/utils": "4.0.0-beta.182",
+        "@crawlee/core": "4.0.0-beta.183",
+        "@crawlee/http-client": "4.0.0-beta.183",
+        "@crawlee/types": "4.0.0-beta.183",
+        "@crawlee/utils": "4.0.0-beta.183",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "csv-stringify": "^6.5.2",
@@ -57,7 +57,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.182"
+        "@crawlee/impit-client": "4.0.0-beta.183"
     },
     "lerna": {
         "command": {
@@ -66,5 +66,5 @@
             }
         }
     },
-    "gitHead": "4b2a19082be4fd17948fb72bef7bb3313db14fce"
+    "gitHead": "3832a8d3392bba9e3e7cc6d06b9be0be4ee1ab98"
 }