@crawlee/core 3.17.1-beta.60 → 3.17.1-beta.62

package/crawlers/statistics.d.ts

@@ -35,7 +35,7 @@ export declare class Statistics {
     /**
      * Statistic instance id.
      */
-    readonly id: number;
+    readonly id: string;
     /**
      * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
      */
@@ -166,13 +166,22 @@ export interface StatisticsOptions {
      * @default false
      */
     saveErrorSnapshots?: boolean;
+    /**
+     * A unique identifier for this statistics instance. This ID is used for persistence
+     * to the key value store, ensuring the same statistics can be loaded after script restarts.
+     *
+     * If not provided, an auto-incremented ID will be used for backward compatibility.
+     * This means statistics may not persist correctly across script restarts
+     * if crawler creation order changes.
+     */
+    id?: string;
 }
 /**
  * Format of the persisted stats
  */
 export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
     requestRetryHistogram: number[];
-    statsId: number;
+    statsId: string;
     requestAvgFailedDurationMillis: number;
     requestAvgFinishedDurationMillis: number;
     requestTotalDurationMillis: number;

package/crawlers/statistics.js

@@ -81,8 +81,8 @@ class Statistics {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: Statistics.id++
-        }); // assign an id while incrementing so it can be saved/restored from KV
+            value: void 0
+        });
         /**
          * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
          */
@@ -120,7 +120,7 @@ class Statistics {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: `SDK_CRAWLER_STATISTICS_${this.id}`
+            value: void 0
         });
         Object.defineProperty(this, "logIntervalMillis", {
             enumerable: true,
@@ -184,10 +184,13 @@ class Statistics {
             config: ow_1.default.optional.object,
             persistenceOptions: ow_1.default.optional.object,
             saveErrorSnapshots: ow_1.default.optional.boolean,
+            id: ow_1.default.optional.any(ow_1.default.number, ow_1.default.string),
         }));
         const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, config = configuration_1.Configuration.getGlobalConfig(), persistenceOptions = {
             enable: true,
-        }, saveErrorSnapshots = false, } = options;
+        }, saveErrorSnapshots = false, id, } = options;
+        this.id = id ?? String(Statistics.id++);
+        this.persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
         this.log = (options.log ?? log_1.log).child({ prefix: 'Statistics' });
         this.errorTracker = new error_tracker_1.ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.errorTrackerRetry = new error_tracker_1.ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "3.17.1-beta.60",
+    "version": "3.17.1-beta.62",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -59,9 +59,9 @@
         "@apify/pseudo_url": "^2.0.30",
         "@apify/timeout": "^0.4.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/memory-storage": "3.17.1-beta.60",
-        "@crawlee/types": "3.17.1-beta.60",
-        "@crawlee/utils": "3.17.1-beta.60",
+        "@crawlee/memory-storage": "3.17.1-beta.62",
+        "@crawlee/types": "3.17.1-beta.62",
+        "@crawlee/utils": "3.17.1-beta.62",
         "@sapphire/async-queue": "^1.5.1",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.2.2",
@@ -84,5 +84,5 @@
             }
         }
     },
-    "gitHead": "5ba2517b5f3bc6ef868b860c6c783a3e7e7d4a93"
+    "gitHead": "a4075c98770711a6fadb4e16d8aef644f8a96690"
 }

package/storages/sitemap_request_list.js

@@ -410,14 +410,21 @@ class SitemapRequestList {
         }
         // Create a new stream, as we have read all the URLs from the current one.
         // Pushing the urls back to the original stream might not be possible if it has been ended.
-        const newStream = this.createNewStream(this.urlQueueStream.readableHighWaterMark);
+        const previousStream = this.urlQueueStream;
+        const newStream = this.createNewStream(previousStream.readableHighWaterMark);
         for (const url of urlQueue) {
             newStream.push(url);
         }
-        if (this.urlQueueStream.writableEnded) {
+        if (previousStream.writableEnded) {
             newStream.end();
         }
         this.urlQueueStream = newStream;
+        // A `pushNextUrl()` call may be blocked on backpressure, waiting for a `readdata` event on the
+        // previous stream. That event is only ever emitted by `readNextUrl()` on the current stream, so
+        // after the swap the waiter would never be notified and the background sitemap loading would hang.
+        // Re-emit `readdata` on the previous stream to release any such pending waiter (its URL has already
+        // been transferred to the new stream above).
+        previousStream.emit('readdata');
         await this.store.setValue(this.persistStateKey, {
             sitemapParsingProgress: {
                 pendingSitemapUrls: Array.from(this.sitemapParsingProgress.pendingSitemapUrls),