@crawlee/basic 4.0.0-beta.79 → 4.0.0-beta.80

package/internals/basic-crawler.d.ts

@@ -367,20 +367,34 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
  * ```
  * @category Crawlers
  */
+/**
+ * Identifies a crawler instance for storage aliasing, `useState()` and status-message events.
+ */
+interface CrawlerIdentity {
+    /**
+     * 0-based instantiation order across all crawlers in the process.
+     * Note that the value can be subject to race conditions between different script invocations.
+     */
+    readonly instanceIndex: number;
+    /** The user-supplied `id` option, or a fallback derived from `instanceIndex`. */
+    readonly id: string;
+    /** Whether `id` came from the user (as opposed to being derived from `instanceIndex`). */
+    readonly hasExplicitId: boolean;
+}
 export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
     #private;
     protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
-    /**
-     * Tracks crawler instances that accessed shared state without having an explicit id.
-     * Used to detect and warn about multiple crawlers sharing the same state.
-     */
-    private static useStateCrawlerIds;
     /**
      * Tracks the number of crawler instances created. The first crawler uses the default
      * request queue; subsequent ones get their own queue via a unique alias so they don't
      * collide.
      */
     private static instanceCount;
+    /**
+     * Tracks crawler instances that accessed shared state without having an explicit id.
+     * Used to detect and warn about multiple crawlers sharing the same state.
+     */
+    private static useStateAnonymousIndices;
     /**
      * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
      */
@@ -479,9 +493,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private _closeEvents?;
     private loggedPerRun;
     private readonly robotsTxtFileCache;
-    private readonly crawlerId;
-    private readonly hasExplicitId;
-    private readonly crawlerInstanceIndex;
+    protected readonly identity: CrawlerIdentity;
     private readonly contextPipelineOptions;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -814,3 +826,4 @@ export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
  */
 export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
 export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export {};

package/internals/basic-crawler.js

@@ -37,90 +37,19 @@ class LazyDefaultHttpClient {
  */
 const SAFE_MIGRATION_WAIT_MILLIS = 20000;
 const deferredCleanupKey = Symbol('deferredCleanup');
-/**
- * Provides a simple framework for parallel crawling of web pages.
- * The URLs to crawl are fed either from a static list of URLs
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
- *
- * `BasicCrawler` is a low-level tool that requires the user to implement the page
- * download and data extraction functionality themselves.
- * If we want a crawler that already facilitates this functionality,
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
- *
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
- * for each {@link Request} object, which represents a single URL to crawl.
- * The {@link Request} objects are fed from the {@link IRequestManager|request manager} provided via the
- * {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is
- * itself a request manager). If no `requestManager` is provided, the crawler opens the default {@link RequestQueue}
- * either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called, or if the `requests`
- * parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
- *
- * To read requests from a read-only source such as a {@link RequestList} or {@link SitemapRequestLoader} while
- * still being able to enqueue new ones, combine the loader with a queue into a {@link RequestManagerTandem} using
- * {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`. The tandem
- * first processes URLs from the loader and automatically enqueues them into the queue, ensuring a single URL is not
- * crawled multiple times.
- *
- * > The legacy {@link BasicCrawlerOptions.requestList|`requestList`} and
- * > {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated. They are still accepted and
- * > folded into a single `requestManager` (combined into a tandem when both are given), but new code should use
- * > `requestManager` directly.
- *
- * The crawler finishes if there are no more {@link Request} objects to crawl.
- *
- * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
- * parameter of the `BasicCrawler` constructor.
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
- * underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
- *
- * **Example usage:**
- *
- * ```javascript
- * import { BasicCrawler, Dataset } from 'crawlee';
- *
- * // Create a crawler instance
- * const crawler = new BasicCrawler({
- *     async requestHandler({ request, sendRequest }) {
- *         // 'request' contains an instance of the Request class
- *         // Here we simply fetch the HTML of the page and store it to a dataset
- *         const { body } = await sendRequest({
- *             url: request.url,
- *             method: request.method,
- *             body: request.payload,
- *             headers: request.headers,
- *         });
- *
- *         await Dataset.pushData({
- *             url: request.url,
- *             html: body,
- *         })
- *     },
- * });
- *
- * // Enqueue the initial requests and run the crawler
- * await crawler.run([
- *     'http://www.example.com/page-1',
- *     'http://www.example.com/page-2',
- * ]);
- * ```
- * @category Crawlers
- */
 export class BasicCrawler {
     static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
-    /**
-     * Tracks crawler instances that accessed shared state without having an explicit id.
-     * Used to detect and warn about multiple crawlers sharing the same state.
-     */
-    static useStateCrawlerIds = new Set();
     /**
      * Tracks the number of crawler instances created. The first crawler uses the default
      * request queue; subsequent ones get their own queue via a unique alias so they don't
      * collide.
      */
     static instanceCount = 0;
+    /**
+     * Tracks crawler instances that accessed shared state without having an explicit id.
+     * Used to detect and warn about multiple crawlers sharing the same state.
+     */
+    static useStateAnonymousIndices = new Set();
     /**
      * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
      */
@@ -233,9 +162,7 @@ export class BasicCrawler {
     _closeEvents;
     loggedPerRun = new Set();
     robotsTxtFileCache;
-    crawlerId;
-    hasExplicitId;
-    crawlerInstanceIndex;
+    identity;
     contextPipelineOptions;
     static optionsShape = {
         contextPipelineBuilder: ow.optional.object,
@@ -313,11 +240,8 @@ export class BasicCrawler {
             this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
             // Initialize the Configuration instance to avoid lazy loading in the components
             serviceLocator.getConfiguration();
-            // Store whether the user explicitly provided an ID
-            this.hasExplicitId = id !== undefined;
-            // Store the user-provided ID, or generate a unique one for tracking purposes (not for state key)
-            this.crawlerId = id ?? cryptoRandomObjectId();
-            this.crawlerInstanceIndex = BasicCrawler.instanceCount++;
+            const instanceIndex = BasicCrawler.instanceCount++;
+            this.identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
             if (requestManager !== undefined) {
                 if (requestList !== undefined || requestQueue !== undefined) {
                     throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
@@ -369,7 +293,7 @@ export class BasicCrawler {
             this.stats = new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
-                ...(this.hasExplicitId ? { id: this.crawlerId } : {}),
+                id: this.identity.id,
                 ...statisticsOptions,
             });
             if (sessionPool && proxyConfiguration) {
@@ -639,7 +563,7 @@ export class BasicCrawler {
         // Setting the status message is not a storage concern, so we intentionally don't route it
         // through the storage client anymore.
         serviceLocator.getEventManager().emit("statusMessage" /* EventType.STATUS_MESSAGE */, {
-            crawlerId: this.crawlerId,
+            crawlerId: this.identity.id,
             message,
             isStatusMessageTerminal: options.isStatusMessageTerminal,
             level: options.level,
@@ -829,7 +753,7 @@ export class BasicCrawler {
     async openOwnedRequestQueue() {
         // The first crawler instance uses the default queue (null identifier);
         // subsequent instances get their own queue via a unique alias so they don't collide.
-        const identifier = this.crawlerInstanceIndex === 0 ? null : { alias: `__default_${this.crawlerInstanceIndex}__` };
+        const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
         const requestQueue = await RequestQueue.open(identifier, { config: serviceLocator.getConfiguration() });
         this.ownedRequestManager = requestQueue;
         return requestQueue;
@@ -872,12 +796,12 @@ export class BasicCrawler {
     }
     async useState(defaultValue = {}) {
         const kvs = await KeyValueStore.open(null, { config: serviceLocator.getConfiguration() });
-        if (this.hasExplicitId) {
-            const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.crawlerId}`;
+        if (this.identity.hasExplicitId) {
+            const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
             return kvs.getAutoSavedValue(stateKey, defaultValue);
         }
-        BasicCrawler.useStateCrawlerIds.add(this.crawlerId);
-        if (BasicCrawler.useStateCrawlerIds.size > 1) {
+        BasicCrawler.useStateAnonymousIndices.add(this.identity.instanceIndex);
+        if (BasicCrawler.useStateAnonymousIndices.size > 1) {
             serviceLocator
                 .getLogger()
                 .warningOnce('Multiple crawler instances are calling useState() without an explicit `id` option. \n' +

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.79",
+    "version": "4.0.0-beta.80",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.79",
-        "@crawlee/http-client": "4.0.0-beta.79",
-        "@crawlee/types": "4.0.0-beta.79",
-        "@crawlee/utils": "4.0.0-beta.79",
+        "@crawlee/core": "4.0.0-beta.80",
+        "@crawlee/http-client": "4.0.0-beta.80",
+        "@crawlee/types": "4.0.0-beta.80",
+        "@crawlee/utils": "4.0.0-beta.80",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
         "ow": "^2.0.0",
@@ -54,7 +54,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.79"
+        "@crawlee/impit-client": "^4.0.0-beta.80"
     },
     "lerna": {
         "command": {
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "e897f0881c8142a1fd32f315263751296b4f1a22"
+    "gitHead": "96c57b4a0c999e4b2bd198792490af28db7aa42d"
 }