@crawlee/playwright 4.0.0-beta.122 → 4.0.0-beta.123

package/internals/adaptive-playwright-crawler.d.ts

@@ -190,6 +190,7 @@ export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<nev
         readonly querySelectorAll: AdaptivePlaywrightCrawlerContext["querySelectorAll"];
         readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
         readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
+        readonly enqueueLinks: AdaptivePlaywrightCrawlerContext["enqueueLinks"];
     }>;
     private adaptCheerioContext;
     private adaptPlaywrightContext;

package/internals/adaptive-playwright-crawler.js

@@ -2,7 +2,7 @@ import { isDeepStrictEqual } from 'node:util';
 import { BasicCrawler } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
 import { CheerioCrawler } from '@crawlee/cheerio';
-import { createStorageTransaction, OwnedOrInjected, parseArgument, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/core';
+import { createStorageTransaction, EnqueueStrategy, OwnedOrInjected, parseArgument, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/core';
 import { extractUrlsFromCheerio } from '@crawlee/utils/internal';
 import { z } from 'zod';
 import { addTimeoutToPromise } from '@apify/timeout';
@@ -218,6 +218,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 get parseWithCheerio() {
                     throw new Error(errorMessage('parseWithCheerio'));
                 },
+                get enqueueLinks() {
+                    throw new Error(errorMessage('enqueueLinks'));
+                },
             }),
         });
     }
@@ -233,9 +236,8 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 return cheerioContext.$(selector);
             },
             enqueueLinks: async (options = {}) => {
-                const urls = options.urls ??
-                    extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
-                return (await this.enqueueLinks({ ...options, urls }, cheerioContext.request));
+                const urls = extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
+                return (await this.enqueueLinks(urls, options, cheerioContext.request));
             },
             response: cheerioContext.response,
         };
@@ -263,19 +265,11 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             },
             enqueueLinks: async (options = {}, timeoutMs = 5000) => {
                 // TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
-                let urls;
-                if (options.urls === undefined) {
-                    const selector = options.selector ?? 'a';
-                    const locator = playwrightContext.page.locator(selector).first();
-                    await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                    urls =
-                        options.urls ??
-                            (await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl));
-                }
-                else {
-                    urls = options.urls;
-                }
-                return (await this.enqueueLinks({ ...options, urls }, playwrightContext.request));
+                const selector = options.selector ?? 'a';
+                const locator = playwrightContext.page.locator(selector).first();
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl);
+                return (await this.enqueueLinks(urls, options, playwrightContext.request));
             },
         };
     }
@@ -423,16 +417,21 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             }
         }
     }
-    async enqueueLinks(options, request) {
+    async enqueueLinks(urls, options, request) {
         const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
             enqueueStrategy: options?.strategy,
             finalRequestUrl: request.loadedUrl,
             originalRequestUrl: request.url,
             userProvidedBaseUrl: options?.baseUrl,
         });
+        const requestsWithDepth = this.addCrawlDepthRequestGenerator(urls, request.crawlDepth + 1);
         // The per-attempt transaction buffers these (the queue policy defaults to `deferred` here),
         // so a discarded attempt's enqueues never reach the queue.
-        return await this.enqueueLinksWithCrawlDepth({ ...options, baseUrl }, request, await this.getRequestManager());
+        return await this.addRequests(requestsWithDepth, {
+            ...options,
+            baseUrl,
+            strategy: options.strategy ?? EnqueueStrategy.SameHostname,
+        });
     }
     createLogProxy(log, logs) {
         return new Proxy(log, {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.122",
+    "version": "4.0.0-beta.123",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.122",
-        "@crawlee/browser": "4.0.0-beta.122",
-        "@crawlee/browser-pool": "4.0.0-beta.122",
-        "@crawlee/cheerio": "4.0.0-beta.122",
-        "@crawlee/core": "4.0.0-beta.122",
-        "@crawlee/types": "4.0.0-beta.122",
-        "@crawlee/utils": "4.0.0-beta.122",
+        "@crawlee/basic": "4.0.0-beta.123",
+        "@crawlee/browser": "4.0.0-beta.123",
+        "@crawlee/browser-pool": "4.0.0-beta.123",
+        "@crawlee/cheerio": "4.0.0-beta.123",
+        "@crawlee/core": "4.0.0-beta.123",
+        "@crawlee/types": "4.0.0-beta.123",
+        "@crawlee/utils": "4.0.0-beta.123",
         "cheerio": "^1.0.0",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
@@ -84,5 +84,5 @@
             }
         }
     },
-    "gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
+    "gitHead": "f77648095c6a3f5ed8815c7620ea765db430ae44"
 }