@crawlee/playwright 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/enqueue-links/click-elements.d.ts

@@ -1,4 +1,4 @@
-import type { GlobInput, IRequestManager, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from '@crawlee/browser';
+import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
@@ -26,64 +26,29 @@ export interface EnqueueLinksByClickingElementsOptions {
      */
     clickOptions?: ClickOptions;
     /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
+     * An array of URL patterns that URLs must match to be enqueued.
      *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `globs` is an empty array or `undefined`, then the function
-     * enqueues all the intercepted navigation requests produced by the page
-     * after clicking on elements matching the provided CSS selector.
-     */
-    globs?: GlobInput[];
-    /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be enqueued.
-     *
-     * The plain objects must include either the `glob` property or the `regexp` property.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
      * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
-     */
-    exclude?: readonly (GlobInput | RegExpInput)[];
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     * If you need case-sensitive matching, use a `RegExp`.
      *
-     * If `regexps` is an empty array or `undefined`, then the function
+     * If `include` is an empty array or `undefined`, then the function
      * enqueues all the intercepted navigation requests produced by the page
      * after clicking on elements matching the provided CSS selector.
      */
-    regexps?: RegExpInput[];
+    include?: UrlPatternInput[];
     /**
-     * *NOTE:* In future versions of SDK the options will be removed.
-     * Please use `globs` or `regexps` instead.
-     *
-     * An array of {@link PseudoUrl} strings or plain objects
-     * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `purl` property, which holds the pseudo-URL pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     * An array of URL patterns. Matching URLs will **not** be enqueued.
      *
-     * With a pseudo-URL string, the matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
-     * If `pseudoUrls` is an empty array or `undefined`, then the function
-     * enqueues all the intercepted navigation requests produced by the page
-     * after clicking on elements matching the provided CSS selector.
-     *
-     * @deprecated prefer using `globs` or `regexps` instead
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
      */
-    pseudoUrls?: PseudoUrlInput[];
+    exclude?: readonly UrlPatternInput[];
     /**
-     * After {@link Request} objects are constructed and filtered by URL patterns (`globs`, `regexps`, `pseudoUrls`),
+     * After request options are filtered by `include`/`exclude` patterns,
      * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
      * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
      * but differ in methods or payloads, or to dynamically update or create `userData`.
@@ -98,8 +63,8 @@ export interface EnqueueLinksByClickingElementsOptions {
      * }
      * ```
      *
-     * Note that `transformRequestFunction` has the highest priority and can overwrite request options
-     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+     * Note that `transformRequestFunction` has the highest priority and can overwrite
+     * the global `label` option.
      *
      * The function receives a {@link RequestOptions} object and can return either:
      * - The modified {@link RequestOptions} object
@@ -160,8 +125,7 @@ export interface EnqueueLinksByClickingElementsOptions {
  * in `href` elements, but rather navigations are triggered in click handlers.
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
  *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
  *
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -183,9 +147,9 @@ export interface EnqueueLinksByClickingElementsOptions {
  *   page,
  *   requestManager,
  *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]'
- *       'https://www.example.com/purses/[.*]'
+ *   include: [
+ *       'https://www.example.com/handbags/*',
+ *       'https://www.example.com/purses/*',
  *   ],
  * });
  * ```

package/internals/enqueue-links/click-elements.js

@@ -1,8 +1,25 @@
 import { URL } from 'node:url';
-import { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, filterRequestOptionsByPatterns, Request as CrawleeRequest, serviceLocator, } from '@crawlee/browser';
-import ow from 'ow';
+import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request as CrawleeRequest, serviceLocator, } from '@crawlee/browser';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 const STARTING_Z_INDEX = 2147400000;
 const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
+const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
+    page: schemas.objectWithKeys(['goto', 'evaluate']),
+    requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
+    selector: z.string(),
+    userData: schemas.anyObject.optional(),
+    clickOptions: schemas.anyObject.optional(),
+    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    transformRequestFunction: schemas.anyFunction.optional(),
+    waitForPageIdleSecs: schemas.anyNumber.default(1),
+    maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
+    label: z.string().optional(),
+    forefront: z.boolean().optional(),
+    skipNavigation: z.boolean().optional(),
+    onSkippedRequest: schemas.anyFunction.optional(),
+});
 /**
  * The function finds elements matching a specific CSS selector in a Playwright page,
  * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -12,8 +29,7 @@ const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
  * in `href` elements, but rather navigations are triggered in click handlers.
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
  *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
  *
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -35,9 +51,9 @@ const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
  *   page,
  *   requestManager,
  *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]'
- *       'https://www.example.com/purses/[.*]'
+ *   include: [
+ *       'https://www.example.com/handbags/*',
+ *       'https://www.example.com/purses/*',
  *   ],
  * });
  * ```
@@ -45,51 +61,13 @@ const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
  */
 export async function enqueueLinksByClickingElements(options) {
-    ow(options, ow.object.exactShape({
-        page: ow.object.hasKeys('goto', 'evaluate'),
-        requestManager: ow.object.hasKeys('fetchNextRequest', 'addRequestsBatched'),
-        selector: ow.string,
-        userData: ow.optional.object,
-        clickOptions: ow.optional.object,
-        pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
-        globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
-        regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
-        exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
-        transformRequestFunction: ow.optional.function,
-        waitForPageIdleSecs: ow.optional.number,
-        maxWaitForPageIdleSecs: ow.optional.number,
-        label: ow.optional.string,
-        forefront: ow.optional.boolean,
-        skipNavigation: ow.optional.boolean,
-        onSkippedRequest: ow.optional.function,
-    }));
-    const { page, requestManager, selector, clickOptions, 
-    // oxlint-disable-next-line typescript/no-deprecated -- still accepted for backwards compat
-    pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, onSkippedRequest, } = options;
+    const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
+    const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
     const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
     const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
-    const urlExcludePatternObjects = [];
-    const urlPatternObjects = [];
-    if (exclude?.length) {
-        for (const excl of exclude) {
-            if (typeof excl === 'string' || 'glob' in excl) {
-                urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
-            }
-            else if (excl instanceof RegExp || 'regexp' in excl) {
-                urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
-            }
-        }
-    }
-    if (pseudoUrls?.length) {
-        serviceLocator.getLogger().deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
-        urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
-    }
-    if (globs?.length) {
-        urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
-    }
-    if (regexps?.length) {
-        urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
-    }
+    const hasOnSkippedRequest = onSkippedRequest !== undefined;
+    const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
+    const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
     const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
         page,
         selector,
@@ -97,17 +75,17 @@ export async function enqueueLinksByClickingElements(options) {
         maxWaitForPageIdleMillis,
         clickOptions,
     });
-    const requestOptions = createRequestOptions(interceptedRequests, options);
+    const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
     const skippedByFilters = [];
-    let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
+    let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
     if (onSkippedRequest && skippedByFilters.length > 0) {
-        await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));
+        await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
     }
     if (transformRequestFunction) {
         const skippedByTransform = [];
-        filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
+        filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
         if (onSkippedRequest && skippedByTransform.length > 0) {
-            await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
+            await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
         }
     }
     const requests = filteredOptions.map((opts) => new CrawleeRequest(opts));

package/internals/playwright-browser-pool.d.ts

@@ -0,0 +1,71 @@
+import type { Configuration } from '@crawlee/browser';
+import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PlaywrightPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
+// @ts-ignore optional peer dependency or compatibility with es2022
+import type { Page } from 'playwright';
+import type { PlaywrightLaunchContext } from './playwright-launcher.js';
+/** A {@link BrowserPool} of Playwright browsers, as built by {@link playwrightBrowserPool}. */
+export type PlaywrightBrowserPool = BrowserPool<{
+    browserPlugins: [PlaywrightPlugin];
+}, [PlaywrightPlugin]>;
+export interface PlaywrightBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PlaywrightPlugin['createController']>, ReturnType<PlaywrightPlugin['createLaunchContext']>, Page> {
+    /** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */
+    launchContext?: PlaywrightLaunchContext;
+    /**
+     * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
+     * Defaults to `true`, and can also be set via {@link Configuration}.
+     */
+    headless?: boolean;
+    /** Configuration to read the browser defaults from. Defaults to the global configuration. */
+    configuration?: Configuration;
+}
+export interface RemotePlaywrightBrowserPoolOptions extends Pick<PlaywrightBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
+}
+/**
+ * Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
+ * given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: playwrightBrowserPool({
+ *         maxOpenPagesPerBrowser: 1,
+ *         launchContext: { launcher: firefox },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function playwrightBrowserPool(options?: PlaywrightBrowserPoolOptions): PlaywrightBrowserPool;
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link playwrightBrowserPool}: connects to a remote browser
+ * service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.
+ *
+ * A {@link PlaywrightCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: remotePlaywrightBrowserPool({
+ *         endpoint: 'wss://production-sfo.browserless.io?token=xxx',
+ *         maxOpenBrowsers: 2,
+ *         browserPoolOptions: { useFingerprints: false },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * @category Browser management
+ */
+export declare function remotePlaywrightBrowserPool(options: RemotePlaywrightBrowserPoolOptions): RemoteBrowserPool<Page>;

package/internals/playwright-browser-pool.js

@@ -0,0 +1,61 @@
+import { PlaywrightLauncher } from './playwright-launcher.js';
+/**
+ * Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
+ * given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: playwrightBrowserPool({
+ *         maxOpenPagesPerBrowser: 1,
+ *         launchContext: { launcher: firefox },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export function playwrightBrowserPool(options = {}) {
+    const { launchContext, headless, configuration, ...poolOptions } = options;
+    return playwrightLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
+}
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link playwrightBrowserPool}: connects to a remote browser
+ * service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.
+ *
+ * A {@link PlaywrightCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: remotePlaywrightBrowserPool({
+ *         endpoint: 'wss://production-sfo.browserless.io?token=xxx',
+ *         maxOpenBrowsers: 2,
+ *         browserPoolOptions: { useFingerprints: false },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * @category Browser management
+ */
+export function remotePlaywrightBrowserPool(options) {
+    const { launchContext, headless, configuration, ...remoteOptions } = options;
+    return playwrightLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
+}
+function playwrightLauncher(launchContext = {}, headless, configuration) {
+    return new PlaywrightLauncher(headless == null
+        ? launchContext
+        : { ...launchContext, launchOptions: { ...launchContext.launchOptions, headless } }, configuration);
+}