@crawlee/puppeteer 3.17.1-beta.6 → 3.17.1-beta.60

package/index.mjs

@@ -52,6 +52,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -82,6 +83,7 @@ export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createPuppeteerRouter = mod.createPuppeteerRouter;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -107,5 +109,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/internals/enqueue-links/click-elements.d.ts

@@ -212,5 +212,7 @@ export declare function isTargetRelevant(page: Page, target: Target): boolean;
  * for large element sets, this will take considerable amount of time.
  * @ignore
  */
-export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
+export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions & {
+    clickCount?: number;
+}): Promise<void>;
 export {};

package/internals/enqueue-links/click-elements.js

@@ -59,7 +59,7 @@ async function enqueueLinksByClickingElements(options) {
         requestQueue: ow_1.default.object.hasKeys('fetchNextRequest', 'addRequest'),
         selector: ow_1.default.string,
         userData: ow_1.default.optional.object,
-        clickOptions: ow_1.default.optional.object.hasKeys('clickCount', 'delay'),
+        clickOptions: ow_1.default.optional.object,
         pseudoUrls: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('purl'))),
         globs: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('glob'))),
         regexps: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.regExp, ow_1.default.object.hasKeys('regexp'))),
@@ -237,6 +237,10 @@ async function preventHistoryNavigation(page) {
  * @ignore
  */
 async function clickElements(page, selector, clickOptions) {
+    // Puppeteer 25 removed the deprecated `clickCount` option in favor of `count`, older versions ignore `count`, so we pass both.
+    if (clickOptions?.clickCount !== undefined && clickOptions.count === undefined) {
+        clickOptions = { ...clickOptions, count: clickOptions.clickCount };
+    }
     const elementHandles = await page.$$(selector);
     log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
     let clickedElementsCount = 0;

package/internals/puppeteer-crawler.d.ts

@@ -1,4 +1,4 @@
-import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, BrowserRequestHandler, GetUserDataFromRequest, LoadedContext, RouterRoutes } from '@crawlee/browser';
+import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, BrowserRequestHandler, GetUserDataFromRequest, LoadedContext, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import { BrowserCrawler, Configuration } from '@crawlee/browser';
 import type { PuppeteerController, PuppeteerPlugin } from '@crawlee/browser-pool';
 import type { Dictionary } from '@crawlee/types';
@@ -239,5 +239,6 @@ export declare class PuppeteerCrawler extends BrowserCrawler<{
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/browser").RouterHandler<Context>;
+export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

package/internals/puppeteer-crawler.js

@@ -127,30 +127,6 @@ Object.defineProperty(PuppeteerCrawler, "optionsShape", {
         browserPoolOptions: ow_1.default.optional.object,
     }
 });
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link PuppeteerCrawler}.
- * Defaults to the {@link PuppeteerCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<PuppeteerCrawlingContext>()`.
- *
- * ```ts
- * import { PuppeteerCrawler, createPuppeteerRouter } from 'crawlee';
- *
- * const router = createPuppeteerRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new PuppeteerCrawler({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-function createPuppeteerRouter(routes) {
-    return browser_1.Router.create(routes);
+function createPuppeteerRouter(routesOrSchemas) {
+    return browser_1.Router.create(routesOrSchemas);
 }

package/internals/utils/puppeteer_utils.js

@@ -373,7 +373,7 @@ async function gotoExtended(page, request, gotoOptions = {}) {
     }
     const { url, method, headers, payload } = request;
     const isEmpty = (o) => !o || Object.keys(o).length === 0;
-    if (method !== 'GET' || payload || !isEmpty(headers)) {
+    if (method !== 'GET' || payload) {
         // This is not deprecated, we use it to log only once.
         log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
             'in recent versions of Puppeteer. Use only when necessary.');
@@ -398,6 +398,19 @@ async function gotoExtended(page, request, gotoOptions = {}) {
         };
         await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, interceptRequestHandler);
     }
+    else if (!isEmpty(headers)) {
+        const extraHeaders = { ...headers };
+        // Chrome bundled with Puppeteer 25+ ignores a `User-Agent` passed via `setExtraHTTPHeaders()`, it has to be set explicitly.
+        for (const name of Object.keys(extraHeaders)) {
+            if (name.toLowerCase() === 'user-agent') {
+                await page.setUserAgent(extraHeaders[name]);
+                delete extraHeaders[name];
+            }
+        }
+        if (!isEmpty(extraHeaders)) {
+            await page.setExtraHTTPHeaders(extraHeaders);
+        }
+    }
     return page.goto(url, gotoOptions);
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/puppeteer",
-    "version": "3.17.1-beta.6",
+    "version": "3.17.1-beta.60",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -55,10 +55,10 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.0",
         "@apify/log": "^2.4.0",
-        "@crawlee/browser": "3.17.1-beta.6",
-        "@crawlee/browser-pool": "3.17.1-beta.6",
-        "@crawlee/types": "3.17.1-beta.6",
-        "@crawlee/utils": "3.17.1-beta.6",
+        "@crawlee/browser": "3.17.1-beta.60",
+        "@crawlee/browser-pool": "3.17.1-beta.60",
+        "@crawlee/types": "3.17.1-beta.60",
+        "@crawlee/utils": "3.17.1-beta.60",
         "cheerio": "1.0.0-rc.12",
         "devtools-protocol": "*",
         "idcac-playwright": "^0.2.0",
@@ -85,5 +85,5 @@
             }
         }
     },
-    "gitHead": "eda1d0870cfa01cfd7d2a4a9d0e4e6b99d18dea1"
+    "gitHead": "5ba2517b5f3bc6ef868b860c6c783a3e7e7d4a93"
 }