@crawlee/playwright 3.17.1-beta.8 → 3.17.1-beta.80

package/index.mjs

@@ -54,6 +54,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -85,6 +86,7 @@ export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createPlaywrightRouter = mod.createPlaywrightRouter;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -109,5 +111,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,4 +1,4 @@
-import type { BrowserHook, LoadedContext, RouterHandler } from '@crawlee/browser';
+import type { BrowserHook, LoadedContext, RequestHandler, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import type { BaseHttpResponseData, EnqueueLinksOptions, GetUserDataFromRequest, RestrictedCrawlingContext, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
 import { Configuration, RequestHandlerResult, Statistics } from '@crawlee/core';
 import type { Awaitable, BatchAddRequestsResult, Dictionary } from '@crawlee/types';
@@ -194,6 +194,11 @@ export declare class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
      */
 // @ts-ignore optional peer dependency or compatibility with es2022
     readonly router: RouterHandler<AdaptivePlaywrightCrawlerContext>;
+    /**
+     * The `requestHandler` never reaches `BrowserCrawler` here — it is kept as `adaptiveRequestHandler` and
+     * invoked through a proxy — so router-aware metadata has to resolve against it.
+     */
+    protected get userRequestHandler(): RequestHandler<PlaywrightCrawlingContext>;
     constructor(options?: AdaptivePlaywrightCrawlerOptions, config?: Configuration);
     /**
      * Returns the number of rendering type detections currently in progress.
@@ -211,5 +216,7 @@ export declare class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
     protected enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: RestrictedCrawlingContext['request'], result: RequestHandlerResult): Promise<BatchAddRequestsResult>;
     private createLogProxy;
 }
-export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
 export {};

package/internals/adaptive-playwright-crawler.js

@@ -94,6 +94,13 @@ const proxyLogMethods = [
  * @experimental
  */
 class AdaptivePlaywrightCrawler extends playwright_crawler_1.PlaywrightCrawler {
+    /**
+     * The `requestHandler` never reaches `BrowserCrawler` here — it is kept as `adaptiveRequestHandler` and
+     * invoked through a proxy — so router-aware metadata has to resolve against it.
+     */
+    get userRequestHandler() {
+        return this.adaptiveRequestHandler;
+    }
     constructor(options = {}, config = core_1.Configuration.getGlobalConfig()) {
         const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statisticsOptions, preventDirectStorageAccess = true, ...rest } = options;
         super(rest, config);
@@ -478,6 +485,6 @@ class AdaptivePlaywrightCrawler extends playwright_crawler_1.PlaywrightCrawler {
     }
 }
 exports.AdaptivePlaywrightCrawler = AdaptivePlaywrightCrawler;
-function createAdaptivePlaywrightRouter(routes) {
-    return core_1.Router.create(routes);
+function createAdaptivePlaywrightRouter(routesOrSchemas) {
+    return core_1.Router.create(routesOrSchemas);
 }

package/internals/enqueue-links/click-elements.js

@@ -57,7 +57,7 @@ async function enqueueLinksByClickingElements(options) {
         requestQueue: ow_1.default.object.hasKeys('fetchNextRequest', 'addRequest'),
         selector: ow_1.default.string,
         userData: ow_1.default.optional.object,
-        clickOptions: ow_1.default.optional.object.hasKeys('clickCount', 'delay'),
+        clickOptions: ow_1.default.optional.object,
         pseudoUrls: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('purl'))),
         globs: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.hasKeys('glob'))),
         regexps: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.regExp, ow_1.default.object.hasKeys('regexp'))),

package/internals/playwright-crawler.d.ts

@@ -1,4 +1,4 @@
-import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, BrowserRequestHandler, GetUserDataFromRequest, LoadedContext, RouterRoutes } from '@crawlee/browser';
+import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, BrowserRequestHandler, GetUserDataFromRequest, LoadedContext, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import { BrowserCrawler, Configuration } from '@crawlee/browser';
 import type { CommonPage, PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool';
 import type { Dictionary } from '@crawlee/types';
@@ -295,5 +295,6 @@ export declare class PlaywrightCrawler extends BrowserCrawler<{
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/browser").RouterHandler<Context>;
+export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

package/internals/playwright-crawler.js

@@ -134,30 +134,6 @@ Object.defineProperty(PlaywrightCrawler, "optionsShape", {
         launcher: ow_1.default.optional.object,
     }
 });
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link PlaywrightCrawler}.
- * Defaults to the {@link PlaywrightCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<PlaywrightCrawlingContext>()`.
- *
- * ```ts
- * import { PlaywrightCrawler, createPlaywrightRouter } from 'crawlee';
- *
- * const router = createPlaywrightRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new PlaywrightCrawler({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-function createPlaywrightRouter(routes) {
-    return browser_1.Router.create(routes);
+function createPlaywrightRouter(routesOrSchemas) {
+    return browser_1.Router.create(routesOrSchemas);
 }

package/internals/utils/playwright-utils.js

@@ -133,7 +133,7 @@ async function gotoExtended(page, request, gotoOptions = {}) {
     (0, ow_1.default)(gotoOptions, ow_1.default.object);
     const { url, method, headers, payload } = request;
     const isEmpty = (o) => !o || Object.keys(o).length === 0;
-    if (method !== 'GET' || payload || !isEmpty(headers)) {
+    if (method !== 'GET' || payload) {
         // This is not deprecated, we use it to log only once.
         log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
             'in recent versions of Playwright. Use only when necessary.');
@@ -162,6 +162,9 @@ async function gotoExtended(page, request, gotoOptions = {}) {
         };
         await page.route('**/*', interceptRequestHandler);
     }
+    else if (!isEmpty(headers)) {
+        await page.setExtraHTTPHeaders(headers);
+    }
     return page.goto(url, gotoOptions);
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "3.17.1-beta.8",
+    "version": "3.17.1-beta.80",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -55,12 +55,12 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.0",
         "@apify/log": "^2.4.0",
-        "@apify/timeout": "^0.3.1",
-        "@crawlee/browser": "3.17.1-beta.8",
-        "@crawlee/browser-pool": "3.17.1-beta.8",
-        "@crawlee/core": "3.17.1-beta.8",
-        "@crawlee/types": "3.17.1-beta.8",
-        "@crawlee/utils": "3.17.1-beta.8",
+        "@apify/timeout": "^0.4.0",
+        "@crawlee/browser": "3.17.1-beta.80",
+        "@crawlee/browser-pool": "3.17.1-beta.80",
+        "@crawlee/core": "3.17.1-beta.80",
+        "@crawlee/types": "3.17.1-beta.80",
+        "@crawlee/utils": "3.17.1-beta.80",
         "cheerio": "1.0.0-rc.12",
         "jquery": "^3.6.0",
         "lodash.isequal": "^4.5.0",
@@ -68,7 +68,8 @@
         "ml-matrix": "^6.11.0",
         "ow": "^0.28.1",
         "string-comparison": "^1.3.0",
-        "tslib": "^2.4.0"
+        "tslib": "^2.4.0",
+        "type-fest": "^4.0.0"
     },
     "peerDependencies": {
         "idcac-playwright": "^0.2.0",
@@ -89,5 +90,5 @@
             }
         }
     },
-    "gitHead": "4f4b8c1d53b9185095ac68017aff0ef2b8182b6b"
+    "gitHead": "5b3c2d52f128643de6f515e87eeb531d55a187f0"
 }