@crawlee/playwright 4.0.0-beta.9 → 4.0.0-beta.91

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/index.d.ts

@@ -6,5 +6,4 @@ export { RenderingTypePredictor } from './internals/utils/rendering-type-predict
 export * as playwrightUtils from './internals/utils/playwright-utils.js';
 export * as playwrightClickElements from './internals/enqueue-links/click-elements.js';
 export type { DirectNavigationOptions as PlaywrightDirectNavigationOptions } from './internals/utils/playwright-utils.js';
-export type { RenderingType } from './internals/utils/rendering-type-prediction.js';
-//# sourceMappingURL=index.d.ts.map
+export type { IRenderingTypePredictor, RenderingType } from './internals/utils/rendering-type-prediction.js';

package/index.js

@@ -5,4 +5,3 @@ export * from './internals/adaptive-playwright-crawler.js';
 export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction.js';
 export * as playwrightUtils from './internals/utils/playwright-utils.js';
 export * as playwrightClickElements from './internals/enqueue-links/click-elements.js';
-//# sourceMappingURL=index.js.map

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,24 +1,16 @@
-import type { BrowserHook, LoadedContext, RouterHandler } from '@crawlee/browser';
-import type { BaseHttpResponseData, GetUserDataFromRequest, RestrictedCrawlingContext, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
-import { Configuration, RequestHandlerResult, Statistics } from '@crawlee/core';
-import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { BrowserHook, LoadedRequest, Request, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
+import type { BasicCrawlerOptions } from '@crawlee/basic';
+import { BasicCrawler } from '@crawlee/basic';
+import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
+import { RequestHandlerResult, Statistics } from '@crawlee/core';
+import type { Dictionary, Awaitable } from '@crawlee/types';
 import { type CheerioRoot } from '@crawlee/utils';
 import { type Cheerio } from 'cheerio';
+import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
-import type { Log } from '@apify/log';
-import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
-import { PlaywrightCrawler } from './playwright-crawler.js';
-import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
-type Result<TResult> = {
-    result: TResult;
-    ok: true;
-    logs?: LogProxyCall[];
-} | {
-    error: unknown;
-    ok: false;
-    logs?: LogProxyCall[];
-};
+import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
+import { type IRenderingTypePredictor } from './utils/rendering-type-prediction.js';
 interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
     httpOnlyRequestHandlerRuns?: number;
     browserRequestHandlerRuns?: number;
@@ -28,25 +20,31 @@ declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
     state: AdaptivePlaywrightCrawlerStatisticState;
     constructor(options?: StatisticsOptions);
     reset(): void;
-    protected _maybeLoadStatistics(): Promise<void>;
+    protected maybeLoadStatistics(): Promise<void>;
     trackHttpOnlyRequestHandlerRun(): void;
     trackBrowserRequestHandlerRun(): void;
     trackRenderingTypeMisprediction(): void;
 }
-export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
+export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
+    request: LoadedRequest<Request<UserData>>;
     /**
      * The HTTP response, either from the HTTP client or from the initial request from playwright's navigation.
      */
-    response: BaseHttpResponseData;
+    response: Response;
     /**
      * Playwright Page object. If accessed in HTTP-only rendering, this will throw an error and make the AdaptivePlaywrightCrawlerContext retry the request in a browser.
      */
     page: Page;
     /**
-     * Wait for an element matching the selector to appear and return a Cheerio object of matched elements.
+     * Wait for an element matching the selector to appear and return a Cheerio object of the first matched element.
      * Timeout defaults to 5s.
      */
-    querySelector<T = any>(selector: string, timeoutMs?: number): Promise<Cheerio<T>>;
+    querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+    /**
+     * Wait for an element matching the selector to appear and return a Cheerio object of all matched elements.
+     * Timeout defaults to 5s.
+     */
+    querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
     /**
      * Wait for an element matching the selector to appear.
      * Timeout defaults to 5s.
@@ -74,36 +72,36 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
      * ```
      */
     parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
 }
-interface AdaptiveHook extends BrowserHook<Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'request' | 'session' | 'proxyInfo' | 'log'> & {
+interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> {
     page?: Page;
-}, PlaywrightGotoOptions> {
+    request: Request;
+    gotoOptions?: PlaywrightGotoOptions;
 }
-export interface AdaptivePlaywrightCrawlerOptions extends Omit<PlaywrightCrawlerOptions, 'requestHandler' | 'preNavigationHooks' | 'postNavigationHooks'> {
-    /**
-     * Function that is called to process each request.
-     *
-     * The function receives the {@link AdaptivePlaywrightCrawlingContext} as an argument, and it must refrain from calling code with side effects,
-     * other than the methods of the crawling context. Any other side effects may be invoked repeatedly by the crawler, which can lead to inconsistent results.
-     *
-     * The function must return a promise, which is then awaited by the crawler.
-     *
-     * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to `option.maxRequestRetries` times.
-     */
-    requestHandler?: (crawlingContext: LoadedContext<AdaptivePlaywrightCrawlerContext>) => Awaitable<void>;
+type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHookContext, ContextExtension>;
+type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
+    request: LoadedRequest<Request>;
+}, ContextExtension>;
+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes>, 'preNavigationHooks' | 'postNavigationHooks'> {
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
      * an exception will be thrown. If it's not caught, the request will be transparently retried in a browser.
+     *
+     * A hook may optionally return a partial object whose properties are merged into the crawling context,
+     * allowing the hook to override context members for subsequent hooks and pipeline stages.
      */
-    preNavigationHooks?: AdaptiveHook[];
+    preNavigationHooks?: AdaptiveHook<ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
      * an exception will be thrown. If it's not caught, the request will be transparently retried in a browser.
+     *
+     * A hook may optionally return a partial object whose properties are merged into the crawling context
+     * (e.g. to override `response` after solving a challenge).
      */
-    postNavigationHooks?: AdaptiveHook[];
+    postNavigationHooks?: AdaptivePostNavigationHook<ContextExtension>[];
     /**
      * Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.
      * Defaults to 0.1 (so 10%).
@@ -115,25 +113,41 @@ export interface AdaptivePlaywrightCrawlerOptions extends Omit<PlaywrightCrawler
      * If no callback is specified, every dataset item is considered valid.
      */
     resultChecker?: (result: RequestHandlerResult) => boolean;
+    /**
+     * An optional callback that decides whether an error thrown during the plain HTTP request handler
+     * should be propagated (instead of falling back to browser navigation).
+     *
+     * If the callback returns `true`, the error is thrown, triggering the standard retry mechanism.
+     * If the callback returns `false` (or is not provided), the error is logged and the crawler
+     * falls back to browser navigation (default behavior).
+     *
+     * @default () => false
+     */
+    shouldPropagateError?: (error: Error, context: PlaywrightCrawlingContext) => Awaitable<boolean>;
     /**
      * An optional callback used in rendering type detection. On each detection, the result of the plain HTTP run is compared to that of the browser one.
-     * If the callback returns true, the results are considered equal and the target site is considered static.
+     * If a callback is provided, the contract is as follows:
+     *   It the callback returns true or 'equal', the results are considered equal and the target site is considered static.
+     *   If it returns false or 'different', the target site is considered client-rendered.
+     *   If it returns 'inconclusive', the detection result won't be used.
      * If no result comparator is specified, but there is a `resultChecker`, any site where the `resultChecker` returns true is considered static.
      * If neither `resultComparator` nor `resultChecker` are specified, a deep comparison of returned dataset items is used as a default.
+     *
+     * For a stricter, ready-made comparator that also takes enqueued requests and key-value store changes into account, see {@link fullResultComparator}.
      */
-    resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean;
+    resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean | 'equal' | 'different' | 'inconclusive';
     /**
-     * A custom rendering type predictor
+     * A custom rendering type predictor. A predictor passed here is borrowed - the crawler never drives its
+     * lifecycle, so set it up yourself (the built-in {@link RenderingTypePredictor} needs `initialize()`).
+     * Omit the option and the crawler builds its own from `renderingTypeDetectionRatio` - and initializes it.
      */
-    renderingTypePredictor?: Pick<RenderingTypePredictor, 'predict' | 'storeResult'>;
+    renderingTypePredictor?: IRenderingTypePredictor;
     /**
      * Prevent direct access to storage in request handlers (only allow using context helpers).
      * Defaults to `true`
      */
     preventDirectStorageAccess?: boolean;
 }
-declare const proxyLogMethods: readonly ["error", "exception", "softFail", "info", "debug", "perf", "warningOnce", "deprecated"];
-type LogProxyCall = [log: Log, method: (typeof proxyLogMethods)[number], ...args: unknown[]];
 /**
  * An extension of {@link PlaywrightCrawler} that uses a more limited request handler interface so that it is able to switch to HTTP-only crawling when it detects it may be possible.
  *
@@ -163,31 +177,71 @@ type LogProxyCall = [log: Log, method: (typeof proxyLogMethods)[number], ...args
  *
  * @experimental
  */
-export declare class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
-    readonly config: Configuration;
-    private adaptiveRequestHandler;
+export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes> {
     private renderingTypePredictor;
     private resultChecker;
+    private shouldPropagateError;
     private resultComparator;
     private preventDirectStorageAccess;
+    private staticContextPipeline;
+    private browserContextPipeline;
+    private individualRequestHandlerTimeoutMillis;
     readonly stats: AdaptivePlaywrightCrawlerStatistics;
+    private resultObjects;
+    private inFlightRenderingTypeDetections;
+    private teardownHooks;
+    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
+    protected _init(): Promise<void>;
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
+        readonly request: LoadedRequest<Request<Dictionary>>;
+        readonly response: Response;
+        readonly page: Page;
+        readonly querySelector: AdaptivePlaywrightCrawlerContext["querySelector"];
+        readonly querySelectorAll: AdaptivePlaywrightCrawlerContext["querySelectorAll"];
+        readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
+        readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
+    }>;
+    private adaptCheerioContext;
+    private adaptPlaywrightContext;
+    private crawlOne;
+    protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
+    private commitResult;
+    private allowStorageAccess;
     /**
-     * Default {@link Router} instance that will be used if we don't specify any {@link AdaptivePlaywrightCrawlerOptions.requestHandler|`requestHandler`}.
-     * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
+     * Reading the pending request count queries the underlying request manager, which counts as storage access.
+     * Since this is internal crawler bookkeeping used to compute the `enqueueLinks` limit (not user-initiated storage
+     * access), it must be allowed even while a request handler runs inside the storage-access guard.
      */
-// @ts-ignore optional peer dependency or compatibility with es2022
-    readonly router: RouterHandler<AdaptivePlaywrightCrawlerContext>;
-    constructor(options?: AdaptivePlaywrightCrawlerOptions, config?: Configuration);
-    protected _runRequestHandler(crawlingContext: PlaywrightCrawlingContext): Promise<void>;
-    protected commitResult(crawlingContext: PlaywrightCrawlingContext, { calls, keyValueStoreChanges }: RequestHandlerResult): Promise<void>;
-    protected allowStorageAccess<R, TArgs extends any[]>(func: (...args: TArgs) => Promise<R>): (...args: TArgs) => Promise<R>;
-    protected runRequestHandlerInBrowser(crawlingContext: PlaywrightCrawlingContext): Promise<{
-        result: Result<RequestHandlerResult>;
-        initialStateCopy?: Record<string, unknown>;
-    }>;
-    protected runRequestHandlerWithPlainHTTP(crawlingContext: PlaywrightCrawlingContext, oldStateCopy?: Dictionary): Promise<Result<RequestHandlerResult>>;
+    protected getPendingRequestCountApproximation(): Promise<number>;
+    private enqueueLinks;
     private createLogProxy;
+    teardown(): Promise<void>;
 }
-export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
+/**
+ * An opt-in {@link AdaptivePlaywrightCrawlerOptions.resultComparator|`resultComparator`} that considers two
+ * request handler results equal only if *all* of their observable effects match - the pushed dataset items, the
+ * enqueued requests, and the key-value store changes. This is stricter than the default comparator, which only
+ * compares dataset items.
+ *
+ * **Beware:** enqueued URLs are compared exactly. The same page rendered in a browser and via plain HTTP often
+ * yields links that differ only in tracking query parameters, for example:
+ * - `https://sdk.apify.com/docs/guides/getting-started`
+ * - `https://sdk.apify.com/docs/guides/getting-started?__hsfp=1136113150&__hssc=7591405.1.173549427712`
+ *
+ * Such links are treated as *different*, which will make the crawler favor browser rendering for those pages.
+ *
+ * **Example usage:**
+ * ```ts
+ * const crawler = new AdaptivePlaywrightCrawler({
+ *     resultComparator: fullResultComparator,
+ *     async requestHandler({ pushData, enqueueLinks }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export declare function fullResultComparator(resultA: RequestHandlerResult, resultB: RequestHandlerResult): boolean;
 export {};
-//# sourceMappingURL=adaptive-playwright-crawler.d.ts.map