@crawlee/playwright 4.0.0-beta.1 → 4.0.0-beta.11

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,24 +1,15 @@
-import type { BrowserHook, LoadedContext, RouterHandler } from '@crawlee/browser';
-import type { BaseHttpResponseData, GetUserDataFromRequest, RestrictedCrawlingContext, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
+import { BasicCrawler } from '@crawlee/basic';
+import type { BasicCrawlerOptions, BrowserHook, LoadedRequest, Request } from '@crawlee/browser';
+import type { BaseHttpResponseData, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
 import { Configuration, RequestHandlerResult, Statistics } from '@crawlee/core';
-import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { Dictionary } from '@crawlee/types';
 import { type CheerioRoot } from '@crawlee/utils';
 import { type Cheerio } from 'cheerio';
+import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
-import type { Log } from '@apify/log';
-import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
-import { PlaywrightCrawler } from './playwright-crawler.js';
+import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
 import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
-type Result<TResult> = {
-    result: TResult;
-    ok: true;
-    logs?: LogProxyCall[];
-} | {
-    error: unknown;
-    ok: false;
-    logs?: LogProxyCall[];
-};
 interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
     httpOnlyRequestHandlerRuns?: number;
     browserRequestHandlerRuns?: number;
@@ -33,7 +24,8 @@ declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
     trackBrowserRequestHandlerRun(): void;
     trackRenderingTypeMisprediction(): void;
 }
-export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
+export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
+    request: LoadedRequest<Request<UserData>>;
     /**
      * The HTTP response, either from the HTTP client or from the initial request from playwright's navigation.
      */
@@ -46,7 +38,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
      * Wait for an element matching the selector to appear and return a Cheerio object of matched elements.
      * Timeout defaults to 5s.
      */
-    querySelector<T = any>(selector: string, timeoutMs?: number): Promise<Cheerio<T>>;
+    querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
     /**
      * Wait for an element matching the selector to appear.
      * Timeout defaults to 5s.
@@ -74,24 +66,14 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
      * ```
      */
     parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    enqueueLinks(options?: EnqueueLinksOptions): Promise<void>;
 }
-interface AdaptiveHook extends BrowserHook<Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'request' | 'session' | 'proxyInfo' | 'log'> & {
+interface AdaptiveHook extends BrowserHook<Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> & {
     page?: Page;
+    request: Request;
 }, PlaywrightGotoOptions> {
 }
-export interface AdaptivePlaywrightCrawlerOptions extends Omit<PlaywrightCrawlerOptions, 'requestHandler' | 'preNavigationHooks' | 'postNavigationHooks'> {
-    /**
-     * Function that is called to process each request.
-     *
-     * The function receives the {@link AdaptivePlaywrightCrawlingContext} as an argument, and it must refrain from calling code with side effects,
-     * other than the methods of the crawling context. Any other side effects may be invoked repeatedly by the crawler, which can lead to inconsistent results.
-     *
-     * The function must return a promise, which is then awaited by the crawler.
-     *
-     * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to `option.maxRequestRetries` times.
-     */
-    requestHandler?: (crawlingContext: LoadedContext<AdaptivePlaywrightCrawlerContext>) => Awaitable<void>;
+export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
@@ -132,8 +114,6 @@ export interface AdaptivePlaywrightCrawlerOptions extends Omit<PlaywrightCrawler
      */
     preventDirectStorageAccess?: boolean;
 }
-declare const proxyLogMethods: readonly ["error", "exception", "softFail", "info", "debug", "perf", "warningOnce", "deprecated"];
-type LogProxyCall = [log: Log, method: (typeof proxyLogMethods)[number], ...args: unknown[]];
 /**
  * An extension of {@link PlaywrightCrawler} that uses a more limited request handler interface so that it is able to switch to HTTP-only crawling when it detects it may be possible.
  *
@@ -163,31 +143,29 @@ type LogProxyCall = [log: Log, method: (typeof proxyLogMethods)[number], ...args
  *
  * @experimental
  */
-export declare class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
+export declare class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ExtendedContext> {
     readonly config: Configuration;
-    private adaptiveRequestHandler;
     private renderingTypePredictor;
     private resultChecker;
     private resultComparator;
     private preventDirectStorageAccess;
+    private staticContextPipeline;
+    private browserContextPipeline;
+    private individualRequestHandlerTimeoutMillis;
     readonly stats: AdaptivePlaywrightCrawlerStatistics;
-    /**
-     * Default {@link Router} instance that will be used if we don't specify any {@link AdaptivePlaywrightCrawlerOptions.requestHandler|`requestHandler`}.
-     * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
-     */
-// @ts-ignore optional peer dependency or compatibility with es2022
-    readonly router: RouterHandler<AdaptivePlaywrightCrawlerContext>;
-    constructor(options?: AdaptivePlaywrightCrawlerOptions, config?: Configuration);
-    protected _runRequestHandler(crawlingContext: PlaywrightCrawlingContext): Promise<void>;
+    private resultObjects;
+    private teardownHooks;
+    constructor(options?: AdaptivePlaywrightCrawlerOptions<ExtendedContext>, config?: Configuration);
+    private adaptCheerioContext;
+    private adaptPlaywrightContext;
+    private crawlOne;
+    protected runRequestHandler(crawlingContext: PlaywrightCrawlingContext): Promise<void>;
     protected commitResult(crawlingContext: PlaywrightCrawlingContext, { calls, keyValueStoreChanges }: RequestHandlerResult): Promise<void>;
     protected allowStorageAccess<R, TArgs extends any[]>(func: (...args: TArgs) => Promise<R>): (...args: TArgs) => Promise<R>;
-    protected runRequestHandlerInBrowser(crawlingContext: PlaywrightCrawlingContext): Promise<{
-        result: Result<RequestHandlerResult>;
-        initialStateCopy?: Record<string, unknown>;
-    }>;
-    protected runRequestHandlerWithPlainHTTP(crawlingContext: PlaywrightCrawlingContext, oldStateCopy?: Dictionary): Promise<Result<RequestHandlerResult>>;
     private createLogProxy;
+    teardown(): Promise<void>;
 }
-export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context>;
 export {};
 //# sourceMappingURL=adaptive-playwright-crawler.d.ts.map

package/internals/adaptive-playwright-crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"adaptive-playwright-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAA0B,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAE1G,OAAO,KAAK,EACR,oBAAoB,EACpB,sBAAsB,EACtB,yBAAyB,EACzB,YAAY,EAEZ,iBAAiB,EACjB,cAAc,EACjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAU,UAAU,EAA4B,MAAM,eAAe,CAAC;AAClH,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,WAAW,EAA0B,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAE,KAAK,OAAO,EAAQ,MAAM,SAAS,CAAC;AAE7C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,KAAK,EACR,wBAAwB,EACxB,yBAAyB,EACzB,qBAAqB,EACxB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAsB,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAElG,KAAK,MAAM,CAAC,OAAO,IACb;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,EAAE,EAAE,IAAI,CAAC;IAAC,IAAI,CAAC,EAAE,YAAY,EAAE,CAAA;CAAE,GACpD;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,EAAE,EAAE,KAAK,CAAC;IAAC,IAAI,CAAC,EAAE,YAAY,EAAE,CAAA;CAAE,CAAC;AAE3D,UAAU,uCAAwC,SAAQ,cAAc;IACpE,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACxC;AAQD,cAAM,mCAAoC,SAAQ,UAAU;IAC/C,KAAK,EAAE,uCAAuC,CAAe;gBAE1D,OAAO,GAAE,iBAAsB;IAKlC,KAAK,IAAI,IAAI;cAOG,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAe9D,8BAA8B,IAAI,IAAI;IAKtC,6BAA6B,IAAI,IAAI;IAKrC,+BAA+B,IAAI,IAAI;CAI1C;AAED,MAAM,WAAW,gCAAgC,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CACtF,SAAQ,yBAAyB,CAAC,QAAQ,CAAC;IAC3C;;OAEG;IACH,QAAQ,EAAE,oBAAoB,CAAC;IAE/B;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;;OAGG;IACH,aAAa,CAAC,CAAC,GAAG,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAElF;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,UAAU,YACN,SAAQ,WAAW,CACf,IAAI,CAAC,gCAAgC,EAAE,IAAI,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,KAAK,CAAC,GAAG;IAAE,IAAI,CAAC,EAAE,IAAI,CAAA;CAAE,EAC5G,qBAAqB,CACxB;CAAG;AAER,MAAM,WAAW,gCACb,SAAQ,IAAI,CAAC,wBAAwB,EAAE,gBAAgB,GAAG,oBAAoB,GAAG,qBAAqB,CAAC;IACvG;;;;;;;;;;OAUG;IACH,cAAc,CAAC,EAAE,CAAC,eAAe,EAAE,aAAa,CAAC,gCAAgC,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;IAEvG;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,YAAY,EAAE,CAAC;IAEpC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,YAAY,EAAE,CAAC;IAErC;;;OAGG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC;IAE1D;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,oBAAoB,KAAK,OAAO,CAAC;IAE7F;;OAEG;IACH,sBAAsB,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,SAAS,GAAG,aAAa,CAAC,CAAC;IAEjF;;;OAGG;IACH,0BAA0B,CAAC,EAAE,OAAO,CAAC;CACxC;AAED,QAAA,MAAM,eAAe,mGASX,CAAC;AAEX,KAAK,YAAY,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;AAE7F;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,yBAA0B,SAAQ,iBAAiB;aAkBtC,MAAM;IAjB5B,OAAO,CAAC,sBAAsB,CAA0D;IACxF,OAAO,CAAC,sBAAsB,CAA0E;IACxG,OAAO,CAAC,aAAa,CAAiE;IACtF,OAAO,CAAC,gBAAgB,CAAoE;IAC5F,OAAO,CAAC,0BAA0B,CAAU;IAC5C,SAAiB,KAAK,EAAE,mCAAmC,CAAC;IAE5D;;;OAGG;IAEH,SAAkB,MAAM,EAAE,aAAa,CAAC,gCAAgC,CAAC,CACnB;gBAGlD,OAAO,GAAE,gCAAqC,EAC5B,MAAM,gBAAkC;cA4CrC,kBAAkB,CAAC,eAAe,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC;cAuEtF,YAAY,CACxB,eAAe,EAAE,yBAAyB,EAC1C,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAAE,oBAAoB,GACtD,OAAO,CAAC,IAAI,CAAC;IAgBhB,SAAS,CAAC,kBAAkB,CAAC,CAAC,EAAE,KAAK,SAAS,GAAG,EAAE,EAC/C,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,GACrC,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC;cAQjB,0BAA0B,CACtC,eAAe,EAAE,yBAAyB,GAC3C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC;cA8FhF,8BAA8B,CAC1C,eAAe,EAAE,yBAAyB,EAC1C,YAAY,CAAC,EAAE,UAAU,GAC1B,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAqGxC,OAAO,CAAC,cAAc;CAYzB;AAED,wBAAgB,8BAA8B,CAC1C,OAAO,SAAS,gCAAgC,GAAG,gCAAgC,EACnF,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,0BAEzC"}
+{"version":3,"file":"adaptive-playwright-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,mBAAmB,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAIjG,OAAO,KAAK,EACR,oBAAoB,EAEpB,eAAe,EACf,mBAAmB,EACnB,sBAAsB,EACtB,YAAY,EAEZ,iBAAiB,EACjB,cAAc,EACjB,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,aAAa,EAEb,oBAAoB,EAEpB,UAAU,EAEb,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAA0B,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAKvC,OAAO,KAAK,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAEhG,OAAO,EAAsB,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAMlG,UAAU,uCAAwC,SAAQ,cAAc;IACpE,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACxC;AAQD,cAAM,mCAAoC,SAAQ,UAAU;IAC/C,KAAK,EAAE,uCAAuC,CAAe;gBAE1D,OAAO,GAAE,iBAAsB;IAKlC,KAAK,IAAI,IAAI;cAOG,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAe9D,8BAA8B,IAAI,IAAI;IAKtC,6BAA6B,IAAI,IAAI;IAKrC,+BAA+B,IAAI,IAAI;CAI1C;AAED,MAAM,WAAW,gCAAgC,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CACtF,SAAQ,eAAe,CAAC,QAAQ,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC1C;;OAEG;IACH,QAAQ,EAAE,oBAAoB,CAAC;IAE/B;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IAE/E;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE9E,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9D;AAED,UAAU,YACN,SAAQ,WAAW,CACf,IAAI,CAAC,gCAAgC,EAAE,IAAI,GAAG,SAAS,GAAG,WAAW,GAAG,KAAK,CAAC,GAAG;IAC7E,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;CACpB,EACD,qBAAqB,CACxB;CAAG;AAER,MAAM,WAAW,gCAAgC,CAC7C,eAAe,SAAS,gCAAgC,GAAG,gCAAgC,CAC7F,SAAQ,IAAI,CACN,mBAAmB,CAAC,gCAAgC,EAAE,eAAe,CAAC,EACtE,oBAAoB,GAAG,qBAAqB,CAC/C;IACD;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,YAAY,EAAE,CAAC;IAEpC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,YAAY,EAAE,CAAC;IAErC;;;OAGG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC;IAE1D;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,oBAAoB,KAAK,OAAO,CAAC;IAE7F;;OAEG;IACH,sBAAsB,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,SAAS,GAAG,aAAa,CAAC,CAAC;IAEjF;;;OAGG;IACH,0BAA0B,CAAC,EAAE,OAAO,CAAC;CACxC;AAeD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,yBAAyB,CAClC,eAAe,SAAS,gCAAgC,GAAG,gCAAgC,CAC7F,SAAQ,YAAY,CAAC,gCAAgC,EAAE,eAAe,CAAC;aAe/C,MAAM;IAd5B,OAAO,CAAC,sBAAsB,CAA0E;IACxG,OAAO,CAAC,aAAa,CAAiE;IACtF,OAAO,CAAC,gBAAgB,CAAoE;IAC5F,OAAO,CAAC,0BAA0B,CAAU;IAC5C,OAAO,CAAC,qBAAqB,CAAoD;IACjF,OAAO,CAAC,sBAAsB,CAAoD;IAClF,OAAO,CAAC,qCAAqC,CAAS;IACtD,SAAiB,KAAK,EAAE,mCAAmC,CAAC;IAC5D,OAAO,CAAC,aAAa,CAAwD;IAE7E,OAAO,CAAC,aAAa,CAAkC;gBAGnD,OAAO,GAAE,gCAAgC,CAAC,eAAe,CAAM,EAC7C,MAAM,gBAAkC;YAuIhD,mBAAmB;YAwCnB,sBAAsB;YA6CtB,QAAQ;cA2DG,iBAAiB,CAAC,eAAe,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC;cAyGrF,YAAY,CACxB,eAAe,EAAE,yBAAyB,EAC1C,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAAE,oBAAoB,GACtD,OAAO,CAAC,IAAI,CAAC;IAgBhB,SAAS,CAAC,kBAAkB,CAAC,CAAC,EAAE,KAAK,SAAS,GAAG,EAAE,EAC/C,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,GACrC,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC;IAQjC,OAAO,CAAC,cAAc;IAaP,QAAQ;CAM1B;AAED,wBAAgB,8BAA8B,CAC1C,OAAO,SAAS,gCAAgC,GAAG,gCAAgC,EACnF,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}

package/internals/adaptive-playwright-crawler.js

@@ -1,8 +1,9 @@
+import { isDeepStrictEqual } from 'node:util';
+import { BasicCrawler } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
-import { Configuration, RequestHandlerResult, Router, Statistics, withCheckedStorageAccess } from '@crawlee/core';
+import { CheerioCrawler } from '@crawlee/cheerio';
+import { Configuration, RequestHandlerError, RequestHandlerResult, Router, Statistics, withCheckedStorageAccess, } from '@crawlee/core';
 import { extractUrlsFromCheerio } from '@crawlee/utils';
-import { load } from 'cheerio';
-import isEqual from 'lodash.isequal';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { PlaywrightCrawler } from './playwright-crawler.js';
 import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
@@ -80,24 +81,32 @@ const proxyLogMethods = [
  *
  * @experimental
  */
-export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
+export class AdaptivePlaywrightCrawler extends BasicCrawler {
     config;
-    adaptiveRequestHandler;
     renderingTypePredictor;
     resultChecker;
     resultComparator;
     preventDirectStorageAccess;
-    /**
-     * Default {@link Router} instance that will be used if we don't specify any {@link AdaptivePlaywrightCrawlerOptions.requestHandler|`requestHandler`}.
-     * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
-     */
-    // @ts-ignore
-    router = Router.create();
+    staticContextPipeline;
+    browserContextPipeline;
+    individualRequestHandlerTimeoutMillis;
+    resultObjects = new WeakMap();
+    teardownHooks = [];
     constructor(options = {}, config = Configuration.getGlobalConfig()) {
-        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, resultComparator, statisticsOptions, preventDirectStorageAccess = true, ...rest } = options;
-        super(rest, config);
+        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, resultComparator, statisticsOptions, preventDirectStorageAccess = true, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks, postNavigationHooks, extendContext, contextPipelineBuilder, ...rest } = options;
+        super({
+            ...rest,
+            // Pass error handlers to the "main" crawler - we only pluck them from `rest` so that they don't go to the sub crawlers
+            errorHandler,
+            failedRequestHandler,
+            // Same for request handler
+            requestHandler,
+            // The builder intentionally returns null so that it crashes the crawler when it tries to use this instead of one of two the specialized context pipelines
+            // (that would be a logical error in this class)
+            contextPipelineBuilder: () => null,
+        }, config);
         this.config = config;
-        this.adaptiveRequestHandler = requestHandler ?? this.router;
+        this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
         this.renderingTypePredictor =
             renderingTypePredictor ?? new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio });
         this.resultChecker = resultChecker ?? (() => true);
@@ -112,10 +121,67 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
                 return (resultA.datasetItems.length === resultB.datasetItems.length &&
                     resultA.datasetItems.every((itemA, i) => {
                         const itemB = resultB.datasetItems[i];
-                        return isEqual(itemA, itemB);
+                        return isDeepStrictEqual(itemA, itemB);
                     }));
             };
         }
+        const staticCrawler = new CheerioCrawler({
+            ...rest,
+            useSessionPool: false,
+            statisticsOptions: {
+                persistenceOptions: { enable: false },
+            },
+            preNavigationHooks: [
+                async (context) => {
+                    for (const hook of preNavigationHooks ?? []) {
+                        await hook(context, undefined);
+                    }
+                },
+            ],
+            postNavigationHooks: [
+                async (context) => {
+                    for (const hook of postNavigationHooks ?? []) {
+                        await hook(context, undefined);
+                    }
+                },
+            ],
+        }, config);
+        const browserCrawler = new PlaywrightCrawler({
+            ...rest,
+            useSessionPool: false,
+            statisticsOptions: {
+                persistenceOptions: { enable: false },
+            },
+            preNavigationHooks: [
+                async (context, gotoOptions) => {
+                    for (const hook of preNavigationHooks ?? []) {
+                        await hook(context, gotoOptions);
+                    }
+                },
+            ],
+            postNavigationHooks: [
+                async (context, gotoOptions) => {
+                    for (const hook of postNavigationHooks ?? []) {
+                        await hook(context, gotoOptions);
+                    }
+                },
+            ],
+        }, config);
+        this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
+        this.staticContextPipeline = staticCrawler.contextPipeline
+            .compose({
+            action: this.adaptCheerioContext.bind(this),
+        })
+            .compose({
+            action: async (context) => extendContext ? await extendContext(context) : context,
+        });
+        this.browserContextPipeline = browserCrawler.contextPipeline
+            .compose({
+            action: this.adaptPlaywrightContext.bind(this),
+        })
+            .compose({
+            action: async (context) => extendContext ? await extendContext(context) : context,
+        });
         this.stats = new AdaptivePlaywrightCrawlerStatistics({
             logMessage: `${this.log.getOptions().prefix} request statistics:`,
             config,
@@ -123,7 +189,112 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
         });
         this.preventDirectStorageAccess = preventDirectStorageAccess;
     }
-    async _runRequestHandler(crawlingContext) {
+    async adaptCheerioContext(cheerioContext) {
+        // Capture the original response to avoid infinite recursion when the getter is copied to the context
+        const originalResponse = cheerioContext.response;
+        const enqueueLinks = this.resultObjects.get(cheerioContext)?.enqueueLinks;
+        if (enqueueLinks === undefined) {
+            throw new Error('Logical error - `this.resultObjects` does not contain the result object');
+        }
+        return {
+            get page() {
+                throw new Error('Page object was used in HTTP-only request handler');
+            },
+            get response() {
+                return {
+                    // TODO remove this once cheerioContext.response is just a Response
+                    complete: true,
+                    headers: originalResponse.headers,
+                    trailers: {},
+                    url: originalResponse.url,
+                    statusCode: originalResponse.statusCode,
+                    redirectUrls: originalResponse.redirectUrls ?? [],
+                };
+            },
+            async querySelector(selector) {
+                return cheerioContext.$(selector);
+            },
+            async enqueueLinks(options = {}) {
+                const urls = options.urls ??
+                    extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
+                await enqueueLinks({ ...options, urls });
+            },
+        };
+    }
+    async adaptPlaywrightContext(playwrightContext) {
+        // Capture the original response to avoid infinite recursion when the getter is copied to the context
+        const originalResponse = playwrightContext.response;
+        const enqueueLinks = this.resultObjects.get(playwrightContext)?.enqueueLinks;
+        if (enqueueLinks === undefined) {
+            throw new Error('Logical error - `this.resultObjects` does not contain the result object');
+        }
+        return {
+            get response() {
+                return {
+                    url: originalResponse.url(),
+                    statusCode: originalResponse.status(),
+                    headers: originalResponse.headers(),
+                    trailers: {},
+                    complete: true,
+                    redirectUrls: [],
+                };
+            },
+            async querySelector(selector, timeoutMs = 5000) {
+                const locator = playwrightContext.page.locator(selector).first();
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                const $ = await playwrightContext.parseWithCheerio();
+                return $(selector);
+            },
+            async enqueueLinks(options = {}, timeoutMs = 5000) {
+                const selector = options.selector ?? 'a';
+                const locator = playwrightContext.page.locator(selector).first();
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                // TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
+                const urls = options.urls ??
+                    (await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl));
+                await enqueueLinks({ ...options, urls });
+            },
+        };
+    }
+    async crawlOne(renderingType, context, useStateFunction) {
+        const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
+        const logs = [];
+        const deferredCleanup = [];
+        const resultBoundContextHelpers = {
+            addRequests: result.addRequests,
+            pushData: result.pushData,
+            useState: this.allowStorageAccess(useStateFunction),
+            getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
+            enqueueLinks: result.enqueueLinks,
+            log: this.createLogProxy(context.log, logs),
+            registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
+        };
+        const subCrawlerContext = { ...context, ...resultBoundContextHelpers };
+        this.resultObjects.set(subCrawlerContext, result);
+        try {
+            const callAdaptiveRequestHandler = async () => {
+                if (renderingType === 'static') {
+                    await this.staticContextPipeline.call(subCrawlerContext, async (finalContext) => await this.requestHandler(finalContext));
+                }
+                else if (renderingType === 'clientOnly') {
+                    await this.browserContextPipeline.call(subCrawlerContext, async (finalContext) => await this.requestHandler(finalContext));
+                }
+            };
+            await addTimeoutToPromise(async () => withCheckedStorageAccess(() => {
+                if (this.preventDirectStorageAccess) {
+                    throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
+                }
+            }, callAdaptiveRequestHandler), this.individualRequestHandlerTimeoutMillis, 'Request handler timed out');
+            return { result, ok: true, logs };
+        }
+        catch (error) {
+            return { error, ok: false, logs };
+        }
+        finally {
+            await Promise.all(deferredCleanup.map((cleanup) => cleanup()));
+        }
+    }
+    async runRequestHandler(crawlingContext) {
         const renderingTypePrediction = this.renderingTypePredictor.predict(crawlingContext.request);
         const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
         if (!shouldDetectRenderingType) {
@@ -132,15 +303,19 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
         if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
             crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
             this.stats.trackHttpOnlyRequestHandlerRun();
-            const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext);
+            const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState);
             if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {
                 crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
                 plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
                 await this.commitResult(crawlingContext, plainHTTPRun.result);
                 return;
             }
+            // Execution will "fall through" and try running the request handler in a browser
             if (!plainHTTPRun.ok) {
-                crawlingContext.log.exception(plainHTTPRun.error, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
+                const actualError = plainHTTPRun.error instanceof RequestHandlerError
+                    ? plainHTTPRun.error.cause
+                    : plainHTTPRun.error;
+                crawlingContext.log.exception(actualError, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
             }
             else {
                 crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
@@ -153,14 +328,30 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
         // a rendering type detection if necessary. Without this measure, the HTTP request handler would run
         // under different conditions, which could change its behavior. Changes done to the crawler state by
         // the HTTP request handler will not be committed to the actual storage.
-        const { result: browserRun, initialStateCopy } = await this.runRequestHandlerInBrowser(crawlingContext);
+        const stateTracker = {
+            stateCopy: null,
+            async getLiveState(defaultValue = {}) {
+                const state = await crawlingContext.useState(defaultValue);
+                if (this.stateCopy === null) {
+                    this.stateCopy = JSON.parse(JSON.stringify(state));
+                }
+                return state;
+            },
+            async getStateCopy(defaultValue = {}) {
+                if (this.stateCopy === null) {
+                    return defaultValue;
+                }
+                return this.stateCopy;
+            },
+        };
+        const browserRun = await this.crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker));
         if (!browserRun.ok) {
             throw browserRun.error;
         }
         await this.commitResult(crawlingContext, browserRun.result);
         if (shouldDetectRenderingType) {
             crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
-            const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext, initialStateCopy);
+            const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker));
             const detectionResult = (() => {
                 if (!plainHTTPRun.ok) {
                     return 'clientOnly';
@@ -188,152 +379,6 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
     allowStorageAccess(func) {
         return async (...args) => withCheckedStorageAccess(() => { }, async () => func(...args));
     }
-    async runRequestHandlerInBrowser(crawlingContext) {
-        const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
-        let initialStateCopy;
-        try {
-            await super._runRequestHandler.call(new Proxy(this, {
-                get: (target, propertyName, receiver) => {
-                    if (propertyName === 'userProvidedRequestHandler') {
-                        return async (playwrightContext) => withCheckedStorageAccess(() => {
-                            if (this.preventDirectStorageAccess) {
-                                throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
-                            }
-                        }, () => this.adaptiveRequestHandler({
-                            id: crawlingContext.id,
-                            session: crawlingContext.session,
-                            proxyInfo: crawlingContext.proxyInfo,
-                            request: crawlingContext.request,
-                            response: {
-                                url: crawlingContext.response.url(),
-                                statusCode: crawlingContext.response.status(),
-                                headers: crawlingContext.response.headers(),
-                                trailers: {},
-                                complete: true,
-                                redirectUrls: [],
-                            },
-                            log: crawlingContext.log,
-                            page: crawlingContext.page,
-                            querySelector: async (selector, timeoutMs = 5_000) => {
-                                const locator = playwrightContext.page.locator(selector).first();
-                                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                                const $ = await playwrightContext.parseWithCheerio();
-                                return $(selector);
-                            },
-                            async waitForSelector(selector, timeoutMs = 5_000) {
-                                const locator = playwrightContext.page.locator(selector).first();
-                                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                            },
-                            async parseWithCheerio(selector, timeoutMs = 5_000) {
-                                if (selector) {
-                                    const locator = playwrightContext.page.locator(selector).first();
-                                    await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                                }
-                                return playwrightContext.parseWithCheerio();
-                            },
-                            async enqueueLinks(options = {}, timeoutMs = 5_000) {
-                                const selector = options.selector ?? 'a';
-                                const locator = playwrightContext.page.locator(selector).first();
-                                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                                const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ??
-                                    playwrightContext.request.loadedUrl ??
-                                    playwrightContext.request.url);
-                                await result.enqueueLinks({ ...options, urls });
-                            },
-                            addRequests: result.addRequests,
-                            pushData: result.pushData,
-                            useState: this.allowStorageAccess(async (defaultValue) => {
-                                const state = await result.useState(defaultValue);
-                                if (initialStateCopy === undefined) {
-                                    initialStateCopy = JSON.parse(JSON.stringify(state));
-                                }
-                                return state;
-                            }),
-                            getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
-                        }));
-                    }
-                    return Reflect.get(target, propertyName, receiver);
-                },
-            }), crawlingContext);
-            return { result: { result, ok: true }, initialStateCopy };
-        }
-        catch (error) {
-            return { result: { error, ok: false }, initialStateCopy };
-        }
-    }
-    async runRequestHandlerWithPlainHTTP(crawlingContext, oldStateCopy) {
-        const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
-        const logs = [];
-        const pageGotoOptions = { timeout: this.navigationTimeoutMillis }; // Irrelevant, but required by BrowserCrawler
-        try {
-            await withCheckedStorageAccess(() => {
-                if (this.preventDirectStorageAccess) {
-                    throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
-                }
-            }, async () => addTimeoutToPromise(async () => {
-                const hookContext = {
-                    id: crawlingContext.id,
-                    session: crawlingContext.session,
-                    proxyInfo: crawlingContext.proxyInfo,
-                    request: crawlingContext.request,
-                    log: this.createLogProxy(crawlingContext.log, logs),
-                };
-                await this._executeHooks(this.preNavigationHooks, {
-                    ...hookContext,
-                    get page() {
-                        throw new Error('Page object was used in HTTP-only pre-navigation hook');
-                    },
-                }, // This is safe because `executeHooks` just passes the context to the hooks which accept the partial context
-                pageGotoOptions);
-                const response = await crawlingContext.sendRequest({});
-                const loadedUrl = response.url;
-                crawlingContext.request.loadedUrl = loadedUrl;
-                const $ = load(response.body);
-                await this.adaptiveRequestHandler({
-                    ...hookContext,
-                    request: crawlingContext.request,
-                    response,
-                    get page() {
-                        throw new Error('Page object was used in HTTP-only request handler');
-                    },
-                    async querySelector(selector, _timeoutMs) {
-                        return $(selector);
-                    },
-                    async waitForSelector(selector, _timeoutMs) {
-                        if ($(selector).get().length === 0) {
-                            throw new Error(`Selector '${selector}' not found.`);
-                        }
-                    },
-                    async parseWithCheerio(selector, _timeoutMs) {
-                        if (selector && $(selector).get().length === 0) {
-                            throw new Error(`Selector '${selector}' not found.`);
-                        }
-                        return $;
-                    },
-                    async enqueueLinks(options = {}) {
-                        const urls = extractUrlsFromCheerio($, options.selector, options.baseUrl ?? loadedUrl);
-                        await result.enqueueLinks({ ...options, urls });
-                    },
-                    addRequests: result.addRequests,
-                    pushData: result.pushData,
-                    useState: async (defaultValue) => {
-                        // return the old state before the browser handler was executed
-                        // when rerunning the handler via HTTP for detection
-                        if (oldStateCopy !== undefined) {
-                            return oldStateCopy ?? defaultValue; // fallback to the default for `null`
-                        }
-                        return this.allowStorageAccess(result.useState)(defaultValue);
-                    },
-                    getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
-                });
-                await this._executeHooks(this.postNavigationHooks, crawlingContext, pageGotoOptions);
-            }, this.requestHandlerTimeoutInnerMillis, 'Request handler timed out'));
-            return { result, logs, ok: true };
-        }
-        catch (error) {
-            return { error, logs, ok: false };
-        }
-    }
     createLogProxy(log, logs) {
         return new Proxy(log, {
             get(target, propertyName, receiver) {
@@ -346,6 +391,12 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
             },
         });
     }
+    async teardown() {
+        await super.teardown();
+        for (const hook of this.teardownHooks) {
+            await hook();
+        }
+    }
 }
 export function createAdaptivePlaywrightRouter(routes) {
     return Router.create(routes);