@crawlee/playwright 4.0.0-beta.99 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/index.d.ts

@@ -1,6 +1,8 @@
 export * from '@crawlee/browser';
+export * from './internals/playwright-browser-pool.js';
 export * from './internals/playwright-crawler.js';
-export * from './internals/playwright-launcher.js';
+export { launchPlaywright } from './internals/playwright-launcher.js';
+export type { PlaywrightLaunchContext } from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';
 export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction.js';
 export * as playwrightUtils from './internals/utils/playwright-utils.js';

package/index.js

@@ -1,6 +1,7 @@
 export * from '@crawlee/browser';
+export * from './internals/playwright-browser-pool.js';
 export * from './internals/playwright-crawler.js';
-export * from './internals/playwright-launcher.js';
+export { launchPlaywright } from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';
 export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction.js';
 export * as playwrightUtils from './internals/utils/playwright-utils.js';

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,32 +1,40 @@
-import type { BrowserHook, LoadedRequest, Request, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
-import type { BasicCrawlerOptions } from '@crawlee/basic';
+import type { BrowserHook, LoadedRequest, CrawlingRequest, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
+import type { BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
-import { RequestHandlerResult, Statistics } from '@crawlee/core';
+import type { StorageTransactionView } from '@crawlee/core';
 import type { Dictionary, Awaitable } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils';
-import { type Cheerio } from 'cheerio';
+import { type Cheerio, type CheerioAPI } from 'cheerio';
 import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
-import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
+import { z } from 'zod';
+import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
 import { type IRenderingTypePredictor } from './utils/rendering-type-prediction.js';
-interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
-    httpOnlyRequestHandlerRuns?: number;
-    browserRequestHandlerRuns?: number;
-    renderingTypeMispredictions?: number;
-}
-declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
-    state: AdaptivePlaywrightCrawlerStatisticState;
-    constructor(options?: StatisticsOptions);
-    reset(): void;
-    protected maybeLoadStatistics(): Promise<void>;
-    trackHttpOnlyRequestHandlerRun(): void;
-    trackBrowserRequestHandlerRun(): void;
-    trackRenderingTypeMisprediction(): void;
-}
-export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
-    request: LoadedRequest<Request<UserData>>;
+declare const adaptiveStatisticStateSchema: z.ZodObject<{
+    httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+    browserRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+    renderingTypeMispredictions: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+/**
+ * The extra statistics fields {@link AdaptivePlaywrightCrawler} tracks on top of the built-in
+ * {@link StatisticState} ones. They are available on `crawler.statistics.state` and are persisted with the rest of
+ * the statistics.
+ */
+export type AdaptivePlaywrightCrawlerStatisticState = z.infer<typeof adaptiveStatisticStateSchema>;
+/**
+ * The {@link AdaptivePlaywrightCrawlerStatisticState} fields as a {@link Statistics} state extension, defaults
+ * and all. A {@link Statistics} instance to be injected into an {@link AdaptivePlaywrightCrawler} has to carry
+ * them - `deserialize.extend()` your own fields onto this one and pass the result as `stateExtension`.
+ */
+export declare const adaptivePlaywrightCrawlerStatisticState: {
+    deserialize: z.ZodObject<{
+        httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+        browserRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+        renderingTypeMispredictions: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strip>;
+};
+export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
+    request: LoadedRequest<CrawlingRequest<UserData>>;
     /**
      * The HTTP response, either from the HTTP client or from the initial request from playwright's navigation.
      */
@@ -71,19 +79,19 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
      * });
      * ```
      */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
     enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
 }
 interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> {
     page?: Page;
-    request: Request;
+    request: CrawlingRequest;
     gotoOptions?: PlaywrightGotoOptions;
 }
 type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHookContext, ContextExtension>;
 type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
-    request: LoadedRequest<Request>;
+    request: LoadedRequest<CrawlingRequest>;
 }, ContextExtension>;
-export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes>, 'preNavigationHooks' | 'postNavigationHooks'> {
+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks' | 'contextPipelineBuilder'>, Pick<PlaywrightCrawlerOptions, 'launchContext' | 'headless' | 'browserPool' | 'remoteBrowser'> {
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
@@ -108,11 +116,12 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
      */
     renderingTypeDetectionRatio?: number;
     /**
-     * An optional callback that is called on dataset items found by the request handler in plain HTTP mode.
+     * An optional callback that is called on the storage writes recorded by the request handler in plain
+     * HTTP mode (exposed as a read-only {@link StorageTransactionView}).
      * If it returns false, the request is retried in a browser.
-     * If no callback is specified, every dataset item is considered valid.
+     * If no callback is specified, every result is considered valid.
      */
-    resultChecker?: (result: RequestHandlerResult) => boolean;
+    resultChecker?: (result: StorageTransactionView) => boolean;
     /**
      * An optional callback that decides whether an error thrown during the plain HTTP request handler
      * should be propagated (instead of falling back to browser navigation).
@@ -135,18 +144,13 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
      *
      * For a stricter, ready-made comparator that also takes enqueued requests and key-value store changes into account, see {@link fullResultComparator}.
      */
-    resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean | 'equal' | 'different' | 'inconclusive';
+    resultComparator?: (resultA: StorageTransactionView, resultB: StorageTransactionView) => boolean | 'equal' | 'different' | 'inconclusive';
     /**
      * A custom rendering type predictor. A predictor passed here is borrowed - the crawler never drives its
      * lifecycle, so set it up yourself (the built-in {@link RenderingTypePredictor} needs `initialize()`).
      * Omit the option and the crawler builds its own from `renderingTypeDetectionRatio` - and initializes it.
      */
     renderingTypePredictor?: IRenderingTypePredictor;
-    /**
-     * Prevent direct access to storage in request handlers (only allow using context helpers).
-     * Defaults to `true`
-     */
-    preventDirectStorageAccess?: boolean;
 }
 /**
  * An extension of {@link PlaywrightCrawler} that uses a more limited request handler interface so that it is able to switch to HTTP-only crawling when it detects it may be possible.
@@ -177,45 +181,31 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
  *
  * @experimental
  */
-export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes> {
-    private renderingTypePredictor;
-    private resultChecker;
-    private shouldPropagateError;
-    private resultComparator;
-    private preventDirectStorageAccess;
-    private staticContextPipeline;
-    private browserContextPipeline;
-    private individualRequestHandlerTimeoutMillis;
-    readonly stats: AdaptivePlaywrightCrawlerStatistics;
-    private resultObjects;
-    private inFlightRenderingTypeDetections;
-    private teardownHooks;
-    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
-    protected _init(): Promise<void>;
-    protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
-        readonly request: LoadedRequest<Request<Dictionary>>;
-        readonly response: Response;
-        readonly page: Page;
-        readonly querySelector: AdaptivePlaywrightCrawlerContext["querySelector"];
-        readonly querySelectorAll: AdaptivePlaywrightCrawlerContext["querySelectorAll"];
-        readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
-        readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
-    }>;
-    private adaptCheerioContext;
-    private adaptPlaywrightContext;
-    private crawlOne;
+export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    #private;
+    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
+    protected init(): Promise<void>;
     protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
-    private commitResult;
-    private allowStorageAccess;
     /**
-     * Reading the pending request count queries the underlying request manager, which counts as storage access.
-     * Since this is internal crawler bookkeeping used to compute the `enqueueLinks` limit (not user-initiated storage
-     * access), it must be allowed even while a request handler runs inside the storage-access guard.
+     * Number of rendering type detections that have not settled yet, including results the predictor is
+     * still persisting.
+     */
+    get inFlightRenderingTypeDetectionCount(): number;
+    /**
+     * Waits for in-flight rendering type detections to settle, bounded by `timeoutMillis` (defaults to the
+     * internal timeout).
+     */
+    drainRenderingDetections({ timeoutMillis }?: {
+        timeoutMillis?: number;
+    }): Promise<void>;
+    /**
+     * Stops the crawler immediately, but not before rendering type detections already under way (and results
+     * the predictor is still persisting) have settled - see
+     * {@link AdaptivePlaywrightCrawler.drainRenderingDetections|`drainRenderingDetections()`}. Requests
+     * that are still running are not waited for, unlike {@link BasicCrawler.stop|`stop()`}.
      */
-    protected getPendingRequestCountApproximation(): Promise<number>;
-    private enqueueLinks;
-    private createLogProxy;
     teardown(): Promise<void>;
+    destroy(): Promise<void>;
 }
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
@@ -243,5 +233,5 @@ export declare function createAdaptivePlaywrightRouter<Context extends AdaptiveP
  * });
  * ```
  */
-export declare function fullResultComparator(resultA: RequestHandlerResult, resultB: RequestHandlerResult): boolean;
+export declare function fullResultComparator(resultA: StorageTransactionView, resultB: StorageTransactionView): boolean;
 export {};