@crawlee/playwright 4.0.0-beta.99 → 4.0.0-rc.0

package/index.d.ts

@@ -1,4 +1,5 @@
 export * from '@crawlee/browser';
+export * from './internals/playwright-browser-pool.js';
 export * from './internals/playwright-crawler.js';
 export * from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';

package/index.js

@@ -1,4 +1,5 @@
 export * from '@crawlee/browser';
+export * from './internals/playwright-browser-pool.js';
 export * from './internals/playwright-crawler.js';
 export * from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,31 +1,40 @@
 import type { BrowserHook, LoadedRequest, Request, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import type { BasicCrawlerOptions } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
-import { RequestHandlerResult, Statistics } from '@crawlee/core';
+import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StorageTransactionView } from '@crawlee/core';
 import type { Dictionary, Awaitable } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils';
+import { type CheerioRoot } from '@crawlee/utils/internal';
 import { type Cheerio } from 'cheerio';
 import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
+import { z } from 'zod';
 import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
 import { type IRenderingTypePredictor } from './utils/rendering-type-prediction.js';
-interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
-    httpOnlyRequestHandlerRuns?: number;
-    browserRequestHandlerRuns?: number;
-    renderingTypeMispredictions?: number;
-}
-declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
-    state: AdaptivePlaywrightCrawlerStatisticState;
-    constructor(options?: StatisticsOptions);
-    reset(): void;
-    protected maybeLoadStatistics(): Promise<void>;
-    trackHttpOnlyRequestHandlerRun(): void;
-    trackBrowserRequestHandlerRun(): void;
-    trackRenderingTypeMisprediction(): void;
-}
-export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
+declare const adaptiveStatisticStateSchema: z.ZodObject<{
+    httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+    browserRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+    renderingTypeMispredictions: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+/**
+ * The extra statistics fields {@link AdaptivePlaywrightCrawler} tracks on top of the built-in
+ * {@link StatisticState} ones. They are available on `crawler.statistics.state` and are persisted with the rest of
+ * the statistics.
+ */
+export type AdaptivePlaywrightCrawlerStatisticState = z.infer<typeof adaptiveStatisticStateSchema>;
+/**
+ * The {@link AdaptivePlaywrightCrawlerStatisticState} fields as a {@link Statistics} state extension, defaults
+ * and all. A {@link Statistics} instance to be injected into an {@link AdaptivePlaywrightCrawler} has to carry
+ * them - `deserialize.extend()` your own fields onto this one and pass the result as `stateExtension`.
+ */
+export declare const adaptivePlaywrightCrawlerStatisticState: {
+    deserialize: z.ZodObject<{
+        httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+        browserRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+        renderingTypeMispredictions: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strip>;
+};
+export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
     request: LoadedRequest<Request<UserData>>;
     /**
      * The HTTP response, either from the HTTP client or from the initial request from playwright's navigation.
@@ -83,7 +92,7 @@ type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHo
 type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
     request: LoadedRequest<Request>;
 }, ContextExtension>;
-export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes>, 'preNavigationHooks' | 'postNavigationHooks'> {
+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks'> {
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
@@ -108,11 +117,12 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
      */
     renderingTypeDetectionRatio?: number;
     /**
-     * An optional callback that is called on dataset items found by the request handler in plain HTTP mode.
+     * An optional callback that is called on the storage writes recorded by the request handler in plain
+     * HTTP mode (exposed as a read-only {@link StorageTransactionView}).
      * If it returns false, the request is retried in a browser.
-     * If no callback is specified, every dataset item is considered valid.
+     * If no callback is specified, every result is considered valid.
      */
-    resultChecker?: (result: RequestHandlerResult) => boolean;
+    resultChecker?: (result: StorageTransactionView) => boolean;
     /**
      * An optional callback that decides whether an error thrown during the plain HTTP request handler
      * should be propagated (instead of falling back to browser navigation).
@@ -135,18 +145,13 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
      *
      * For a stricter, ready-made comparator that also takes enqueued requests and key-value store changes into account, see {@link fullResultComparator}.
      */
-    resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean | 'equal' | 'different' | 'inconclusive';
+    resultComparator?: (resultA: StorageTransactionView, resultB: StorageTransactionView) => boolean | 'equal' | 'different' | 'inconclusive';
     /**
      * A custom rendering type predictor. A predictor passed here is borrowed - the crawler never drives its
      * lifecycle, so set it up yourself (the built-in {@link RenderingTypePredictor} needs `initialize()`).
      * Omit the option and the crawler builds its own from `renderingTypeDetectionRatio` - and initializes it.
      */
     renderingTypePredictor?: IRenderingTypePredictor;
-    /**
-     * Prevent direct access to storage in request handlers (only allow using context helpers).
-     * Defaults to `true`
-     */
-    preventDirectStorageAccess?: boolean;
 }
 /**
  * An extension of {@link PlaywrightCrawler} that uses a more limited request handler interface so that it is able to switch to HTTP-only crawling when it detects it may be possible.
@@ -177,21 +182,10 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
  *
  * @experimental
  */
-export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes> {
-    private renderingTypePredictor;
-    private resultChecker;
-    private shouldPropagateError;
-    private resultComparator;
-    private preventDirectStorageAccess;
-    private staticContextPipeline;
-    private browserContextPipeline;
-    private individualRequestHandlerTimeoutMillis;
-    readonly stats: AdaptivePlaywrightCrawlerStatistics;
-    private resultObjects;
-    private inFlightRenderingTypeDetections;
-    private teardownHooks;
-    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
-    protected _init(): Promise<void>;
+export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    #private;
+    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
+    protected init(): Promise<void>;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
         readonly request: LoadedRequest<Request<Dictionary>>;
         readonly response: Response;
@@ -200,19 +194,18 @@ export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<nev
         readonly querySelectorAll: AdaptivePlaywrightCrawlerContext["querySelectorAll"];
         readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
         readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
+        readonly enqueueLinks: AdaptivePlaywrightCrawlerContext["enqueueLinks"];
     }>;
     private adaptCheerioContext;
     private adaptPlaywrightContext;
-    private crawlOne;
-    protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
-    private commitResult;
-    private allowStorageAccess;
     /**
-     * Reading the pending request count queries the underlying request manager, which counts as storage access.
-     * Since this is internal crawler bookkeeping used to compute the `enqueueLinks` limit (not user-initiated storage
-     * access), it must be allowed even while a request handler runs inside the storage-access guard.
+     * Runs one request handler attempt inside its own {@link StorageTransaction}, wrapping the inner
+     * (static or browser) context pipeline. The transaction is pushed to `transactions` *at creation
+     * time, before the `try`* - the `ok: false` branch of the returned {@link Result} carries no
+     * result, and failed attempts are routine here. The caller owns the outcome and disposal.
      */
-    protected getPendingRequestCountApproximation(): Promise<number>;
+    private crawlOne;
+    protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
     private enqueueLinks;
     private createLogProxy;
     teardown(): Promise<void>;
@@ -243,5 +236,5 @@ export declare function createAdaptivePlaywrightRouter<Context extends AdaptiveP
  * });
  * ```
  */
-export declare function fullResultComparator(resultA: RequestHandlerResult, resultB: RequestHandlerResult): boolean;
+export declare function fullResultComparator(resultA: StorageTransactionView, resultB: StorageTransactionView): boolean;
 export {};