@crawlee/playwright 4.0.0-beta.24 → 4.0.0-beta.241

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +4 -3
  3. package/index.js +2 -2
  4. package/internals/adaptive-playwright-crawler.d.ts +144 -74
  5. package/internals/adaptive-playwright-crawler.js +447 -292
  6. package/internals/enqueue-links/click-elements.d.ts +36 -64
  7. package/internals/enqueue-links/click-elements.js +65 -67
  8. package/internals/playwright-browser-pool.d.ts +55 -0
  9. package/internals/playwright-browser-pool.js +34 -0
  10. package/internals/playwright-crawler.d.ts +179 -149
  11. package/internals/playwright-crawler.js +93 -79
  12. package/internals/playwright-launcher.d.ts +30 -20
  13. package/internals/playwright-launcher.js +24 -18
  14. package/internals/utils/playwright-utils.d.ts +64 -56
  15. package/internals/utils/playwright-utils.js +117 -138
  16. package/internals/utils/rendering-type-prediction.d.ts +34 -14
  17. package/internals/utils/rendering-type-prediction.js +87 -35
  18. package/package.json +17 -20
  19. package/index.d.ts.map +0 -1
  20. package/index.js.map +0 -1
  21. package/internals/adaptive-playwright-crawler.d.ts.map +0 -1
  22. package/internals/adaptive-playwright-crawler.js.map +0 -1
  23. package/internals/enqueue-links/click-elements.d.ts.map +0 -1
  24. package/internals/enqueue-links/click-elements.js.map +0 -1
  25. package/internals/playwright-crawler.d.ts.map +0 -1
  26. package/internals/playwright-crawler.js.map +0 -1
  27. package/internals/playwright-launcher.d.ts.map +0 -1
  28. package/internals/playwright-launcher.js.map +0 -1
  29. package/internals/utils/playwright-utils.d.ts.map +0 -1
  30. package/internals/utils/playwright-utils.js.map +0 -1
  31. package/internals/utils/rendering-type-prediction.d.ts.map +0 -1
  32. package/internals/utils/rendering-type-prediction.js.map +0 -1
package/README.md CHANGED
@@ -1,23 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
14
  </p>
15
15
 
16
- <p align=center>
17
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
18
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
19
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
20
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
21
21
  </p>
22
22
 
23
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
34
34
 
35
35
  We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
36
36
 
37
- > Crawlee requires **Node.js 16 or higher**.
37
+ > Crawlee requires **Node.js 22.13 or higher**.
38
38
 
39
39
  ### With Crawlee CLI
40
40
 
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
89
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
90
90
 
91
91
  ```bash
92
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
93
93
  ```
94
94
 
95
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
98
98
  {
99
99
  "overrides": {
100
100
  "apify": {
101
- "@crawlee/core": "3.12.3-beta.13",
102
- "@crawlee/types": "3.12.3-beta.13",
103
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
104
104
  }
105
105
  }
106
106
  }
package/index.d.ts CHANGED
@@ -1,10 +1,11 @@
1
1
  export * from '@crawlee/browser';
2
+ export * from './internals/playwright-browser-pool.js';
2
3
  export * from './internals/playwright-crawler.js';
3
- export * from './internals/playwright-launcher.js';
4
+ export { launchPlaywright } from './internals/playwright-launcher.js';
5
+ export type { PlaywrightLaunchContext } from './internals/playwright-launcher.js';
4
6
  export * from './internals/adaptive-playwright-crawler.js';
5
7
  export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction.js';
6
8
  export * as playwrightUtils from './internals/utils/playwright-utils.js';
7
9
  export * as playwrightClickElements from './internals/enqueue-links/click-elements.js';
8
10
  export type { DirectNavigationOptions as PlaywrightDirectNavigationOptions } from './internals/utils/playwright-utils.js';
9
- export type { RenderingType } from './internals/utils/rendering-type-prediction.js';
10
- //# sourceMappingURL=index.d.ts.map
11
+ export type { IRenderingTypePredictor, RenderingType } from './internals/utils/rendering-type-prediction.js';
package/index.js CHANGED
@@ -1,8 +1,8 @@
1
1
  export * from '@crawlee/browser';
2
+ export * from './internals/playwright-browser-pool.js';
2
3
  export * from './internals/playwright-crawler.js';
3
- export * from './internals/playwright-launcher.js';
4
+ export { launchPlaywright } from './internals/playwright-launcher.js';
4
5
  export * from './internals/adaptive-playwright-crawler.js';
5
6
  export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction.js';
6
7
  export * as playwrightUtils from './internals/utils/playwright-utils.js';
7
8
  export * as playwrightClickElements from './internals/enqueue-links/click-elements.js';
8
- //# sourceMappingURL=index.js.map
@@ -1,32 +1,40 @@
1
+ import type { BrowserHook, LoadedRequest, CrawlingRequest, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
2
+ import type { BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes } from '@crawlee/basic';
1
3
  import { BasicCrawler } from '@crawlee/basic';
2
- import type { BasicCrawlerOptions, BrowserHook, LoadedRequest, Request } from '@crawlee/browser';
3
- import type { CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RestrictedCrawlingContext, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
4
- import { Configuration, RequestHandlerResult, Statistics } from '@crawlee/core';
5
- import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
6
- import { type CheerioRoot } from '@crawlee/utils';
7
- import { type Cheerio } from 'cheerio';
4
+ import type { StorageTransactionView } from '@crawlee/core';
5
+ import type { Dictionary, Awaitable } from '@crawlee/types';
6
+ import { type Cheerio, type CheerioAPI } from 'cheerio';
8
7
  import type { AnyNode } from 'domhandler';
9
8
  // @ts-ignore optional peer dependency or compatibility with es2022
10
9
  import type { Page } from 'playwright';
11
- import type { SetRequired } from 'type-fest';
12
- import type { PlaywrightGotoOptions } from './playwright-crawler.js';
13
- import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
14
- interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
15
- httpOnlyRequestHandlerRuns?: number;
16
- browserRequestHandlerRuns?: number;
17
- renderingTypeMispredictions?: number;
18
- }
19
- declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
20
- state: AdaptivePlaywrightCrawlerStatisticState;
21
- constructor(options?: StatisticsOptions);
22
- reset(): void;
23
- protected _maybeLoadStatistics(): Promise<void>;
24
- trackHttpOnlyRequestHandlerRun(): void;
25
- trackBrowserRequestHandlerRun(): void;
26
- trackRenderingTypeMisprediction(): void;
27
- }
28
- export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
29
- request: LoadedRequest<Request<UserData>>;
10
+ import { z } from 'zod';
11
+ import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
12
+ import { type IRenderingTypePredictor } from './utils/rendering-type-prediction.js';
13
+ declare const adaptiveStatisticStateSchema: z.ZodObject<{
14
+ httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
15
+ browserRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
16
+ renderingTypeMispredictions: z.ZodDefault<z.ZodNumber>;
17
+ }, z.core.$strip>;
18
+ /**
19
+ * The extra statistics fields {@link AdaptivePlaywrightCrawler} tracks on top of the built-in
20
+ * {@link StatisticState} ones. They are available on `crawler.statistics.state` and are persisted with the rest of
21
+ * the statistics.
22
+ */
23
+ export type AdaptivePlaywrightCrawlerStatisticState = z.infer<typeof adaptiveStatisticStateSchema>;
24
+ /**
25
+ * The {@link AdaptivePlaywrightCrawlerStatisticState} fields as a {@link Statistics} state extension, defaults
26
+ * and all. A {@link Statistics} instance to be injected into an {@link AdaptivePlaywrightCrawler} has to carry
27
+ * them - `deserialize.extend()` your own fields onto this one and pass the result as `stateExtension`.
28
+ */
29
+ export declare const adaptivePlaywrightCrawlerStatisticState: {
30
+ deserialize: z.ZodObject<{
31
+ httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
32
+ browserRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
33
+ renderingTypeMispredictions: z.ZodDefault<z.ZodNumber>;
34
+ }, z.core.$strip>;
35
+ };
36
+ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
37
+ request: LoadedRequest<CrawlingRequest<UserData>>;
30
38
  /**
31
39
  * The HTTP response, either from the HTTP client or from the initial request from playwright's navigation.
32
40
  */
@@ -36,10 +44,16 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
36
44
  */
37
45
  page: Page;
38
46
  /**
39
- * Wait for an element matching the selector to appear and return a Cheerio object of matched elements.
47
+ * Wait for an element matching the selector to appear and return a Cheerio object of all matched elements.
48
+ * Throws when nothing matches - in the HTTP-only run, that makes the crawler retry the request in a browser.
40
49
  * Timeout defaults to 5s.
41
50
  */
42
- querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
51
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
52
+ /**
53
+ * Like {@link AdaptivePlaywrightCrawlerContext.querySelectorAll|`querySelectorAll`}, but returns only the
54
+ * first match.
55
+ */
56
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
43
57
  /**
44
58
  * Wait for an element matching the selector to appear.
45
59
  * Timeout defaults to 5s.
@@ -66,38 +80,71 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
66
80
  * });
67
81
  * ```
68
82
  */
69
- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
70
- enqueueLinks(options?: EnqueueLinksOptions): Promise<void>;
83
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
84
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
71
85
  }
72
- interface AdaptiveHook extends BrowserHook<Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> & {
86
+ interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log' | 'extendTimeout'> {
73
87
  page?: Page;
74
- request: Request;
75
- }, PlaywrightGotoOptions> {
88
+ request: CrawlingRequest;
89
+ gotoOptions?: PlaywrightGotoOptions;
76
90
  }
77
- export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {
91
+ type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHookContext, ContextExtension>;
92
+ type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
93
+ request: LoadedRequest<CrawlingRequest>;
94
+ }, ContextExtension>;
95
+ /** Passed to {@link AdaptivePlaywrightCrawlerOptions.resultChecker} and `resultComparator` alongside the results. */
96
+ export interface AdaptiveResultContext {
97
+ /** The request whose handler produced the results. */
98
+ request: CrawlingRequest;
99
+ }
100
+ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks' | 'contextPipelineBuilder'>, Pick<PlaywrightCrawlerOptions, 'launchContext' | 'headless' | 'browserPool' | 'remoteBrowser'> {
101
+ /**
102
+ * Timeout for the navigation phase of each attempt, in seconds, shared by the `preNavigationHooks`, the
103
+ * navigation and the `postNavigationHooks`. Applies to both the HTTP-only and the browser attempt. When not
104
+ * set, each attempt keeps its own default: 30 seconds for HTTP-only and 60 seconds for the browser.
105
+ */
106
+ navigationTimeoutSecs?: number;
78
107
  /**
79
108
  * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
80
109
  * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
81
110
  * an exception will be thrown. If it's not caught, the request will be transparently retried in a browser.
111
+ *
112
+ * A hook may optionally return a partial object whose properties are merged into the crawling context,
113
+ * allowing the hook to override context members for subsequent hooks and pipeline stages.
82
114
  */
83
- preNavigationHooks?: AdaptiveHook[];
115
+ preNavigationHooks?: AdaptiveHook<ContextExtension>[];
84
116
  /**
85
117
  * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
86
118
  * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
87
119
  * an exception will be thrown. If it's not caught, the request will be transparently retried in a browser.
120
+ *
121
+ * A hook may optionally return a partial object whose properties are merged into the crawling context
122
+ * (e.g. to override `response` after solving a challenge).
88
123
  */
89
- postNavigationHooks?: AdaptiveHook[];
124
+ postNavigationHooks?: AdaptivePostNavigationHook<ContextExtension>[];
90
125
  /**
91
126
  * Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.
92
127
  * Defaults to 0.1 (so 10%).
93
128
  */
94
129
  renderingTypeDetectionRatio?: number;
95
130
  /**
96
- * An optional callback that is called on dataset items found by the request handler in plain HTTP mode.
131
+ * An optional callback that is called on the storage writes recorded by the request handler in plain
132
+ * HTTP mode (exposed as a read-only {@link StorageTransactionView}).
97
133
  * If it returns false, the request is retried in a browser.
98
- * If no callback is specified, every dataset item is considered valid.
134
+ * If no callback is specified, every result is considered valid.
99
135
  */
100
- resultChecker?: (result: RequestHandlerResult) => boolean;
136
+ resultChecker?: (result: StorageTransactionView, context: AdaptiveResultContext) => boolean;
137
+ /**
138
+ * An optional callback that decides whether an error thrown during the plain HTTP request handler
139
+ * should be propagated (instead of falling back to browser navigation).
140
+ *
141
+ * If the callback returns `true`, the error is thrown, triggering the standard retry mechanism.
142
+ * If the callback returns `false` (or is not provided), the error is logged and the crawler
143
+ * falls back to browser navigation (default behavior).
144
+ *
145
+ * @default () => false
146
+ */
147
+ shouldPropagateError?: (error: Error, context: PlaywrightCrawlingContext) => Awaitable<boolean>;
101
148
  /**
102
149
  * An optional callback used in rendering type detection. On each detection, the result of the plain HTTP run is compared to that of the browser one.
103
150
  * If a callback is provided, the contract is as follows:
@@ -106,17 +153,16 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
106
153
  * If it returns 'inconclusive', the detection result won't be used.
107
154
  * If no result comparator is specified, but there is a `resultChecker`, any site where the `resultChecker` returns true is considered static.
108
155
  * If neither `resultComparator` nor `resultChecker` are specified, a deep comparison of returned dataset items is used as a default.
156
+ *
157
+ * For a stricter, ready-made comparator that also takes enqueued requests and key-value store changes into account, see {@link fullResultComparator}.
109
158
  */
110
- resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean | 'equal' | 'different' | 'inconclusive';
111
- /**
112
- * A custom rendering type predictor
113
- */
114
- renderingTypePredictor?: Pick<RenderingTypePredictor, 'predict' | 'storeResult' | 'initialize'>;
159
+ resultComparator?: (resultA: StorageTransactionView, resultB: StorageTransactionView, context: AdaptiveResultContext) => boolean | 'equal' | 'different' | 'inconclusive';
115
160
  /**
116
- * Prevent direct access to storage in request handlers (only allow using context helpers).
117
- * Defaults to `true`
161
+ * A custom rendering type predictor. A predictor passed here is borrowed - the crawler never drives its
162
+ * lifecycle, so set it up yourself (the built-in {@link RenderingTypePredictor} needs `initialize()`).
163
+ * Omit the option and the crawler builds its own from `renderingTypeDetectionRatio` - and initializes it.
118
164
  */
119
- preventDirectStorageAccess?: boolean;
165
+ renderingTypePredictor?: IRenderingTypePredictor;
120
166
  }
121
167
  /**
122
168
  * An extension of {@link PlaywrightCrawler} that uses a more limited request handler interface so that it is able to switch to HTTP-only crawling when it detects it may be possible.
@@ -126,9 +172,9 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
126
172
  * ```javascript
127
173
  * const crawler = new AdaptivePlaywrightCrawler({
128
174
  * renderingTypeDetectionRatio: 0.1,
129
- * async requestHandler({ querySelector, pushData, enqueueLinks, request, log }) {
175
+ * async requestHandler({ querySelectorAll, pushData, enqueueLinks, request, log }) {
130
176
  * // This function is called to extract data from a single web page
131
- * const $prices = await querySelector('span.price')
177
+ * const $prices = await querySelectorAll('span.price')
132
178
  *
133
179
  * await pushData({
134
180
  * url: request.url,
@@ -144,34 +190,58 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
144
190
  * 'http://www.example.com/page-2',
145
191
  * ]);
146
192
  * ```
147
- *
148
- * @experimental
149
193
  */
150
- export declare class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ExtendedContext> {
151
- readonly config: Configuration;
152
- private renderingTypePredictor;
153
- private resultChecker;
154
- private resultComparator;
155
- private preventDirectStorageAccess;
156
- private staticContextPipeline;
157
- private browserContextPipeline;
158
- private individualRequestHandlerTimeoutMillis;
159
- readonly stats: AdaptivePlaywrightCrawlerStatistics;
160
- private resultObjects;
161
- private teardownHooks;
162
- constructor(options?: AdaptivePlaywrightCrawlerOptions<ExtendedContext>, config?: Configuration);
163
- protected _init(): Promise<void>;
164
- private adaptCheerioContext;
165
- private adaptPlaywrightContext;
166
- private crawlOne;
194
+ export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
195
+ #private;
196
+ constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
197
+ protected init(): Promise<void>;
167
198
  protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
168
- protected commitResult(crawlingContext: CrawlingContext, { calls, keyValueStoreChanges }: RequestHandlerResult): Promise<void>;
169
- protected allowStorageAccess<R, TArgs extends any[]>(func: (...args: TArgs) => Promise<R>): (...args: TArgs) => Promise<R>;
170
- protected enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: RestrictedCrawlingContext['request'], result: RequestHandlerResult): Promise<BatchAddRequestsResult>;
171
- private createLogProxy;
199
+ /**
200
+ * Number of rendering type detections that have not settled yet, including results the predictor is
201
+ * still persisting.
202
+ */
203
+ get inFlightRenderingTypeDetectionCount(): number;
204
+ /**
205
+ * Waits for in-flight rendering type detections to settle, bounded by `timeoutMillis` (defaults to the
206
+ * internal timeout).
207
+ */
208
+ drainRenderingDetections({ timeoutMillis }?: {
209
+ timeoutMillis?: number;
210
+ }): Promise<void>;
211
+ /**
212
+ * Stops the crawler immediately, but not before rendering type detections already under way (and results
213
+ * the predictor is still persisting) have settled - see
214
+ * {@link AdaptivePlaywrightCrawler.drainRenderingDetections|`drainRenderingDetections()`}. Requests
215
+ * that are still running are not waited for, unlike {@link BasicCrawler.stop|`stop()`}.
216
+ */
172
217
  teardown(): Promise<void>;
218
+ destroy(): Promise<void>;
173
219
  }
174
- // @ts-ignore optional peer dependency or compatibility with es2022
175
- export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context>;
220
+ export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
221
+ export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
222
+ export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
223
+ /**
224
+ * An opt-in {@link AdaptivePlaywrightCrawlerOptions.resultComparator|`resultComparator`} that considers two
225
+ * request handler results equal only if *all* of their observable effects match - the pushed dataset items, the
226
+ * enqueued requests, and the key-value store changes. This is stricter than the default comparator, which only
227
+ * compares dataset items.
228
+ *
229
+ * **Beware:** enqueued URLs are compared exactly. The same page rendered in a browser and via plain HTTP often
230
+ * yields links that differ only in tracking query parameters, for example:
231
+ * - `https://sdk.apify.com/docs/guides/getting-started`
232
+ * - `https://sdk.apify.com/docs/guides/getting-started?__hsfp=1136113150&__hssc=7591405.1.173549427712`
233
+ *
234
+ * Such links are treated as *different*, which will make the crawler favor browser rendering for those pages.
235
+ *
236
+ * **Example usage:**
237
+ * ```ts
238
+ * const crawler = new AdaptivePlaywrightCrawler({
239
+ * resultComparator: fullResultComparator,
240
+ * async requestHandler({ pushData, enqueueLinks }) {
241
+ * // ...
242
+ * },
243
+ * });
244
+ * ```
245
+ */
246
+ export declare function fullResultComparator(resultA: StorageTransactionView, resultB: StorageTransactionView): boolean;
176
247
  export {};
177
- //# sourceMappingURL=adaptive-playwright-crawler.d.ts.map