@crawlee/puppeteer 4.0.0-beta.25 → 4.0.0-beta.250
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +4 -6
- package/index.js +2 -4
- package/internals/enqueue-links/click-elements.d.ts +45 -72
- package/internals/enqueue-links/click-elements.js +66 -67
- package/internals/puppeteer-browser-pool.d.ts +44 -0
- package/internals/puppeteer-browser-pool.js +34 -0
- package/internals/puppeteer-crawler.d.ts +164 -120
- package/internals/puppeteer-crawler.js +72 -77
- package/internals/puppeteer-launcher.d.ts +31 -19
- package/internals/puppeteer-launcher.js +20 -15
- package/internals/utils/{puppeteer_request_interception.d.ts → puppeteer-request-interception.d.ts} +0 -1
- package/internals/utils/{puppeteer_request_interception.js → puppeteer-request-interception.js} +8 -8
- package/internals/utils/{puppeteer_utils.d.ts → puppeteer-utils.d.ts} +29 -79
- package/internals/utils/{puppeteer_utils.js → puppeteer-utils.js} +83 -182
- package/package.json +15 -19
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/enqueue-links/click-elements.d.ts.map +0 -1
- package/internals/enqueue-links/click-elements.js.map +0 -1
- package/internals/puppeteer-crawler.d.ts.map +0 -1
- package/internals/puppeteer-crawler.js.map +0 -1
- package/internals/puppeteer-launcher.d.ts.map +0 -1
- package/internals/puppeteer-launcher.js.map +0 -1
- package/internals/utils/puppeteer_request_interception.d.ts.map +0 -1
- package/internals/utils/puppeteer_request_interception.js.map +0 -1
- package/internals/utils/puppeteer_utils.d.ts.map +0 -1
- package/internals/utils/puppeteer_utils.js.map +0 -1
|
@@ -3,6 +3,7 @@ import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
|
3
3
|
import { PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
4
4
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
5
5
|
import type { Browser } from 'puppeteer';
|
|
6
|
+
import { z } from 'zod';
|
|
6
7
|
/**
|
|
7
8
|
* Apify extends the launch options of Puppeteer.
|
|
8
9
|
* You can use any of the Puppeteer compatible
|
|
@@ -66,30 +67,42 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
|
|
|
66
67
|
* @ignore
|
|
67
68
|
*/
|
|
68
69
|
export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
|
|
69
|
-
readonly
|
|
70
|
+
readonly configuration: Configuration;
|
|
71
|
+
/**
|
|
72
|
+
* @internal
|
|
73
|
+
*/
|
|
70
74
|
protected static optionsShape: {
|
|
75
|
+
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
76
|
+
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
77
|
+
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
78
|
+
browserPerProxy: z.ZodOptional<z.ZodBoolean>;
|
|
79
|
+
ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
|
|
80
|
+
userDataDir: z.ZodOptional<z.ZodString>;
|
|
71
81
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
75
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
76
|
-
useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
82
|
+
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
83
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
77
84
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
85
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
86
|
+
};
|
|
87
|
+
/** @internal */
|
|
88
|
+
protected static optionsSchema: z.ZodObject<{
|
|
89
|
+
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
90
|
+
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
91
|
+
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
92
|
+
browserPerProxy: z.ZodOptional<z.ZodBoolean>;
|
|
93
|
+
ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
|
|
94
|
+
userDataDir: z.ZodOptional<z.ZodString>;
|
|
83
95
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
84
|
-
launchOptions: import("
|
|
96
|
+
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
97
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
85
98
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
86
|
-
|
|
87
|
-
}
|
|
99
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
100
|
+
}, z.core.$strict>;
|
|
88
101
|
/**
|
|
89
102
|
* All `PuppeteerLauncher` parameters are passed via an launchContext object.
|
|
90
103
|
*/
|
|
91
|
-
constructor(launchContext?: PuppeteerLaunchContext,
|
|
92
|
-
protected
|
|
104
|
+
constructor(launchContext?: PuppeteerLaunchContext, configuration?: Configuration);
|
|
105
|
+
protected getDefaultHeadlessOption(): boolean;
|
|
93
106
|
}
|
|
94
107
|
/**
|
|
95
108
|
* Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
|
|
@@ -119,9 +132,8 @@ export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin,
|
|
|
119
132
|
* @param [launchContext]
|
|
120
133
|
* All `PuppeteerLauncher` parameters are passed via an launchContext object.
|
|
121
134
|
* If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions` property.
|
|
122
|
-
* @param [
|
|
135
|
+
* @param [configuration]
|
|
123
136
|
* @returns
|
|
124
137
|
* Promise that resolves to Puppeteer's `Browser` instance.
|
|
125
138
|
*/
|
|
126
|
-
export declare function launchPuppeteer(launchContext?: PuppeteerLaunchContext,
|
|
127
|
-
//# sourceMappingURL=puppeteer-launcher.d.ts.map
|
|
139
|
+
export declare function launchPuppeteer(launchContext?: PuppeteerLaunchContext, configuration?: Configuration): Promise<Browser>;
|
|
@@ -1,31 +1,37 @@
|
|
|
1
1
|
import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
2
|
-
import { PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
3
|
-
import
|
|
2
|
+
import { PuppeteerPlugin, RemotePuppeteerPlugin } from '@crawlee/browser-pool';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
4
|
+
import { z } from 'zod';
|
|
4
5
|
/**
|
|
5
6
|
* `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
|
|
6
7
|
* @ignore
|
|
7
8
|
*/
|
|
8
9
|
export class PuppeteerLauncher extends BrowserLauncher {
|
|
9
|
-
|
|
10
|
+
configuration;
|
|
11
|
+
/**
|
|
12
|
+
* @internal
|
|
13
|
+
*/
|
|
10
14
|
static optionsShape = {
|
|
11
15
|
...BrowserLauncher.optionsShape,
|
|
12
|
-
launcher:
|
|
16
|
+
launcher: schemas.anyObject.optional(),
|
|
13
17
|
};
|
|
18
|
+
/** @internal */
|
|
19
|
+
static optionsSchema = z.strictObject(PuppeteerLauncher.optionsShape);
|
|
14
20
|
/**
|
|
15
21
|
* All `PuppeteerLauncher` parameters are passed via an launchContext object.
|
|
16
22
|
*/
|
|
17
|
-
constructor(launchContext = {},
|
|
18
|
-
|
|
19
|
-
const { launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = launchContext;
|
|
23
|
+
constructor(launchContext = {}, configuration = Configuration.getGlobalConfiguration()) {
|
|
24
|
+
const { launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = parseArgument(launchContext, PuppeteerLauncher.optionsSchema, 'PuppeteerLaunchContext');
|
|
20
25
|
super({
|
|
21
26
|
...browserLauncherOptions,
|
|
22
27
|
launcher,
|
|
23
|
-
},
|
|
24
|
-
this.
|
|
28
|
+
}, configuration);
|
|
29
|
+
this.configuration = configuration;
|
|
25
30
|
this.Plugin = PuppeteerPlugin;
|
|
31
|
+
this.RemotePlugin = RemotePuppeteerPlugin;
|
|
26
32
|
}
|
|
27
|
-
|
|
28
|
-
const headless = super.
|
|
33
|
+
getDefaultHeadlessOption() {
|
|
34
|
+
const headless = super.getDefaultHeadlessOption();
|
|
29
35
|
return headless ? 'new' : headless;
|
|
30
36
|
}
|
|
31
37
|
}
|
|
@@ -57,12 +63,11 @@ export class PuppeteerLauncher extends BrowserLauncher {
|
|
|
57
63
|
* @param [launchContext]
|
|
58
64
|
* All `PuppeteerLauncher` parameters are passed via an launchContext object.
|
|
59
65
|
* If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions` property.
|
|
60
|
-
* @param [
|
|
66
|
+
* @param [configuration]
|
|
61
67
|
* @returns
|
|
62
68
|
* Promise that resolves to Puppeteer's `Browser` instance.
|
|
63
69
|
*/
|
|
64
|
-
export async function launchPuppeteer(launchContext,
|
|
65
|
-
const puppeteerLauncher = new PuppeteerLauncher(launchContext,
|
|
70
|
+
export async function launchPuppeteer(launchContext, configuration = Configuration.getGlobalConfiguration()) {
|
|
71
|
+
const puppeteerLauncher = new PuppeteerLauncher(launchContext, configuration);
|
|
66
72
|
return puppeteerLauncher.launch();
|
|
67
73
|
}
|
|
68
|
-
//# sourceMappingURL=puppeteer-launcher.js.map
|
package/internals/utils/{puppeteer_request_interception.d.ts → puppeteer-request-interception.d.ts}
RENAMED
|
@@ -58,4 +58,3 @@ export declare function addInterceptRequestHandler(page: Page, handler: Intercep
|
|
|
58
58
|
* @param handler Request interception handler.
|
|
59
59
|
*/
|
|
60
60
|
export declare function removeInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
|
|
61
|
-
//# sourceMappingURL=puppeteer_request_interception.d.ts.map
|
package/internals/utils/{puppeteer_request_interception.js → puppeteer-request-interception.js}
RENAMED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
-
import
|
|
3
|
-
import
|
|
2
|
+
import { serviceLocator } from '@crawlee/browser';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
4
|
+
const pageSchema = schemas.objectWithKeys(['goto', 'evaluate']);
|
|
4
5
|
// We use weak maps here so that the content gets discarded after page gets closed.
|
|
5
6
|
const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
|
|
6
7
|
const pageInterceptRequestMasterHandlerMap = new WeakMap(); // Maps page to master request interception handler.
|
|
@@ -137,8 +138,8 @@ async function handleRequest(request, interceptRequestHandlers) {
|
|
|
137
138
|
* @param handler Request interception handler.
|
|
138
139
|
*/
|
|
139
140
|
export async function addInterceptRequestHandler(page, handler) {
|
|
140
|
-
|
|
141
|
-
|
|
141
|
+
parseArgument(page, pageSchema);
|
|
142
|
+
parseArgument(handler, schemas.anyFunction);
|
|
142
143
|
if (!pageInterceptRequestHandlersMap.has(page)) {
|
|
143
144
|
pageInterceptRequestHandlersMap.set(page, []);
|
|
144
145
|
}
|
|
@@ -174,8 +175,8 @@ export async function addInterceptRequestHandler(page, handler) {
|
|
|
174
175
|
* @param handler Request interception handler.
|
|
175
176
|
*/
|
|
176
177
|
export async function removeInterceptRequestHandler(page, handler) {
|
|
177
|
-
|
|
178
|
-
|
|
178
|
+
parseArgument(page, pageSchema);
|
|
179
|
+
parseArgument(handler, schemas.anyFunction);
|
|
179
180
|
const handlersArray = pageInterceptRequestHandlersMap.get(page).filter((item) => item !== handler);
|
|
180
181
|
pageInterceptRequestHandlersMap.set(page, handlersArray);
|
|
181
182
|
if (handlersArray.length === 0) {
|
|
@@ -194,7 +195,7 @@ export async function removeInterceptRequestHandler(page, handler) {
|
|
|
194
195
|
interceptedRequestsInProgress.removeListener('delete', onDelete);
|
|
195
196
|
}
|
|
196
197
|
catch (error) {
|
|
197
|
-
|
|
198
|
+
serviceLocator.getLogger().debug('Error while disabling request interception', { error });
|
|
198
199
|
}
|
|
199
200
|
}
|
|
200
201
|
};
|
|
@@ -207,4 +208,3 @@ async function disableRequestInterception(page) {
|
|
|
207
208
|
const requestHandler = pageInterceptRequestMasterHandlerMap.get(page);
|
|
208
209
|
page.off('request', requestHandler);
|
|
209
210
|
}
|
|
210
|
-
//# sourceMappingURL=puppeteer_request_interception.js.map
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* **Example usage:**
|
|
6
6
|
*
|
|
7
7
|
* ```javascript
|
|
8
|
-
* import { launchPuppeteer,
|
|
8
|
+
* import { launchPuppeteer, puppeteerUtils } from 'crawlee';
|
|
9
9
|
*
|
|
10
10
|
* // Open https://www.example.com in Puppeteer
|
|
11
11
|
* const browser = await launchPuppeteer();
|
|
@@ -13,21 +13,21 @@
|
|
|
13
13
|
* await page.goto('https://www.example.com');
|
|
14
14
|
*
|
|
15
15
|
* // Inject jQuery into a page
|
|
16
|
-
* await
|
|
16
|
+
* await puppeteerUtils.injectJQuery(page);
|
|
17
17
|
* ```
|
|
18
18
|
* @module puppeteerUtils
|
|
19
19
|
*/
|
|
20
20
|
import type { Request } from '@crawlee/browser';
|
|
21
21
|
import { Configuration } from '@crawlee/browser';
|
|
22
22
|
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
23
|
-
import {
|
|
24
|
-
import type {
|
|
23
|
+
import type { Cheerio, CheerioAPI } from 'cheerio';
|
|
24
|
+
import type { AnyNode } from 'domhandler';
|
|
25
25
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
26
|
-
import type { HTTPResponse, Page
|
|
26
|
+
import type { HTTPResponse, Page } from 'puppeteer';
|
|
27
27
|
import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
|
|
28
28
|
import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
|
|
29
|
-
import type { InterceptHandler } from './
|
|
30
|
-
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './
|
|
29
|
+
import type { InterceptHandler } from './puppeteer-request-interception.js';
|
|
30
|
+
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer-request-interception.js';
|
|
31
31
|
export interface DirectNavigationOptions {
|
|
32
32
|
/**
|
|
33
33
|
* Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The
|
|
@@ -101,7 +101,7 @@ export declare function injectFile(page: Page, filePath: string, options?: Injec
|
|
|
101
101
|
*
|
|
102
102
|
* **Example usage:**
|
|
103
103
|
* ```javascript
|
|
104
|
-
* await
|
|
104
|
+
* await puppeteerUtils.injectJQuery(page);
|
|
105
105
|
* const title = await page.evaluate(() => {
|
|
106
106
|
* return $('head title').text();
|
|
107
107
|
* });
|
|
@@ -122,14 +122,14 @@ export declare function injectJQuery(page: Page, options?: {
|
|
|
122
122
|
*
|
|
123
123
|
* **Example usage:**
|
|
124
124
|
* ```javascript
|
|
125
|
-
* const $ = await
|
|
125
|
+
* const $ = await puppeteerUtils.parseWithCheerio(page);
|
|
126
126
|
* const title = $('title').text();
|
|
127
127
|
* ```
|
|
128
128
|
*
|
|
129
129
|
* @param page Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
|
|
130
130
|
* @param ignoreShadowRoots
|
|
131
131
|
*/
|
|
132
|
-
export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<
|
|
132
|
+
export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;
|
|
133
133
|
/**
|
|
134
134
|
* Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
|
|
135
135
|
* This is useful to speed up crawling of websites, since it reduces the amount
|
|
@@ -156,13 +156,13 @@ export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean
|
|
|
156
156
|
*
|
|
157
157
|
* **Example usage**
|
|
158
158
|
* ```javascript
|
|
159
|
-
* import { launchPuppeteer,
|
|
159
|
+
* import { launchPuppeteer, puppeteerUtils } from 'crawlee';
|
|
160
160
|
*
|
|
161
161
|
* const browser = await launchPuppeteer();
|
|
162
162
|
* const page = await browser.newPage();
|
|
163
163
|
*
|
|
164
164
|
* // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
|
|
165
|
-
* await
|
|
165
|
+
* await puppeteerUtils.blockRequests(page, {
|
|
166
166
|
* extraUrlPatterns: ['adsbygoogle.js'],
|
|
167
167
|
* });
|
|
168
168
|
*
|
|
@@ -173,33 +173,6 @@ export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean
|
|
|
173
173
|
* @param [options]
|
|
174
174
|
*/
|
|
175
175
|
export declare function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;
|
|
176
|
-
/**
|
|
177
|
-
* @internal
|
|
178
|
-
*/
|
|
179
|
-
export declare function sendCDPCommand<T extends keyof ProtocolMapping.Commands>(page: Page, command: T, ...args: ProtocolMapping.Commands[T]['paramsType']): Promise<ProtocolMapping.Commands[T]['returnType']>;
|
|
180
|
-
/**
|
|
181
|
-
* `blockResources()` has a high impact on performance in recent versions of Puppeteer.
|
|
182
|
-
* Until this resolves, please use `utils.puppeteer.blockRequests()`.
|
|
183
|
-
* @deprecated
|
|
184
|
-
*/
|
|
185
|
-
export declare const blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
|
|
186
|
-
/**
|
|
187
|
-
* *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
|
|
188
|
-
* performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
|
|
189
|
-
*
|
|
190
|
-
* Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
|
|
191
|
-
* *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
|
|
192
|
-
* This issue should be resolved or atleast mitigated in future iterations of this feature.
|
|
193
|
-
* @param page
|
|
194
|
-
* Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
|
|
195
|
-
* @param cache
|
|
196
|
-
* Object in which responses are stored
|
|
197
|
-
* @param responseUrlRules
|
|
198
|
-
* List of rules that are used to check if the response should be cached.
|
|
199
|
-
* String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
|
|
200
|
-
* @deprecated
|
|
201
|
-
*/
|
|
202
|
-
export declare function cacheResponses(page: Page, cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
|
|
203
176
|
/**
|
|
204
177
|
* Compiles a Puppeteer script into an async function that may be executed at any time
|
|
205
178
|
* by providing it with the following object:
|
|
@@ -305,9 +278,9 @@ export interface SaveSnapshotOptions {
|
|
|
305
278
|
keyValueStoreName?: string | null;
|
|
306
279
|
/**
|
|
307
280
|
* Configuration of the crawler that will be used to save the snapshot.
|
|
308
|
-
* @default Configuration.
|
|
281
|
+
* @default Configuration.getGlobalConfiguration()
|
|
309
282
|
*/
|
|
310
|
-
|
|
283
|
+
configuration?: Configuration;
|
|
311
284
|
}
|
|
312
285
|
/**
|
|
313
286
|
* Saves a full screenshot and HTML of the current page into a Key-Value store.
|
|
@@ -315,8 +288,6 @@ export interface SaveSnapshotOptions {
|
|
|
315
288
|
* @param [options]
|
|
316
289
|
*/
|
|
317
290
|
export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
|
|
318
|
-
export declare function closeCookieModals(page: Page): Promise<void>;
|
|
319
|
-
/** @internal */
|
|
320
291
|
export interface PuppeteerContextUtils {
|
|
321
292
|
/**
|
|
322
293
|
* Injects a JavaScript file into current `page`.
|
|
@@ -366,6 +337,15 @@ export interface PuppeteerContextUtils {
|
|
|
366
337
|
* ```
|
|
367
338
|
*/
|
|
368
339
|
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
340
|
+
/**
|
|
341
|
+
* Like {@link PuppeteerContextUtils.waitForSelector|`waitForSelector`}, but returns the matched elements
|
|
342
|
+
* as a Cheerio object. Throws when nothing matches.
|
|
343
|
+
*/
|
|
344
|
+
querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
345
|
+
/**
|
|
346
|
+
* Like {@link PuppeteerContextUtils.querySelectorAll|`querySelectorAll`}, but returns only the first match.
|
|
347
|
+
*/
|
|
348
|
+
querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
369
349
|
/**
|
|
370
350
|
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
|
|
371
351
|
* When provided with the `selector` argument, it waits for it to be available first.
|
|
@@ -378,7 +358,7 @@ export interface PuppeteerContextUtils {
|
|
|
378
358
|
* });
|
|
379
359
|
* ```
|
|
380
360
|
*/
|
|
381
|
-
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<
|
|
361
|
+
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
|
|
382
362
|
/**
|
|
383
363
|
* The function finds elements matching a specific CSS selector in a Puppeteer page,
|
|
384
364
|
* clicks all those elements using a mouse move and a left mouse button click and intercepts
|
|
@@ -388,8 +368,7 @@ export interface PuppeteerContextUtils {
|
|
|
388
368
|
* in `href` elements, but rather navigations are triggered in click handlers.
|
|
389
369
|
* If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
|
|
390
370
|
*
|
|
391
|
-
* Optionally, the function allows you to filter the target links' URLs using an array of
|
|
392
|
-
* and override settings of the enqueued {@link Request} objects.
|
|
371
|
+
* Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
|
|
393
372
|
*
|
|
394
373
|
* **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
|
|
395
374
|
* such as changing the Z-index of elements being clicked and their visibility. Therefore,
|
|
@@ -410,9 +389,9 @@ export interface PuppeteerContextUtils {
|
|
|
410
389
|
* async requestHandler({ enqueueLinksByClickingElements }) {
|
|
411
390
|
* await enqueueLinksByClickingElements({
|
|
412
391
|
* selector: 'a.product-detail',
|
|
413
|
-
*
|
|
414
|
-
* 'https://www.example.com/handbags/**'
|
|
415
|
-
* 'https://www.example.com/purses/**'
|
|
392
|
+
* include: [
|
|
393
|
+
* 'https://www.example.com/handbags/**',
|
|
394
|
+
* 'https://www.example.com/purses/**',
|
|
416
395
|
* ],
|
|
417
396
|
* });
|
|
418
397
|
* });
|
|
@@ -420,7 +399,7 @@ export interface PuppeteerContextUtils {
|
|
|
420
399
|
*
|
|
421
400
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
422
401
|
*/
|
|
423
|
-
enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | '
|
|
402
|
+
enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
|
|
424
403
|
/**
|
|
425
404
|
* Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
|
|
426
405
|
* This is useful to speed up crawling of websites, since it reduces the amount
|
|
@@ -550,34 +529,5 @@ export interface PuppeteerContextUtils {
|
|
|
550
529
|
* Saves a full screenshot and HTML of the current page into a Key-Value store.
|
|
551
530
|
*/
|
|
552
531
|
saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
|
|
553
|
-
/**
|
|
554
|
-
* Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
|
|
555
|
-
*
|
|
556
|
-
* Note that this method requires the idcac-playwright package to be installed.
|
|
557
|
-
* Crawlee does not include it by default due to licensing issues.
|
|
558
|
-
*
|
|
559
|
-
* To use this method, please install the package manually by running:
|
|
560
|
-
*
|
|
561
|
-
* ```bash
|
|
562
|
-
* npm install idcac-playwright
|
|
563
|
-
* ```
|
|
564
|
-
*/
|
|
565
|
-
closeCookieModals(): Promise<void>;
|
|
566
532
|
}
|
|
567
533
|
export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler };
|
|
568
|
-
/** @internal */
|
|
569
|
-
export declare const puppeteerUtils: {
|
|
570
|
-
injectFile: typeof injectFile;
|
|
571
|
-
injectJQuery: typeof injectJQuery;
|
|
572
|
-
enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
|
|
573
|
-
blockRequests: typeof blockRequests;
|
|
574
|
-
compileScript: typeof compileScript;
|
|
575
|
-
gotoExtended: typeof gotoExtended;
|
|
576
|
-
addInterceptRequestHandler: typeof addInterceptRequestHandler;
|
|
577
|
-
removeInterceptRequestHandler: typeof removeInterceptRequestHandler;
|
|
578
|
-
infiniteScroll: typeof infiniteScroll;
|
|
579
|
-
saveSnapshot: typeof saveSnapshot;
|
|
580
|
-
parseWithCheerio: typeof parseWithCheerio;
|
|
581
|
-
closeCookieModals: typeof closeCookieModals;
|
|
582
|
-
};
|
|
583
|
-
//# sourceMappingURL=puppeteer_utils.d.ts.map
|