@crawlee/browser 4.0.0-beta.210 → 4.0.0-beta.212
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -381,35 +381,14 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
|
|
|
381
381
|
/** @internal */
|
|
382
382
|
protected getNavigationTimeoutMillis(): number;
|
|
383
383
|
protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
|
|
384
|
-
private containsSelectors;
|
|
385
|
-
private isRequestBlocked;
|
|
386
|
-
private preparePage;
|
|
387
|
-
private prepareNavigation;
|
|
388
384
|
private navigate;
|
|
389
|
-
private finalizeNavigation;
|
|
390
|
-
/**
|
|
391
|
-
* Copies cookies from the live browser page into the session cookie jar.
|
|
392
|
-
*/
|
|
393
|
-
private persistCookiesFromPage;
|
|
394
385
|
/**
|
|
395
386
|
* Runs the user request handler, then re-reads browser cookies so login flows /
|
|
396
387
|
* `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
|
|
397
388
|
* @internal
|
|
398
389
|
*/
|
|
399
390
|
protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
|
|
400
|
-
private handleBlockedRequestByContent;
|
|
401
|
-
private restoreRequestState;
|
|
402
|
-
private applyCookies;
|
|
403
|
-
/**
|
|
404
|
-
* Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
|
|
405
|
-
*/
|
|
406
|
-
private handleNavigationTimeout;
|
|
407
|
-
/**
|
|
408
|
-
* Transforms proxy-related errors to `SessionError`.
|
|
409
|
-
*/
|
|
410
|
-
private throwIfProxyError;
|
|
411
391
|
protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
|
|
412
|
-
private processResponse;
|
|
413
392
|
/**
|
|
414
393
|
* Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
|
|
415
394
|
* crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
|
|
@@ -116,7 +116,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
116
116
|
}
|
|
117
117
|
return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
118
118
|
});
|
|
119
|
-
let pipeline = contextPipelineBuilder().compose(this
|
|
119
|
+
let pipeline = contextPipelineBuilder().compose(this.#prepareNavigation.bind(this));
|
|
120
120
|
for (const hook of this.#preNavigationHooks) {
|
|
121
121
|
pipeline = pipeline.compose(windowGuard(hook));
|
|
122
122
|
}
|
|
@@ -125,9 +125,9 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
125
125
|
pipeline = pipeline.compose(windowGuard(hook));
|
|
126
126
|
}
|
|
127
127
|
return pipeline
|
|
128
|
-
.compose(skipGuard(this
|
|
129
|
-
.compose(this
|
|
130
|
-
.compose(this
|
|
128
|
+
.compose(skipGuard(this.#finalizeNavigation.bind(this)))
|
|
129
|
+
.compose(this.#handleBlockedRequestByContent.bind(this))
|
|
130
|
+
.compose(this.#restoreRequestState.bind(this));
|
|
131
131
|
},
|
|
132
132
|
extendContext,
|
|
133
133
|
});
|
|
@@ -147,27 +147,27 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
147
147
|
return this.#navigationTimeoutMillis;
|
|
148
148
|
}
|
|
149
149
|
buildContextPipeline() {
|
|
150
|
-
return ContextPipeline.create().compose(this
|
|
150
|
+
return ContextPipeline.create().compose(this.#preparePage.bind(this));
|
|
151
151
|
}
|
|
152
|
-
async containsSelectors(page, selectors) {
|
|
152
|
+
async #containsSelectors(page, selectors) {
|
|
153
153
|
const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
|
|
154
154
|
.map((x, i) => [x, selectors[i]])
|
|
155
155
|
.filter(([x]) => x !== null)
|
|
156
156
|
.map(([, selector]) => selector);
|
|
157
157
|
return foundSelectors.length > 0 ? foundSelectors : null;
|
|
158
158
|
}
|
|
159
|
-
async isRequestBlocked(crawlingContext) {
|
|
159
|
+
async #isRequestBlocked(crawlingContext) {
|
|
160
160
|
const { page, response } = crawlingContext;
|
|
161
161
|
// Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
|
|
162
|
-
if ((await this
|
|
162
|
+
if ((await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
|
|
163
163
|
await sleep(5000);
|
|
164
164
|
// here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
|
|
165
|
-
const foundSelectors = await this
|
|
165
|
+
const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
166
166
|
if (!foundSelectors)
|
|
167
167
|
return false;
|
|
168
168
|
return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
|
|
169
169
|
}
|
|
170
|
-
const foundSelectors = await this
|
|
170
|
+
const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
171
171
|
const statusCode = response?.status() ?? 0;
|
|
172
172
|
if (foundSelectors)
|
|
173
173
|
return `Found selectors: ${foundSelectors.join(', ')}`;
|
|
@@ -175,7 +175,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
175
175
|
return `Received blocked status code: ${statusCode}`;
|
|
176
176
|
return false;
|
|
177
177
|
}
|
|
178
|
-
async preparePage(crawlingContext, onCleanup) {
|
|
178
|
+
async #preparePage(crawlingContext, onCleanup) {
|
|
179
179
|
const page = await this.browserPool.newPage({
|
|
180
180
|
id: crawlingContext.id,
|
|
181
181
|
session: crawlingContext.session,
|
|
@@ -232,16 +232,19 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
232
232
|
},
|
|
233
233
|
};
|
|
234
234
|
}
|
|
235
|
-
async prepareNavigation(crawlingContext) {
|
|
235
|
+
async #prepareNavigation(crawlingContext) {
|
|
236
236
|
if (crawlingContext.request.skipNavigation) {
|
|
237
237
|
return {
|
|
238
238
|
request: new Proxy(crawlingContext.request, {
|
|
239
|
-
get(target, propertyName
|
|
239
|
+
get(target, propertyName) {
|
|
240
240
|
if (propertyName === 'loadedUrl') {
|
|
241
241
|
throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
242
242
|
}
|
|
243
|
-
|
|
243
|
+
// `target` as receiver and bound methods, or `#` members throw on the proxy
|
|
244
|
+
const value = Reflect.get(target, propertyName, target);
|
|
245
|
+
return typeof value === 'function' ? value.bind(target) : value;
|
|
244
246
|
},
|
|
247
|
+
set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
|
|
245
248
|
}),
|
|
246
249
|
get response() {
|
|
247
250
|
throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
|
|
@@ -274,22 +277,22 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
274
277
|
}
|
|
275
278
|
const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
|
|
276
279
|
const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
|
|
277
|
-
await this
|
|
280
|
+
await this.#applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
|
|
278
281
|
let response;
|
|
279
282
|
try {
|
|
280
283
|
response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
|
|
281
284
|
}
|
|
282
285
|
catch (error) {
|
|
283
|
-
await this
|
|
286
|
+
await this.#handleNavigationTimeout(crawlingContext, error);
|
|
284
287
|
crawlingContext.request.state = RequestState.ERROR;
|
|
285
|
-
this
|
|
288
|
+
this.#throwIfProxyError(error);
|
|
286
289
|
throw error;
|
|
287
290
|
}
|
|
288
291
|
tryCancel();
|
|
289
292
|
crawlingContext.request.state = RequestState.AFTER_NAV;
|
|
290
293
|
return { response };
|
|
291
294
|
}
|
|
292
|
-
async finalizeNavigation(crawlingContext) {
|
|
295
|
+
async #finalizeNavigation(crawlingContext) {
|
|
293
296
|
tryCancel();
|
|
294
297
|
let response;
|
|
295
298
|
try {
|
|
@@ -299,17 +302,17 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
299
302
|
// `preparePage` installs a throwing getter for `response`; reaching this branch means
|
|
300
303
|
// navigation produced no response and no hook overrode it. Treat as undefined.
|
|
301
304
|
}
|
|
302
|
-
await this
|
|
305
|
+
await this.#processResponse(response, crawlingContext);
|
|
303
306
|
tryCancel();
|
|
304
307
|
// Persist cookies from the navigation response before the user handler runs.
|
|
305
308
|
// Cookies set during `requestHandler` are saved again afterwards.
|
|
306
|
-
await this
|
|
309
|
+
await this.#persistCookiesFromPage(crawlingContext);
|
|
307
310
|
return { request: crawlingContext.request };
|
|
308
311
|
}
|
|
309
312
|
/**
|
|
310
313
|
* Copies cookies from the live browser page into the session cookie jar.
|
|
311
314
|
*/
|
|
312
|
-
async persistCookiesFromPage(crawlingContext) {
|
|
315
|
+
async #persistCookiesFromPage(crawlingContext) {
|
|
313
316
|
if (!this.#saveResponseCookies || !crawlingContext.session) {
|
|
314
317
|
return;
|
|
315
318
|
}
|
|
@@ -340,7 +343,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
340
343
|
finally {
|
|
341
344
|
if (!crawlingContext.request.skipNavigation) {
|
|
342
345
|
try {
|
|
343
|
-
await this
|
|
346
|
+
await this.#persistCookiesFromPage(crawlingContext);
|
|
344
347
|
}
|
|
345
348
|
catch {
|
|
346
349
|
// Page may already be closed on some failure paths; ignore.
|
|
@@ -348,19 +351,19 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
348
351
|
}
|
|
349
352
|
}
|
|
350
353
|
}
|
|
351
|
-
async handleBlockedRequestByContent(crawlingContext) {
|
|
354
|
+
async #handleBlockedRequestByContent(crawlingContext) {
|
|
352
355
|
if (this.retryOnBlocked) {
|
|
353
|
-
const error = await this
|
|
356
|
+
const error = await this.#isRequestBlocked(crawlingContext);
|
|
354
357
|
if (error)
|
|
355
358
|
throw new SessionError(error);
|
|
356
359
|
}
|
|
357
360
|
return {};
|
|
358
361
|
}
|
|
359
|
-
async restoreRequestState(crawlingContext) {
|
|
362
|
+
async #restoreRequestState(crawlingContext) {
|
|
360
363
|
crawlingContext.request.state = RequestState.REQUEST_HANDLER;
|
|
361
364
|
return {};
|
|
362
365
|
}
|
|
363
|
-
async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
|
|
366
|
+
async #applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
|
|
364
367
|
const sessionCookie = session
|
|
365
368
|
? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
|
|
366
369
|
: [];
|
|
@@ -374,7 +377,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
374
377
|
/**
|
|
375
378
|
* Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
|
|
376
379
|
*/
|
|
377
|
-
async handleNavigationTimeout(crawlingContext, error) {
|
|
380
|
+
async #handleNavigationTimeout(crawlingContext, error) {
|
|
378
381
|
const { session, page } = crawlingContext;
|
|
379
382
|
// Fire-and-forget: no user code will run on this page after a failed navigation.
|
|
380
383
|
// Swallow rejections: the page may already be detached.
|
|
@@ -389,12 +392,12 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
389
392
|
/**
|
|
390
393
|
* Transforms proxy-related errors to `SessionError`.
|
|
391
394
|
*/
|
|
392
|
-
throwIfProxyError(error) {
|
|
395
|
+
#throwIfProxyError(error) {
|
|
393
396
|
if (this.isProxyError(error)) {
|
|
394
397
|
throw new SessionError(this.getMessageFromError(error));
|
|
395
398
|
}
|
|
396
399
|
}
|
|
397
|
-
async processResponse(response, crawlingContext) {
|
|
400
|
+
async #processResponse(response, crawlingContext) {
|
|
398
401
|
const { session, request, page } = crawlingContext;
|
|
399
402
|
if (typeof response === 'object' && typeof response.status === 'function') {
|
|
400
403
|
const status = response.status();
|
|
@@ -84,6 +84,7 @@ type LauncherRemoteBrowserPoolOptions = Omit<RemoteBrowserPoolOptions, 'browserP
|
|
|
84
84
|
* @ignore
|
|
85
85
|
*/
|
|
86
86
|
export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Launcher = Plugin['library'], T extends Constructor<Plugin> = Constructor<Plugin>, LaunchOptions extends Dictionary<any> | undefined = Partial<Parameters<Plugin['launch']>[0]>, LaunchResult extends ReturnType<Plugin['launch']> = ReturnType<Plugin['launch']>> {
|
|
87
|
+
#private;
|
|
87
88
|
readonly configuration: Configuration;
|
|
88
89
|
launcher: Launcher;
|
|
89
90
|
proxyUrl?: string;
|
|
@@ -140,11 +141,6 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
140
141
|
* @internal
|
|
141
142
|
*/
|
|
142
143
|
createRemoteBrowserPool<Page>(options: LauncherRemoteBrowserPoolOptions): RemoteBrowserPool<Page>;
|
|
143
|
-
/**
|
|
144
|
-
* A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
|
|
145
|
-
* explicitly requested user agent wins.
|
|
146
|
-
*/
|
|
147
|
-
private resolveFingerprinting;
|
|
148
144
|
/**
|
|
149
145
|
* Launches a browser instance based on the plugin.
|
|
150
146
|
* @returns Browser instance.
|
|
@@ -152,11 +148,5 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
152
148
|
launch(): LaunchResult;
|
|
153
149
|
createLaunchOptions(): Dictionary;
|
|
154
150
|
protected getDefaultHeadlessOption(): boolean;
|
|
155
|
-
private getChromeExecutablePath;
|
|
156
|
-
/**
|
|
157
|
-
* Gets a typical path to Chrome executable, depending on the current operating system.
|
|
158
|
-
*/
|
|
159
|
-
private getTypicalChromeExecutablePath;
|
|
160
|
-
private validateProxyUrlProtocol;
|
|
161
151
|
}
|
|
162
152
|
export {};
|
|
@@ -61,7 +61,7 @@ export class BrowserLauncher {
|
|
|
61
61
|
constructor(launchContext, configuration = Configuration.getGlobalConfiguration()) {
|
|
62
62
|
this.configuration = configuration;
|
|
63
63
|
const { launcher, proxyUrl, useChrome, userAgent, launchOptions = {}, ...otherLaunchContextProps } = launchContext;
|
|
64
|
-
this
|
|
64
|
+
this.#validateProxyUrlProtocol(proxyUrl);
|
|
65
65
|
// those need to be reassigned otherwise they are {} in types
|
|
66
66
|
this.launcher = launcher;
|
|
67
67
|
this.proxyUrl = proxyUrl;
|
|
@@ -91,7 +91,7 @@ export class BrowserLauncher {
|
|
|
91
91
|
// parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the
|
|
92
92
|
// caller-facing hook types get pinned down.
|
|
93
93
|
return new BrowserPool({
|
|
94
|
-
...this
|
|
94
|
+
...this.#resolveFingerprinting(options),
|
|
95
95
|
browserPlugins: [this.createBrowserPlugin()],
|
|
96
96
|
});
|
|
97
97
|
}
|
|
@@ -104,14 +104,14 @@ export class BrowserLauncher {
|
|
|
104
104
|
return new RemoteBrowserPool({
|
|
105
105
|
...options,
|
|
106
106
|
browserPlugins: [this.createBrowserPlugin()],
|
|
107
|
-
browserPoolOptions: this
|
|
107
|
+
browserPoolOptions: this.#resolveFingerprinting(options.browserPoolOptions ?? {}),
|
|
108
108
|
});
|
|
109
109
|
}
|
|
110
110
|
/**
|
|
111
111
|
* A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
|
|
112
112
|
* explicitly requested user agent wins.
|
|
113
113
|
*/
|
|
114
|
-
resolveFingerprinting(options) {
|
|
114
|
+
#resolveFingerprinting(options) {
|
|
115
115
|
if (!this.userAgent) {
|
|
116
116
|
return options;
|
|
117
117
|
}
|
|
@@ -148,20 +148,20 @@ export class BrowserLauncher {
|
|
|
148
148
|
launchOptions.headless = this.getDefaultHeadlessOption();
|
|
149
149
|
}
|
|
150
150
|
if (this.useChrome && !launchOptions.executablePath) {
|
|
151
|
-
launchOptions.executablePath = this
|
|
151
|
+
launchOptions.executablePath = this.#getChromeExecutablePath();
|
|
152
152
|
}
|
|
153
153
|
return launchOptions;
|
|
154
154
|
}
|
|
155
155
|
getDefaultHeadlessOption() {
|
|
156
156
|
return this.configuration.headless && !this.configuration.xvfb;
|
|
157
157
|
}
|
|
158
|
-
getChromeExecutablePath() {
|
|
159
|
-
return this.configuration.chromeExecutablePath ?? this
|
|
158
|
+
#getChromeExecutablePath() {
|
|
159
|
+
return this.configuration.chromeExecutablePath ?? this.#getTypicalChromeExecutablePath();
|
|
160
160
|
}
|
|
161
161
|
/**
|
|
162
162
|
* Gets a typical path to Chrome executable, depending on the current operating system.
|
|
163
163
|
*/
|
|
164
|
-
getTypicalChromeExecutablePath() {
|
|
164
|
+
#getTypicalChromeExecutablePath() {
|
|
165
165
|
/**
|
|
166
166
|
* Returns path of Chrome executable by its OS environment variable to deal with non-english language OS.
|
|
167
167
|
* Taking also into account the old [chrome 380177 issue](https://bugs.chromium.org/p/chromium/issues/detail?id=380177).
|
|
@@ -189,7 +189,7 @@ export class BrowserLauncher {
|
|
|
189
189
|
return '/usr/bin/google-chrome';
|
|
190
190
|
}
|
|
191
191
|
}
|
|
192
|
-
validateProxyUrlProtocol(proxyUrl) {
|
|
192
|
+
#validateProxyUrlProtocol(proxyUrl) {
|
|
193
193
|
if (!proxyUrl)
|
|
194
194
|
return;
|
|
195
195
|
if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.212",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^1.0.1",
|
|
51
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/basic": "4.0.0-beta.212",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.212",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.212",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.212",
|
|
55
55
|
"tslib": "^2.8.1",
|
|
56
56
|
"type-fest": "^4.41.0",
|
|
57
57
|
"zod": "^4.5.4"
|
|
@@ -75,5 +75,5 @@
|
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
},
|
|
78
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "171a63a5d7f2a6d74cbce58479204ac20d30b87e"
|
|
79
79
|
}
|