@crawlee/browser 4.0.0-beta.183 → 4.0.0-beta.185
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest,
|
|
1
|
+
import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequestHandler, RouterHandler } from '@crawlee/basic';
|
|
2
2
|
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
3
3
|
import type { CommonPage, CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';
|
|
4
4
|
import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
|
|
@@ -34,7 +34,7 @@ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
|
34
34
|
/**
|
|
35
35
|
* The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
|
|
36
36
|
*/
|
|
37
|
-
request: LoadedRequest<
|
|
37
|
+
request: LoadedRequest<CrawlingRequest<UserData>>;
|
|
38
38
|
/**
|
|
39
39
|
* The HTTP response object returned by the browser's navigation.
|
|
40
40
|
*/
|
|
@@ -107,7 +107,7 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
107
107
|
* To make this work, we should **always**
|
|
108
108
|
* let our function throw exceptions rather than catch them.
|
|
109
109
|
* The exceptions are logged to the request using the
|
|
110
|
-
* {@link
|
|
110
|
+
* {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
|
|
111
111
|
*/
|
|
112
112
|
requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
|
|
113
113
|
/**
|
|
@@ -116,9 +116,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
116
116
|
if (browserPool) {
|
|
117
117
|
assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });
|
|
118
118
|
}
|
|
119
|
-
const skipGuard = (action) => ({
|
|
120
|
-
action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
|
|
121
|
-
});
|
|
119
|
+
const skipGuard = (action) => async (ctx) => ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {});
|
|
122
120
|
super({
|
|
123
121
|
...basicCrawlerOptions,
|
|
124
122
|
contextPipelineBuilder: () => {
|
|
@@ -133,7 +131,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
133
131
|
}
|
|
134
132
|
return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
135
133
|
});
|
|
136
|
-
let pipeline = contextPipelineBuilder().compose(
|
|
134
|
+
let pipeline = contextPipelineBuilder().compose(this.prepareNavigation.bind(this));
|
|
137
135
|
for (const hook of this.#preNavigationHooks) {
|
|
138
136
|
pipeline = pipeline.compose(windowGuard(hook));
|
|
139
137
|
}
|
|
@@ -143,8 +141,8 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
143
141
|
}
|
|
144
142
|
return pipeline
|
|
145
143
|
.compose(skipGuard(this.finalizeNavigation.bind(this)))
|
|
146
|
-
.compose(
|
|
147
|
-
.compose(
|
|
144
|
+
.compose(this.handleBlockedRequestByContent.bind(this))
|
|
145
|
+
.compose(this.restoreRequestState.bind(this));
|
|
148
146
|
},
|
|
149
147
|
extendContext,
|
|
150
148
|
});
|
|
@@ -164,19 +162,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
164
162
|
return this.#navigationTimeoutMillis;
|
|
165
163
|
}
|
|
166
164
|
buildContextPipeline() {
|
|
167
|
-
return ContextPipeline.create().compose(
|
|
168
|
-
action: this.preparePage.bind(this),
|
|
169
|
-
cleanup: async (context) => {
|
|
170
|
-
context.registerDeferredCleanup(async () => {
|
|
171
|
-
const error = !context.session.isUsable()
|
|
172
|
-
? new SessionError('Session is no longer usable')
|
|
173
|
-
: undefined;
|
|
174
|
-
await this.browserPool
|
|
175
|
-
.closePage(context.page, { error })
|
|
176
|
-
.catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
|
|
177
|
-
});
|
|
178
|
-
},
|
|
179
|
-
});
|
|
165
|
+
return ContextPipeline.create().compose(this.preparePage.bind(this));
|
|
180
166
|
}
|
|
181
167
|
async containsSelectors(page, selectors) {
|
|
182
168
|
const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
|
|
@@ -204,12 +190,22 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
204
190
|
return `Received blocked status code: ${statusCode}`;
|
|
205
191
|
return false;
|
|
206
192
|
}
|
|
207
|
-
async preparePage(crawlingContext) {
|
|
193
|
+
async preparePage(crawlingContext, onCleanup) {
|
|
208
194
|
const page = await this.browserPool.newPage({
|
|
209
195
|
id: crawlingContext.id,
|
|
210
196
|
session: crawlingContext.session,
|
|
211
197
|
});
|
|
212
198
|
tryCancel();
|
|
199
|
+
onCleanup(() => {
|
|
200
|
+
crawlingContext.registerDeferredCleanup(async () => {
|
|
201
|
+
const error = !crawlingContext.session.isUsable()
|
|
202
|
+
? new SessionError('Session is no longer usable')
|
|
203
|
+
: undefined;
|
|
204
|
+
await this.browserPool
|
|
205
|
+
.closePage(page, { error })
|
|
206
|
+
.catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
|
|
207
|
+
});
|
|
208
|
+
});
|
|
213
209
|
const addRequests = crawlingContext.addRequests;
|
|
214
210
|
const extractLinks = async (options) => {
|
|
215
211
|
return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.185",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^1.0.1",
|
|
51
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/basic": "4.0.0-beta.185",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.185",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.185",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.185",
|
|
55
55
|
"tslib": "^2.8.1",
|
|
56
56
|
"type-fest": "^4.41.0",
|
|
57
57
|
"zod": "^4.5.4"
|
|
@@ -75,5 +75,5 @@
|
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
},
|
|
78
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "642c670956a5a60384d5ed5e67ee809927cc419b"
|
|
79
79
|
}
|