@crawlee/playwright 4.0.0-beta.96 → 4.0.0-beta.98
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -100,6 +100,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
100
100
|
errorHandler,
|
|
101
101
|
failedRequestHandler,
|
|
102
102
|
requestHandler,
|
|
103
|
+
requestHandlerTimeoutSecs,
|
|
103
104
|
contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
|
|
104
105
|
});
|
|
105
106
|
this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
|
|
@@ -290,11 +291,15 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
290
291
|
await this.browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
|
|
291
292
|
}
|
|
292
293
|
};
|
|
294
|
+
// this crawler overrides `runRequestHandler` and times each rendering-type run itself, so it has
|
|
295
|
+
// to resolve any per-route override too - otherwise routes would be silently ignored here
|
|
296
|
+
const routeTimeoutSecs = this.requestHandler.getTimeoutSecs?.(context.request.label);
|
|
297
|
+
const timeoutMillis = routeTimeoutSecs === undefined ? this.individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;
|
|
293
298
|
await addTimeoutToPromise(async () => withCheckedStorageAccess(() => {
|
|
294
299
|
if (this.preventDirectStorageAccess) {
|
|
295
300
|
throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
|
|
296
301
|
}
|
|
297
|
-
}, callAdaptiveRequestHandler),
|
|
302
|
+
}, callAdaptiveRequestHandler), timeoutMillis, 'Request handler timed out');
|
|
298
303
|
return { result, ok: true, logs };
|
|
299
304
|
}
|
|
300
305
|
catch (error) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/playwright",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.98",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,14 +48,14 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
|
-
"@apify/timeout": "^0.
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
54
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
55
|
-
"@crawlee/cheerio": "4.0.0-beta.
|
|
56
|
-
"@crawlee/core": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@apify/timeout": "^0.4.4",
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.98",
|
|
53
|
+
"@crawlee/browser": "4.0.0-beta.98",
|
|
54
|
+
"@crawlee/browser-pool": "4.0.0-beta.98",
|
|
55
|
+
"@crawlee/cheerio": "4.0.0-beta.98",
|
|
56
|
+
"@crawlee/core": "4.0.0-beta.98",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.98",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.98",
|
|
59
59
|
"cheerio": "^1.0.0",
|
|
60
60
|
"idcac-playwright": "^0.1.3",
|
|
61
61
|
"jquery": "^3.7.1",
|
|
@@ -85,5 +85,5 @@
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
},
|
|
88
|
-
"gitHead": "
|
|
88
|
+
"gitHead": "3b8cd86b13e253ab5fc71e631e12a68f7465cee5"
|
|
89
89
|
}
|