@crawlee/playwright 4.0.0-beta.279 → 4.0.0-beta.280
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -326,17 +326,32 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
326
326
|
for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(attemptBoundContextHelpers))) {
|
|
327
327
|
Object.defineProperty(subCrawlerContext, key, { ...descriptor, configurable: false });
|
|
328
328
|
}
|
|
329
|
+
// The timeout below does not stop the request handler, and the pipeline closes the page only after the handler
|
|
330
|
+
// settles. Racing the handler against this lets a timed-out attempt release its page right away.
|
|
331
|
+
const attemptFailed = new AbortController();
|
|
329
332
|
try {
|
|
330
333
|
// Any failure - middleware or handler - ends up in the `ok: false` branch below.
|
|
331
334
|
const rethrow = (error) => {
|
|
332
335
|
throw error;
|
|
333
336
|
};
|
|
337
|
+
const abandonableRequestHandler = async (crawlingContext) => {
|
|
338
|
+
// The attempt may have failed before the handler starts, and a late abort listener would never fire.
|
|
339
|
+
attemptFailed.signal.throwIfAborted();
|
|
340
|
+
return Promise.race([
|
|
341
|
+
this.requestHandler(crawlingContext),
|
|
342
|
+
new Promise((_resolve, reject) => {
|
|
343
|
+
attemptFailed.signal.addEventListener('abort', () => reject(attemptFailed.signal.reason), {
|
|
344
|
+
once: true,
|
|
345
|
+
});
|
|
346
|
+
}),
|
|
347
|
+
]);
|
|
348
|
+
};
|
|
334
349
|
const callAdaptiveRequestHandler = async () => {
|
|
335
350
|
if (renderingType === 'static') {
|
|
336
351
|
await this.#staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this), rethrow);
|
|
337
352
|
}
|
|
338
353
|
else if (renderingType === 'clientOnly') {
|
|
339
|
-
await this.#browserContextPipeline.call(subCrawlerContext,
|
|
354
|
+
await this.#browserContextPipeline.call(subCrawlerContext, abandonableRequestHandler, rethrow);
|
|
340
355
|
}
|
|
341
356
|
};
|
|
342
357
|
// this crawler overrides `runRequestHandler` and times each rendering-type run itself, so it has
|
|
@@ -347,6 +362,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
347
362
|
return { ok: true, transaction, contextModifications, logs };
|
|
348
363
|
}
|
|
349
364
|
catch (error) {
|
|
365
|
+
attemptFailed.abort(error);
|
|
350
366
|
return { ok: false, error, transaction, contextModifications, logs };
|
|
351
367
|
}
|
|
352
368
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/playwright",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.280",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -49,14 +49,14 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^3.0.1",
|
|
51
51
|
"@apify/timeout": "^1.0.1",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
54
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
55
|
-
"@crawlee/cheerio": "4.0.0-beta.
|
|
56
|
-
"@crawlee/core": "4.0.0-beta.
|
|
57
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
58
|
-
"@crawlee/types": "4.0.0-beta.
|
|
59
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.280",
|
|
53
|
+
"@crawlee/browser": "4.0.0-beta.280",
|
|
54
|
+
"@crawlee/browser-pool": "4.0.0-beta.280",
|
|
55
|
+
"@crawlee/cheerio": "4.0.0-beta.280",
|
|
56
|
+
"@crawlee/core": "4.0.0-beta.280",
|
|
57
|
+
"@crawlee/http-client": "4.0.0-beta.280",
|
|
58
|
+
"@crawlee/types": "4.0.0-beta.280",
|
|
59
|
+
"@crawlee/utils": "4.0.0-beta.280",
|
|
60
60
|
"cheerio": "^1.2.0",
|
|
61
61
|
"domhandler": "^6.0.1",
|
|
62
62
|
"jquery": "^3.7.1",
|
|
@@ -81,5 +81,5 @@
|
|
|
81
81
|
}
|
|
82
82
|
}
|
|
83
83
|
},
|
|
84
|
-
"gitHead": "
|
|
84
|
+
"gitHead": "8c8d5e57a8015e3147dbe4e50843d924e63f4212"
|
|
85
85
|
}
|