@crawlee/basic 4.0.0-beta.261 → 4.0.0-beta.263
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -9,6 +9,7 @@ import type { TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
|
|
|
9
9
|
import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
10
10
|
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
11
11
|
import type { FinalStatistics } from './autoscaling/system-status.js';
|
|
12
|
+
import type { ContextMiddleware } from './crawlers/context-pipeline.js';
|
|
12
13
|
import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
13
14
|
import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
|
|
14
15
|
import type { IStatistics, StatisticState } from './crawlers/statistics.js';
|
|
@@ -65,6 +66,10 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
65
66
|
* **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
|
|
66
67
|
* those, use a `postNavigationHook` or the `requestHandler` instead.
|
|
67
68
|
*
|
|
69
|
+
* The second argument is the same {@link CleanupRegistrar|`onCleanup`} registrar used by
|
|
70
|
+
* {@link ContextPipeline.compose}. Use it to register teardown callbacks that run after the
|
|
71
|
+
* request handler or a pipeline failure, in reverse registration order.
|
|
72
|
+
*
|
|
68
73
|
* **Example usage:**
|
|
69
74
|
*
|
|
70
75
|
* ```javascript
|
|
@@ -83,7 +88,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
83
88
|
* });
|
|
84
89
|
* ```
|
|
85
90
|
*/
|
|
86
|
-
extendContext?:
|
|
91
|
+
extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
|
|
87
92
|
/**
|
|
88
93
|
* *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
|
|
89
94
|
*
|
|
@@ -278,7 +283,9 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
278
283
|
*
|
|
279
284
|
* Currently supports:
|
|
280
285
|
* - [**Cloudflare** Bot Management](https://www.cloudflare.com/products/bot-management/)
|
|
286
|
+
* - [**AWS WAF** Challenge and CAPTCHA](https://docs.aws.amazon.com/waf/latest/developerguide/waf-captcha-and-challenge.html)
|
|
281
287
|
* - [**Google Search** Rate Limiting](https://www.google.com/sorry/)
|
|
288
|
+
* - **Imperva Incapsula**
|
|
282
289
|
*/
|
|
283
290
|
retryOnBlocked?: boolean;
|
|
284
291
|
/**
|
|
@@ -685,7 +685,7 @@ export class BasicCrawler {
|
|
|
685
685
|
let contextPipeline;
|
|
686
686
|
if (extendContext !== undefined) {
|
|
687
687
|
contextPipeline = ContextPipeline.create()
|
|
688
|
-
.compose(
|
|
688
|
+
.compose(extendContext)
|
|
689
689
|
.chain(subclassPipeline);
|
|
690
690
|
}
|
|
691
691
|
else {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.263",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.263",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.263",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.263",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.263",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.263"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "7303c694793b8f5cc17f5b16328f435fae121668"
|
|
70
70
|
}
|