@crawlee/basic 4.0.0-beta.261 → 4.0.0-beta.263

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,6 +9,7 @@ import type { TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
9
9
  import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
10
10
  import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
11
11
  import type { FinalStatistics } from './autoscaling/system-status.js';
12
+ import type { ContextMiddleware } from './crawlers/context-pipeline.js';
12
13
  import { ContextPipeline } from './crawlers/context-pipeline.js';
13
14
  import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
14
15
  import type { IStatistics, StatisticState } from './crawlers/statistics.js';
@@ -65,6 +66,10 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
65
66
  * **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
66
67
  * those, use a `postNavigationHook` or the `requestHandler` instead.
67
68
  *
69
+ * The second argument is the same {@link CleanupRegistrar|`onCleanup`} registrar used by
70
+ * {@link ContextPipeline.compose}. Use it to register teardown callbacks that run after the
71
+ * request handler or a pipeline failure, in reverse registration order.
72
+ *
68
73
  * **Example usage:**
69
74
  *
70
75
  * ```javascript
@@ -83,7 +88,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
83
88
  * });
84
89
  * ```
85
90
  */
86
- extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;
91
+ extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
87
92
  /**
88
93
  * *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
89
94
  *
@@ -278,7 +283,9 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
278
283
  *
279
284
  * Currently supports:
280
285
  * - [**Cloudflare** Bot Management](https://www.cloudflare.com/products/bot-management/)
286
+ * - [**AWS WAF** Challenge and CAPTCHA](https://docs.aws.amazon.com/waf/latest/developerguide/waf-captcha-and-challenge.html)
281
287
  * - [**Google Search** Rate Limiting](https://www.google.com/sorry/)
288
+ * - **Imperva Incapsula**
282
289
  */
283
290
  retryOnBlocked?: boolean;
284
291
  /**
@@ -685,7 +685,7 @@ export class BasicCrawler {
685
685
  let contextPipeline;
686
686
  if (extendContext !== undefined) {
687
687
  contextPipeline = ContextPipeline.create()
688
- .compose(async (context) => await extendContext(context))
688
+ .compose(extendContext)
689
689
  .chain(subclassPipeline);
690
690
  }
691
691
  else {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.261",
3
+ "version": "4.0.0-beta.263",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -42,10 +42,10 @@
42
42
  "@apify/datastructures": "^3.0.1",
43
43
  "@apify/timeout": "^1.0.1",
44
44
  "@apify/utilities": "^3.1.0",
45
- "@crawlee/core": "4.0.0-beta.261",
46
- "@crawlee/http-client": "4.0.0-beta.261",
47
- "@crawlee/types": "4.0.0-beta.261",
48
- "@crawlee/utils": "4.0.0-beta.261",
45
+ "@crawlee/core": "4.0.0-beta.263",
46
+ "@crawlee/http-client": "4.0.0-beta.263",
47
+ "@crawlee/types": "4.0.0-beta.263",
48
+ "@crawlee/utils": "4.0.0-beta.263",
49
49
  "@sapphire/async-queue": "^1.5.5",
50
50
  "@standard-schema/spec": "^1.1.0",
51
51
  "csv-stringify": "^6.9.0",
@@ -57,7 +57,7 @@
57
57
  "zod": "^4.6.7"
58
58
  },
59
59
  "optionalDependencies": {
60
- "@crawlee/impit-client": "4.0.0-beta.261"
60
+ "@crawlee/impit-client": "4.0.0-beta.263"
61
61
  },
62
62
  "lerna": {
63
63
  "command": {
@@ -66,5 +66,5 @@
66
66
  }
67
67
  }
68
68
  },
69
- "gitHead": "24e897c21c5f7ba19846bbca3eba4def5331a081"
69
+ "gitHead": "7303c694793b8f5cc17f5b16328f435fae121668"
70
70
  }