@crawlee/basic 4.0.0-beta.261 → 4.0.0-beta.262
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -9,6 +9,7 @@ import type { TaskLoopPredicates } from './autoscaling/autoscaled-pool.js';
|
|
|
9
9
|
import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
10
10
|
import { ConcurrencySystem } from './autoscaling/concurrency-system.js';
|
|
11
11
|
import type { FinalStatistics } from './autoscaling/system-status.js';
|
|
12
|
+
import type { ContextMiddleware } from './crawlers/context-pipeline.js';
|
|
12
13
|
import { ContextPipeline } from './crawlers/context-pipeline.js';
|
|
13
14
|
import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler-commons.js';
|
|
14
15
|
import type { IStatistics, StatisticState } from './crawlers/statistics.js';
|
|
@@ -65,6 +66,10 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
65
66
|
* **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
|
|
66
67
|
* those, use a `postNavigationHook` or the `requestHandler` instead.
|
|
67
68
|
*
|
|
69
|
+
* The second argument is the same {@link CleanupRegistrar|`onCleanup`} registrar used by
|
|
70
|
+
* {@link ContextPipeline.compose}. Use it to register teardown callbacks that run after the
|
|
71
|
+
* request handler or a pipeline failure, in reverse registration order.
|
|
72
|
+
*
|
|
68
73
|
* **Example usage:**
|
|
69
74
|
*
|
|
70
75
|
* ```javascript
|
|
@@ -83,7 +88,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
83
88
|
* });
|
|
84
89
|
* ```
|
|
85
90
|
*/
|
|
86
|
-
extendContext?:
|
|
91
|
+
extendContext?: ContextMiddleware<CrawlingContext, ContextExtension>;
|
|
87
92
|
/**
|
|
88
93
|
* *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
|
|
89
94
|
*
|
|
@@ -685,7 +685,7 @@ export class BasicCrawler {
|
|
|
685
685
|
let contextPipeline;
|
|
686
686
|
if (extendContext !== undefined) {
|
|
687
687
|
contextPipeline = ContextPipeline.create()
|
|
688
|
-
.compose(
|
|
688
|
+
.compose(extendContext)
|
|
689
689
|
.chain(subclassPipeline);
|
|
690
690
|
}
|
|
691
691
|
else {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.262",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.1.0",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.262",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.262",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.262",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.262",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.1.0",
|
|
51
51
|
"csv-stringify": "^6.9.0",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.6.7"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.262"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "56d3158b87fd8f83d195634092f6fbfe3b0e4616"
|
|
70
70
|
}
|