@crawlee/http 4.0.0-beta.91 → 4.0.0-beta.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -68,7 +68,7 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
|
|
|
68
68
|
* ]
|
|
69
69
|
* ```
|
|
70
70
|
*
|
|
71
|
-
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
71
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
|
|
72
72
|
*
|
|
73
73
|
* ## Example usage
|
|
74
74
|
*
|
|
@@ -87,7 +87,7 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
|
|
|
87
87
|
* ]
|
|
88
88
|
* ```
|
|
89
89
|
*
|
|
90
|
-
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
90
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
|
|
91
91
|
*
|
|
92
92
|
* ## Example usage
|
|
93
93
|
*
|
|
@@ -214,8 +214,8 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
|
|
|
214
214
|
* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
|
|
215
215
|
* For details, see {@link HttpCrawlerOptions.requestHandler}.
|
|
216
216
|
*
|
|
217
|
-
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
218
|
-
*
|
|
217
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
|
|
218
|
+
* {@link ConcurrencySystem}.
|
|
219
219
|
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
220
220
|
* constructor, or, for finer control, by injecting a pre-configured
|
|
221
221
|
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
@@ -294,7 +294,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
294
294
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
295
295
|
maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
296
296
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
297
|
-
|
|
297
|
+
taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
298
298
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
299
299
|
concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
300
300
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -75,8 +75,8 @@ export const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS = {
|
|
|
75
75
|
* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
|
|
76
76
|
* For details, see {@link HttpCrawlerOptions.requestHandler}.
|
|
77
77
|
*
|
|
78
|
-
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
79
|
-
*
|
|
78
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
|
|
79
|
+
* {@link ConcurrencySystem}.
|
|
80
80
|
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
81
81
|
* constructor, or, for finer control, by injecting a pre-configured
|
|
82
82
|
* {@link ConcurrencySystem|`concurrencySystem`}.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.93",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,11 +49,11 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.3.2",
|
|
51
51
|
"@apify/utilities": "^2.15.5",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/core": "4.0.0-beta.
|
|
54
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
55
|
-
"@crawlee/types": "4.0.0-beta.
|
|
56
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.93",
|
|
53
|
+
"@crawlee/core": "4.0.0-beta.93",
|
|
54
|
+
"@crawlee/http-client": "4.0.0-beta.93",
|
|
55
|
+
"@crawlee/types": "4.0.0-beta.93",
|
|
56
|
+
"@crawlee/utils": "4.0.0-beta.93",
|
|
57
57
|
"@types/content-type": "^1.1.8",
|
|
58
58
|
"cheerio": "^1.0.0",
|
|
59
59
|
"content-type": "^1.0.5",
|
|
@@ -70,5 +70,5 @@
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"gitHead": "
|
|
73
|
+
"gitHead": "b9d21e80d94d01e21d4e1c19191610d0157dd172"
|
|
74
74
|
}
|