@crawlee/core 4.0.0-beta.18 → 4.0.0-beta.181
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/byte_utils.d.ts +17 -0
- package/byte_utils.js +42 -0
- package/configuration.d.ts +90 -226
- package/configuration.js +164 -225
- package/debug.d.ts +36 -0
- package/debug.js +70 -0
- package/errors.d.ts +40 -19
- package/errors.js +53 -28
- package/events/event_manager.d.ts +35 -9
- package/events/event_manager.js +12 -13
- package/events/index.d.ts +1 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -3
- package/events/local_event_manager.js +39 -13
- package/events/system_info.d.ts +38 -0
- package/events/system_info.js +1 -0
- package/index.d.ts +7 -11
- package/index.js +8 -10
- package/internal.d.ts +8 -0
- package/internal.js +9 -0
- package/iterables.d.ts +79 -0
- package/iterables.js +134 -0
- package/log.d.ts +77 -1
- package/log.js +134 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +44 -0
- package/memory-storage/memory-storage.js +160 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +106 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +199 -0
- package/memory-storage/resource-clients/request-queue.d.ts +77 -0
- package/memory-storage/resource-clients/request-queue.js +414 -0
- package/memory-storage/utils.d.ts +16 -0
- package/memory-storage/utils.js +41 -0
- package/owned_or_injected.d.ts +58 -0
- package/owned_or_injected.js +98 -0
- package/package.json +18 -19
- package/proxy_configuration.d.ts +22 -128
- package/proxy_configuration.js +32 -144
- package/recoverable_state.d.ts +85 -51
- package/recoverable_state.js +178 -72
- package/request.d.ts +60 -17
- package/request.js +130 -69
- package/serialization.d.ts +0 -1
- package/serialization.js +15 -15
- package/service_locator.d.ts +165 -0
- package/service_locator.js +254 -0
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +114 -56
- package/storages/dataset.js +292 -149
- package/storages/index.d.ts +7 -9
- package/storages/index.js +5 -9
- package/storages/key_value_store.d.ts +187 -50
- package/storages/key_value_store.js +453 -172
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +22 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +54 -129
- package/storages/request_list.js +272 -245
- package/storages/request_loader.d.ts +127 -0
- package/storages/request_loader.js +36 -0
- package/storages/request_manager.d.ts +109 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +50 -23
- package/storages/request_manager_tandem.js +132 -57
- package/storages/request_queue.d.ts +286 -48
- package/storages/request_queue.js +757 -218
- package/storages/storage_instance_manager.d.ts +87 -0
- package/storages/storage_instance_manager.js +256 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/transaction.d.ts +260 -0
- package/storages/transaction.js +287 -0
- package/storages/utils.d.ts +59 -11
- package/storages/utils.js +75 -15
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +0 -6
- package/typedefs.js +0 -1
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +23 -18
- package/validators.js +14 -18
- package/autoscaling/autoscaled_pool.d.ts +0 -300
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js +0 -533
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts +0 -4
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js +0 -4
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts +0 -189
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js +0 -285
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts +0 -181
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js +0 -169
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts +0 -43
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js +0 -111
- package/cookie_utils.js.map +0 -1
- package/crawlers/context_pipeline.d.ts +0 -61
- package/crawlers/context_pipeline.d.ts.map +0 -1
- package/crawlers/context_pipeline.js +0 -99
- package/crawlers/context_pipeline.js.map +0 -1
- package/crawlers/crawler_commons.d.ts +0 -195
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js +0 -108
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js +0 -12
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts +0 -60
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js +0 -118
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts +0 -55
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js +0 -309
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts +0 -7
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js +0 -7
- package/crawlers/index.js.map +0 -1
- package/crawlers/internals/types.d.ts +0 -8
- package/crawlers/internals/types.d.ts.map +0 -1
- package/crawlers/internals/types.js +0 -2
- package/crawlers/internals/types.js.map +0 -1
- package/crawlers/statistics.d.ts +0 -204
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js +0 -346
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts +0 -249
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js +0 -269
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts +0 -3
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js +0 -3
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts +0 -73
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js +0 -212
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -91
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -44
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -20
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -102
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/recoverable_state.d.ts.map +0 -1
- package/recoverable_state.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts +0 -134
- package/router.d.ts.map +0 -1
- package/router.js +0 -164
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts +0 -4
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js +0 -4
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts +0 -8
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js +0 -12
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts +0 -6
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js +0 -6
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts +0 -203
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js +0 -278
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts +0 -258
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js +0 -387
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts +0 -13
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js +0 -14
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_list_adapter.d.ts +0 -58
- package/storages/request_list_adapter.d.ts.map +0 -1
- package/storages/request_list_adapter.js +0 -81
- package/storages/request_list_adapter.js.map +0 -1
- package/storages/request_manager_tandem.d.ts.map +0 -1
- package/storages/request_manager_tandem.js.map +0 -1
- package/storages/request_provider.d.ts +0 -371
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -585
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts +0 -245
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js +0 -430
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
<h1 align="center">
|
|
2
2
|
<a href="https://crawlee.dev">
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true"
|
|
5
|
-
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500"
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
|
|
5
|
+
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
|
|
6
6
|
</picture>
|
|
7
7
|
</a>
|
|
8
|
-
<br
|
|
8
|
+
<br />
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
-
<p align=center>
|
|
13
|
-
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift"
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
<p align=center>
|
|
17
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg"
|
|
18
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg"
|
|
19
|
-
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord"
|
|
20
|
-
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status"
|
|
16
|
+
<p align="center">
|
|
17
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
|
|
19
|
+
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
|
|
20
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
|
|
21
21
|
</p>
|
|
22
22
|
|
|
23
23
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
|
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
89
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
90
90
|
|
|
91
91
|
```bash
|
|
92
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
98
98
|
{
|
|
99
99
|
"overrides": {
|
|
100
100
|
"apify": {
|
|
101
|
-
"@crawlee/core": "
|
|
102
|
-
"@crawlee/types": "
|
|
103
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
}
|
package/byte_utils.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checks if the given value is a Node.js Stream or a Web API ReadableStream.
|
|
3
|
+
* @ignore
|
|
4
|
+
*/
|
|
5
|
+
export declare function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;
|
|
6
|
+
/**
|
|
7
|
+
* Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
|
|
8
|
+
* @ignore
|
|
9
|
+
*/
|
|
10
|
+
export declare function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;
|
|
11
|
+
/**
|
|
12
|
+
* Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
|
|
13
|
+
* the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
|
|
14
|
+
* as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
|
|
15
|
+
* @ignore
|
|
16
|
+
*/
|
|
17
|
+
export declare function toBuffer(value: Buffer | ArrayBuffer | ArrayBufferView): Buffer;
|
package/byte_utils.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checks if the given value is a Node.js Stream or a Web API ReadableStream.
|
|
3
|
+
* @ignore
|
|
4
|
+
*/
|
|
5
|
+
export function isStream(value) {
|
|
6
|
+
if (typeof value !== 'object' || value === null) {
|
|
7
|
+
return false;
|
|
8
|
+
}
|
|
9
|
+
// A Node.js Readable is both pipeable and async-iterable; a Web ReadableStream exposes pipeTo.
|
|
10
|
+
// Requiring async-iterability for the `pipe` branch rejects plain `{ pipe }` ducks that would
|
|
11
|
+
// otherwise blow up later in the storage backends' drain loop with a cryptic TypeError.
|
|
12
|
+
const isNodeStream = typeof value.pipe === 'function' && typeof value[Symbol.asyncIterator] === 'function';
|
|
13
|
+
const isWebStream = typeof value.pipeTo === 'function';
|
|
14
|
+
return isNodeStream || isWebStream;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
|
|
18
|
+
* @ignore
|
|
19
|
+
*/
|
|
20
|
+
export function isBuffer(value) {
|
|
21
|
+
return (value != null &&
|
|
22
|
+
typeof value === 'object' &&
|
|
23
|
+
(Buffer.isBuffer(value) ||
|
|
24
|
+
value instanceof ArrayBuffer ||
|
|
25
|
+
ArrayBuffer.isView(value) ||
|
|
26
|
+
value.constructor?.name === 'Buffer'));
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
|
|
30
|
+
* the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
|
|
31
|
+
* as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
|
|
32
|
+
* @ignore
|
|
33
|
+
*/
|
|
34
|
+
export function toBuffer(value) {
|
|
35
|
+
if (Buffer.isBuffer(value)) {
|
|
36
|
+
return value;
|
|
37
|
+
}
|
|
38
|
+
if (value instanceof ArrayBuffer) {
|
|
39
|
+
return Buffer.from(value);
|
|
40
|
+
}
|
|
41
|
+
return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
|
|
42
|
+
}
|
package/configuration.d.ts
CHANGED
|
@@ -1,151 +1,62 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
export
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
*
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
* @default 0.95
|
|
57
|
-
*/
|
|
58
|
-
maxUsedCpuRatio?: number;
|
|
59
|
-
/**
|
|
60
|
-
* Sets the ratio, defining the amount of system memory that could be used by the {@link AutoscaledPool}.
|
|
61
|
-
* When the memory usage is more than the provided ratio, the memory is considered overloaded.
|
|
62
|
-
*
|
|
63
|
-
* Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable.
|
|
64
|
-
* @default 0.25
|
|
65
|
-
*/
|
|
66
|
-
availableMemoryRatio?: number;
|
|
67
|
-
/**
|
|
68
|
-
* Sets the amount of system memory in megabytes to be used by the {@link AutoscaledPool}.
|
|
69
|
-
* By default, the maximum memory is set to one quarter of total system memory.
|
|
70
|
-
*
|
|
71
|
-
* Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable.
|
|
72
|
-
*/
|
|
73
|
-
memoryMbytes?: number;
|
|
74
|
-
/**
|
|
75
|
-
* Defines the interval of emitting the `persistState` event.
|
|
76
|
-
*
|
|
77
|
-
* Alternative to `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` environment variable.
|
|
78
|
-
* @default 60_000
|
|
79
|
-
*/
|
|
80
|
-
persistStateIntervalMillis?: number;
|
|
81
|
-
/**
|
|
82
|
-
Defines the interval of emitting the `systemInfo` event.
|
|
83
|
-
@default 1_000
|
|
84
|
-
*/
|
|
85
|
-
systemInfoIntervalMillis?: number;
|
|
86
|
-
/**
|
|
87
|
-
* Defines the default input key, i.e. the key that is used to get the crawler input value
|
|
88
|
-
* from the default {@link KeyValueStore} associated with the current crawler run.
|
|
89
|
-
*
|
|
90
|
-
* Alternative to `CRAWLEE_INPUT_KEY` environment variable.
|
|
91
|
-
* @default 'INPUT'
|
|
92
|
-
*/
|
|
93
|
-
inputKey?: string;
|
|
94
|
-
/**
|
|
95
|
-
* Defines whether web browsers launched by Crawlee will run in the headless mode.
|
|
96
|
-
*
|
|
97
|
-
* Alternative to `CRAWLEE_HEADLESS` environment variable.
|
|
98
|
-
* @default true
|
|
99
|
-
*/
|
|
100
|
-
headless?: boolean;
|
|
101
|
-
/**
|
|
102
|
-
* Defines whether to run X virtual framebuffer on the web browsers launched by Crawlee.
|
|
103
|
-
*
|
|
104
|
-
* Alternative to `CRAWLEE_XVFB` environment variable.
|
|
105
|
-
* @default false
|
|
106
|
-
*/
|
|
107
|
-
xvfb?: boolean;
|
|
108
|
-
/**
|
|
109
|
-
* Defines a path to Chrome executable.
|
|
110
|
-
*
|
|
111
|
-
* Alternative to `CRAWLEE_CHROME_EXECUTABLE_PATH` environment variable.
|
|
112
|
-
*/
|
|
113
|
-
chromeExecutablePath?: string;
|
|
114
|
-
/**
|
|
115
|
-
* Defines a path to default browser executable.
|
|
116
|
-
*
|
|
117
|
-
* Alternative to `CRAWLEE_DEFAULT_BROWSER_PATH` environment variable.
|
|
118
|
-
*/
|
|
119
|
-
defaultBrowserPath?: string;
|
|
120
|
-
/**
|
|
121
|
-
* Defines whether to disable browser sandbox by adding `--no-sandbox` flag to `launchOptions`.
|
|
122
|
-
*
|
|
123
|
-
* Alternative to `CRAWLEE_DISABLE_BROWSER_SANDBOX` environment variable.
|
|
124
|
-
*/
|
|
125
|
-
disableBrowserSandbox?: boolean;
|
|
126
|
-
/**
|
|
127
|
-
* Sets the log level to the given value.
|
|
128
|
-
*
|
|
129
|
-
* Alternative to `CRAWLEE_LOG_LEVEL` environment variable.
|
|
130
|
-
* @default 'INFO'
|
|
131
|
-
*/
|
|
132
|
-
logLevel?: LogLevel | LogLevel[keyof LogLevel];
|
|
133
|
-
/**
|
|
134
|
-
* Defines whether the storage client used should persist the data it stores.
|
|
135
|
-
*
|
|
136
|
-
* Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable.
|
|
137
|
-
*/
|
|
138
|
-
persistStorage?: boolean;
|
|
139
|
-
/**
|
|
140
|
-
* Used in place of `isContainerized()` when collecting system metrics.
|
|
141
|
-
*
|
|
142
|
-
* Alternative to `CRAWLEE_CONTAINERIZED` environment variable.
|
|
143
|
-
*/
|
|
144
|
-
containerized?: boolean;
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { LogLevel } from './log.js';
|
|
3
|
+
export interface ConfigField<T extends z.ZodType = z.ZodType> {
|
|
4
|
+
schema: T;
|
|
5
|
+
envVar?: string | string[];
|
|
6
|
+
}
|
|
7
|
+
export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
|
|
8
|
+
/** Zod preprocessor treating `'0'` and `'false'` as falsy. */
|
|
9
|
+
export declare const coerceBoolean: z.ZodPreprocess<z.ZodBoolean, unknown>;
|
|
10
|
+
export declare const coerceNumber: z.ZodPreprocess<z.ZodNumber, unknown>;
|
|
11
|
+
export declare const crawleeConfigFields: {
|
|
12
|
+
/** @default true */
|
|
13
|
+
purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
|
|
14
|
+
/** @default 0.95 */
|
|
15
|
+
maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
16
|
+
/** @default 0.25 */
|
|
17
|
+
availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
18
|
+
memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
19
|
+
/** @default 60_000 */
|
|
20
|
+
persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
21
|
+
/**
|
|
22
|
+
* Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
|
|
23
|
+
* the request handler timeout (twice it, and never below 5 minutes).
|
|
24
|
+
*/
|
|
25
|
+
internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
26
|
+
/** @default 1_000 */
|
|
27
|
+
systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
28
|
+
/** @default 'INPUT' */
|
|
29
|
+
inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
30
|
+
/** @default true */
|
|
31
|
+
headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
|
|
32
|
+
/** @default false */
|
|
33
|
+
xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
|
|
34
|
+
chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
|
|
35
|
+
defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
|
|
36
|
+
/** @default false */
|
|
37
|
+
disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
|
|
38
|
+
logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>, unknown>>>;
|
|
39
|
+
/** @default true */
|
|
40
|
+
persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
|
|
41
|
+
/** @default './storage' */
|
|
42
|
+
storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
43
|
+
containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
|
|
44
|
+
};
|
|
45
|
+
export type FieldsInput<F extends Record<string, ConfigField>> = {
|
|
46
|
+
[K in keyof F]?: z.output<F[K]['schema']>;
|
|
47
|
+
};
|
|
48
|
+
export type FieldsOutput<F extends Record<string, ConfigField>> = {
|
|
49
|
+
[K in keyof F]: z.output<F[K]['schema']>;
|
|
50
|
+
};
|
|
51
|
+
export type ConfigurationInput = FieldsInput<typeof crawleeConfigFields>;
|
|
52
|
+
export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;
|
|
53
|
+
/** @deprecated Use {@link ConfigurationInput} instead. */
|
|
54
|
+
export type ConfigurationOptions = ConfigurationInput;
|
|
55
|
+
export interface Configuration extends ResolvedConfigValues {
|
|
145
56
|
}
|
|
146
57
|
/**
|
|
147
58
|
* `Configuration` is a value object holding Crawlee configuration. By default, there is a
|
|
148
|
-
* global singleton instance of this class available via `Configuration.
|
|
59
|
+
* global singleton instance of this class available via `Configuration.getGlobalConfiguration()`.
|
|
149
60
|
* Places that depend on a configurable behaviour depend on this class, as they have the global
|
|
150
61
|
* instance as the default value.
|
|
151
62
|
*
|
|
@@ -154,14 +65,10 @@ export interface ConfigurationOptions {
|
|
|
154
65
|
* import { BasicCrawler, Configuration } from 'crawlee';
|
|
155
66
|
*
|
|
156
67
|
* // Get the global configuration
|
|
157
|
-
* const config = Configuration.
|
|
158
|
-
* //
|
|
159
|
-
*
|
|
160
|
-
* config.
|
|
161
|
-
*
|
|
162
|
-
* // No need to pass the configuration to the crawler,
|
|
163
|
-
* // as it's using the global configuration by default
|
|
164
|
-
* const crawler = new BasicCrawler();
|
|
68
|
+
* const config = Configuration.getGlobalConfiguration();
|
|
69
|
+
* // Access configuration values directly as properties
|
|
70
|
+
* console.log(config.headless);
|
|
71
|
+
* console.log(config.persistStateIntervalMillis);
|
|
165
72
|
* ```
|
|
166
73
|
*
|
|
167
74
|
* *Using custom configuration:*
|
|
@@ -171,15 +78,14 @@ export interface ConfigurationOptions {
|
|
|
171
78
|
* // Create a new configuration
|
|
172
79
|
* const config = new Configuration({ persistStateIntervalMillis: 30_000 });
|
|
173
80
|
* // Pass the configuration to the crawler
|
|
174
|
-
* const crawler = new BasicCrawler({
|
|
81
|
+
* const crawler = new BasicCrawler({ configuration: config });
|
|
175
82
|
* ```
|
|
176
83
|
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
* so the options provided in constructor will override those. In other words, the precedence is:
|
|
84
|
+
* Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
|
|
85
|
+
* The priority order for resolving values is (highest to lowest):
|
|
180
86
|
*
|
|
181
87
|
* ```text
|
|
182
|
-
*
|
|
88
|
+
* constructor options > environment variables > crawlee.json > schema defaults
|
|
183
89
|
* ```
|
|
184
90
|
*
|
|
185
91
|
* ## Supported Configuration Options
|
|
@@ -189,102 +95,60 @@ export interface ConfigurationOptions {
|
|
|
189
95
|
* `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
|
|
190
96
|
* `logLevel` | `CRAWLEE_LOG_LEVEL` | -
|
|
191
97
|
* `headless` | `CRAWLEE_HEADLESS` | `true`
|
|
192
|
-
* `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
|
|
193
|
-
* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
|
|
194
|
-
* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
|
|
195
98
|
* `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
|
|
99
|
+
* `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
|
|
196
100
|
* `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
|
|
197
101
|
* `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
|
|
102
|
+
* `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`
|
|
198
103
|
*
|
|
199
104
|
* ## Advanced Configuration Options
|
|
200
105
|
*
|
|
201
106
|
* Key | Environment Variable | Default Value
|
|
202
107
|
* ---|---|---
|
|
203
108
|
* `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
|
|
204
|
-
* `xvfb` | `CRAWLEE_XVFB` |
|
|
109
|
+
* `xvfb` | `CRAWLEE_XVFB` | `false`
|
|
205
110
|
* `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
|
|
206
111
|
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
|
|
207
112
|
* `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
|
|
208
113
|
* `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
|
|
209
|
-
* `containerized | `CRAWLEE_CONTAINERIZED | -
|
|
114
|
+
* `containerized` | `CRAWLEE_CONTAINERIZED` | -
|
|
210
115
|
*/
|
|
211
116
|
export declare class Configuration {
|
|
117
|
+
#private;
|
|
212
118
|
/**
|
|
213
|
-
*
|
|
214
|
-
|
|
215
|
-
protected static ENV_MAP: Dictionary;
|
|
216
|
-
protected static BOOLEAN_VARS: string[];
|
|
217
|
-
protected static INTEGER_VARS: string[];
|
|
218
|
-
protected static COMMA_SEPARATED_LIST_VARS: string[];
|
|
219
|
-
protected static DEFAULTS: Dictionary;
|
|
220
|
-
/**
|
|
221
|
-
* Provides access to the current-instance-scoped Configuration without passing it around in parameters.
|
|
222
|
-
* @internal
|
|
223
|
-
*/
|
|
224
|
-
static storage: AsyncLocalStorage<Configuration>;
|
|
225
|
-
protected options: Map<keyof ConfigurationOptions, ConfigurationOptions[keyof ConfigurationOptions]>;
|
|
226
|
-
protected services: Map<string, unknown>;
|
|
227
|
-
/** @internal */
|
|
228
|
-
static globalConfig?: Configuration;
|
|
229
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
230
|
-
readonly storageManagers: Map<Constructor, StorageManager<import("./storages/storage_manager.js").IStorage>>;
|
|
231
|
-
/**
|
|
232
|
-
* Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
|
|
233
|
-
*/
|
|
234
|
-
constructor(options?: ConfigurationOptions);
|
|
235
|
-
/**
|
|
236
|
-
* Returns configured value. First checks the environment variables, then provided configuration,
|
|
237
|
-
* fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
|
|
238
|
-
* in the above section.
|
|
239
|
-
*/
|
|
240
|
-
get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U;
|
|
241
|
-
protected _castEnvValue(key: keyof ConfigurationOptions, value: number | string | boolean): string | number | boolean | string[];
|
|
242
|
-
/**
|
|
243
|
-
* Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
|
|
244
|
-
* To reset a value, we can omit the `value` argument or pass `undefined` there.
|
|
119
|
+
* Field definitions for this configuration class.
|
|
120
|
+
* Subclasses override this to register additional fields.
|
|
245
121
|
*/
|
|
246
|
-
|
|
122
|
+
protected static fields: Record<string, ConfigField>;
|
|
247
123
|
/**
|
|
248
|
-
*
|
|
249
|
-
*
|
|
124
|
+
* Creates new `Configuration` instance with provided options.
|
|
125
|
+
* Constructor options take precedence over environment variables, which take precedence
|
|
126
|
+
* over crawlee.json values, which take precedence over schema defaults.
|
|
250
127
|
*/
|
|
251
|
-
|
|
128
|
+
constructor(options?: ConfigurationInput);
|
|
252
129
|
/**
|
|
253
|
-
* Returns
|
|
254
|
-
* this {@link Configuration} instance. Only first call of this method will create the client, following calls will
|
|
255
|
-
* return the same client instance.
|
|
130
|
+
* Returns the global configuration instance. It will respect the environment variables.
|
|
256
131
|
*
|
|
257
|
-
*
|
|
258
|
-
* multiple instances, one for each variant of the options.
|
|
259
|
-
* @internal
|
|
260
|
-
*/
|
|
261
|
-
getStorageClient(): StorageClient;
|
|
262
|
-
getEventManager(): EventManager;
|
|
263
|
-
/**
|
|
264
|
-
* Creates an instance of MemoryStorage using options as defined in the environment variables or in this `Configuration` instance.
|
|
265
|
-
* @internal
|
|
132
|
+
* Delegates to the global ServiceLocator, making it the single source of truth for service management.
|
|
266
133
|
*/
|
|
267
|
-
|
|
268
|
-
useStorageClient(client: StorageClient): void;
|
|
269
|
-
static useStorageClient(client: StorageClient): void;
|
|
270
|
-
useEventManager(events: EventManager): void;
|
|
134
|
+
static getGlobalConfiguration(): Configuration;
|
|
271
135
|
/**
|
|
272
|
-
*
|
|
136
|
+
* Resolves all field values once using the priority chain:
|
|
137
|
+
* constructor options > env vars > crawlee.json > schema defaults.
|
|
273
138
|
*/
|
|
274
|
-
static
|
|
139
|
+
private static resolveAll;
|
|
275
140
|
/**
|
|
276
|
-
*
|
|
141
|
+
* Registers getters (and throwing setters) on the instance for each field.
|
|
277
142
|
*/
|
|
278
|
-
|
|
143
|
+
private registerAccessors;
|
|
279
144
|
/**
|
|
280
|
-
*
|
|
145
|
+
* Reads the first defined env var value for a field definition.
|
|
146
|
+
* Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
|
|
147
|
+
* (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
|
|
281
148
|
*/
|
|
282
|
-
static
|
|
149
|
+
private static readEnvVar;
|
|
283
150
|
/**
|
|
284
|
-
*
|
|
285
|
-
* if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
|
|
151
|
+
* Loads config options from crawlee.json in the current working directory.
|
|
286
152
|
*/
|
|
287
|
-
static
|
|
288
|
-
protected buildOptions(options: ConfigurationOptions): void;
|
|
153
|
+
private static loadFileOptions;
|
|
289
154
|
}
|
|
290
|
-
//# sourceMappingURL=configuration.d.ts.map
|