@crawlee/core 4.0.0-beta.12 → 4.0.0-beta.121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/autoscaling/autoscaled_pool.d.ts +67 -172
- package/autoscaling/autoscaled_pool.js +165 -320
- package/autoscaling/client_load_signal.d.ts +55 -0
- package/autoscaling/client_load_signal.js +73 -0
- package/autoscaling/concurrency_system.d.ts +268 -0
- package/autoscaling/concurrency_system.js +351 -0
- package/autoscaling/cpu_load_signal.d.ts +43 -0
- package/autoscaling/cpu_load_signal.js +47 -0
- package/autoscaling/event_loop_load_signal.d.ts +51 -0
- package/autoscaling/event_loop_load_signal.js +60 -0
- package/autoscaling/index.d.ts +6 -1
- package/autoscaling/index.js +6 -1
- package/autoscaling/load_signal.d.ts +99 -0
- package/autoscaling/load_signal.js +104 -0
- package/autoscaling/memory_load_signal.d.ts +47 -0
- package/autoscaling/memory_load_signal.js +106 -0
- package/autoscaling/snapshotter.d.ts +58 -163
- package/autoscaling/snapshotter.js +45 -263
- package/autoscaling/system_status.d.ts +62 -84
- package/autoscaling/system_status.js +92 -122
- package/autoscaling/weighted_avg.d.ts +5 -0
- package/autoscaling/weighted_avg.js +14 -0
- package/byte_utils.d.ts +17 -0
- package/byte_utils.js +42 -0
- package/configuration.d.ts +96 -223
- package/configuration.js +170 -222
- package/cookie_utils.d.ts +4 -3
- package/cookie_utils.js +22 -13
- package/crawlers/context_pipeline.d.ts +10 -1
- package/crawlers/context_pipeline.js +31 -8
- package/crawlers/crawler_commons.d.ts +90 -83
- package/crawlers/crawler_commons.js +1 -116
- package/crawlers/error_snapshotter.d.ts +2 -5
- package/crawlers/error_snapshotter.js +7 -8
- package/crawlers/error_tracker.d.ts +0 -1
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +1 -3
- package/crawlers/index.js +0 -3
- package/crawlers/internals/types.d.ts +0 -1
- package/crawlers/internals/types.js +0 -1
- package/crawlers/statistics.d.ts +143 -59
- package/crawlers/statistics.js +243 -153
- package/debug.d.ts +36 -0
- package/debug.js +70 -0
- package/enqueue_links/enqueue_links.d.ts +59 -68
- package/enqueue_links/enqueue_links.js +57 -62
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +40 -27
- package/enqueue_links/shared.js +90 -68
- package/errors.d.ts +72 -4
- package/errors.js +89 -5
- package/events/event_manager.d.ts +35 -9
- package/events/event_manager.js +10 -12
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -3
- package/events/local_event_manager.js +39 -13
- package/http.d.ts +9 -0
- package/http.js +28 -0
- package/index.d.ts +7 -4
- package/index.js +6 -3
- package/iterables.d.ts +79 -0
- package/iterables.js +134 -0
- package/log.d.ts +82 -3
- package/log.js +106 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +38 -0
- package/memory-storage/memory-storage.js +130 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +114 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +204 -0
- package/memory-storage/resource-clients/request-queue.d.ts +77 -0
- package/memory-storage/resource-clients/request-queue.js +422 -0
- package/memory-storage/utils.d.ts +16 -0
- package/memory-storage/utils.js +41 -0
- package/owned_or_injected.d.ts +58 -0
- package/owned_or_injected.js +98 -0
- package/package.json +13 -12
- package/proxy_configuration.d.ts +24 -132
- package/proxy_configuration.js +24 -143
- package/recoverable_state.d.ts +140 -0
- package/recoverable_state.js +212 -0
- package/request.d.ts +86 -17
- package/request.js +120 -41
- package/router.d.ts +193 -21
- package/router.js +188 -43
- package/serialization.d.ts +0 -1
- package/serialization.js +9 -11
- package/service_locator.d.ts +165 -0
- package/service_locator.js +253 -0
- package/session_pool/consts.d.ts +1 -2
- package/session_pool/consts.js +1 -2
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +9 -0
- package/session_pool/fingerprint.js +30 -0
- package/session_pool/index.d.ts +0 -2
- package/session_pool/index.js +0 -2
- package/session_pool/session.d.ts +35 -89
- package/session_pool/session.js +82 -142
- package/session_pool/session_pool.d.ts +69 -90
- package/session_pool/session_pool.js +151 -150
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +114 -54
- package/storages/dataset.js +285 -144
- package/storages/index.d.ts +10 -8
- package/storages/index.js +8 -8
- package/storages/key_value_store.d.ts +185 -42
- package/storages/key_value_store.js +424 -151
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +22 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +52 -116
- package/storages/request_list.js +159 -133
- package/storages/request_loader.d.ts +101 -0
- package/storages/request_loader.js +1 -0
- package/storages/request_manager.d.ts +33 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +97 -0
- package/storages/request_manager_tandem.js +197 -0
- package/storages/request_queue.d.ts +290 -47
- package/storages/request_queue.js +757 -216
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +37 -88
- package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +137 -143
- package/storages/storage_instance_manager.d.ts +87 -0
- package/storages/storage_instance_manager.js +256 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/throttling_request_manager.d.ts +216 -0
- package/storages/throttling_request_manager.js +453 -0
- package/storages/transaction.d.ts +252 -0
- package/storages/transaction.js +251 -0
- package/storages/utils.d.ts +58 -11
- package/storages/utils.js +64 -13
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +0 -6
- package/typedefs.js +0 -1
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +8 -1
- package/validators.js +10 -3
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/context_pipeline.d.ts.map +0 -1
- package/crawlers/context_pipeline.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js +0 -12
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/internals/types.d.ts.map +0 -1
- package/crawlers/internals/types.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts +0 -13
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js +0 -14
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_provider.d.ts +0 -308
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -555
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.121",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
},
|
|
39
39
|
"homepage": "https://crawlee.dev",
|
|
40
40
|
"scripts": {
|
|
41
|
-
"build": "
|
|
41
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
42
42
|
"clean": "rimraf ./dist",
|
|
43
43
|
"compile": "tsc -p tsconfig.build.json",
|
|
44
44
|
"copy": "tsx ../../scripts/copy.ts"
|
|
@@ -50,25 +50,26 @@
|
|
|
50
50
|
"@apify/consts": "^2.41.0",
|
|
51
51
|
"@apify/datastructures": "^2.0.3",
|
|
52
52
|
"@apify/log": "^2.5.18",
|
|
53
|
-
"@apify/
|
|
54
|
-
"@apify/timeout": "^0.3.2",
|
|
53
|
+
"@apify/timeout": "^0.4.4",
|
|
55
54
|
"@apify/utilities": "^2.15.5",
|
|
56
|
-
"@crawlee/
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
55
|
+
"@crawlee/fs-storage": "4.0.0-beta.121",
|
|
56
|
+
"@crawlee/types": "4.0.0-beta.121",
|
|
57
|
+
"@crawlee/utils": "4.0.0-beta.121",
|
|
59
58
|
"@sapphire/async-queue": "^1.5.5",
|
|
59
|
+
"@sapphire/shapeshift": "^4.0.0",
|
|
60
60
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
61
|
+
"content-type": "^1.0.5",
|
|
61
62
|
"csv-stringify": "^6.5.2",
|
|
62
|
-
"fs-extra": "^11.3.0",
|
|
63
|
-
"got-scraping": "^4.1.1",
|
|
64
63
|
"json5": "^2.2.3",
|
|
64
|
+
"mime-types": "^3.0.1",
|
|
65
65
|
"minimatch": "^10.0.1",
|
|
66
66
|
"ow": "^2.0.0",
|
|
67
67
|
"stream-json": "^1.9.1",
|
|
68
68
|
"tldts": "^7.0.6",
|
|
69
|
-
"tough-cookie": "^
|
|
69
|
+
"tough-cookie": "^6.0.0",
|
|
70
70
|
"tslib": "^2.8.1",
|
|
71
|
-
"type-fest": "^4.41.0"
|
|
71
|
+
"type-fest": "^4.41.0",
|
|
72
|
+
"zod": "^4.1.0"
|
|
72
73
|
},
|
|
73
74
|
"lerna": {
|
|
74
75
|
"command": {
|
|
@@ -77,5 +78,5 @@
|
|
|
77
78
|
}
|
|
78
79
|
}
|
|
79
80
|
},
|
|
80
|
-
"gitHead": "
|
|
81
|
+
"gitHead": "5027317de626f5ba6de5047ae9341a898258cc5a"
|
|
81
82
|
}
|
package/proxy_configuration.d.ts
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
|
+
import type { ProxyInfo } from '@crawlee/types';
|
|
1
2
|
import type { Request } from './request.js';
|
|
2
3
|
export interface ProxyConfigurationFunction {
|
|
3
4
|
(options?: {
|
|
4
5
|
request?: Request;
|
|
5
6
|
}): string | null | Promise<string | null>;
|
|
6
7
|
}
|
|
8
|
+
type UrlList = (string | null)[];
|
|
7
9
|
export interface ProxyConfigurationOptions {
|
|
8
10
|
/**
|
|
9
11
|
* An array of custom proxy URLs to be rotated.
|
|
10
12
|
* Custom proxies are not compatible with Apify Proxy and an attempt to use both
|
|
11
13
|
* configuration options will cause an error to be thrown on initialize.
|
|
12
14
|
*/
|
|
13
|
-
proxyUrls?:
|
|
15
|
+
proxyUrls?: UrlList;
|
|
14
16
|
/**
|
|
15
17
|
* Custom function that allows you to generate the new proxy URL dynamically. It gets an optional parameter with the `Request` object when applicable.
|
|
16
18
|
* Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous.
|
|
@@ -18,111 +20,26 @@ export interface ProxyConfigurationOptions {
|
|
|
18
20
|
* This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
|
|
19
21
|
*/
|
|
20
22
|
newUrlFunction?: ProxyConfigurationFunction;
|
|
21
|
-
/**
|
|
22
|
-
* An array of custom proxy URLs to be rotated stratified in tiers.
|
|
23
|
-
* This is a more advanced version of `proxyUrls` that allows you to define a hierarchy of proxy URLs
|
|
24
|
-
* If everything goes well, all the requests will be sent through the first proxy URL in the list.
|
|
25
|
-
* Whenever the crawler encounters a problem with the current proxy on the given domain, it will switch to the higher tier for this domain.
|
|
26
|
-
* The crawler probes lower-level proxies at intervals to check if it can make the tier downshift.
|
|
27
|
-
*
|
|
28
|
-
* This feature is useful when you have a set of proxies with different performance characteristics (speed, price, antibot performance etc.) and you want to use the best one for each domain.
|
|
29
|
-
*
|
|
30
|
-
* Use `null` as a proxy URL to disable the proxy for the given tier.
|
|
31
|
-
*/
|
|
32
|
-
tieredProxyUrls?: (string | null)[][];
|
|
33
|
-
}
|
|
34
|
-
export interface TieredProxy {
|
|
35
|
-
proxyUrl: string | null;
|
|
36
|
-
proxyTier?: number;
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* The main purpose of the ProxyInfo object is to provide information
|
|
40
|
-
* about the current proxy connection used by the crawler for the request.
|
|
41
|
-
* Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
|
|
42
|
-
*
|
|
43
|
-
* **Example usage:**
|
|
44
|
-
*
|
|
45
|
-
* ```javascript
|
|
46
|
-
* const proxyConfiguration = new ProxyConfiguration({
|
|
47
|
-
* proxyUrls: ['...', '...'] // List of Proxy URLs to rotate
|
|
48
|
-
* });
|
|
49
|
-
*
|
|
50
|
-
* // Getting proxyInfo object by calling class method directly
|
|
51
|
-
* const proxyInfo = await proxyConfiguration.newProxyInfo();
|
|
52
|
-
*
|
|
53
|
-
* // In crawler
|
|
54
|
-
* const crawler = new CheerioCrawler({
|
|
55
|
-
* // ...
|
|
56
|
-
* proxyConfiguration,
|
|
57
|
-
* requestHandler({ proxyInfo }) {
|
|
58
|
-
* // Getting used proxy URL
|
|
59
|
-
* const proxyUrl = proxyInfo.url;
|
|
60
|
-
* }
|
|
61
|
-
* })
|
|
62
|
-
*
|
|
63
|
-
* ```
|
|
64
|
-
*/
|
|
65
|
-
export interface ProxyInfo {
|
|
66
|
-
/**
|
|
67
|
-
* The URL of the proxy.
|
|
68
|
-
*/
|
|
69
|
-
url: string;
|
|
70
|
-
/**
|
|
71
|
-
* Username for the proxy.
|
|
72
|
-
*/
|
|
73
|
-
username?: string;
|
|
74
|
-
/**
|
|
75
|
-
* User's password for the proxy.
|
|
76
|
-
*/
|
|
77
|
-
password: string;
|
|
78
|
-
/**
|
|
79
|
-
* Hostname of your proxy.
|
|
80
|
-
*/
|
|
81
|
-
hostname: string;
|
|
82
|
-
/**
|
|
83
|
-
* Proxy port.
|
|
84
|
-
*/
|
|
85
|
-
port: number | string;
|
|
86
|
-
/**
|
|
87
|
-
* Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
|
|
88
|
-
*/
|
|
89
|
-
proxyTier?: number;
|
|
90
|
-
/**
|
|
91
|
-
* When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content.
|
|
92
|
-
*
|
|
93
|
-
* @default false
|
|
94
|
-
*/
|
|
95
|
-
ignoreTlsErrors?: boolean;
|
|
96
23
|
}
|
|
97
|
-
interface
|
|
24
|
+
interface NewUrlOptions {
|
|
98
25
|
request?: Request;
|
|
99
|
-
proxyTier?: number;
|
|
100
26
|
}
|
|
101
27
|
/**
|
|
102
|
-
*
|
|
28
|
+
* Minimal contract that any object passed to a crawler as its `proxyConfiguration`
|
|
29
|
+
* option must satisfy.
|
|
30
|
+
*
|
|
31
|
+
* Implement this interface to plug a custom proxy-provisioning strategy into any Crawlee
|
|
32
|
+
* crawler — for example a remote proxy service or a thin wrapper around the built-in
|
|
33
|
+
* `ProxyConfiguration` with different rotation rules. *
|
|
103
34
|
*
|
|
104
|
-
*
|
|
35
|
+
* @category Scaling
|
|
105
36
|
*/
|
|
106
|
-
|
|
107
|
-
private histogram;
|
|
108
|
-
private currentTier;
|
|
109
|
-
constructor(tieredProxyUrls: (string | null)[][]);
|
|
37
|
+
export interface IProxyConfiguration {
|
|
110
38
|
/**
|
|
111
|
-
*
|
|
39
|
+
* Creates a new {@link ProxyInfo} object describing the proxy to use for the given
|
|
40
|
+
* request. Returns `undefined` when no proxy should be used.
|
|
112
41
|
*/
|
|
113
|
-
|
|
114
|
-
/**
|
|
115
|
-
* Increases the error score for the given proxy tier. This raises the chance of picking a different proxy tier for the subsequent requests.
|
|
116
|
-
*
|
|
117
|
-
* The error score is increased by 10 for the given tier. This means that this tier will be disadvantaged for the next 10 requests (every new request prediction decreases the error score by 1).
|
|
118
|
-
* @param tier The proxy tier to mark as problematic.
|
|
119
|
-
*/
|
|
120
|
-
addError(tier: number): void;
|
|
121
|
-
/**
|
|
122
|
-
* Returns the best proxy tier for the next request based on the error history for different proxy tiers.
|
|
123
|
-
* @returns The proxy tier prediction
|
|
124
|
-
*/
|
|
125
|
-
predictTier(): number;
|
|
42
|
+
newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
|
|
126
43
|
}
|
|
127
44
|
/**
|
|
128
45
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
@@ -152,16 +69,9 @@ declare class ProxyTierTracker {
|
|
|
152
69
|
* ```
|
|
153
70
|
* @category Scaling
|
|
154
71
|
*/
|
|
155
|
-
export declare class ProxyConfiguration {
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
protected proxyUrls?: string[];
|
|
159
|
-
protected tieredProxyUrls?: (string | null)[][];
|
|
160
|
-
protected usedProxyUrls: Map<string, string>;
|
|
161
|
-
protected newUrlFunction?: ProxyConfigurationFunction;
|
|
162
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
163
|
-
protected log: import("@apify/log").Log;
|
|
164
|
-
protected domainTiers: Map<string, ProxyTierTracker>;
|
|
72
|
+
export declare class ProxyConfiguration implements IProxyConfiguration {
|
|
73
|
+
#private;
|
|
74
|
+
readonly isManInTheMiddle = false;
|
|
165
75
|
/**
|
|
166
76
|
* Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
|
|
167
77
|
* blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
@@ -192,38 +102,20 @@ export declare class ProxyConfiguration {
|
|
|
192
102
|
*
|
|
193
103
|
* @return Represents information about used proxy and its configuration.
|
|
194
104
|
*/
|
|
195
|
-
newProxyInfo(options?:
|
|
196
|
-
/**
|
|
197
|
-
* Given a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
|
|
198
|
-
* @param options Options for the tiered proxy rotation
|
|
199
|
-
* @returns An object with the proxy URL and the proxy tier used.
|
|
200
|
-
*/
|
|
201
|
-
protected _handleTieredUrl(options?: TieredProxyOptions): TieredProxy;
|
|
202
|
-
/**
|
|
203
|
-
* Given a `Request` object, this function returns the tier of the proxy that should be used for the request.
|
|
204
|
-
*
|
|
205
|
-
* This returns `null` if `tieredProxyUrls` option is not set.
|
|
206
|
-
*/
|
|
207
|
-
protected predictProxyTier(request: Request): number | null;
|
|
105
|
+
newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
|
|
208
106
|
/**
|
|
209
107
|
* Returns a new proxy URL based on provided configuration options.
|
|
210
108
|
*
|
|
211
109
|
* @return A string with a proxy URL, including authentication credentials and port number.
|
|
212
110
|
* For example, `http://bob:password123@proxy.example.com:8000`
|
|
213
111
|
*/
|
|
214
|
-
newUrl(options?:
|
|
215
|
-
|
|
216
|
-
* Handles custom url rotation
|
|
217
|
-
*/
|
|
218
|
-
protected _handleProxyUrlsList(): string;
|
|
112
|
+
newUrl(options?: NewUrlOptions): Promise<string | undefined>;
|
|
113
|
+
private handleProxyUrlsList;
|
|
219
114
|
/**
|
|
220
115
|
* Calls the custom newUrlFunction and checks format of its return value
|
|
221
116
|
*/
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
protected _throwCannotCombineCustomMethods(): never;
|
|
226
|
-
protected _throwNoOptionsProvided(): never;
|
|
117
|
+
private callNewUrlFunction;
|
|
118
|
+
private throwCannotCombineCustomMethods;
|
|
119
|
+
private throwNoOptionsProvided;
|
|
227
120
|
}
|
|
228
121
|
export {};
|
|
229
|
-
//# sourceMappingURL=proxy_configuration.d.ts.map
|
package/proxy_configuration.js
CHANGED
|
@@ -1,54 +1,4 @@
|
|
|
1
1
|
import ow from 'ow';
|
|
2
|
-
import log from '@apify/log';
|
|
3
|
-
/**
|
|
4
|
-
* Internal class for tracking the proxy tier history for a specific domain.
|
|
5
|
-
*
|
|
6
|
-
* Predicts the best proxy tier for the next request based on the error history for different proxy tiers.
|
|
7
|
-
*/
|
|
8
|
-
class ProxyTierTracker {
|
|
9
|
-
histogram;
|
|
10
|
-
currentTier;
|
|
11
|
-
constructor(tieredProxyUrls) {
|
|
12
|
-
this.histogram = tieredProxyUrls.map(() => 0);
|
|
13
|
-
this.currentTier = 0;
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* Processes a single step of the algorithm and updates the current tier prediction based on the error history.
|
|
17
|
-
*/
|
|
18
|
-
processStep() {
|
|
19
|
-
this.histogram.forEach((x, i) => {
|
|
20
|
-
if (this.currentTier === i)
|
|
21
|
-
return;
|
|
22
|
-
if (x > 0)
|
|
23
|
-
this.histogram[i]--;
|
|
24
|
-
});
|
|
25
|
-
const left = this.currentTier > 0 ? this.histogram[this.currentTier - 1] : Infinity;
|
|
26
|
-
const right = this.currentTier < this.histogram.length - 1 ? this.histogram[this.currentTier + 1] : Infinity;
|
|
27
|
-
if (this.histogram[this.currentTier] > Math.min(left, right)) {
|
|
28
|
-
this.currentTier = left <= right ? this.currentTier - 1 : this.currentTier + 1;
|
|
29
|
-
}
|
|
30
|
-
else if (this.histogram[this.currentTier] === left) {
|
|
31
|
-
this.currentTier--;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
/**
|
|
35
|
-
* Increases the error score for the given proxy tier. This raises the chance of picking a different proxy tier for the subsequent requests.
|
|
36
|
-
*
|
|
37
|
-
* The error score is increased by 10 for the given tier. This means that this tier will be disadvantaged for the next 10 requests (every new request prediction decreases the error score by 1).
|
|
38
|
-
* @param tier The proxy tier to mark as problematic.
|
|
39
|
-
*/
|
|
40
|
-
addError(tier) {
|
|
41
|
-
this.histogram[tier] += 10;
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* Returns the best proxy tier for the next request based on the error history for different proxy tiers.
|
|
45
|
-
* @returns The proxy tier prediction
|
|
46
|
-
*/
|
|
47
|
-
predictTier() {
|
|
48
|
-
this.processStep();
|
|
49
|
-
return this.currentTier;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
2
|
/**
|
|
53
3
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
54
4
|
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
@@ -79,13 +29,9 @@ class ProxyTierTracker {
|
|
|
79
29
|
*/
|
|
80
30
|
export class ProxyConfiguration {
|
|
81
31
|
isManInTheMiddle = false;
|
|
82
|
-
nextCustomUrlIndex = 0;
|
|
83
|
-
proxyUrls;
|
|
84
|
-
|
|
85
|
-
usedProxyUrls = new Map();
|
|
86
|
-
newUrlFunction;
|
|
87
|
-
log = log.child({ prefix: 'ProxyConfiguration' });
|
|
88
|
-
domainTiers = new Map();
|
|
32
|
+
#nextCustomUrlIndex = 0;
|
|
33
|
+
#proxyUrls;
|
|
34
|
+
#newUrlFunction;
|
|
89
35
|
/**
|
|
90
36
|
* Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
|
|
91
37
|
* blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
@@ -108,19 +54,21 @@ export class ProxyConfiguration {
|
|
|
108
54
|
*/
|
|
109
55
|
constructor(options = {}) {
|
|
110
56
|
const { validateRequired, ...rest } = options;
|
|
57
|
+
if ('tieredProxyUrls' in rest) {
|
|
58
|
+
throw new Error('The `tieredProxyUrls` option has been removed in Crawlee v4. ' +
|
|
59
|
+
'See the v4 upgrading guide for the recommended migration to named sessions.');
|
|
60
|
+
}
|
|
111
61
|
ow(rest, ow.object.exactShape({
|
|
112
|
-
proxyUrls: ow.optional.array.nonEmpty.ofType(ow.string.url),
|
|
62
|
+
proxyUrls: ow.optional.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
|
|
113
63
|
newUrlFunction: ow.optional.function,
|
|
114
|
-
tieredProxyUrls: ow.optional.array.nonEmpty.ofType(ow.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null))),
|
|
115
64
|
}));
|
|
116
|
-
const { proxyUrls, newUrlFunction
|
|
117
|
-
if (
|
|
118
|
-
this.
|
|
65
|
+
const { proxyUrls, newUrlFunction } = options;
|
|
66
|
+
if (proxyUrls && newUrlFunction)
|
|
67
|
+
this.throwCannotCombineCustomMethods();
|
|
119
68
|
if (!proxyUrls && !newUrlFunction && validateRequired)
|
|
120
|
-
this.
|
|
121
|
-
this
|
|
122
|
-
this
|
|
123
|
-
this.tieredProxyUrls = tieredProxyUrls;
|
|
69
|
+
this.throwNoOptionsProvided();
|
|
70
|
+
this.#proxyUrls = proxyUrls;
|
|
71
|
+
this.#newUrlFunction = newUrlFunction;
|
|
124
72
|
}
|
|
125
73
|
/**
|
|
126
74
|
* This function creates a new {@link ProxyInfo} info object.
|
|
@@ -132,16 +80,7 @@ export class ProxyConfiguration {
|
|
|
132
80
|
* @return Represents information about used proxy and its configuration.
|
|
133
81
|
*/
|
|
134
82
|
async newProxyInfo(options) {
|
|
135
|
-
|
|
136
|
-
let tier;
|
|
137
|
-
if (this.tieredProxyUrls) {
|
|
138
|
-
const { proxyUrl, proxyTier } = this._handleTieredUrl(options);
|
|
139
|
-
url = proxyUrl ?? undefined;
|
|
140
|
-
tier = proxyTier;
|
|
141
|
-
}
|
|
142
|
-
else {
|
|
143
|
-
url = await this.newUrl(options);
|
|
144
|
-
}
|
|
83
|
+
const url = await this.newUrl(options);
|
|
145
84
|
if (!url)
|
|
146
85
|
return undefined;
|
|
147
86
|
const { username, password, port, hostname } = new URL(url);
|
|
@@ -151,59 +90,8 @@ export class ProxyConfiguration {
|
|
|
151
90
|
password: decodeURIComponent(password),
|
|
152
91
|
hostname,
|
|
153
92
|
port: port,
|
|
154
|
-
proxyTier: tier,
|
|
155
93
|
};
|
|
156
94
|
}
|
|
157
|
-
/**
|
|
158
|
-
* Given a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
|
|
159
|
-
* @param options Options for the tiered proxy rotation
|
|
160
|
-
* @returns An object with the proxy URL and the proxy tier used.
|
|
161
|
-
*/
|
|
162
|
-
_handleTieredUrl(options) {
|
|
163
|
-
if (!this.tieredProxyUrls)
|
|
164
|
-
throw new Error('Tiered proxy URLs are not set');
|
|
165
|
-
if (!options || (!options?.request && options?.proxyTier === undefined)) {
|
|
166
|
-
const allProxyUrls = this.tieredProxyUrls.flat();
|
|
167
|
-
return {
|
|
168
|
-
proxyUrl: allProxyUrls[this.nextCustomUrlIndex++ % allProxyUrls.length],
|
|
169
|
-
};
|
|
170
|
-
}
|
|
171
|
-
let tierPrediction = options.proxyTier;
|
|
172
|
-
if (typeof tierPrediction !== 'number') {
|
|
173
|
-
tierPrediction = this.predictProxyTier(options.request);
|
|
174
|
-
}
|
|
175
|
-
const proxyTier = this.tieredProxyUrls[tierPrediction];
|
|
176
|
-
return {
|
|
177
|
-
proxyUrl: proxyTier[this.nextCustomUrlIndex++ % proxyTier.length],
|
|
178
|
-
proxyTier: tierPrediction,
|
|
179
|
-
};
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Given a `Request` object, this function returns the tier of the proxy that should be used for the request.
|
|
183
|
-
*
|
|
184
|
-
* This returns `null` if `tieredProxyUrls` option is not set.
|
|
185
|
-
*/
|
|
186
|
-
predictProxyTier(request) {
|
|
187
|
-
if (!this.tieredProxyUrls)
|
|
188
|
-
return null;
|
|
189
|
-
const domain = new URL(request.url).hostname;
|
|
190
|
-
if (!this.domainTiers.has(domain)) {
|
|
191
|
-
this.domainTiers.set(domain, new ProxyTierTracker(this.tieredProxyUrls));
|
|
192
|
-
}
|
|
193
|
-
request.userData.__crawlee ??= {};
|
|
194
|
-
const tracker = this.domainTiers.get(domain);
|
|
195
|
-
if (typeof request.userData.__crawlee.lastProxyTier === 'number') {
|
|
196
|
-
tracker.addError(request.userData.__crawlee.lastProxyTier);
|
|
197
|
-
}
|
|
198
|
-
const tierPrediction = tracker.predictTier();
|
|
199
|
-
if (typeof request.userData.__crawlee.lastProxyTier === 'number' &&
|
|
200
|
-
request.userData.__crawlee.lastProxyTier !== tierPrediction) {
|
|
201
|
-
log.debug(`Changing proxy tier for domain "${domain}" from ${request.userData.__crawlee.lastProxyTier} to ${tierPrediction}.`);
|
|
202
|
-
}
|
|
203
|
-
request.userData.__crawlee.lastProxyTier = tierPrediction;
|
|
204
|
-
request.userData.__crawlee.forefront = true;
|
|
205
|
-
return tierPrediction;
|
|
206
|
-
}
|
|
207
95
|
/**
|
|
208
96
|
* Returns a new proxy URL based on provided configuration options.
|
|
209
97
|
*
|
|
@@ -211,25 +99,19 @@ export class ProxyConfiguration {
|
|
|
211
99
|
* For example, `http://bob:password123@proxy.example.com:8000`
|
|
212
100
|
*/
|
|
213
101
|
async newUrl(options) {
|
|
214
|
-
if (this
|
|
215
|
-
return (await this.
|
|
216
|
-
}
|
|
217
|
-
if (this.tieredProxyUrls) {
|
|
218
|
-
return this._handleTieredUrl(options).proxyUrl ?? undefined;
|
|
102
|
+
if (this.#newUrlFunction) {
|
|
103
|
+
return (await this.callNewUrlFunction({ request: options?.request })) ?? undefined;
|
|
219
104
|
}
|
|
220
|
-
return this.
|
|
105
|
+
return this.handleProxyUrlsList() ?? undefined;
|
|
221
106
|
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
*/
|
|
225
|
-
_handleProxyUrlsList() {
|
|
226
|
-
return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
|
|
107
|
+
handleProxyUrlsList() {
|
|
108
|
+
return this.#proxyUrls[this.#nextCustomUrlIndex++ % this.#proxyUrls.length];
|
|
227
109
|
}
|
|
228
110
|
/**
|
|
229
111
|
* Calls the custom newUrlFunction and checks format of its return value
|
|
230
112
|
*/
|
|
231
|
-
async
|
|
232
|
-
const proxyUrl = await this
|
|
113
|
+
async callNewUrlFunction(options) {
|
|
114
|
+
const proxyUrl = await this.#newUrlFunction(options);
|
|
233
115
|
try {
|
|
234
116
|
if (proxyUrl) {
|
|
235
117
|
new URL(proxyUrl); // eslint-disable-line no-new
|
|
@@ -240,11 +122,10 @@ export class ProxyConfiguration {
|
|
|
240
122
|
throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
|
|
241
123
|
}
|
|
242
124
|
}
|
|
243
|
-
|
|
125
|
+
throwCannotCombineCustomMethods() {
|
|
244
126
|
throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
|
|
245
127
|
}
|
|
246
|
-
|
|
128
|
+
throwNoOptionsProvided() {
|
|
247
129
|
throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
|
|
248
130
|
}
|
|
249
131
|
}
|
|
250
|
-
//# sourceMappingURL=proxy_configuration.js.map
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import type { Configuration, CrawleeLogger } from '@crawlee/core';
|
|
2
|
+
import { KeyValueStore } from '@crawlee/core';
|
|
3
|
+
import type { Awaitable } from '@crawlee/types';
|
|
4
|
+
import type { StandardSchemaV1 } from '@standard-schema/spec';
|
|
5
|
+
/**
|
|
6
|
+
* One direction of the conversion between the state model and its persisted form - either a plain function, or a
|
|
7
|
+
* [Standard Schema](https://standardschema.dev) whose validated output is the result.
|
|
8
|
+
*
|
|
9
|
+
* A schema that fails to validate makes {@link RecoverableState} throw a {@link StateValidationError}. Zod
|
|
10
|
+
* codecs work directly, as their validation *is* the decode direction; use `(state) => codec.encode(state)` for the
|
|
11
|
+
* other one.
|
|
12
|
+
*/
|
|
13
|
+
export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;
|
|
14
|
+
export interface RecoverableStatePersistenceOptions {
|
|
15
|
+
/**
|
|
16
|
+
* The key under which the state is stored in the KeyValueStore
|
|
17
|
+
*/
|
|
18
|
+
persistStateKey: string;
|
|
19
|
+
/**
|
|
20
|
+
* Flag to enable or disable state persistence
|
|
21
|
+
*/
|
|
22
|
+
persistenceEnabled?: boolean;
|
|
23
|
+
/**
|
|
24
|
+
* The KeyValueStore to persist into, defaulting to the default store. Accepts a pending
|
|
25
|
+
* {@link KeyValueStore.open} so that callers do not have to be async to point at a specific store.
|
|
26
|
+
*/
|
|
27
|
+
keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
|
|
28
|
+
/**
|
|
29
|
+
* Time limit for a single load or save of the state, in milliseconds.
|
|
30
|
+
* @default 60_000
|
|
31
|
+
*/
|
|
32
|
+
persistenceTimeoutMillis?: number;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Options for configuring the RecoverableState
|
|
36
|
+
*/
|
|
37
|
+
export interface RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
|
|
38
|
+
/**
|
|
39
|
+
* The state used when no persisted state is found, and the state {@link RecoverableState.reset} restores.
|
|
40
|
+
*
|
|
41
|
+
* A plain value is deep-copied with `structuredClone` each time it is used, so pass a factory for a state
|
|
42
|
+
* that `structuredClone` cannot rebuild - one holding class instances, say, or one derived from a schema.
|
|
43
|
+
*/
|
|
44
|
+
defaultState: TStateModel | (() => TStateModel);
|
|
45
|
+
/**
|
|
46
|
+
* A logger instance for logging operations related to state persistence
|
|
47
|
+
*/
|
|
48
|
+
logger?: CrawleeLogger;
|
|
49
|
+
/**
|
|
50
|
+
* Configuration instance to use when opening the KeyValueStore
|
|
51
|
+
*/
|
|
52
|
+
configuration?: Configuration;
|
|
53
|
+
/**
|
|
54
|
+
* Optional conversion of the state to a plain JSON-serializable value before it is persisted.
|
|
55
|
+
* If not provided, the state is persisted as is.
|
|
56
|
+
*/
|
|
57
|
+
serialize?: StateConversion<TStateModel, TPersistedState>;
|
|
58
|
+
/**
|
|
59
|
+
* Optional conversion of a persisted value back to the state model, and the place to validate a record before
|
|
60
|
+
* trusting it. If not provided, the persisted value is used as is.
|
|
61
|
+
*/
|
|
62
|
+
deserialize?: StateConversion<TPersistedState, TStateModel>;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* A class for managing persistent recoverable state using a plain JavaScript object.
|
|
66
|
+
*
|
|
67
|
+
* This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
|
|
68
|
+
* across migrations or restarts. It manages the loading, saving, and resetting of state data,
|
|
69
|
+
* with optional persistence capabilities.
|
|
70
|
+
*
|
|
71
|
+
* The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
|
|
72
|
+
* The class automatically hooks into the event system to persist state when needed.
|
|
73
|
+
*/
|
|
74
|
+
export declare class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
|
|
75
|
+
#private;
|
|
76
|
+
/**
|
|
77
|
+
* Initialize a new recoverable state object.
|
|
78
|
+
*
|
|
79
|
+
* @param options Configuration options for the recoverable state
|
|
80
|
+
*/
|
|
81
|
+
constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);
|
|
82
|
+
/**
|
|
83
|
+
* Initialize the recoverable state.
|
|
84
|
+
*
|
|
85
|
+
* If persistence is enabled, this method loads the saved state and registers the object to listen for
|
|
86
|
+
* PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
|
|
87
|
+
* is no record to restore.
|
|
88
|
+
*
|
|
89
|
+
* Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
|
|
90
|
+
* listener is registered again and the record reloaded.
|
|
91
|
+
*
|
|
92
|
+
* @returns The loaded state object
|
|
93
|
+
*/
|
|
94
|
+
initialize(): Promise<TStateModel>;
|
|
95
|
+
/**
|
|
96
|
+
* Clean up resources used by the recoverable state.
|
|
97
|
+
*
|
|
98
|
+
* If persistence is enabled, this method deregisters the object from PERSIST_STATE events
|
|
99
|
+
* and persists the current state one last time, warning rather than throwing if that write fails - cleanup
|
|
100
|
+
* runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
|
|
101
|
+
* state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
|
|
102
|
+
* persistence window.
|
|
103
|
+
*/
|
|
104
|
+
teardown(): Promise<void>;
|
|
105
|
+
/**
|
|
106
|
+
* Get the current state.
|
|
107
|
+
*
|
|
108
|
+
* Throws until the state has been established, by either {@link RecoverableState.initialize} or the
|
|
109
|
+
* synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
|
|
110
|
+
* constructor gets a usable state right away.
|
|
111
|
+
*/
|
|
112
|
+
get currentValue(): TStateModel;
|
|
113
|
+
/**
|
|
114
|
+
* Reset the in-memory state to the default values, leaving any persisted record alone.
|
|
115
|
+
*
|
|
116
|
+
* Use {@link RecoverableState.resetStore} to clear the persisted record as well.
|
|
117
|
+
*/
|
|
118
|
+
reset(): void;
|
|
119
|
+
/**
|
|
120
|
+
* Clear the persisted state record, leaving the in-memory state alone.
|
|
121
|
+
*
|
|
122
|
+
* This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
|
|
123
|
+
* from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
|
|
124
|
+
* one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
|
|
125
|
+
* or {@link RecoverableState.teardown} before clearing the record.
|
|
126
|
+
*
|
|
127
|
+
* A no-op if persistence is disabled or no KeyValueStore is available yet.
|
|
128
|
+
*/
|
|
129
|
+
resetStore(): Promise<void>;
|
|
130
|
+
/**
|
|
131
|
+
* Persist the current state to the KeyValueStore.
|
|
132
|
+
*
|
|
133
|
+
* This method is typically called in response to a PERSIST_STATE event, but can also be called
|
|
134
|
+
* directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
|
|
135
|
+
* there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
|
|
136
|
+
*
|
|
137
|
+
* @param eventData Optional data associated with a PERSIST_STATE event
|
|
138
|
+
*/
|
|
139
|
+
persistState(eventData?: Record<string, unknown>): Promise<void>;
|
|
140
|
+
}
|