@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/autoscaling/autoscaled_pool.d.ts +14 -15
- package/autoscaling/autoscaled_pool.js +37 -36
- package/autoscaling/client_load_signal.d.ts +25 -0
- package/autoscaling/client_load_signal.js +36 -0
- package/autoscaling/cpu_load_signal.d.ts +28 -0
- package/autoscaling/cpu_load_signal.js +24 -0
- package/autoscaling/event_loop_load_signal.d.ts +23 -0
- package/autoscaling/event_loop_load_signal.js +35 -0
- package/autoscaling/index.d.ts +5 -1
- package/autoscaling/index.js +5 -1
- package/autoscaling/load_signal.d.ts +99 -0
- package/autoscaling/load_signal.js +135 -0
- package/autoscaling/memory_load_signal.d.ts +43 -0
- package/autoscaling/memory_load_signal.js +102 -0
- package/autoscaling/snapshotter.d.ts +29 -91
- package/autoscaling/snapshotter.js +65 -204
- package/autoscaling/system_status.d.ts +22 -37
- package/autoscaling/system_status.js +49 -87
- package/configuration.d.ts +88 -228
- package/configuration.js +162 -224
- package/cookie_utils.d.ts +4 -3
- package/cookie_utils.js +20 -13
- package/crawlers/context_pipeline.d.ts +70 -0
- package/crawlers/context_pipeline.js +122 -0
- package/crawlers/crawler_commons.d.ts +20 -30
- package/crawlers/crawler_commons.js +12 -21
- package/crawlers/crawler_utils.d.ts +2 -3
- package/crawlers/crawler_utils.js +1 -2
- package/crawlers/error_snapshotter.d.ts +3 -3
- package/crawlers/error_snapshotter.js +2 -3
- package/crawlers/error_tracker.d.ts +2 -2
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +1 -2
- package/crawlers/index.js +1 -2
- package/crawlers/internals/types.d.ts +7 -0
- package/crawlers/internals/types.js +1 -0
- package/crawlers/statistics.d.ts +28 -23
- package/crawlers/statistics.js +38 -34
- package/enqueue_links/enqueue_links.d.ts +44 -22
- package/enqueue_links/enqueue_links.js +51 -30
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +25 -9
- package/enqueue_links/shared.js +69 -38
- package/errors.d.ts +53 -4
- package/errors.js +70 -5
- package/events/event_manager.d.ts +34 -8
- package/events/event_manager.js +8 -10
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -5
- package/events/local_event_manager.js +37 -40
- package/index.d.ts +4 -3
- package/index.js +3 -2
- package/log.d.ts +82 -3
- package/log.js +102 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +46 -0
- package/memory-storage/memory-storage.js +136 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +113 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +203 -0
- package/memory-storage/resource-clients/request-queue.d.ts +96 -0
- package/memory-storage/resource-clients/request-queue.js +421 -0
- package/memory-storage/utils.d.ts +17 -0
- package/memory-storage/utils.js +42 -0
- package/package.json +12 -10
- package/proxy_configuration.d.ts +29 -152
- package/proxy_configuration.js +21 -173
- package/recoverable_state.d.ts +120 -0
- package/recoverable_state.js +141 -0
- package/request.d.ts +84 -13
- package/request.js +107 -28
- package/router.d.ts +118 -16
- package/router.js +124 -30
- package/serialization.d.ts +0 -1
- package/serialization.js +1 -2
- package/service_locator.d.ts +156 -0
- package/service_locator.js +238 -0
- package/session_pool/consts.d.ts +1 -2
- package/session_pool/consts.js +1 -2
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +9 -0
- package/session_pool/fingerprint.js +30 -0
- package/session_pool/index.d.ts +0 -2
- package/session_pool/index.js +0 -2
- package/session_pool/session.d.ts +37 -75
- package/session_pool/session.js +49 -102
- package/session_pool/session_pool.d.ts +85 -90
- package/session_pool/session_pool.js +131 -120
- package/storages/access_checking.d.ts +0 -1
- package/storages/access_checking.js +5 -2
- package/storages/dataset.d.ts +101 -51
- package/storages/dataset.js +168 -126
- package/storages/index.d.ts +8 -7
- package/storages/index.js +6 -7
- package/storages/key_value_store.d.ts +165 -36
- package/storages/key_value_store.js +269 -122
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +23 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +53 -97
- package/storages/request_list.js +99 -75
- package/storages/request_loader.d.ts +96 -0
- package/storages/request_loader.js +1 -0
- package/storages/request_manager.d.ts +33 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +106 -0
- package/storages/request_manager_tandem.js +197 -0
- package/storages/request_queue.d.ts +290 -47
- package/storages/request_queue.js +628 -215
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +28 -24
- package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +49 -46
- package/storages/storage_instance_manager.d.ts +91 -0
- package/storages/storage_instance_manager.js +258 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/utils.d.ts +51 -6
- package/storages/utils.js +60 -9
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +1 -2
- package/typedefs.js +0 -1
- package/validators.d.ts +8 -1
- package/validators.js +10 -3
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_provider.d.ts +0 -307
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -555
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
package/proxy_configuration.d.ts
CHANGED
|
@@ -1,129 +1,45 @@
|
|
|
1
|
+
import type { ProxyInfo } from '@crawlee/types';
|
|
1
2
|
import type { Request } from './request.js';
|
|
2
3
|
export interface ProxyConfigurationFunction {
|
|
3
|
-
(
|
|
4
|
+
(options?: {
|
|
4
5
|
request?: Request;
|
|
5
6
|
}): string | null | Promise<string | null>;
|
|
6
7
|
}
|
|
8
|
+
type UrlList = (string | null)[];
|
|
7
9
|
export interface ProxyConfigurationOptions {
|
|
8
10
|
/**
|
|
9
11
|
* An array of custom proxy URLs to be rotated.
|
|
10
12
|
* Custom proxies are not compatible with Apify Proxy and an attempt to use both
|
|
11
13
|
* configuration options will cause an error to be thrown on initialize.
|
|
12
14
|
*/
|
|
13
|
-
proxyUrls?:
|
|
15
|
+
proxyUrls?: UrlList;
|
|
14
16
|
/**
|
|
15
|
-
* Custom function that allows you to generate the new proxy URL dynamically. It gets
|
|
17
|
+
* Custom function that allows you to generate the new proxy URL dynamically. It gets an optional parameter with the `Request` object when applicable.
|
|
16
18
|
* Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous.
|
|
17
19
|
*
|
|
18
20
|
* This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
|
|
19
21
|
*/
|
|
20
22
|
newUrlFunction?: ProxyConfigurationFunction;
|
|
21
|
-
/**
|
|
22
|
-
* An array of custom proxy URLs to be rotated stratified in tiers.
|
|
23
|
-
* This is a more advanced version of `proxyUrls` that allows you to define a hierarchy of proxy URLs
|
|
24
|
-
* If everything goes well, all the requests will be sent through the first proxy URL in the list.
|
|
25
|
-
* Whenever the crawler encounters a problem with the current proxy on the given domain, it will switch to the higher tier for this domain.
|
|
26
|
-
* The crawler probes lower-level proxies at intervals to check if it can make the tier downshift.
|
|
27
|
-
*
|
|
28
|
-
* This feature is useful when you have a set of proxies with different performance characteristics (speed, price, antibot performance etc.) and you want to use the best one for each domain.
|
|
29
|
-
*
|
|
30
|
-
* Use `null` as a proxy URL to disable the proxy for the given tier.
|
|
31
|
-
*/
|
|
32
|
-
tieredProxyUrls?: (string | null)[][];
|
|
33
|
-
}
|
|
34
|
-
export interface TieredProxy {
|
|
35
|
-
proxyUrl: string | null;
|
|
36
|
-
proxyTier?: number;
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* The main purpose of the ProxyInfo object is to provide information
|
|
40
|
-
* about the current proxy connection used by the crawler for the request.
|
|
41
|
-
* Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
|
|
42
|
-
*
|
|
43
|
-
* **Example usage:**
|
|
44
|
-
*
|
|
45
|
-
* ```javascript
|
|
46
|
-
* const proxyConfiguration = new ProxyConfiguration({
|
|
47
|
-
* proxyUrls: ['...', '...'] // List of Proxy URLs to rotate
|
|
48
|
-
* });
|
|
49
|
-
*
|
|
50
|
-
* // Getting proxyInfo object by calling class method directly
|
|
51
|
-
* const proxyInfo = await proxyConfiguration.newProxyInfo();
|
|
52
|
-
*
|
|
53
|
-
* // In crawler
|
|
54
|
-
* const crawler = new CheerioCrawler({
|
|
55
|
-
* // ...
|
|
56
|
-
* proxyConfiguration,
|
|
57
|
-
* requestHandler({ proxyInfo }) {
|
|
58
|
-
* // Getting used proxy URL
|
|
59
|
-
* const proxyUrl = proxyInfo.url;
|
|
60
|
-
*
|
|
61
|
-
* // Getting ID of used Session
|
|
62
|
-
* const sessionIdentifier = proxyInfo.sessionId;
|
|
63
|
-
* }
|
|
64
|
-
* })
|
|
65
|
-
*
|
|
66
|
-
* ```
|
|
67
|
-
*/
|
|
68
|
-
export interface ProxyInfo {
|
|
69
|
-
/**
|
|
70
|
-
* The identifier of used {@link Session}, if used.
|
|
71
|
-
*/
|
|
72
|
-
sessionId?: string;
|
|
73
|
-
/**
|
|
74
|
-
* The URL of the proxy.
|
|
75
|
-
*/
|
|
76
|
-
url: string;
|
|
77
|
-
/**
|
|
78
|
-
* Username for the proxy.
|
|
79
|
-
*/
|
|
80
|
-
username?: string;
|
|
81
|
-
/**
|
|
82
|
-
* User's password for the proxy.
|
|
83
|
-
*/
|
|
84
|
-
password: string;
|
|
85
|
-
/**
|
|
86
|
-
* Hostname of your proxy.
|
|
87
|
-
*/
|
|
88
|
-
hostname: string;
|
|
89
|
-
/**
|
|
90
|
-
* Proxy port.
|
|
91
|
-
*/
|
|
92
|
-
port: number | string;
|
|
93
|
-
/**
|
|
94
|
-
* Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
|
|
95
|
-
*/
|
|
96
|
-
proxyTier?: number;
|
|
97
23
|
}
|
|
98
|
-
interface
|
|
24
|
+
interface NewUrlOptions {
|
|
99
25
|
request?: Request;
|
|
100
|
-
proxyTier?: number;
|
|
101
26
|
}
|
|
102
27
|
/**
|
|
103
|
-
*
|
|
28
|
+
* Minimal contract that any object passed to a crawler as its `proxyConfiguration`
|
|
29
|
+
* option must satisfy.
|
|
30
|
+
*
|
|
31
|
+
* Implement this interface to plug a custom proxy-provisioning strategy into any Crawlee
|
|
32
|
+
* crawler — for example a remote proxy service or a thin wrapper around the built-in
|
|
33
|
+
* `ProxyConfiguration` with different rotation rules. *
|
|
104
34
|
*
|
|
105
|
-
*
|
|
35
|
+
* @category Scaling
|
|
106
36
|
*/
|
|
107
|
-
|
|
108
|
-
private histogram;
|
|
109
|
-
private currentTier;
|
|
110
|
-
constructor(tieredProxyUrls: (string | null)[][]);
|
|
37
|
+
export interface IProxyConfiguration {
|
|
111
38
|
/**
|
|
112
|
-
*
|
|
39
|
+
* Creates a new {@link ProxyInfo} object describing the proxy to use for the given
|
|
40
|
+
* request. Returns `undefined` when no proxy should be used.
|
|
113
41
|
*/
|
|
114
|
-
|
|
115
|
-
/**
|
|
116
|
-
* Increases the error score for the given proxy tier. This raises the chance of picking a different proxy tier for the subsequent requests.
|
|
117
|
-
*
|
|
118
|
-
* The error score is increased by 10 for the given tier. This means that this tier will be disadvantaged for the next 10 requests (every new request prediction decreases the error score by 1).
|
|
119
|
-
* @param tier The proxy tier to mark as problematic.
|
|
120
|
-
*/
|
|
121
|
-
addError(tier: number): void;
|
|
122
|
-
/**
|
|
123
|
-
* Returns the best proxy tier for the next request based on the error history for different proxy tiers.
|
|
124
|
-
* @returns The proxy tier prediction
|
|
125
|
-
*/
|
|
126
|
-
predictTier(): number;
|
|
42
|
+
newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
|
|
127
43
|
}
|
|
128
44
|
/**
|
|
129
45
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
@@ -153,16 +69,11 @@ declare class ProxyTierTracker {
|
|
|
153
69
|
* ```
|
|
154
70
|
* @category Scaling
|
|
155
71
|
*/
|
|
156
|
-
export declare class ProxyConfiguration {
|
|
157
|
-
isManInTheMiddle
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
protected usedProxyUrls: Map<string, string>;
|
|
162
|
-
protected newUrlFunction?: ProxyConfigurationFunction;
|
|
163
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
164
|
-
protected log: import("@apify/log").Log;
|
|
165
|
-
protected domainTiers: Map<string, ProxyTierTracker>;
|
|
72
|
+
export declare class ProxyConfiguration implements IProxyConfiguration {
|
|
73
|
+
readonly isManInTheMiddle = false;
|
|
74
|
+
private nextCustomUrlIndex;
|
|
75
|
+
private proxyUrls?;
|
|
76
|
+
private newUrlFunction?;
|
|
166
77
|
/**
|
|
167
78
|
* Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
|
|
168
79
|
* blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
@@ -190,57 +101,23 @@ export declare class ProxyConfiguration {
|
|
|
190
101
|
* the currently used proxy via the requestHandler parameter `proxyInfo`.
|
|
191
102
|
* Use it if you want to work with a rich representation of a proxy URL.
|
|
192
103
|
* If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
|
|
193
|
-
* @param [sessionId]
|
|
194
|
-
* Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
|
|
195
|
-
* you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
|
|
196
|
-
* When the provided sessionId is a number, it's converted to a string. Property sessionId of
|
|
197
|
-
* {@link ProxyInfo} is always returned as a type string.
|
|
198
104
|
*
|
|
199
|
-
* All the HTTP requests going through the proxy with the same session identifier
|
|
200
|
-
* will use the same target proxy server (i.e. the same IP address).
|
|
201
|
-
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
|
|
202
105
|
* @return Represents information about used proxy and its configuration.
|
|
203
106
|
*/
|
|
204
|
-
newProxyInfo(
|
|
205
|
-
/**
|
|
206
|
-
* Given a session identifier and a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
|
|
207
|
-
* @param _sessionId Session identifier
|
|
208
|
-
* @param options Options for the tiered proxy rotation
|
|
209
|
-
* @returns An object with the proxy URL and the proxy tier used.
|
|
210
|
-
*/
|
|
211
|
-
protected _handleTieredUrl(_sessionId: string, options?: TieredProxyOptions): TieredProxy;
|
|
107
|
+
newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
|
|
212
108
|
/**
|
|
213
|
-
*
|
|
109
|
+
* Returns a new proxy URL based on provided configuration options.
|
|
214
110
|
*
|
|
215
|
-
* This returns `null` if `tieredProxyUrls` option is not set.
|
|
216
|
-
*/
|
|
217
|
-
protected predictProxyTier(request: Request): number | null;
|
|
218
|
-
/**
|
|
219
|
-
* Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
|
|
220
|
-
* @param [sessionId]
|
|
221
|
-
* Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
|
|
222
|
-
* you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
|
|
223
|
-
* When the provided sessionId is a number, it's converted to a string.
|
|
224
|
-
*
|
|
225
|
-
* All the HTTP requests going through the proxy with the same session identifier
|
|
226
|
-
* will use the same target proxy server (i.e. the same IP address).
|
|
227
|
-
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
|
|
228
111
|
* @return A string with a proxy URL, including authentication credentials and port number.
|
|
229
112
|
* For example, `http://bob:password123@proxy.example.com:8000`
|
|
230
113
|
*/
|
|
231
|
-
newUrl(
|
|
232
|
-
|
|
233
|
-
* Handles custom url rotation with session
|
|
234
|
-
*/
|
|
235
|
-
protected _handleCustomUrl(sessionId?: string): string;
|
|
114
|
+
newUrl(options?: NewUrlOptions): Promise<string | undefined>;
|
|
115
|
+
private handleProxyUrlsList;
|
|
236
116
|
/**
|
|
237
117
|
* Calls the custom newUrlFunction and checks format of its return value
|
|
238
118
|
*/
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
protected _throwCannotCombineCustomMethods(): never;
|
|
243
|
-
protected _throwNoOptionsProvided(): never;
|
|
119
|
+
private callNewUrlFunction;
|
|
120
|
+
private throwCannotCombineCustomMethods;
|
|
121
|
+
private throwNoOptionsProvided;
|
|
244
122
|
}
|
|
245
123
|
export {};
|
|
246
|
-
//# sourceMappingURL=proxy_configuration.d.ts.map
|
package/proxy_configuration.js
CHANGED
|
@@ -1,55 +1,4 @@
|
|
|
1
1
|
import ow from 'ow';
|
|
2
|
-
import log from '@apify/log';
|
|
3
|
-
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
4
|
-
/**
|
|
5
|
-
* Internal class for tracking the proxy tier history for a specific domain.
|
|
6
|
-
*
|
|
7
|
-
* Predicts the best proxy tier for the next request based on the error history for different proxy tiers.
|
|
8
|
-
*/
|
|
9
|
-
class ProxyTierTracker {
|
|
10
|
-
histogram;
|
|
11
|
-
currentTier;
|
|
12
|
-
constructor(tieredProxyUrls) {
|
|
13
|
-
this.histogram = tieredProxyUrls.map(() => 0);
|
|
14
|
-
this.currentTier = 0;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Processes a single step of the algorithm and updates the current tier prediction based on the error history.
|
|
18
|
-
*/
|
|
19
|
-
processStep() {
|
|
20
|
-
this.histogram.forEach((x, i) => {
|
|
21
|
-
if (this.currentTier === i)
|
|
22
|
-
return;
|
|
23
|
-
if (x > 0)
|
|
24
|
-
this.histogram[i]--;
|
|
25
|
-
});
|
|
26
|
-
const left = this.currentTier > 0 ? this.histogram[this.currentTier - 1] : Infinity;
|
|
27
|
-
const right = this.currentTier < this.histogram.length - 1 ? this.histogram[this.currentTier + 1] : Infinity;
|
|
28
|
-
if (this.histogram[this.currentTier] > Math.min(left, right)) {
|
|
29
|
-
this.currentTier = left <= right ? this.currentTier - 1 : this.currentTier + 1;
|
|
30
|
-
}
|
|
31
|
-
else if (this.histogram[this.currentTier] === left) {
|
|
32
|
-
this.currentTier--;
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Increases the error score for the given proxy tier. This raises the chance of picking a different proxy tier for the subsequent requests.
|
|
37
|
-
*
|
|
38
|
-
* The error score is increased by 10 for the given tier. This means that this tier will be disadvantaged for the next 10 requests (every new request prediction decreases the error score by 1).
|
|
39
|
-
* @param tier The proxy tier to mark as problematic.
|
|
40
|
-
*/
|
|
41
|
-
addError(tier) {
|
|
42
|
-
this.histogram[tier] += 10;
|
|
43
|
-
}
|
|
44
|
-
/**
|
|
45
|
-
* Returns the best proxy tier for the next request based on the error history for different proxy tiers.
|
|
46
|
-
* @returns The proxy tier prediction
|
|
47
|
-
*/
|
|
48
|
-
predictTier() {
|
|
49
|
-
this.processStep();
|
|
50
|
-
return this.currentTier;
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
2
|
/**
|
|
54
3
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
55
4
|
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
@@ -82,11 +31,7 @@ export class ProxyConfiguration {
|
|
|
82
31
|
isManInTheMiddle = false;
|
|
83
32
|
nextCustomUrlIndex = 0;
|
|
84
33
|
proxyUrls;
|
|
85
|
-
tieredProxyUrls;
|
|
86
|
-
usedProxyUrls = new Map();
|
|
87
34
|
newUrlFunction;
|
|
88
|
-
log = log.child({ prefix: 'ProxyConfiguration' });
|
|
89
|
-
domainTiers = new Map();
|
|
90
35
|
/**
|
|
91
36
|
* Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
|
|
92
37
|
* blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
@@ -109,19 +54,21 @@ export class ProxyConfiguration {
|
|
|
109
54
|
*/
|
|
110
55
|
constructor(options = {}) {
|
|
111
56
|
const { validateRequired, ...rest } = options;
|
|
57
|
+
if ('tieredProxyUrls' in rest) {
|
|
58
|
+
throw new Error('The `tieredProxyUrls` option has been removed in Crawlee v4. ' +
|
|
59
|
+
'See the v4 upgrading guide for the recommended migration to named sessions.');
|
|
60
|
+
}
|
|
112
61
|
ow(rest, ow.object.exactShape({
|
|
113
|
-
proxyUrls: ow.optional.array.nonEmpty.ofType(ow.string.url),
|
|
62
|
+
proxyUrls: ow.optional.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
|
|
114
63
|
newUrlFunction: ow.optional.function,
|
|
115
|
-
tieredProxyUrls: ow.optional.array.nonEmpty.ofType(ow.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null))),
|
|
116
64
|
}));
|
|
117
|
-
const { proxyUrls, newUrlFunction
|
|
118
|
-
if (
|
|
119
|
-
this.
|
|
65
|
+
const { proxyUrls, newUrlFunction } = options;
|
|
66
|
+
if (proxyUrls && newUrlFunction)
|
|
67
|
+
this.throwCannotCombineCustomMethods();
|
|
120
68
|
if (!proxyUrls && !newUrlFunction && validateRequired)
|
|
121
|
-
this.
|
|
69
|
+
this.throwNoOptionsProvided();
|
|
122
70
|
this.proxyUrls = proxyUrls;
|
|
123
71
|
this.newUrlFunction = newUrlFunction;
|
|
124
|
-
this.tieredProxyUrls = tieredProxyUrls;
|
|
125
72
|
}
|
|
126
73
|
/**
|
|
127
74
|
* This function creates a new {@link ProxyInfo} info object.
|
|
@@ -129,140 +76,42 @@ export class ProxyConfiguration {
|
|
|
129
76
|
* the currently used proxy via the requestHandler parameter `proxyInfo`.
|
|
130
77
|
* Use it if you want to work with a rich representation of a proxy URL.
|
|
131
78
|
* If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
|
|
132
|
-
* @param [sessionId]
|
|
133
|
-
* Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
|
|
134
|
-
* you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
|
|
135
|
-
* When the provided sessionId is a number, it's converted to a string. Property sessionId of
|
|
136
|
-
* {@link ProxyInfo} is always returned as a type string.
|
|
137
79
|
*
|
|
138
|
-
* All the HTTP requests going through the proxy with the same session identifier
|
|
139
|
-
* will use the same target proxy server (i.e. the same IP address).
|
|
140
|
-
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
|
|
141
80
|
* @return Represents information about used proxy and its configuration.
|
|
142
81
|
*/
|
|
143
|
-
async newProxyInfo(
|
|
144
|
-
|
|
145
|
-
sessionId = `${sessionId}`;
|
|
146
|
-
let url;
|
|
147
|
-
let tier;
|
|
148
|
-
if (this.tieredProxyUrls) {
|
|
149
|
-
const { proxyUrl, proxyTier } = this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options);
|
|
150
|
-
url = proxyUrl ?? undefined;
|
|
151
|
-
tier = proxyTier;
|
|
152
|
-
}
|
|
153
|
-
else {
|
|
154
|
-
url = await this.newUrl(sessionId, options);
|
|
155
|
-
}
|
|
82
|
+
async newProxyInfo(options) {
|
|
83
|
+
const url = await this.newUrl(options);
|
|
156
84
|
if (!url)
|
|
157
85
|
return undefined;
|
|
158
86
|
const { username, password, port, hostname } = new URL(url);
|
|
159
87
|
return {
|
|
160
|
-
sessionId,
|
|
161
88
|
url,
|
|
162
89
|
username: decodeURIComponent(username),
|
|
163
90
|
password: decodeURIComponent(password),
|
|
164
91
|
hostname,
|
|
165
92
|
port: port,
|
|
166
|
-
proxyTier: tier,
|
|
167
93
|
};
|
|
168
94
|
}
|
|
169
95
|
/**
|
|
170
|
-
*
|
|
171
|
-
* @param _sessionId Session identifier
|
|
172
|
-
* @param options Options for the tiered proxy rotation
|
|
173
|
-
* @returns An object with the proxy URL and the proxy tier used.
|
|
174
|
-
*/
|
|
175
|
-
_handleTieredUrl(_sessionId, options) {
|
|
176
|
-
if (!this.tieredProxyUrls)
|
|
177
|
-
throw new Error('Tiered proxy URLs are not set');
|
|
178
|
-
if (!options || (!options?.request && options?.proxyTier === undefined)) {
|
|
179
|
-
const allProxyUrls = this.tieredProxyUrls.flat();
|
|
180
|
-
return {
|
|
181
|
-
proxyUrl: allProxyUrls[this.nextCustomUrlIndex++ % allProxyUrls.length],
|
|
182
|
-
};
|
|
183
|
-
}
|
|
184
|
-
let tierPrediction = options.proxyTier;
|
|
185
|
-
if (typeof tierPrediction !== 'number') {
|
|
186
|
-
tierPrediction = this.predictProxyTier(options.request);
|
|
187
|
-
}
|
|
188
|
-
const proxyTier = this.tieredProxyUrls[tierPrediction];
|
|
189
|
-
return {
|
|
190
|
-
proxyUrl: proxyTier[this.nextCustomUrlIndex++ % proxyTier.length],
|
|
191
|
-
proxyTier: tierPrediction,
|
|
192
|
-
};
|
|
193
|
-
}
|
|
194
|
-
/**
|
|
195
|
-
* Given a `Request` object, this function returns the tier of the proxy that should be used for the request.
|
|
196
|
-
*
|
|
197
|
-
* This returns `null` if `tieredProxyUrls` option is not set.
|
|
198
|
-
*/
|
|
199
|
-
predictProxyTier(request) {
|
|
200
|
-
if (!this.tieredProxyUrls)
|
|
201
|
-
return null;
|
|
202
|
-
const domain = new URL(request.url).hostname;
|
|
203
|
-
if (!this.domainTiers.has(domain)) {
|
|
204
|
-
this.domainTiers.set(domain, new ProxyTierTracker(this.tieredProxyUrls));
|
|
205
|
-
}
|
|
206
|
-
request.userData.__crawlee ??= {};
|
|
207
|
-
const tracker = this.domainTiers.get(domain);
|
|
208
|
-
if (typeof request.userData.__crawlee.lastProxyTier === 'number') {
|
|
209
|
-
tracker.addError(request.userData.__crawlee.lastProxyTier);
|
|
210
|
-
}
|
|
211
|
-
const tierPrediction = tracker.predictTier();
|
|
212
|
-
if (typeof request.userData.__crawlee.lastProxyTier === 'number' &&
|
|
213
|
-
request.userData.__crawlee.lastProxyTier !== tierPrediction) {
|
|
214
|
-
log.debug(`Changing proxy tier for domain "${domain}" from ${request.userData.__crawlee.lastProxyTier} to ${tierPrediction}.`);
|
|
215
|
-
}
|
|
216
|
-
request.userData.__crawlee.lastProxyTier = tierPrediction;
|
|
217
|
-
request.userData.__crawlee.forefront = true;
|
|
218
|
-
return tierPrediction;
|
|
219
|
-
}
|
|
220
|
-
/**
|
|
221
|
-
* Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
|
|
222
|
-
* @param [sessionId]
|
|
223
|
-
* Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
|
|
224
|
-
* you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
|
|
225
|
-
* When the provided sessionId is a number, it's converted to a string.
|
|
96
|
+
* Returns a new proxy URL based on provided configuration options.
|
|
226
97
|
*
|
|
227
|
-
* All the HTTP requests going through the proxy with the same session identifier
|
|
228
|
-
* will use the same target proxy server (i.e. the same IP address).
|
|
229
|
-
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
|
|
230
98
|
* @return A string with a proxy URL, including authentication credentials and port number.
|
|
231
99
|
* For example, `http://bob:password123@proxy.example.com:8000`
|
|
232
100
|
*/
|
|
233
|
-
async newUrl(
|
|
234
|
-
if (typeof sessionId === 'number')
|
|
235
|
-
sessionId = `${sessionId}`;
|
|
101
|
+
async newUrl(options) {
|
|
236
102
|
if (this.newUrlFunction) {
|
|
237
|
-
return (await this.
|
|
238
|
-
}
|
|
239
|
-
if (this.tieredProxyUrls) {
|
|
240
|
-
return this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options).proxyUrl ?? undefined;
|
|
103
|
+
return (await this.callNewUrlFunction({ request: options?.request })) ?? undefined;
|
|
241
104
|
}
|
|
242
|
-
return this.
|
|
105
|
+
return this.handleProxyUrlsList() ?? undefined;
|
|
243
106
|
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
*/
|
|
247
|
-
_handleCustomUrl(sessionId) {
|
|
248
|
-
let customUrlToUse;
|
|
249
|
-
if (!sessionId) {
|
|
250
|
-
return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
|
|
251
|
-
}
|
|
252
|
-
if (this.usedProxyUrls.has(sessionId)) {
|
|
253
|
-
customUrlToUse = this.usedProxyUrls.get(sessionId);
|
|
254
|
-
}
|
|
255
|
-
else {
|
|
256
|
-
customUrlToUse = this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
|
|
257
|
-
this.usedProxyUrls.set(sessionId, customUrlToUse);
|
|
258
|
-
}
|
|
259
|
-
return customUrlToUse;
|
|
107
|
+
handleProxyUrlsList() {
|
|
108
|
+
return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
|
|
260
109
|
}
|
|
261
110
|
/**
|
|
262
111
|
* Calls the custom newUrlFunction and checks format of its return value
|
|
263
112
|
*/
|
|
264
|
-
async
|
|
265
|
-
const proxyUrl = await this.newUrlFunction(
|
|
113
|
+
async callNewUrlFunction(options) {
|
|
114
|
+
const proxyUrl = await this.newUrlFunction(options);
|
|
266
115
|
try {
|
|
267
116
|
if (proxyUrl) {
|
|
268
117
|
new URL(proxyUrl); // eslint-disable-line no-new
|
|
@@ -273,11 +122,10 @@ export class ProxyConfiguration {
|
|
|
273
122
|
throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
|
|
274
123
|
}
|
|
275
124
|
}
|
|
276
|
-
|
|
125
|
+
throwCannotCombineCustomMethods() {
|
|
277
126
|
throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
|
|
278
127
|
}
|
|
279
|
-
|
|
128
|
+
throwNoOptionsProvided() {
|
|
280
129
|
throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
|
|
281
130
|
}
|
|
282
131
|
}
|
|
283
|
-
//# sourceMappingURL=proxy_configuration.js.map
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import type { Configuration, CrawleeLogger } from '@crawlee/core';
|
|
2
|
+
export interface RecoverableStatePersistenceOptions {
|
|
3
|
+
/**
|
|
4
|
+
* The key under which the state is stored in the KeyValueStore
|
|
5
|
+
*/
|
|
6
|
+
persistStateKey: string;
|
|
7
|
+
/**
|
|
8
|
+
* Flag to enable or disable state persistence
|
|
9
|
+
*/
|
|
10
|
+
persistenceEnabled?: boolean;
|
|
11
|
+
/**
|
|
12
|
+
* The name of the KeyValueStore to use for persistence.
|
|
13
|
+
* If neither a name nor an id are supplied, the default store will be used.
|
|
14
|
+
*/
|
|
15
|
+
persistStateKvsName?: string;
|
|
16
|
+
/**
|
|
17
|
+
* The identifier of the KeyValueStore to use for persistence.
|
|
18
|
+
* If neither a name nor an id are supplied, the default store will be used.
|
|
19
|
+
*/
|
|
20
|
+
persistStateKvsId?: string;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Options for configuring the RecoverableState
|
|
24
|
+
*/
|
|
25
|
+
export interface RecoverableStateOptions<TStateModel = Record<string, unknown>> extends RecoverableStatePersistenceOptions {
|
|
26
|
+
/**
|
|
27
|
+
* The default state used if no persisted state is found.
|
|
28
|
+
* A deep copy is made each time the state is used.
|
|
29
|
+
*/
|
|
30
|
+
defaultState: TStateModel;
|
|
31
|
+
/**
|
|
32
|
+
* A logger instance for logging operations related to state persistence
|
|
33
|
+
*/
|
|
34
|
+
logger?: CrawleeLogger;
|
|
35
|
+
/**
|
|
36
|
+
* Configuration instance to use
|
|
37
|
+
*/
|
|
38
|
+
config?: Configuration;
|
|
39
|
+
/**
|
|
40
|
+
* Optional function to transform the state to a JSON string before persistence.
|
|
41
|
+
* If not provided, JSON.stringify will be used.
|
|
42
|
+
*/
|
|
43
|
+
serialize?: (state: TStateModel) => string;
|
|
44
|
+
/**
|
|
45
|
+
* Optional function to transform a JSON-serialized object back to the state model.
|
|
46
|
+
* If not provided, JSON.parse is used.
|
|
47
|
+
* It is advisable to perform validation in this function and to throw an exception if it fails.
|
|
48
|
+
*/
|
|
49
|
+
deserialize?: (serializedState: string) => TStateModel;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* A class for managing persistent recoverable state using a plain JavaScript object.
|
|
53
|
+
*
|
|
54
|
+
* This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
|
|
55
|
+
* across migrations or restarts. It manages the loading, saving, and resetting of state data,
|
|
56
|
+
* with optional persistence capabilities.
|
|
57
|
+
*
|
|
58
|
+
* The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
|
|
59
|
+
* The class automatically hooks into the event system to persist state when needed.
|
|
60
|
+
*/
|
|
61
|
+
export declare class RecoverableState<TStateModel = Record<string, unknown>> {
|
|
62
|
+
private readonly defaultState;
|
|
63
|
+
private state;
|
|
64
|
+
private readonly persistenceEnabled;
|
|
65
|
+
private readonly persistStateKey;
|
|
66
|
+
private readonly persistStateKvsName?;
|
|
67
|
+
private readonly persistStateKvsId?;
|
|
68
|
+
private keyValueStore;
|
|
69
|
+
private readonly log;
|
|
70
|
+
private readonly serialize;
|
|
71
|
+
private readonly deserialize;
|
|
72
|
+
/**
|
|
73
|
+
* Initialize a new recoverable state object.
|
|
74
|
+
*
|
|
75
|
+
* @param options Configuration options for the recoverable state
|
|
76
|
+
*/
|
|
77
|
+
constructor(options: RecoverableStateOptions<TStateModel>);
|
|
78
|
+
/**
|
|
79
|
+
* Initialize the recoverable state.
|
|
80
|
+
*
|
|
81
|
+
* This method must be called before using the recoverable state. It loads the saved state
|
|
82
|
+
* if persistence is enabled and registers the object to listen for PERSIST_STATE events.
|
|
83
|
+
*
|
|
84
|
+
* @returns The loaded state object
|
|
85
|
+
*/
|
|
86
|
+
initialize(): Promise<TStateModel>;
|
|
87
|
+
/**
|
|
88
|
+
* Clean up resources used by the recoverable state.
|
|
89
|
+
*
|
|
90
|
+
* If persistence is enabled, this method deregisters the object from PERSIST_STATE events
|
|
91
|
+
* and persists the current state one last time.
|
|
92
|
+
*/
|
|
93
|
+
teardown(): Promise<void>;
|
|
94
|
+
/**
|
|
95
|
+
* Get the current state.
|
|
96
|
+
*/
|
|
97
|
+
get currentValue(): TStateModel;
|
|
98
|
+
/**
|
|
99
|
+
* Reset the state to the default values and clear any persisted state.
|
|
100
|
+
*
|
|
101
|
+
* Resets the current state to the default state and, if persistence is enabled,
|
|
102
|
+
* clears the persisted state from the KeyValueStore.
|
|
103
|
+
*/
|
|
104
|
+
reset(): Promise<void>;
|
|
105
|
+
/**
|
|
106
|
+
* Persist the current state to the KeyValueStore.
|
|
107
|
+
*
|
|
108
|
+
* This method is typically called in response to a PERSIST_STATE event, but can also be called
|
|
109
|
+
* directly when needed.
|
|
110
|
+
*
|
|
111
|
+
* @param eventData Optional data associated with a PERSIST_STATE event
|
|
112
|
+
*/
|
|
113
|
+
persistState(eventData?: {
|
|
114
|
+
isMigrating: boolean;
|
|
115
|
+
}): Promise<void>;
|
|
116
|
+
/**
|
|
117
|
+
* Load the saved state from the KeyValueStore
|
|
118
|
+
*/
|
|
119
|
+
private loadSavedState;
|
|
120
|
+
}
|