@crawlee/core 4.0.0-beta.15 → 4.0.0-beta.150
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/autoscaling/autoscaled_pool.d.ts +73 -177
- package/autoscaling/autoscaled_pool.js +182 -329
- package/autoscaling/concurrency_system.d.ts +268 -0
- package/autoscaling/concurrency_system.js +364 -0
- package/autoscaling/cpu_load_signal.d.ts +43 -0
- package/autoscaling/cpu_load_signal.js +47 -0
- package/autoscaling/event_loop_load_signal.d.ts +51 -0
- package/autoscaling/event_loop_load_signal.js +60 -0
- package/autoscaling/index.d.ts +6 -1
- package/autoscaling/index.js +6 -1
- package/autoscaling/load_signal.d.ts +98 -0
- package/autoscaling/load_signal.js +103 -0
- package/autoscaling/memory_load_signal.d.ts +47 -0
- package/autoscaling/memory_load_signal.js +106 -0
- package/autoscaling/snapshotter.d.ts +58 -163
- package/autoscaling/snapshotter.js +45 -263
- package/autoscaling/storage_backend_load_signal.d.ts +56 -0
- package/autoscaling/storage_backend_load_signal.js +73 -0
- package/autoscaling/system_status.d.ts +67 -89
- package/autoscaling/system_status.js +92 -122
- package/autoscaling/weighted_avg.d.ts +5 -0
- package/autoscaling/weighted_avg.js +14 -0
- package/byte_utils.d.ts +17 -0
- package/byte_utils.js +42 -0
- package/configuration.d.ts +96 -223
- package/configuration.js +170 -222
- package/cookie_utils.d.ts +3 -2
- package/cookie_utils.js +18 -7
- package/crawlers/context_pipeline.d.ts +10 -1
- package/crawlers/context_pipeline.js +31 -8
- package/crawlers/crawler_commons.d.ts +90 -126
- package/crawlers/crawler_commons.js +1 -108
- package/crawlers/error_snapshotter.d.ts +2 -5
- package/crawlers/error_snapshotter.js +7 -8
- package/crawlers/error_tracker.d.ts +0 -1
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +1 -3
- package/crawlers/index.js +0 -3
- package/crawlers/internals/types.d.ts +0 -1
- package/crawlers/internals/types.js +0 -1
- package/crawlers/statistics.d.ts +187 -64
- package/crawlers/statistics.js +354 -165
- package/debug.d.ts +36 -0
- package/debug.js +70 -0
- package/enqueue_links/enqueue_links.d.ts +61 -154
- package/enqueue_links/enqueue_links.js +40 -232
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +55 -28
- package/enqueue_links/shared.js +116 -69
- package/errors.d.ts +72 -4
- package/errors.js +89 -5
- package/events/event_manager.d.ts +35 -9
- package/events/event_manager.js +12 -13
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -3
- package/events/local_event_manager.js +39 -13
- package/http.d.ts +9 -0
- package/http.js +28 -0
- package/index.d.ts +7 -5
- package/index.js +8 -4
- package/iterables.d.ts +79 -0
- package/iterables.js +134 -0
- package/log.d.ts +77 -1
- package/log.js +104 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +44 -0
- package/memory-storage/memory-storage.js +160 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +106 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +199 -0
- package/memory-storage/resource-clients/request-queue.d.ts +77 -0
- package/memory-storage/resource-clients/request-queue.js +407 -0
- package/memory-storage/utils.d.ts +16 -0
- package/memory-storage/utils.js +41 -0
- package/owned_or_injected.d.ts +58 -0
- package/owned_or_injected.js +98 -0
- package/package.json +13 -12
- package/proxy_configuration.d.ts +22 -128
- package/proxy_configuration.js +32 -144
- package/recoverable_state.d.ts +83 -51
- package/recoverable_state.js +163 -72
- package/request.d.ts +57 -16
- package/request.js +130 -69
- package/router.d.ts +193 -21
- package/router.js +188 -43
- package/serialization.d.ts +0 -1
- package/serialization.js +15 -15
- package/service_locator.d.ts +165 -0
- package/service_locator.js +253 -0
- package/session_pool/consts.d.ts +1 -2
- package/session_pool/consts.js +1 -2
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +9 -0
- package/session_pool/fingerprint.js +30 -0
- package/session_pool/index.d.ts +0 -2
- package/session_pool/index.js +0 -2
- package/session_pool/session.d.ts +39 -89
- package/session_pool/session.js +102 -159
- package/session_pool/session_pool.d.ts +67 -91
- package/session_pool/session_pool.js +196 -187
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +109 -56
- package/storages/dataset.js +284 -149
- package/storages/index.d.ts +9 -9
- package/storages/index.js +7 -9
- package/storages/key_value_store.d.ts +183 -48
- package/storages/key_value_store.js +444 -171
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +22 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +53 -115
- package/storages/request_list.js +194 -167
- package/storages/request_loader.d.ts +133 -0
- package/storages/request_loader.js +36 -0
- package/storages/request_manager.d.ts +109 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +55 -23
- package/storages/request_manager_tandem.js +139 -57
- package/storages/request_queue.d.ts +286 -48
- package/storages/request_queue.js +757 -218
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +43 -91
- package/storages/sitemap_request_loader.js +439 -0
- package/storages/storage_instance_manager.d.ts +87 -0
- package/storages/storage_instance_manager.js +256 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/throttling_request_manager.d.ts +222 -0
- package/storages/throttling_request_manager.js +780 -0
- package/storages/transaction.d.ts +252 -0
- package/storages/transaction.js +251 -0
- package/storages/utils.d.ts +59 -11
- package/storages/utils.js +75 -15
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +0 -6
- package/typedefs.js +0 -1
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +22 -18
- package/validators.js +13 -18
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/context_pipeline.d.ts.map +0 -1
- package/crawlers/context_pipeline.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js +0 -12
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/internals/types.d.ts.map +0 -1
- package/crawlers/internals/types.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -140
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -40
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -20
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -85
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/recoverable_state.d.ts.map +0 -1
- package/recoverable_state.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts +0 -13
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js +0 -14
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_list_adapter.d.ts +0 -58
- package/storages/request_list_adapter.d.ts.map +0 -1
- package/storages/request_list_adapter.js +0 -81
- package/storages/request_list_adapter.js.map +0 -1
- package/storages/request_manager_tandem.d.ts.map +0 -1
- package/storages/request_manager_tandem.js.map +0 -1
- package/storages/request_provider.d.ts +0 -371
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -585
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js +0 -430
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
package/enqueue_links/shared.js
CHANGED
|
@@ -1,16 +1,45 @@
|
|
|
1
1
|
import { URL } from 'node:url';
|
|
2
2
|
import { Minimatch } from 'minimatch';
|
|
3
|
-
import {
|
|
3
|
+
import { z } from 'zod';
|
|
4
4
|
import { Request } from '../request.js';
|
|
5
|
-
|
|
5
|
+
import { schemas } from '../validators.js';
|
|
6
6
|
const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
|
|
7
7
|
/**
|
|
8
|
-
* To
|
|
9
|
-
*
|
|
10
|
-
* by the `enqueueLinks()` function.
|
|
8
|
+
* To keep high performance when the same patterns are passed on every `enqueueLinks()` call,
|
|
9
|
+
* each glob/regexp is only compiled once and then kept in a cache.
|
|
11
10
|
* @ignore
|
|
12
11
|
*/
|
|
13
12
|
const enqueueLinksPatternCache = new Map();
|
|
13
|
+
/**
|
|
14
|
+
* Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
|
|
15
|
+
* @internal
|
|
16
|
+
*/
|
|
17
|
+
export const urlPatternSchema = z.union([
|
|
18
|
+
z.string(),
|
|
19
|
+
z.instanceof(RegExp),
|
|
20
|
+
schemas.objectWithKeys(['glob']),
|
|
21
|
+
schemas.objectWithKeys(['regexp']),
|
|
22
|
+
]);
|
|
23
|
+
/**
|
|
24
|
+
* Builds the `{ request, reason }` argument passed to a {@link SkippedRequestCallback}, constructing the
|
|
25
|
+
* `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a
|
|
26
|
+
* real callback and building a `Request` isn't free.
|
|
27
|
+
* @ignore
|
|
28
|
+
*/
|
|
29
|
+
export function createSkippedRequestArgs(source, reason) {
|
|
30
|
+
const sourceReason = typeof source === 'string' ? undefined : source.skippedReason;
|
|
31
|
+
let request;
|
|
32
|
+
return {
|
|
33
|
+
reason: sourceReason ?? reason,
|
|
34
|
+
get request() {
|
|
35
|
+
request ??=
|
|
36
|
+
source instanceof Request
|
|
37
|
+
? source
|
|
38
|
+
: new Request(typeof source === 'string' ? { url: source } : source);
|
|
39
|
+
return request;
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
14
43
|
/**
|
|
15
44
|
* @ignore
|
|
16
45
|
*/
|
|
@@ -21,28 +50,6 @@ export function updateEnqueueLinksPatternCache(item, pattern) {
|
|
|
21
50
|
enqueueLinksPatternCache.delete(key);
|
|
22
51
|
}
|
|
23
52
|
}
|
|
24
|
-
/**
|
|
25
|
-
* Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
|
|
26
|
-
* to construct RegExps from PseudoUrl strings.
|
|
27
|
-
* @ignore
|
|
28
|
-
*/
|
|
29
|
-
export function constructRegExpObjectsFromPseudoUrls(pseudoUrls) {
|
|
30
|
-
return pseudoUrls.map((item) => {
|
|
31
|
-
// Get pseudoUrl object from cache.
|
|
32
|
-
let regexpObject = enqueueLinksPatternCache.get(item);
|
|
33
|
-
if (regexpObject)
|
|
34
|
-
return regexpObject;
|
|
35
|
-
if (typeof item === 'string') {
|
|
36
|
-
regexpObject = { regexp: purlToRegExp(item) };
|
|
37
|
-
}
|
|
38
|
-
else {
|
|
39
|
-
const { purl, ...requestOptions } = item;
|
|
40
|
-
regexpObject = { regexp: purlToRegExp(purl), ...requestOptions };
|
|
41
|
-
}
|
|
42
|
-
updateEnqueueLinksPatternCache(item, regexpObject);
|
|
43
|
-
return regexpObject;
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
53
|
/**
|
|
47
54
|
* Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
|
|
48
55
|
* to construct Glob objects from Glob pattern strings.
|
|
@@ -72,8 +79,7 @@ export function constructGlobObjectsFromGlobs(globs) {
|
|
|
72
79
|
globObject = { glob: validateGlobPattern(item) };
|
|
73
80
|
}
|
|
74
81
|
else {
|
|
75
|
-
|
|
76
|
-
globObject = { glob: validateGlobPattern(glob), ...requestOptions };
|
|
82
|
+
globObject = { glob: validateGlobPattern(item.glob) };
|
|
77
83
|
}
|
|
78
84
|
updateEnqueueLinksPatternCache(item, globObject);
|
|
79
85
|
return globObject;
|
|
@@ -103,62 +109,70 @@ export function constructRegExpObjectsFromRegExps(regexps) {
|
|
|
103
109
|
regexpObject = { regexp: item };
|
|
104
110
|
}
|
|
105
111
|
else {
|
|
106
|
-
regexpObject = item;
|
|
112
|
+
regexpObject = { regexp: item.regexp };
|
|
107
113
|
}
|
|
108
114
|
updateEnqueueLinksPatternCache(item, regexpObject);
|
|
109
115
|
return regexpObject;
|
|
110
116
|
});
|
|
111
117
|
}
|
|
112
118
|
/**
|
|
119
|
+
* Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
|
|
120
|
+
* from a mixed array of glob strings, glob objects, RegExp instances, and regexp objects.
|
|
113
121
|
* @ignore
|
|
114
122
|
*/
|
|
115
|
-
export function
|
|
116
|
-
const
|
|
117
|
-
const
|
|
123
|
+
export function constructUrlPatternObjects(patterns) {
|
|
124
|
+
const result = [];
|
|
125
|
+
for (const item of patterns) {
|
|
126
|
+
if (typeof item === 'string' || 'glob' in item) {
|
|
127
|
+
result.push(...constructGlobObjectsFromGlobs([item]));
|
|
128
|
+
}
|
|
129
|
+
else if (item instanceof RegExp || 'regexp' in item) {
|
|
130
|
+
result.push(...constructRegExpObjectsFromRegExps([item]));
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return result;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Filters request options by URL patterns.
|
|
137
|
+
*
|
|
138
|
+
* When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
|
|
139
|
+
* @ignore
|
|
140
|
+
*/
|
|
141
|
+
export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedRequestOptions) {
|
|
142
|
+
const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
|
|
143
|
+
const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
|
|
118
144
|
return requestOptions
|
|
119
|
-
.
|
|
120
|
-
.
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
onSkippedUrl?.(url);
|
|
145
|
+
.filter((opts) => {
|
|
146
|
+
const matchesExclude = excludeMatchers.some(({ match }) => match(opts.url));
|
|
147
|
+
if (matchesExclude) {
|
|
148
|
+
onSkippedRequestOptions?.(opts);
|
|
124
149
|
}
|
|
125
|
-
return !
|
|
150
|
+
return !matchesExclude;
|
|
126
151
|
})
|
|
127
|
-
.map((
|
|
128
|
-
if (!
|
|
129
|
-
return
|
|
130
|
-
}
|
|
131
|
-
for (const
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
const request = typeof opts === 'string'
|
|
135
|
-
? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
|
|
136
|
-
: { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
|
|
137
|
-
return new Request(request);
|
|
152
|
+
.map((opts) => {
|
|
153
|
+
if (!includeMatchers) {
|
|
154
|
+
return { ...opts, enqueueStrategy: strategy };
|
|
155
|
+
}
|
|
156
|
+
for (const { match } of includeMatchers) {
|
|
157
|
+
if (match(opts.url)) {
|
|
158
|
+
return { ...opts, enqueueStrategy: strategy };
|
|
138
159
|
}
|
|
139
160
|
}
|
|
140
161
|
// didn't match any positive pattern
|
|
141
|
-
|
|
162
|
+
onSkippedRequestOptions?.(opts);
|
|
142
163
|
return null;
|
|
143
164
|
})
|
|
144
|
-
.filter((
|
|
165
|
+
.filter((opts) => opts !== null);
|
|
145
166
|
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
167
|
+
function isAbsoluteUrl(url) {
|
|
168
|
+
try {
|
|
169
|
+
// eslint-disable-next-line no-new
|
|
170
|
+
new URL(url);
|
|
171
|
+
return true;
|
|
149
172
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
for (const request of requests) {
|
|
153
|
-
const matchingPattern = patternMatchers.find(({ match }) => match(request.url));
|
|
154
|
-
if (matchingPattern !== undefined) {
|
|
155
|
-
filtered.push(request);
|
|
156
|
-
}
|
|
157
|
-
else {
|
|
158
|
-
onSkippedUrl?.(request.url);
|
|
159
|
-
}
|
|
173
|
+
catch {
|
|
174
|
+
return false;
|
|
160
175
|
}
|
|
161
|
-
return filtered;
|
|
162
176
|
}
|
|
163
177
|
/**
|
|
164
178
|
* @ignore
|
|
@@ -177,7 +191,12 @@ export function createRequestOptions(sources, options = {}) {
|
|
|
177
191
|
}
|
|
178
192
|
})
|
|
179
193
|
.map((requestOptions) => {
|
|
180
|
-
|
|
194
|
+
// Leave already-absolute URLs untouched - re-deriving them via `new URL()` would normalize them
|
|
195
|
+
// (e.g. adding a trailing slash to a bare domain), which is surprising for URLs that didn't need
|
|
196
|
+
// resolving against `baseUrl` in the first place.
|
|
197
|
+
if (!isAbsoluteUrl(requestOptions.url)) {
|
|
198
|
+
requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
|
|
199
|
+
}
|
|
181
200
|
requestOptions.userData ??= options.userData ?? {};
|
|
182
201
|
if (typeof options.label === 'string') {
|
|
183
202
|
requestOptions.userData = {
|
|
@@ -188,6 +207,9 @@ export function createRequestOptions(sources, options = {}) {
|
|
|
188
207
|
if (options.skipNavigation) {
|
|
189
208
|
requestOptions.skipNavigation = true;
|
|
190
209
|
}
|
|
210
|
+
if (options.sessionId) {
|
|
211
|
+
requestOptions.sessionId = options.sessionId;
|
|
212
|
+
}
|
|
191
213
|
return requestOptions;
|
|
192
214
|
});
|
|
193
215
|
}
|
|
@@ -207,6 +229,31 @@ function createPatternObjectMatcher(urlPatternObject) {
|
|
|
207
229
|
else {
|
|
208
230
|
match = () => false;
|
|
209
231
|
}
|
|
210
|
-
return {
|
|
232
|
+
return { match };
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Applies a {@link RequestTransform} function to a list of request options.
|
|
236
|
+
* Options for which the transform returns a falsy value are removed from the list.
|
|
237
|
+
* @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
|
|
238
|
+
* @ignore
|
|
239
|
+
* @internal
|
|
240
|
+
*/
|
|
241
|
+
export function applyRequestTransform(requestOptions, transformFn, onSkipped) {
|
|
242
|
+
return requestOptions
|
|
243
|
+
.map((opts) => {
|
|
244
|
+
const transformed = transformFn(opts);
|
|
245
|
+
if (transformed === 'skip') {
|
|
246
|
+
onSkipped?.(opts);
|
|
247
|
+
return null;
|
|
248
|
+
}
|
|
249
|
+
if (transformed === 'unchanged') {
|
|
250
|
+
return opts;
|
|
251
|
+
}
|
|
252
|
+
if (!transformed) {
|
|
253
|
+
onSkipped?.(opts);
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
return transformed;
|
|
257
|
+
})
|
|
258
|
+
.filter((r) => r !== null);
|
|
211
259
|
}
|
|
212
|
-
//# sourceMappingURL=shared.js.map
|
package/errors.d.ts
CHANGED
|
@@ -14,6 +14,37 @@ export declare class CriticalError extends NonRetryableError {
|
|
|
14
14
|
*/
|
|
15
15
|
export declare class MissingRouteError extends CriticalError {
|
|
16
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* A schema validation issue, structurally compatible with `StandardSchemaV1.Issue`. Declared here so that
|
|
19
|
+
* error types do not have to depend on `@standard-schema/spec`.
|
|
20
|
+
*/
|
|
21
|
+
export interface SchemaIssue {
|
|
22
|
+
readonly message: string;
|
|
23
|
+
readonly path?: readonly (PropertyKey | {
|
|
24
|
+
key: PropertyKey;
|
|
25
|
+
})[];
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
|
|
29
|
+
*
|
|
30
|
+
* As the `userData` does not change between attempts, this error is non-retryable.
|
|
31
|
+
*/
|
|
32
|
+
export declare class RequestValidationError extends NonRetryableError {
|
|
33
|
+
readonly label: string | symbol;
|
|
34
|
+
readonly issues: readonly SchemaIssue[];
|
|
35
|
+
constructor(label: string | symbol, issues: readonly SchemaIssue[]);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Thrown by {@link RecoverableState} when a persisted state record does not match its `stateSchema`.
|
|
39
|
+
*
|
|
40
|
+
* Whether a corrupt record should abort the run or be discarded in favour of the defaults depends on what the
|
|
41
|
+
* state is for, so {@link RecoverableState.initialize} always throws and leaves the choice to the caller.
|
|
42
|
+
*/
|
|
43
|
+
export declare class StateValidationError extends Error {
|
|
44
|
+
readonly persistStateKey: string;
|
|
45
|
+
readonly issues: readonly SchemaIssue[];
|
|
46
|
+
constructor(persistStateKey: string, issues: readonly SchemaIssue[]);
|
|
47
|
+
}
|
|
17
48
|
/**
|
|
18
49
|
* Errors of `RetryRequestError` type will always be retried by the crawler.
|
|
19
50
|
*
|
|
@@ -23,13 +54,39 @@ export declare class RetryRequestError extends Error {
|
|
|
23
54
|
constructor(message?: string);
|
|
24
55
|
}
|
|
25
56
|
/**
|
|
26
|
-
*
|
|
57
|
+
* Thrown when a domain has rate-limited us and the request should simply be attempted again later.
|
|
27
58
|
*
|
|
28
|
-
*
|
|
59
|
+
* The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
|
|
60
|
+
* nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
|
|
61
|
+
* the domain's backoff expires, so retries are paced rather than immediate.
|
|
29
62
|
*/
|
|
30
|
-
export declare class
|
|
63
|
+
export declare class RequestThrottledError extends RetryRequestError {
|
|
31
64
|
constructor(message?: string);
|
|
32
65
|
}
|
|
66
|
+
/**
|
|
67
|
+
* Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
|
|
68
|
+
* abandoned rather than kept waiting.
|
|
69
|
+
*
|
|
70
|
+
* Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
|
|
71
|
+
* The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
|
|
72
|
+
* resumes them once the domain recovers.
|
|
73
|
+
*/
|
|
74
|
+
export declare class PersistentRateLimitError extends CriticalError {
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
|
|
78
|
+
*
|
|
79
|
+
* The retry counts towards the `maxRequestRetries` limit, just like any other error.
|
|
80
|
+
*/
|
|
81
|
+
export declare class SessionError extends Error {
|
|
82
|
+
constructor(message?: string);
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Thrown when a requested session is not found in the referenced SessionPool.
|
|
86
|
+
*/
|
|
87
|
+
export declare class MissingSessionError extends Error {
|
|
88
|
+
constructor(sessionId?: string);
|
|
89
|
+
}
|
|
33
90
|
export declare class ContextPipelineInterruptedError extends Error {
|
|
34
91
|
constructor(message?: string);
|
|
35
92
|
}
|
|
@@ -42,4 +99,15 @@ export declare class ContextPipelineCleanupError extends CriticalError {
|
|
|
42
99
|
export declare class RequestHandlerError extends Error {
|
|
43
100
|
constructor(error: unknown, options?: ErrorOptions);
|
|
44
101
|
}
|
|
45
|
-
|
|
102
|
+
/**
|
|
103
|
+
* Thrown when attempting to set a different service instance after one has already been retrieved.
|
|
104
|
+
*/
|
|
105
|
+
export declare class ServiceConflictError extends Error {
|
|
106
|
+
constructor(serviceName: string, newValue: unknown, existingValue: unknown);
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Thrown by crawlers when `skipNavigation` is used on a request.
|
|
110
|
+
* Subclasses can catch this error to skip their own navigation-dependent logic.
|
|
111
|
+
*/
|
|
112
|
+
export declare class NavigationSkippedError extends NonRetryableError {
|
|
113
|
+
}
|
package/errors.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { inspectValue } from './debug.js';
|
|
1
2
|
/**
|
|
2
3
|
* Errors of `NonRetryableError` type will never be retried by the crawler.
|
|
3
4
|
*/
|
|
@@ -14,6 +15,45 @@ export class CriticalError extends NonRetryableError {
|
|
|
14
15
|
*/
|
|
15
16
|
export class MissingRouteError extends CriticalError {
|
|
16
17
|
}
|
|
18
|
+
function formatIssues(issues) {
|
|
19
|
+
return issues
|
|
20
|
+
.map((issue) => {
|
|
21
|
+
const path = (issue.path ?? [])
|
|
22
|
+
.map((segment) => (typeof segment === 'object' ? segment.key : segment))
|
|
23
|
+
.join('.');
|
|
24
|
+
return `- ${path ? `${path}: ` : ''}${issue.message}`;
|
|
25
|
+
})
|
|
26
|
+
.join('\n');
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
|
|
30
|
+
*
|
|
31
|
+
* As the `userData` does not change between attempts, this error is non-retryable.
|
|
32
|
+
*/
|
|
33
|
+
export class RequestValidationError extends NonRetryableError {
|
|
34
|
+
label;
|
|
35
|
+
issues;
|
|
36
|
+
constructor(label, issues) {
|
|
37
|
+
super(`Request userData for label '${String(label)}' failed schema validation:\n${formatIssues(issues)}`);
|
|
38
|
+
this.label = label;
|
|
39
|
+
this.issues = issues;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Thrown by {@link RecoverableState} when a persisted state record does not match its `stateSchema`.
|
|
44
|
+
*
|
|
45
|
+
* Whether a corrupt record should abort the run or be discarded in favour of the defaults depends on what the
|
|
46
|
+
* state is for, so {@link RecoverableState.initialize} always throws and leaves the choice to the caller.
|
|
47
|
+
*/
|
|
48
|
+
export class StateValidationError extends Error {
|
|
49
|
+
persistStateKey;
|
|
50
|
+
issues;
|
|
51
|
+
constructor(persistStateKey, issues) {
|
|
52
|
+
super(`State persisted under key '${persistStateKey}' failed schema validation:\n${formatIssues(issues)}`);
|
|
53
|
+
this.persistStateKey = persistStateKey;
|
|
54
|
+
this.issues = issues;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
17
57
|
/**
|
|
18
58
|
* Errors of `RetryRequestError` type will always be retried by the crawler.
|
|
19
59
|
*
|
|
@@ -25,13 +65,43 @@ export class RetryRequestError extends Error {
|
|
|
25
65
|
}
|
|
26
66
|
}
|
|
27
67
|
/**
|
|
28
|
-
*
|
|
68
|
+
* Thrown when a domain has rate-limited us and the request should simply be attempted again later.
|
|
69
|
+
*
|
|
70
|
+
* The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
|
|
71
|
+
* nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
|
|
72
|
+
* the domain's backoff expires, so retries are paced rather than immediate.
|
|
73
|
+
*/
|
|
74
|
+
export class RequestThrottledError extends RetryRequestError {
|
|
75
|
+
constructor(message) {
|
|
76
|
+
super(message ?? 'Request is being retried later because its domain is rate-limiting us');
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
|
|
81
|
+
* abandoned rather than kept waiting.
|
|
82
|
+
*
|
|
83
|
+
* Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
|
|
84
|
+
* The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
|
|
85
|
+
* resumes them once the domain recovers.
|
|
86
|
+
*/
|
|
87
|
+
export class PersistentRateLimitError extends CriticalError {
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
|
|
29
91
|
*
|
|
30
|
-
*
|
|
92
|
+
* The retry counts towards the `maxRequestRetries` limit, just like any other error.
|
|
31
93
|
*/
|
|
32
|
-
export class SessionError extends
|
|
94
|
+
export class SessionError extends Error {
|
|
33
95
|
constructor(message) {
|
|
34
|
-
super(`Detected a session error,
|
|
96
|
+
super(`Detected a session error, retiring session... ${message ? `\n${message}` : ''}`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Thrown when a requested session is not found in the referenced SessionPool.
|
|
101
|
+
*/
|
|
102
|
+
export class MissingSessionError extends Error {
|
|
103
|
+
constructor(sessionId) {
|
|
104
|
+
super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
|
|
35
105
|
}
|
|
36
106
|
}
|
|
37
107
|
export class ContextPipelineInterruptedError extends Error {
|
|
@@ -54,4 +124,18 @@ export class RequestHandlerError extends Error {
|
|
|
54
124
|
super(undefined, { cause: error, ...options });
|
|
55
125
|
}
|
|
56
126
|
}
|
|
57
|
-
|
|
127
|
+
/**
|
|
128
|
+
* Thrown when attempting to set a different service instance after one has already been retrieved.
|
|
129
|
+
*/
|
|
130
|
+
export class ServiceConflictError extends Error {
|
|
131
|
+
constructor(serviceName, newValue, existingValue) {
|
|
132
|
+
super(`Service ${serviceName} is already in use. ` +
|
|
133
|
+
`Existing value: ${inspectValue(existingValue)}, attempted new value: ${inspectValue(newValue)}.`);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Thrown by crawlers when `skipNavigation` is used on a request.
|
|
138
|
+
* Subclasses can catch this error to skip their own navigation-dependent logic.
|
|
139
|
+
*/
|
|
140
|
+
export class NavigationSkippedError extends NonRetryableError {
|
|
141
|
+
}
|
|
@@ -1,28 +1,55 @@
|
|
|
1
1
|
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
|
|
2
2
|
import type { BetterIntervalID } from '@apify/utilities';
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
export interface EventManagerOptions {
|
|
4
|
+
/** Interval between emitted `persistState` events in milliseconds. */
|
|
5
|
+
persistStateIntervalMillis: number;
|
|
6
|
+
}
|
|
7
|
+
export declare enum EventType {
|
|
5
8
|
PERSIST_STATE = "persistState",
|
|
6
9
|
SYSTEM_INFO = "systemInfo",
|
|
7
10
|
MIGRATING = "migrating",
|
|
8
11
|
ABORTING = "aborting",
|
|
9
|
-
EXIT = "exit"
|
|
12
|
+
EXIT = "exit",
|
|
13
|
+
STATUS_MESSAGE = "statusMessage"
|
|
14
|
+
}
|
|
15
|
+
export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit' | 'statusMessage';
|
|
16
|
+
/**
|
|
17
|
+
* Payload emitted with the {@link EventType.STATUS_MESSAGE|`statusMessage`} event.
|
|
18
|
+
*
|
|
19
|
+
* The crawler broadcasts these whenever it wants to report its progress (e.g. periodically, or on
|
|
20
|
+
* start/finish). Consumers such as the Apify SDK can listen for the event and propagate the message
|
|
21
|
+
* to the platform. This keeps the crawler decoupled from any specific status-reporting backend.
|
|
22
|
+
*/
|
|
23
|
+
export interface EventStatusMessageData {
|
|
24
|
+
/**
|
|
25
|
+
* Identifies the crawler that emitted the message.
|
|
26
|
+
*
|
|
27
|
+
* Either the user-provided `id` from the crawler options, or a randomly generated one.
|
|
28
|
+
* Since a single event manager may be shared by multiple crawlers, consumers can use this
|
|
29
|
+
* to attribute the message to a specific crawler instance.
|
|
30
|
+
*/
|
|
31
|
+
crawlerId: string;
|
|
32
|
+
/** The human-readable status message. */
|
|
33
|
+
message: string;
|
|
34
|
+
/** Whether this is the final status message of the run. */
|
|
35
|
+
isStatusMessageTerminal?: boolean;
|
|
36
|
+
/** The log level the message was logged with. */
|
|
37
|
+
level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
|
|
10
38
|
}
|
|
11
|
-
export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit';
|
|
12
39
|
interface Intervals {
|
|
13
40
|
persistState?: BetterIntervalID;
|
|
14
41
|
systemInfo?: BetterIntervalID;
|
|
15
42
|
}
|
|
16
43
|
export declare abstract class EventManager {
|
|
17
|
-
|
|
44
|
+
#private;
|
|
18
45
|
protected events: AsyncEventEmitter<{}>;
|
|
19
46
|
protected initialized: boolean;
|
|
20
47
|
protected intervals: Intervals;
|
|
21
48
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
22
|
-
protected log: import("@
|
|
23
|
-
constructor(
|
|
49
|
+
protected log: import("@crawlee/types").CrawleeLogger;
|
|
50
|
+
constructor(options: EventManagerOptions);
|
|
24
51
|
/**
|
|
25
|
-
* Initializes the event manager by
|
|
52
|
+
* Initializes the event manager by starting the `persistState` event interval.
|
|
26
53
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
27
54
|
*/
|
|
28
55
|
init(): Promise<void>;
|
|
@@ -49,4 +76,3 @@ export declare abstract class EventManager {
|
|
|
49
76
|
waitForAllListenersToComplete(): Promise<boolean>;
|
|
50
77
|
}
|
|
51
78
|
export {};
|
|
52
|
-
//# sourceMappingURL=event_manager.d.ts.map
|
package/events/event_manager.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
|
|
2
|
-
import log from '@apify/log';
|
|
3
2
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
4
|
-
import {
|
|
3
|
+
import { serviceLocator } from '../service_locator.js';
|
|
5
4
|
export var EventType;
|
|
6
5
|
(function (EventType) {
|
|
7
6
|
EventType["PERSIST_STATE"] = "persistState";
|
|
@@ -9,30 +8,31 @@ export var EventType;
|
|
|
9
8
|
EventType["MIGRATING"] = "migrating";
|
|
10
9
|
EventType["ABORTING"] = "aborting";
|
|
11
10
|
EventType["EXIT"] = "exit";
|
|
11
|
+
EventType["STATUS_MESSAGE"] = "statusMessage";
|
|
12
12
|
})(EventType || (EventType = {}));
|
|
13
13
|
export class EventManager {
|
|
14
|
-
config;
|
|
15
14
|
events = new AsyncEventEmitter();
|
|
16
15
|
initialized = false;
|
|
17
16
|
intervals = {};
|
|
18
|
-
log =
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
this.
|
|
17
|
+
log = serviceLocator.getLogger().child({ prefix: 'Events' });
|
|
18
|
+
#persistStateIntervalMillis;
|
|
19
|
+
constructor(options) {
|
|
20
|
+
this.#persistStateIntervalMillis = options.persistStateIntervalMillis;
|
|
21
|
+
// One MIGRATING listener per RequestQueue, and ThrottlingRequestManager opens one per domain.
|
|
22
|
+
this.events.setMaxListeners(150);
|
|
22
23
|
}
|
|
23
24
|
/**
|
|
24
|
-
* Initializes the event manager by
|
|
25
|
+
* Initializes the event manager by starting the `persistState` event interval.
|
|
25
26
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
26
27
|
*/
|
|
27
28
|
async init() {
|
|
28
29
|
if (this.initialized) {
|
|
29
30
|
return;
|
|
30
31
|
}
|
|
31
|
-
const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
|
|
32
32
|
this.intervals.persistState = betterSetInterval((intervalCallback) => {
|
|
33
|
-
this.emit(
|
|
33
|
+
this.emit(EventType.PERSIST_STATE, { isMigrating: false });
|
|
34
34
|
intervalCallback();
|
|
35
|
-
}, persistStateIntervalMillis);
|
|
35
|
+
}, this.#persistStateIntervalMillis);
|
|
36
36
|
this.initialized = true;
|
|
37
37
|
}
|
|
38
38
|
/**
|
|
@@ -46,7 +46,7 @@ export class EventManager {
|
|
|
46
46
|
betterClearInterval(this.intervals.persistState);
|
|
47
47
|
this.initialized = false;
|
|
48
48
|
// Emit final PERSIST_STATE event
|
|
49
|
-
this.emit(
|
|
49
|
+
this.emit(EventType.PERSIST_STATE, { isMigrating: false });
|
|
50
50
|
// Wait for PERSIST_STATE to process
|
|
51
51
|
await this.waitForAllListenersToComplete();
|
|
52
52
|
}
|
|
@@ -86,4 +86,3 @@ export class EventManager {
|
|
|
86
86
|
return this.events.waitForAllListenersToComplete();
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
|
-
//# sourceMappingURL=event_manager.js.map
|
package/events/index.d.ts
CHANGED
package/events/index.js
CHANGED
|
@@ -1,7 +1,19 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Configuration } from '../configuration.js';
|
|
2
|
+
import { EventManager, type EventManagerOptions } from './event_manager.js';
|
|
3
|
+
export interface LocalEventManagerOptions extends EventManagerOptions {
|
|
4
|
+
/** Interval between emitted `systemInfo` events in milliseconds. */
|
|
5
|
+
systemInfoIntervalMillis: number;
|
|
6
|
+
}
|
|
2
7
|
export declare class LocalEventManager extends EventManager {
|
|
8
|
+
#private;
|
|
9
|
+
constructor(options: LocalEventManagerOptions);
|
|
10
|
+
/**
|
|
11
|
+
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|
|
12
|
+
* Uses the global configuration from the service locator if none is provided.
|
|
13
|
+
*/
|
|
14
|
+
static fromConfiguration(configuration?: Configuration): LocalEventManager;
|
|
3
15
|
/**
|
|
4
|
-
* Initializes the EventManager and sets up periodic `systemInfo`
|
|
16
|
+
* Initializes the EventManager and sets up periodic `systemInfo` events.
|
|
5
17
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
6
18
|
*/
|
|
7
19
|
init(): Promise<void>;
|
|
@@ -23,5 +35,5 @@ export declare class LocalEventManager extends EventManager {
|
|
|
23
35
|
private createSystemInfo;
|
|
24
36
|
private createCpuInfo;
|
|
25
37
|
private createMemoryInfo;
|
|
38
|
+
private getMemoryInfo;
|
|
26
39
|
}
|
|
27
|
-
//# sourceMappingURL=local_event_manager.d.ts.map
|