@crawlee/core 4.0.0-beta.26 → 4.0.0-beta.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +1 -3
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +1 -7
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +0 -10
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +12 -18
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +4 -70
- package/configuration.d.ts.map +1 -1
- package/configuration.js +5 -97
- package/configuration.js.map +1 -1
- package/crawlers/statistics.d.ts +2 -11
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +14 -13
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +13 -8
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +20 -21
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +19 -8
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +45 -38
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +6 -0
- package/errors.d.ts.map +1 -1
- package/errors.js +9 -0
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +7 -4
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +5 -7
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -2
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +24 -6
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +1 -0
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/package.json +5 -5
- package/recoverable_state.d.ts +1 -2
- package/recoverable_state.d.ts.map +1 -1
- package/recoverable_state.js +4 -6
- package/recoverable_state.js.map +1 -1
- package/request.d.ts +8 -0
- package/request.d.ts.map +1 -1
- package/request.js +17 -0
- package/request.js.map +1 -1
- package/service_locator.d.ts +139 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +252 -0
- package/service_locator.js.map +1 -0
- package/session_pool/session_pool.d.ts +2 -4
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +9 -10
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +4 -3
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +5 -4
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +1 -2
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +3 -5
- package/storages/request_list.js.map +1 -1
- package/storages/request_provider.d.ts +4 -2
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +8 -7
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +1 -1
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +2 -2
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +1 -1
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +4 -5
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +0 -5
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +3 -3
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts +5 -7
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +11 -21
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +4 -3
- package/storages/utils.js.map +1 -1
package/enqueue_links/shared.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { URL } from 'node:url';
|
|
2
2
|
import { Minimatch } from 'minimatch';
|
|
3
3
|
import { purlToRegExp } from '@apify/pseudo_url';
|
|
4
|
-
import { Request } from '../request.js';
|
|
5
4
|
export { tryAbsoluteURL } from '@crawlee/utils';
|
|
6
5
|
const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
|
|
7
6
|
/**
|
|
@@ -110,55 +109,37 @@ export function constructRegExpObjectsFromRegExps(regexps) {
|
|
|
110
109
|
});
|
|
111
110
|
}
|
|
112
111
|
/**
|
|
112
|
+
* Filters request options by URL patterns and merges pattern-level options (label, userData, method, payload, headers)
|
|
113
|
+
* from the first matching pattern into each RequestOptions entry.
|
|
114
|
+
*
|
|
115
|
+
* When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
|
|
113
116
|
* @ignore
|
|
114
117
|
*/
|
|
115
|
-
export function
|
|
116
|
-
const
|
|
117
|
-
const
|
|
118
|
+
export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedUrl) {
|
|
119
|
+
const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
|
|
120
|
+
const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
|
|
118
121
|
return requestOptions
|
|
119
|
-
.map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
|
|
120
122
|
.filter(({ url }) => {
|
|
121
|
-
const
|
|
122
|
-
if (
|
|
123
|
+
const matchesExclude = excludeMatchers.some(({ match }) => match(url));
|
|
124
|
+
if (matchesExclude) {
|
|
123
125
|
onSkippedUrl?.(url);
|
|
124
126
|
}
|
|
125
|
-
return !
|
|
127
|
+
return !matchesExclude;
|
|
126
128
|
})
|
|
127
|
-
.map((
|
|
128
|
-
if (!
|
|
129
|
-
return
|
|
130
|
-
}
|
|
131
|
-
for (const
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
const request = typeof opts === 'string'
|
|
135
|
-
? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
|
|
136
|
-
: { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
|
|
137
|
-
return new Request(request);
|
|
129
|
+
.map((opts) => {
|
|
130
|
+
if (!includeMatchers) {
|
|
131
|
+
return { ...opts, enqueueStrategy: strategy };
|
|
132
|
+
}
|
|
133
|
+
for (const { match, glob, regexp, ...patternOptions } of includeMatchers) {
|
|
134
|
+
if (match(opts.url)) {
|
|
135
|
+
return { ...opts, ...patternOptions, enqueueStrategy: strategy };
|
|
138
136
|
}
|
|
139
137
|
}
|
|
140
138
|
// didn't match any positive pattern
|
|
141
|
-
onSkippedUrl?.(url);
|
|
139
|
+
onSkippedUrl?.(opts.url);
|
|
142
140
|
return null;
|
|
143
141
|
})
|
|
144
|
-
.filter((
|
|
145
|
-
}
|
|
146
|
-
export function filterRequestsByPatterns(requests, patterns, onSkippedUrl) {
|
|
147
|
-
if (!patterns?.length) {
|
|
148
|
-
return requests;
|
|
149
|
-
}
|
|
150
|
-
const filtered = [];
|
|
151
|
-
const patternMatchers = patterns?.map(createPatternObjectMatcher);
|
|
152
|
-
for (const request of requests) {
|
|
153
|
-
const matchingPattern = patternMatchers.find(({ match }) => match(request.url));
|
|
154
|
-
if (matchingPattern !== undefined) {
|
|
155
|
-
filtered.push(request);
|
|
156
|
-
}
|
|
157
|
-
else {
|
|
158
|
-
onSkippedUrl?.(request.url);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
return filtered;
|
|
142
|
+
.filter((opts) => opts !== null);
|
|
162
143
|
}
|
|
163
144
|
/**
|
|
164
145
|
* @ignore
|
|
@@ -209,4 +190,30 @@ function createPatternObjectMatcher(urlPatternObject) {
|
|
|
209
190
|
}
|
|
210
191
|
return { ...urlPatternObject, match };
|
|
211
192
|
}
|
|
193
|
+
/**
|
|
194
|
+
* Applies a {@link RequestTransform} function to a list of request options.
|
|
195
|
+
* Options for which the transform returns a falsy value are removed from the list.
|
|
196
|
+
* @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
|
|
197
|
+
* @ignore
|
|
198
|
+
* @internal
|
|
199
|
+
*/
|
|
200
|
+
export function applyRequestTransform(requestOptions, transformFn, onSkipped) {
|
|
201
|
+
return requestOptions
|
|
202
|
+
.map((opts) => {
|
|
203
|
+
const transformed = transformFn(opts);
|
|
204
|
+
if (transformed === 'skip') {
|
|
205
|
+
onSkipped?.(opts);
|
|
206
|
+
return null;
|
|
207
|
+
}
|
|
208
|
+
if (transformed === 'unchanged') {
|
|
209
|
+
return opts;
|
|
210
|
+
}
|
|
211
|
+
if (!transformed) {
|
|
212
|
+
onSkipped?.(opts);
|
|
213
|
+
return null;
|
|
214
|
+
}
|
|
215
|
+
return transformed;
|
|
216
|
+
})
|
|
217
|
+
.filter((r) => r !== null);
|
|
218
|
+
}
|
|
212
219
|
//# sourceMappingURL=shared.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAG/B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAG/B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAKjD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AAuC3C;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAC1C,IAA8C,EAC9C,OAAkC;IAElC,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oCAAoC,CAAC,UAAqC;IACtF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAAC,KAA2B;IACrE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAClG,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAAC,OAA+B;IAC7E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,8BAA8B,CAC1C,cAAgC,EAChC,eAA+C,EAC/C,kBAAsC,EAAE,EACxC,QAA0C,EAC1C,YAAoC;IAEpC,MAAM,eAAe,GAAG,eAAe,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxE,MAAM,eAAe,GAAG,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE9G,OAAO,cAAc;SAChB,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvE,IAAI,cAAc,EAAE,CAAC;YACjB,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,CAAC,cAAc,CAAC;IAC3B,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,IAAI,CAAC,eAAe,EAAE,CAAC;YACnB,OAAO,EAAE,GAAG,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;QAClD,CAAC;QAED,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,IAAI,eAAe,EAAE,CAAC;YACvE,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClB,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;YACrE,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,YAAY,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAChC,OAAsD,EACtD,UAAuG,EAAE;IAEzG,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACT,OAAO,GAAG,KAAK,QAAQ;QACnB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE;QACjD,CAAC,CAAE,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAqB,CAC1E;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,gBAAkC;IAClE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;IAC1C,IAAI,KAAK,CAAC;IACV,IAAI,MAAM,EAAE,CAAC;QACT,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,IAAI,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;SAAM,CAAC;QACJ,KAAK,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;IACxB,CAAC;IACD,OAAO,EAAE,GAAG,gBAAgB,EAAE,KAAK,EAAE,CAAC;AAC1C,CAAC;AAeD;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACjC,cAAgC,EAChC,WAA6B,EAC7B,SAAoD;IAEpD,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YACzB,SAAS,EAAE,CAAC,IAAI,CAAC,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,SAAS,EAAE,CAAC,IAAI,CAAC,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,WAAW,CAAC;IACvB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAuB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AACxD,CAAC"}
|
package/errors.d.ts
CHANGED
|
@@ -42,4 +42,10 @@ export declare class ContextPipelineCleanupError extends CriticalError {
|
|
|
42
42
|
export declare class RequestHandlerError extends Error {
|
|
43
43
|
constructor(error: unknown, options?: ErrorOptions);
|
|
44
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* Thrown when attempting to set a different service instance after one has already been retrieved.
|
|
47
|
+
*/
|
|
48
|
+
export declare class ServiceConflictError extends Error {
|
|
49
|
+
constructor(serviceName: string, newValue: unknown, existingValue: unknown);
|
|
50
|
+
}
|
|
45
51
|
//# sourceMappingURL=errors.d.ts.map
|
package/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,iBAAiB;gBACnC,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,+BAAgC,SAAQ,KAAK;gBAC1C,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,kCAAmC,SAAQ,KAAK;gBAC7C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,2BAA4B,SAAQ,aAAa;gBAC9C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,mBAAoB,SAAQ,KAAK;gBAC9B,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD"}
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,iBAAiB;gBACnC,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,+BAAgC,SAAQ,KAAK;gBAC1C,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,kCAAmC,SAAQ,KAAK;gBAC7C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,2BAA4B,SAAQ,aAAa;gBAC9C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,mBAAoB,SAAQ,KAAK;gBAC9B,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED;;GAEG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;gBAC/B,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO;CAM7E"}
|
package/errors.js
CHANGED
|
@@ -54,4 +54,13 @@ export class RequestHandlerError extends Error {
|
|
|
54
54
|
super(undefined, { cause: error, ...options });
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Thrown when attempting to set a different service instance after one has already been retrieved.
|
|
59
|
+
*/
|
|
60
|
+
export class ServiceConflictError extends Error {
|
|
61
|
+
constructor(serviceName, newValue, existingValue) {
|
|
62
|
+
super(`Service ${serviceName} is already in use. ` +
|
|
63
|
+
`Existing value: ${existingValue}, attempted new value: ${newValue}.`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
57
66
|
//# sourceMappingURL=errors.js.map
|
package/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,MAAM,OAAO,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACxC,YAAY,OAAgB;QACxB,KAAK,CAAC,OAAO,IAAI,gDAAgD,CAAC,CAAC;IACvE,CAAC;CACJ;AAED;;;;GAIG;AACH,MAAM,OAAO,YAAa,SAAQ,iBAAiB;IAC/C,YAAY,OAAgB;QACxB,KAAK,CAAC,iDAAiD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;CACJ;AAED,MAAM,OAAO,+BAAgC,SAAQ,KAAK;IACtD,YAAY,OAAgB;QACxB,KAAK,CAAC,kEAAkE,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9G,CAAC;CACJ;AAED,MAAM,OAAO,kCAAmC,SAAQ,KAAK;IACzD,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,MAAM,OAAO,2BAA4B,SAAQ,aAAa;IAC1D,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC1C,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,MAAM,OAAO,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACxC,YAAY,OAAgB;QACxB,KAAK,CAAC,OAAO,IAAI,gDAAgD,CAAC,CAAC;IACvE,CAAC;CACJ;AAED;;;;GAIG;AACH,MAAM,OAAO,YAAa,SAAQ,iBAAiB;IAC/C,YAAY,OAAgB;QACxB,KAAK,CAAC,iDAAiD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;CACJ;AAED,MAAM,OAAO,+BAAgC,SAAQ,KAAK;IACtD,YAAY,OAAgB;QACxB,KAAK,CAAC,kEAAkE,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9G,CAAC;CACJ;AAED,MAAM,OAAO,kCAAmC,SAAQ,KAAK;IACzD,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,MAAM,OAAO,2BAA4B,SAAQ,aAAa;IAC1D,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC1C,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC3C,YAAY,WAAmB,EAAE,QAAiB,EAAE,aAAsB;QACtE,KAAK,CACD,WAAW,WAAW,sBAAsB;YACxC,mBAAmB,aAAa,0BAA0B,QAAQ,GAAG,CAC5E,CAAC;IACN,CAAC;CACJ"}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
|
|
2
2
|
import type { BetterIntervalID } from '@apify/utilities';
|
|
3
|
-
|
|
3
|
+
export interface EventManagerOptions {
|
|
4
|
+
/** Interval between emitted `persistState` events in milliseconds. */
|
|
5
|
+
persistStateIntervalMillis: number;
|
|
6
|
+
}
|
|
4
7
|
export declare const enum EventType {
|
|
5
8
|
PERSIST_STATE = "persistState",
|
|
6
9
|
SYSTEM_INFO = "systemInfo",
|
|
@@ -14,15 +17,15 @@ interface Intervals {
|
|
|
14
17
|
systemInfo?: BetterIntervalID;
|
|
15
18
|
}
|
|
16
19
|
export declare abstract class EventManager {
|
|
17
|
-
readonly config: Configuration;
|
|
18
20
|
protected events: AsyncEventEmitter<{}>;
|
|
19
21
|
protected initialized: boolean;
|
|
20
22
|
protected intervals: Intervals;
|
|
21
23
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
22
24
|
protected log: import("@apify/log").Log;
|
|
23
|
-
|
|
25
|
+
private persistStateIntervalMillis;
|
|
26
|
+
constructor(options: EventManagerOptions);
|
|
24
27
|
/**
|
|
25
|
-
* Initializes the event manager by
|
|
28
|
+
* Initializes the event manager by starting the `persistState` event interval.
|
|
26
29
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
27
30
|
*/
|
|
28
31
|
init(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAGpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,
|
|
1
|
+
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAGpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,MAAM,WAAW,mBAAmB;IAChC,sEAAsE;IACtE,0BAA0B,EAAE,MAAM,CAAC;CACtC;AAED,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAC9B,SAAS,CAAC,MAAM,wBAA2B;IAC3C,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;IAChD,OAAO,CAAC,0BAA0B,CAAS;gBAE/B,OAAO,EAAE,mBAAmB;IAKxC;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IAeX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;IAI3D;;OAEG;IACG,6BAA6B;CAGtC"}
|
package/events/event_manager.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
|
|
2
2
|
import log from '@apify/log';
|
|
3
3
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
4
|
-
import { Configuration } from '../configuration.js';
|
|
5
4
|
export var EventType;
|
|
6
5
|
(function (EventType) {
|
|
7
6
|
EventType["PERSIST_STATE"] = "persistState";
|
|
@@ -11,28 +10,27 @@ export var EventType;
|
|
|
11
10
|
EventType["EXIT"] = "exit";
|
|
12
11
|
})(EventType || (EventType = {}));
|
|
13
12
|
export class EventManager {
|
|
14
|
-
config;
|
|
15
13
|
events = new AsyncEventEmitter();
|
|
16
14
|
initialized = false;
|
|
17
15
|
intervals = {};
|
|
18
16
|
log = log.child({ prefix: 'Events' });
|
|
19
|
-
|
|
20
|
-
|
|
17
|
+
persistStateIntervalMillis;
|
|
18
|
+
constructor(options) {
|
|
19
|
+
this.persistStateIntervalMillis = options.persistStateIntervalMillis;
|
|
21
20
|
this.events.setMaxListeners(50);
|
|
22
21
|
}
|
|
23
22
|
/**
|
|
24
|
-
* Initializes the event manager by
|
|
23
|
+
* Initializes the event manager by starting the `persistState` event interval.
|
|
25
24
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
26
25
|
*/
|
|
27
26
|
async init() {
|
|
28
27
|
if (this.initialized) {
|
|
29
28
|
return;
|
|
30
29
|
}
|
|
31
|
-
const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
|
|
32
30
|
this.intervals.persistState = betterSetInterval((intervalCallback) => {
|
|
33
31
|
this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
|
|
34
32
|
intervalCallback();
|
|
35
|
-
}, persistStateIntervalMillis);
|
|
33
|
+
}, this.persistStateIntervalMillis);
|
|
36
34
|
this.initialized = true;
|
|
37
35
|
}
|
|
38
36
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAO1E,MAAM,CAAN,IAAkB,SAMjB;AAND,WAAkB,SAAS;IACvB,2CAA8B,CAAA;IAC9B,uCAA0B,CAAA;IAC1B,oCAAuB,CAAA;IACvB,kCAAqB,CAAA;IACrB,0BAAa,CAAA;AACjB,CAAC,EANiB,SAAS,KAAT,SAAS,QAM1B;AASD,MAAM,OAAgB,YAAY;IACpB,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;IACjC,WAAW,GAAG,KAAK,CAAC;IACpB,SAAS,GAAc,EAAE,CAAC;IAC1B,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IACxC,0BAA0B,CAAS;IAE3C,YAAY,OAA4B;QACpC,IAAI,CAAC,0BAA0B,GAAG,OAAO,CAAC,0BAA0B,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,YAAY,GAAG,iBAAiB,CAAC,CAAC,gBAA+B,EAAE,EAAE;YAChF,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,gBAAgB,EAAE,CAAC;QACvB,CAAC,EAAE,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAEpC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB,iCAAiC;QACjC,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QAE3D,oCAAoC;QACpC,MAAM,IAAI,CAAC,6BAA6B,EAAE,CAAC;IAC/C,CAAC;IAED,EAAE,CAAC,KAAoB,EAAE,QAAiC;QACtD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CAAC,KAAoB,EAAE,QAAkC;QACxD,IAAI,QAAQ,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC1C,CAAC;IACL,CAAC;IAED,IAAI,CAAC,KAAoB,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,aAAa;QACT,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAoB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAoB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAA+B,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,6BAA6B;QAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,6BAA6B,EAAE,CAAC;IACvD,CAAC;CACJ"}
|
|
@@ -1,7 +1,19 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Configuration } from '../configuration.js';
|
|
2
|
+
import { EventManager, type EventManagerOptions } from './event_manager.js';
|
|
3
|
+
export interface LocalEventManagerOptions extends EventManagerOptions {
|
|
4
|
+
/** Interval between emitted `systemInfo` events in milliseconds. */
|
|
5
|
+
systemInfoIntervalMillis: number;
|
|
6
|
+
}
|
|
2
7
|
export declare class LocalEventManager extends EventManager {
|
|
8
|
+
private systemInfoIntervalMillis;
|
|
9
|
+
constructor(options: LocalEventManagerOptions);
|
|
10
|
+
/**
|
|
11
|
+
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|
|
12
|
+
* Uses the global configuration from the service locator if none is provided.
|
|
13
|
+
*/
|
|
14
|
+
static fromConfig(config?: Configuration): LocalEventManager;
|
|
3
15
|
/**
|
|
4
|
-
* Initializes the EventManager and sets up periodic `systemInfo`
|
|
16
|
+
* Initializes the EventManager and sets up periodic `systemInfo` events.
|
|
5
17
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
6
18
|
*/
|
|
7
19
|
init(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAa,MAAM,oBAAoB,CAAC;AAEvF,MAAM,WAAW,wBAAyB,SAAQ,mBAAmB;IACjE,oEAAoE;IACpE,wBAAwB,EAAE,MAAM,CAAC;CACpC;AAED,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,wBAAwB,CAAS;gBAE7B,OAAO,EAAE,wBAAwB;IAK7C;;;OAGG;IACH,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,aAAa,GAAG,iBAAiB;IAS5D;;;OAGG;IACY,IAAI;IAcnB;;OAEG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD;;OAEG;IACG,sBAAsB;IAK5B;;OAEG;YACW,gBAAgB;YAQhB,aAAa;YASb,gBAAgB;CAYjC"}
|
|
@@ -1,10 +1,26 @@
|
|
|
1
|
-
import { getCurrentCpuTicksV2, getMemoryInfo, isContainerized } from '@crawlee/utils';
|
|
2
1
|
import log from '@apify/log';
|
|
3
2
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
3
|
+
import { serviceLocator } from '../service_locator.js';
|
|
4
4
|
import { EventManager } from './event_manager.js';
|
|
5
5
|
export class LocalEventManager extends EventManager {
|
|
6
|
+
systemInfoIntervalMillis;
|
|
7
|
+
constructor(options) {
|
|
8
|
+
super(options);
|
|
9
|
+
this.systemInfoIntervalMillis = options.systemInfoIntervalMillis;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|
|
13
|
+
* Uses the global configuration from the service locator if none is provided.
|
|
14
|
+
*/
|
|
15
|
+
static fromConfig(config) {
|
|
16
|
+
const resolvedConfig = config ?? serviceLocator.getConfiguration();
|
|
17
|
+
return new LocalEventManager({
|
|
18
|
+
persistStateIntervalMillis: resolvedConfig.get('persistStateIntervalMillis'),
|
|
19
|
+
systemInfoIntervalMillis: resolvedConfig.get('systemInfoIntervalMillis'),
|
|
20
|
+
});
|
|
21
|
+
}
|
|
6
22
|
/**
|
|
7
|
-
* Initializes the EventManager and sets up periodic `systemInfo`
|
|
23
|
+
* Initializes the EventManager and sets up periodic `systemInfo` events.
|
|
8
24
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
9
25
|
*/
|
|
10
26
|
async init() {
|
|
@@ -12,9 +28,8 @@ export class LocalEventManager extends EventManager {
|
|
|
12
28
|
return;
|
|
13
29
|
}
|
|
14
30
|
await super.init();
|
|
15
|
-
const systemInfoIntervalMillis = this.config.get('systemInfoIntervalMillis');
|
|
16
31
|
this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
|
|
17
|
-
this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), systemInfoIntervalMillis);
|
|
32
|
+
this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.systemInfoIntervalMillis);
|
|
18
33
|
}
|
|
19
34
|
/**
|
|
20
35
|
* @inheritDoc
|
|
@@ -31,7 +46,7 @@ export class LocalEventManager extends EventManager {
|
|
|
31
46
|
*/
|
|
32
47
|
async emitSystemInfoEvent(intervalCallback) {
|
|
33
48
|
const info = await this.createSystemInfo({
|
|
34
|
-
maxUsedCpuRatio:
|
|
49
|
+
maxUsedCpuRatio: serviceLocator.getConfiguration().get('maxUsedCpuRatio'),
|
|
35
50
|
});
|
|
36
51
|
this.events.emit("systemInfo" /* EventType.SYSTEM_INFO */, info);
|
|
37
52
|
intervalCallback();
|
|
@@ -40,7 +55,8 @@ export class LocalEventManager extends EventManager {
|
|
|
40
55
|
* @internal
|
|
41
56
|
*/
|
|
42
57
|
async isContainerizedWrapper() {
|
|
43
|
-
|
|
58
|
+
const { isContainerized } = await import('@crawlee/utils');
|
|
59
|
+
return serviceLocator.getConfiguration().get('containerized', await isContainerized());
|
|
44
60
|
}
|
|
45
61
|
/**
|
|
46
62
|
* Creates a SystemInfo object based on local metrics.
|
|
@@ -53,6 +69,7 @@ export class LocalEventManager extends EventManager {
|
|
|
53
69
|
};
|
|
54
70
|
}
|
|
55
71
|
async createCpuInfo(options) {
|
|
72
|
+
const { getCurrentCpuTicksV2 } = await import('@crawlee/utils');
|
|
56
73
|
const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper());
|
|
57
74
|
return {
|
|
58
75
|
cpuCurrentUsage: usedCpuRatio * 100,
|
|
@@ -61,6 +78,7 @@ export class LocalEventManager extends EventManager {
|
|
|
61
78
|
}
|
|
62
79
|
async createMemoryInfo() {
|
|
63
80
|
try {
|
|
81
|
+
const { getMemoryInfo } = await import('@crawlee/utils');
|
|
64
82
|
const memInfo = await getMemoryInfo(await this.isContainerizedWrapper());
|
|
65
83
|
return {
|
|
66
84
|
memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,YAAY,CAAC;AAC7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAI1E,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAuC,MAAM,oBAAoB,CAAC;AAOvF,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IACvC,wBAAwB,CAAS;IAEzC,YAAY,OAAiC;QACzC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,wBAAwB,CAAC;IACrE,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,UAAU,CAAC,MAAsB;QACpC,MAAM,cAAc,GAAG,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;QAEnE,OAAO,IAAI,iBAAiB,CAAC;YACzB,0BAA0B,EAAE,cAAc,CAAC,GAAG,CAAC,4BAA4B,CAAC;YAC5E,wBAAwB,EAAE,cAAc,CAAC,GAAG,CAAC,0BAA0B,CAAC;SAC3E,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,iBAAiB,CACzC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EACnC,IAAI,CAAC,wBAAwB,CAChC,CAAC;IACN,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,cAAc,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,iBAAiB,CAAC;SAC5E,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC3D,OAAO,cAAc,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,eAAe,EAAE,CAAC,CAAC;IAC3F,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAChE,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;QACrF,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YACzD,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;YACzE,OAAO;gBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}
|
package/index.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export * from './errors.js';
|
|
2
2
|
export * from './autoscaling/index.js';
|
|
3
3
|
export * from './configuration.js';
|
|
4
|
+
export * from './service_locator.js';
|
|
4
5
|
export * from './crawlers/index.js';
|
|
5
6
|
export * from './enqueue_links/index.js';
|
|
6
7
|
export * from './events/index.js';
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}
|
package/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export * from './errors.js';
|
|
2
2
|
export * from './autoscaling/index.js';
|
|
3
3
|
export * from './configuration.js';
|
|
4
|
+
export * from './service_locator.js';
|
|
4
5
|
export * from './crawlers/index.js';
|
|
5
6
|
export * from './enqueue_links/index.js';
|
|
6
7
|
export * from './events/index.js';
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.28",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,9 +53,9 @@
|
|
|
53
53
|
"@apify/pseudo_url": "^2.0.59",
|
|
54
54
|
"@apify/timeout": "^0.3.2",
|
|
55
55
|
"@apify/utilities": "^2.15.5",
|
|
56
|
-
"@crawlee/memory-storage": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
56
|
+
"@crawlee/memory-storage": "4.0.0-beta.28",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.28",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.28",
|
|
59
59
|
"@sapphire/async-queue": "^1.5.5",
|
|
60
60
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
61
61
|
"csv-stringify": "^6.5.2",
|
|
@@ -76,5 +76,5 @@
|
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
78
|
},
|
|
79
|
-
"gitHead": "
|
|
79
|
+
"gitHead": "34fe46a70d4d6a6ddb49b0814469dbde1e7f0a9c"
|
|
80
80
|
}
|
package/recoverable_state.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Configuration } from '@crawlee/core';
|
|
1
|
+
import type { Configuration } from '@crawlee/core';
|
|
2
2
|
import type { Log } from '@apify/log';
|
|
3
3
|
export interface RecoverableStatePersistenceOptions {
|
|
4
4
|
/**
|
|
@@ -68,7 +68,6 @@ export declare class RecoverableState<TStateModel = Record<string, unknown>> {
|
|
|
68
68
|
private readonly persistStateKvsId?;
|
|
69
69
|
private keyValueStore;
|
|
70
70
|
private readonly log;
|
|
71
|
-
private readonly config;
|
|
72
71
|
private readonly serialize;
|
|
73
72
|
private readonly deserialize;
|
|
74
73
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recoverable_state.d.ts","sourceRoot":"","sources":["../src/recoverable_state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"recoverable_state.d.ts","sourceRoot":"","sources":["../src/recoverable_state.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,MAAM,WAAW,kCAAkC;IAC/C;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC1E,SAAQ,kCAAkC;IAC1C;;;OAGG;IACH,YAAY,EAAE,WAAW,CAAC;IAE1B;;OAEG;IACH,MAAM,CAAC,EAAE,GAAG,CAAC;IAEb;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;;OAGG;IACH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,MAAM,CAAC;IAE3C;;;;OAIG;IACH,WAAW,CAAC,EAAE,CAAC,eAAe,EAAE,MAAM,KAAK,WAAW,CAAC;CAC1D;AAED;;;;;;;;;GASG;AACH,qBAAa,gBAAgB,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAC/D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAc;IAC3C,OAAO,CAAC,KAAK,CAA4B;IACzC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAC7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAS;IAC5C,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAC1B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiC;IAC3D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2C;IAEvE;;;;OAIG;gBACS,OAAO,EAAE,uBAAuB,CAAC,WAAW,CAAC;IAazD;;;;;;;OAOG;IACG,UAAU,IAAI,OAAO,CAAC,WAAW,CAAC;IAuBxC;;;;;OAKG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B;;OAEG;IACH,IAAI,YAAY,IAAI,WAAW,CAM9B;IAED;;;;;OAKG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAY5B;;;;;;;OAOG;IACG,YAAY,CAAC,SAAS,CAAC,EAAE;QAAE,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAcvE;;OAEG;YACW,cAAc;CAY/B"}
|
package/recoverable_state.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { KeyValueStore, serviceLocator } from '@crawlee/core';
|
|
2
2
|
import log from '@apify/log';
|
|
3
3
|
/**
|
|
4
4
|
* A class for managing persistent recoverable state using a plain JavaScript object.
|
|
@@ -19,7 +19,6 @@ export class RecoverableState {
|
|
|
19
19
|
persistStateKvsId;
|
|
20
20
|
keyValueStore = null;
|
|
21
21
|
log;
|
|
22
|
-
config;
|
|
23
22
|
serialize;
|
|
24
23
|
deserialize;
|
|
25
24
|
/**
|
|
@@ -34,7 +33,6 @@ export class RecoverableState {
|
|
|
34
33
|
this.persistStateKvsName = options.persistStateKvsName;
|
|
35
34
|
this.persistStateKvsId = options.persistStateKvsId;
|
|
36
35
|
this.log = options.logger ?? log.child({ prefix: 'RecoverableState' });
|
|
37
|
-
this.config = options.config ?? Configuration.getGlobalConfig();
|
|
38
36
|
this.serialize = options.serialize ?? JSON.stringify;
|
|
39
37
|
this.deserialize = options.deserialize ?? JSON.parse;
|
|
40
38
|
this.persistState = this.persistState.bind(this);
|
|
@@ -56,11 +54,11 @@ export class RecoverableState {
|
|
|
56
54
|
return this.currentValue;
|
|
57
55
|
}
|
|
58
56
|
this.keyValueStore = await KeyValueStore.open(this.persistStateKvsName ?? this.persistStateKvsId, {
|
|
59
|
-
config:
|
|
57
|
+
config: serviceLocator.getConfiguration(),
|
|
60
58
|
});
|
|
61
59
|
await this.loadSavedState();
|
|
62
60
|
// Register for persist state events
|
|
63
|
-
const eventManager =
|
|
61
|
+
const eventManager = serviceLocator.getEventManager();
|
|
64
62
|
eventManager.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
|
|
65
63
|
return this.currentValue;
|
|
66
64
|
}
|
|
@@ -74,7 +72,7 @@ export class RecoverableState {
|
|
|
74
72
|
if (!this.persistenceEnabled || !this.persistState) {
|
|
75
73
|
return;
|
|
76
74
|
}
|
|
77
|
-
const eventManager =
|
|
75
|
+
const eventManager = serviceLocator.getEventManager();
|
|
78
76
|
eventManager.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
|
|
79
77
|
await this.persistState();
|
|
80
78
|
}
|
package/recoverable_state.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recoverable_state.js","sourceRoot":"","sources":["../src/recoverable_state.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"recoverable_state.js","sourceRoot":"","sources":["../src/recoverable_state.ts"],"names":[],"mappings":"AACA,OAAO,EAAa,aAAa,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAGzE,OAAO,GAAG,MAAM,YAAY,CAAC;AA6D7B;;;;;;;;;GASG;AACH,MAAM,OAAO,gBAAgB;IACR,YAAY,CAAc;IACnC,KAAK,GAAuB,IAAI,CAAC;IACxB,kBAAkB,CAAU;IAC5B,eAAe,CAAS;IACxB,mBAAmB,CAAU;IAC7B,iBAAiB,CAAU;IACpC,aAAa,GAAyB,IAAI,CAAC;IAClC,GAAG,CAAM;IACT,SAAS,CAAiC;IAC1C,WAAW,CAA2C;IAEvE;;;;OAIG;IACH,YAAY,OAA6C;QACrD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,KAAK,CAAC;QAC9D,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC9F,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE;SAC5C,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,oCAAoC;QACpC,MAAM,YAAY,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QACtD,YAAY,CAAC,EAAE,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAE5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACjD,OAAO;QACX,CAAC;QAED,MAAM,YAAY,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QACtD,YAAY,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,YAAY,CAAC,SAAoC;QACnD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uDAAuD,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAChF,WAAW,EAAE,YAAY,EAAE,wHAAwH;aACtJ,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5E,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YACpD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,WAAqB,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;CACJ"}
|
package/request.d.ts
CHANGED
|
@@ -123,6 +123,14 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
|
123
123
|
get state(): RequestState;
|
|
124
124
|
/** Describes the request's current lifecycle state. */
|
|
125
125
|
set state(value: RequestState);
|
|
126
|
+
/**
|
|
127
|
+
* Reason for skipping this request.
|
|
128
|
+
*/
|
|
129
|
+
get skippedReason(): SkippedRequestReason | undefined;
|
|
130
|
+
/**
|
|
131
|
+
* Reason for skipping this request.
|
|
132
|
+
*/
|
|
133
|
+
set skippedReason(value: SkippedRequestReason | undefined);
|
|
126
134
|
private get enqueueStrategy();
|
|
127
135
|
private set enqueueStrategy(value);
|
|
128
136
|
/**
|
package/request.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA4BxD,oBAAY,YAAY;IACpB,WAAW,IAAA;IACX,UAAU,IAAA;IACV,SAAS,IAAA;IACT,eAAe,IAAA;IACf,IAAI,IAAA;IACJ,aAAa,IAAA;IACb,KAAK,IAAA;IACL,OAAO,IAAA;CACV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,cAAM,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;IA2H7C;;;OAGG;IACI,mBAAmB,IAAI,OAAO;IAQrC,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAMhC;IAED;;;OAGG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,EAG3B;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAMrC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED,8HAA8H;IAC9H,IAAI,UAAU,IAAI,MAAM,GAAG,SAAS,CAEnC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAMvC;IAED,uDAAuD;IACvD,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,EAM5B;IAED,OAAO,KAAK,eAAe,GAE1B;IAED,OAAO,KAAK,eAAe,QAM1B;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAc,EACd,OAAO,EACP,eAAuB,EACvB,oBAA4B,GAC/B,EAAE,uBAAuB;IAiB1B,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAGlD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,oBAAoB,CAAC;IAErC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gBAAgB;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IAErB,gBAAgB;IAChB,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,cAAc,CAAC;AAE/G,gBAAgB;AAChB,MAAM,WAAW,cAAc;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA4BxD,oBAAY,YAAY;IACpB,WAAW,IAAA;IACX,UAAU,IAAA;IACV,SAAS,IAAA;IACT,eAAe,IAAA;IACf,IAAI,IAAA;IACJ,aAAa,IAAA;IACb,KAAK,IAAA;IACL,OAAO,IAAA;CACV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,cAAM,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;IA2H7C;;;OAGG;IACI,mBAAmB,IAAI,OAAO;IAQrC,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAMhC;IAED;;;OAGG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,EAG3B;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAMrC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED,8HAA8H;IAC9H,IAAI,UAAU,IAAI,MAAM,GAAG,SAAS,CAEnC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAMvC;IAED,uDAAuD;IACvD,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,EAM5B;IAED;;OAEG;IACH,IAAI,aAAa,IAAI,oBAAoB,GAAG,SAAS,CAEpD;IAED;;OAEG;IACH,IAAI,aAAa,CAAC,KAAK,EAAE,oBAAoB,GAAG,SAAS,EAMxD;IAED,OAAO,KAAK,eAAe,GAE1B;IAED,OAAO,KAAK,eAAe,QAM1B;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAc,EACd,OAAO,EACP,eAAuB,EACvB,oBAA4B,GAC/B,EAAE,uBAAuB;IAiB1B,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAGlD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,oBAAoB,CAAC;IAErC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gBAAgB;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IAErB,gBAAgB;IAChB,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,cAAc,CAAC;AAE/G,gBAAgB;AAChB,MAAM,WAAW,cAAc;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
package/request.js
CHANGED
|
@@ -286,6 +286,23 @@ class CrawleeRequest {
|
|
|
286
286
|
this.userData.__crawlee.state = value;
|
|
287
287
|
}
|
|
288
288
|
}
|
|
289
|
+
/**
|
|
290
|
+
* Reason for skipping this request.
|
|
291
|
+
*/
|
|
292
|
+
get skippedReason() {
|
|
293
|
+
return this.userData.__crawlee?.skippedReason;
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* Reason for skipping this request.
|
|
297
|
+
*/
|
|
298
|
+
set skippedReason(value) {
|
|
299
|
+
if (!this.userData.__crawlee) {
|
|
300
|
+
this.userData.__crawlee = { skippedReason: value };
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
this.userData.__crawlee.skippedReason = value;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
289
306
|
get enqueueStrategy() {
|
|
290
307
|
return this.userData.__crawlee?.enqueueStrategy;
|
|
291
308
|
}
|