@crawlee/core 4.0.0-beta.4 → 4.0.0-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/autoscaling/autoscaled_pool.d.ts +3 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -9
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -13
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +18 -29
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +5 -78
- package/configuration.d.ts.map +1 -1
- package/configuration.js +6 -102
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +1 -1
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +8 -8
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +71 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +123 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +15 -23
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +0 -8
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +15 -15
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +21 -18
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +30 -18
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +41 -23
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +24 -7
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +66 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +18 -0
- package/errors.d.ts.map +1 -1
- package/errors.js +35 -0
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +8 -5
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +7 -9
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +33 -39
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +2 -1
- package/index.d.ts.map +1 -1
- package/index.js +2 -1
- package/index.js.map +1 -1
- package/log.d.ts +82 -2
- package/log.d.ts.map +1 -1
- package/log.js +102 -0
- package/log.js.map +1 -1
- package/package.json +6 -7
- package/proxy_configuration.d.ts +17 -94
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +18 -54
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +121 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +137 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +48 -6
- package/request.d.ts.map +1 -1
- package/request.js +62 -16
- package/request.js.map +1 -1
- package/service_locator.d.ts +162 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +247 -0
- package/service_locator.js.map +1 -0
- package/session_pool/session.d.ts +9 -31
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +17 -21
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +27 -54
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +54 -69
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +53 -3
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +78 -6
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +2 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +2 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +71 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +95 -12
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +9 -9
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +11 -8
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +76 -9
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +92 -54
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +1 -3
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +2 -4
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +3 -3
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +4 -5
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +5 -5
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +10 -7
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts +10 -8
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +12 -22
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +4 -3
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/request.js
CHANGED
|
@@ -2,10 +2,9 @@ import crypto from 'node:crypto';
|
|
|
2
2
|
import util from 'node:util';
|
|
3
3
|
import ow from 'ow';
|
|
4
4
|
import { normalizeUrl } from '@apify/utilities';
|
|
5
|
-
import {
|
|
5
|
+
import { serviceLocator } from './service_locator.js';
|
|
6
6
|
import { keys } from './typedefs.js';
|
|
7
7
|
// new properties on the Request object breaks serialization
|
|
8
|
-
const log = defaultLog.child({ prefix: 'Request' });
|
|
9
8
|
const requestOptionalPredicates = {
|
|
10
9
|
id: ow.optional.string,
|
|
11
10
|
loadedUrl: ow.optional.string.url,
|
|
@@ -24,6 +23,7 @@ const requestOptionalPredicates = {
|
|
|
24
23
|
keepUrlFragment: ow.optional.boolean,
|
|
25
24
|
useExtendedUniqueKey: ow.optional.boolean,
|
|
26
25
|
skipNavigation: ow.optional.boolean,
|
|
26
|
+
crawlDepth: ow.optional.number.greaterThanOrEqual(0),
|
|
27
27
|
state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
|
|
28
28
|
};
|
|
29
29
|
export var RequestState;
|
|
@@ -68,7 +68,7 @@ export var RequestState;
|
|
|
68
68
|
* ```
|
|
69
69
|
* @category Sources
|
|
70
70
|
*/
|
|
71
|
-
|
|
71
|
+
class CrawleeRequest {
|
|
72
72
|
/** Request ID */
|
|
73
73
|
id;
|
|
74
74
|
/** URL of the web page to crawl. */
|
|
@@ -101,7 +101,12 @@ export class Request {
|
|
|
101
101
|
headers;
|
|
102
102
|
/** Private store for the custom user data assigned to the request. */
|
|
103
103
|
_userData = {};
|
|
104
|
-
/**
|
|
104
|
+
/**
|
|
105
|
+
* Custom user data assigned to the request.
|
|
106
|
+
*
|
|
107
|
+
* All data stored in `userData` must be JSON-serializable.
|
|
108
|
+
* Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
|
|
109
|
+
*/
|
|
105
110
|
userData = {};
|
|
106
111
|
/**
|
|
107
112
|
* ISO datetime string that indicates the time when the request has been processed.
|
|
@@ -130,7 +135,7 @@ export class Request {
|
|
|
130
135
|
ow(value, `RequestOptions.${prop}`, predicate);
|
|
131
136
|
}
|
|
132
137
|
});
|
|
133
|
-
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, } = options;
|
|
138
|
+
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
|
|
134
139
|
let { method = 'GET' } = options;
|
|
135
140
|
method = method.toUpperCase();
|
|
136
141
|
if (method === 'GET' && payload)
|
|
@@ -139,7 +144,8 @@ export class Request {
|
|
|
139
144
|
this.url = url;
|
|
140
145
|
this.loadedUrl = loadedUrl;
|
|
141
146
|
this.uniqueKey =
|
|
142
|
-
uniqueKey ||
|
|
147
|
+
uniqueKey ||
|
|
148
|
+
CrawleeRequest.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
|
|
143
149
|
this.method = method;
|
|
144
150
|
this.payload = payload;
|
|
145
151
|
this.noRetry = noRetry;
|
|
@@ -191,11 +197,24 @@ export class Request {
|
|
|
191
197
|
this.skipNavigation = skipNavigation;
|
|
192
198
|
if (maxRetries != null)
|
|
193
199
|
this.maxRetries = maxRetries;
|
|
200
|
+
if (crawlDepth != null)
|
|
201
|
+
this.userData.__crawlee.crawlDepth ??= crawlDepth;
|
|
194
202
|
// If it's already set, don't override it (for instance when fetching from storage)
|
|
195
203
|
if (enqueueStrategy) {
|
|
196
204
|
this.enqueueStrategy ??= enqueueStrategy;
|
|
197
205
|
}
|
|
198
206
|
}
|
|
207
|
+
/**
|
|
208
|
+
* Converts the Crawlee Request object to a `fetch` API Request object.
|
|
209
|
+
* @returns The native `fetch` API Request object.
|
|
210
|
+
*/
|
|
211
|
+
intoFetchAPIRequest() {
|
|
212
|
+
return new Request(this.url, {
|
|
213
|
+
method: this.method,
|
|
214
|
+
headers: this.headers,
|
|
215
|
+
body: this.payload,
|
|
216
|
+
});
|
|
217
|
+
}
|
|
199
218
|
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
200
219
|
get skipNavigation() {
|
|
201
220
|
return this.userData.__crawlee?.skipNavigation ?? false;
|
|
@@ -209,6 +228,21 @@ export class Request {
|
|
|
209
228
|
this.userData.__crawlee.skipNavigation = value;
|
|
210
229
|
}
|
|
211
230
|
}
|
|
231
|
+
/**
|
|
232
|
+
* Depth of the request in the current crawl tree.
|
|
233
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
234
|
+
*/
|
|
235
|
+
get crawlDepth() {
|
|
236
|
+
return this.userData.__crawlee?.crawlDepth ?? 0;
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Depth of the request in the current crawl tree.
|
|
240
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
241
|
+
*/
|
|
242
|
+
set crawlDepth(value) {
|
|
243
|
+
this.userData.__crawlee ??= {};
|
|
244
|
+
this.userData.__crawlee.crawlDepth = value;
|
|
245
|
+
}
|
|
212
246
|
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
|
|
213
247
|
get sessionRotationCount() {
|
|
214
248
|
return this.userData.__crawlee?.sessionRotationCount ?? 0;
|
|
@@ -256,6 +290,23 @@ export class Request {
|
|
|
256
290
|
this.userData.__crawlee.state = value;
|
|
257
291
|
}
|
|
258
292
|
}
|
|
293
|
+
/**
|
|
294
|
+
* Reason for skipping this request.
|
|
295
|
+
*/
|
|
296
|
+
get skippedReason() {
|
|
297
|
+
return this.userData.__crawlee?.skippedReason;
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Reason for skipping this request.
|
|
301
|
+
*/
|
|
302
|
+
set skippedReason(value) {
|
|
303
|
+
if (!this.userData.__crawlee) {
|
|
304
|
+
this.userData.__crawlee = { skippedReason: value };
|
|
305
|
+
}
|
|
306
|
+
else {
|
|
307
|
+
this.userData.__crawlee.skippedReason = value;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
259
310
|
get enqueueStrategy() {
|
|
260
311
|
return this.userData.__crawlee?.enqueueStrategy;
|
|
261
312
|
}
|
|
@@ -317,27 +368,21 @@ export class Request {
|
|
|
317
368
|
}
|
|
318
369
|
this.errorMessages.push(message);
|
|
319
370
|
}
|
|
320
|
-
// TODO: only for better BC, remove in v4
|
|
321
|
-
_computeUniqueKey(options) {
|
|
322
|
-
return Request.computeUniqueKey(options);
|
|
323
|
-
}
|
|
324
|
-
// TODO: only for better BC, remove in v4
|
|
325
|
-
_hashPayload(payload) {
|
|
326
|
-
return Request.hashPayload(payload);
|
|
327
|
-
}
|
|
328
371
|
/** @internal */
|
|
329
372
|
static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, }) {
|
|
330
373
|
const normalizedMethod = method.toUpperCase();
|
|
331
374
|
const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
|
|
332
375
|
if (!useExtendedUniqueKey) {
|
|
333
376
|
if (normalizedMethod !== 'GET' && payload) {
|
|
334
|
-
|
|
377
|
+
serviceLocator
|
|
378
|
+
.getLogger()
|
|
379
|
+
.warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
|
|
335
380
|
'This is fine. Just letting you know that if your requests point to the same URL ' +
|
|
336
381
|
'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
|
|
337
382
|
}
|
|
338
383
|
return normalizedUrl;
|
|
339
384
|
}
|
|
340
|
-
const payloadHash = payload ?
|
|
385
|
+
const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
|
|
341
386
|
return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
|
|
342
387
|
}
|
|
343
388
|
/** @internal */
|
|
@@ -345,4 +390,5 @@ export class Request {
|
|
|
345
390
|
return crypto.createHash('sha256').update(payload).digest('base64').replace(/[+/=]/g, '').substring(0, 8);
|
|
346
391
|
}
|
|
347
392
|
}
|
|
393
|
+
export { CrawleeRequest as Request };
|
|
348
394
|
//# sourceMappingURL=request.js.map
|
package/request.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAErC,4DAA4D;AAC5D,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACxC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACzB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACzC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACnC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACpD,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;CACrE,CAAC;AAEF,MAAM,CAAN,IAAY,YASX;AATD,WAAY,YAAY;IACpB,6DAAW,CAAA;IACX,2DAAU,CAAA;IACV,yDAAS,CAAA;IACT,qEAAe,CAAA;IACf,+CAAI,CAAA;IACJ,iEAAa,CAAA;IACb,iDAAK,CAAA;IACL,qDAAO,CAAA;AACX,CAAC,EATW,YAAY,KAAZ,YAAY,QASvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,cAAc;IAChB,iBAAiB;IACjB,EAAE,CAAU;IAEZ,oCAAoC;IACpC,GAAG,CAAS;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAU;IAEnB;;;OAGG;IACH,SAAS,CAAS;IAElB,yCAAyC;IACzC,MAAM,CAAqB;IAE3B,oDAAoD;IACpD,OAAO,CAAU;IAEjB,8FAA8F;IAC9F,OAAO,CAAU;IAEjB,2FAA2F;IAC3F,UAAU,CAAS;IAEnB,0DAA0D;IAC1D,aAAa,CAAW;IAExB,wEAAwE;IACxE,OAAO,CAA0B;IAEjC,sEAAsE;IAC9D,SAAS,GAAwB,EAAE,CAAC;IAE5C;;;;;OAKG;IACH,QAAQ,GAAa,EAAc,CAAC;IAEpC;;;OAGG;IACH,SAAS,CAAU;IAEnB;;OAEG;IACH,YAAY,OAAiC;QACzC,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACzC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,0CAA0C;YAC1C,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACjB,OAAO;YACX,CAAC;YAED,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE,CAAC;gBACZ,EAAE,CAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;YACpE,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,oBAAoB,GAAG,CAAC,EACxB,UAAU,EACV,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,KAAK,EACL,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,cAAc,EACd,eAAe,EACf,UAAU,GACb,GAAG,OAMH,CAAC;QAEF,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEjC,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS;YACV,SAAS;gBACT,cAAc,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAI,SAAqB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;QAEzG,IAAI,KAAK,EAAE,CAAC;YACR,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC1B,SAAS,EAAE;gBACP,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,QAAQ,EAAE;gBACrC,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACjB;YACD,QAAQ,EAAE;gBACN,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;gBACzB,GAAG,EAAE,CAAC,KAA0B,EAAE,EAAE;oBAChC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE;wBAC3B,SAAS,EAAE;4BACP,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;4BAC/B,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;wBACD,MAAM,EAAE;4BACJ,KAAK,EAAE,GAAG,EAAE;gCACR,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACnD,OAAO;wCACH,GAAG,IAAI,CAAC,SAAS;wCACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;qCACtC,CAAC;gCACN,CAAC;gCAED,OAAO,IAAI,CAAC,SAAS,CAAC;4BAC1B,CAAC;4BACD,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;qBACJ,CAAC,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,UAAU,EAAE,IAAI;aACnB;SACJ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,cAAc,IAAI,IAAI;YAAE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACjE,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QACrD,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,KAAK,UAAU,CAAC;QAE1E,mFAAmF;QACnF,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,mBAAmB;QACtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,OAAO;SACrB,CAAC,CAAC;IACP,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,IAAI,KAAK,CAAC;IAC5D,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAc;QAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QACxE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;QACnD,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAa;QACvB,IAAI,CAAC,QAAuB,CAAC,SAAS,KAAK,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;IAC/C,CAAC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,oBAAoB,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAa;QAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;QAC9E,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,oBAAoB,GAAG,KAAK,CAAC;QACzD,CAAC;IACL,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAyB;QAC9B,IAAI,CAAC,QAAuB,CAAC,KAAK,GAAG,KAAK,CAAC;IAChD,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC;IAC/C,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAyB;QACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACpE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,IAAI,YAAY,CAAC,WAAW,CAAC;IACtE,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAmB;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAC/D,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,IAAI,aAAa,CAAC,KAAuC;QACrD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACvE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,GAAG,KAAK,CAAC;QAClD,CAAC;IACL,CAAC;IAED,IAAY,eAAe;QACvB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;IACpD,CAAC;IAED,IAAY,eAAe,CAAC,KAAsC;QAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;QACzE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,eAAe,GAAG,KAAK,CAAC;QACpD,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClB,OAAO,GAAG,MAAM,CAAC;YACrB,CAAC;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE,CAAC;gBACzC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,CAAC,CAAC,8BAA8B;wBAC9B,cAAc,CAAC,KAAK,CAAC;YAC/B,CAAC;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;gBACxD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;YAC7D,CAAC;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE,CAAC;gBACrE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;gBAC3C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,GAAG,yDAAyD,CAAC;gBACxE,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,GAAG,WAAW,CAAC;QAC1B,CAAC;aAAM,CAAC;YACJ,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAM,GAAG,KAAK,EACd,OAAO,EACP,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,GACN;QACtB,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACxB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE,CAAC;gBACxC,cAAc;qBACT,SAAS,EAAE;qBACX,WAAW,CACR,uBAAuB,gBAAgB,2BAA2B;oBAC9D,kFAAkF;oBAClF,iHAAiH,CACxH,CAAC;YACV,CAAC;YACD,OAAO,aAAa,CAAC;QACzB,CAAC;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAmB;QAClC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9G,CAAC;CACJ;AAiJD,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import type { StorageClient } from '@crawlee/types';
|
|
2
|
+
import { Configuration } from './configuration.js';
|
|
3
|
+
import type { EventManager } from './events/event_manager.js';
|
|
4
|
+
import type { CrawleeLogger } from './log.js';
|
|
5
|
+
import type { IStorage, StorageManager } from './storages/storage_manager.js';
|
|
6
|
+
import type { Constructor } from './typedefs.js';
|
|
7
|
+
interface ServiceLocatorInterface {
|
|
8
|
+
/**
|
|
9
|
+
* Get the configuration.
|
|
10
|
+
* Creates a default Configuration instance if none has been set.
|
|
11
|
+
*/
|
|
12
|
+
getConfiguration(): Configuration;
|
|
13
|
+
/**
|
|
14
|
+
* Set the configuration.
|
|
15
|
+
*
|
|
16
|
+
* @param configuration The configuration to set
|
|
17
|
+
* @throws {ServiceConflictError} If a different configuration has already been retrieved
|
|
18
|
+
*/
|
|
19
|
+
setConfiguration(configuration: Configuration): void;
|
|
20
|
+
/**
|
|
21
|
+
* Get the event manager.
|
|
22
|
+
* Creates a default LocalEventManager instance if none has been set.
|
|
23
|
+
*/
|
|
24
|
+
getEventManager(): EventManager;
|
|
25
|
+
/**
|
|
26
|
+
* Set the event manager.
|
|
27
|
+
*
|
|
28
|
+
* @param eventManager The event manager to set
|
|
29
|
+
* @throws {ServiceConflictError} If a different event manager has already been retrieved
|
|
30
|
+
*/
|
|
31
|
+
setEventManager(eventManager: EventManager): void;
|
|
32
|
+
/**
|
|
33
|
+
* Get the storage client.
|
|
34
|
+
* Creates a default MemoryStorage instance if none has been set.
|
|
35
|
+
*/
|
|
36
|
+
getStorageClient(): StorageClient;
|
|
37
|
+
/**
|
|
38
|
+
* Set the storage client.
|
|
39
|
+
*
|
|
40
|
+
* @param storageClient The storage client to set
|
|
41
|
+
* @throws {ServiceConflictError} If a different storage client has already been retrieved
|
|
42
|
+
*/
|
|
43
|
+
setStorageClient(storageClient: StorageClient): void;
|
|
44
|
+
/**
|
|
45
|
+
* Get the logger.
|
|
46
|
+
* Returns the default `@apify/log` logger if none has been set.
|
|
47
|
+
*/
|
|
48
|
+
getLogger(): CrawleeLogger;
|
|
49
|
+
/**
|
|
50
|
+
* Set the logger.
|
|
51
|
+
*
|
|
52
|
+
* @param logger The logger to set
|
|
53
|
+
* @throws {ServiceConflictError} If a different logger has already been retrieved
|
|
54
|
+
*/
|
|
55
|
+
setLogger(logger: CrawleeLogger): void;
|
|
56
|
+
/**
|
|
57
|
+
* Get a child logger with the given prefix.
|
|
58
|
+
* Equivalent to `getLogger().child({ prefix })`.
|
|
59
|
+
*/
|
|
60
|
+
getChildLog(prefix: string): CrawleeLogger;
|
|
61
|
+
getStorageManager(constructor: Constructor<IStorage>): StorageManager | undefined;
|
|
62
|
+
setStorageManager(constructor: Constructor<IStorage>, storageManager: StorageManager): void;
|
|
63
|
+
/**
|
|
64
|
+
* Clears all storage manager caches.
|
|
65
|
+
* @internal
|
|
66
|
+
*/
|
|
67
|
+
clearStorageManagerCache(): void;
|
|
68
|
+
/**
|
|
69
|
+
* Resets the service locator to its initial state.
|
|
70
|
+
* Used mainly for testing purposes.
|
|
71
|
+
* @internal
|
|
72
|
+
*/
|
|
73
|
+
reset(): void;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Service locator for managing the services used by Crawlee.
|
|
77
|
+
*
|
|
78
|
+
* All services are initialized to their default value lazily.
|
|
79
|
+
*
|
|
80
|
+
* There are two primary usage patterns:
|
|
81
|
+
*
|
|
82
|
+
* **1. Global service locator (for default services):**
|
|
83
|
+
* ```typescript
|
|
84
|
+
* import { serviceLocator, BasicCrawler } from 'crawlee';
|
|
85
|
+
*
|
|
86
|
+
* // Optionally configure global services before creating crawlers
|
|
87
|
+
* serviceLocator.setStorageClient(myCustomClient);
|
|
88
|
+
*
|
|
89
|
+
* // Crawler uses global services
|
|
90
|
+
* const crawler = new BasicCrawler({ ... });
|
|
91
|
+
* ```
|
|
92
|
+
*
|
|
93
|
+
* **2. Per-crawler services (recommended for isolation):**
|
|
94
|
+
* ```typescript
|
|
95
|
+
* import { BasicCrawler, Configuration, LocalEventManager } from 'crawlee';
|
|
96
|
+
* import { MemoryStorage } from '@crawlee/memory-storage';
|
|
97
|
+
*
|
|
98
|
+
* const crawler = new BasicCrawler({
|
|
99
|
+
* requestHandler: async ({ request }) => { ... },
|
|
100
|
+
* configuration: new Configuration({ ... }), // custom config
|
|
101
|
+
* storageClient: new MemoryStorage(), // custom storage
|
|
102
|
+
* eventManager: LocalEventManager.fromConfig(), // custom events
|
|
103
|
+
* });
|
|
104
|
+
* // Crawler has its own isolated ServiceLocator instance
|
|
105
|
+
* ```
|
|
106
|
+
*/
|
|
107
|
+
export declare class ServiceLocator implements ServiceLocatorInterface {
|
|
108
|
+
private configuration?;
|
|
109
|
+
private eventManager?;
|
|
110
|
+
private storageClient?;
|
|
111
|
+
private logger?;
|
|
112
|
+
/**
|
|
113
|
+
* Storage managers for Dataset, KeyValueStore, and RequestQueue.
|
|
114
|
+
* Manages caching and lifecycle of storage instances.
|
|
115
|
+
*/
|
|
116
|
+
private storageManagers;
|
|
117
|
+
/**
|
|
118
|
+
* Creates a new ServiceLocator instance.
|
|
119
|
+
*
|
|
120
|
+
* @param configuration Optional configuration instance to use
|
|
121
|
+
* @param eventManager Optional event manager instance to use
|
|
122
|
+
* @param storageClient Optional storage client instance to use
|
|
123
|
+
* @param logger Optional logger instance to use
|
|
124
|
+
*/
|
|
125
|
+
constructor(configuration?: Configuration, eventManager?: EventManager, storageClient?: StorageClient, logger?: CrawleeLogger);
|
|
126
|
+
getConfiguration(): Configuration;
|
|
127
|
+
setConfiguration(configuration: Configuration): void;
|
|
128
|
+
getEventManager(): EventManager;
|
|
129
|
+
setEventManager(eventManager: EventManager): void;
|
|
130
|
+
getStorageClient(): StorageClient;
|
|
131
|
+
setStorageClient(storageClient: StorageClient): void;
|
|
132
|
+
getLogger(): CrawleeLogger;
|
|
133
|
+
setLogger(logger: CrawleeLogger): void;
|
|
134
|
+
getChildLog(prefix: string): CrawleeLogger;
|
|
135
|
+
getStorageManager(constructor: Constructor<IStorage>): StorageManager | undefined;
|
|
136
|
+
setStorageManager(constructor: Constructor<IStorage>, storageManager: StorageManager): void;
|
|
137
|
+
clearStorageManagerCache(): void;
|
|
138
|
+
reset(): void;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Wraps all methods on `target` so that any code they invoke will see the given
|
|
142
|
+
* `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
|
|
143
|
+
*
|
|
144
|
+
* Walks the prototype chain and replaces each method on the *instance* (not the prototype)
|
|
145
|
+
* with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
|
|
146
|
+
*
|
|
147
|
+
* The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
|
|
148
|
+
* wrapped method — including `super` calls, since the prototype methods execute within the
|
|
149
|
+
* context established by the instance-level wrapper.
|
|
150
|
+
*
|
|
151
|
+
* @internal
|
|
152
|
+
* @returns Scope control functions: `run` executes a callback within the scoped context,
|
|
153
|
+
* `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
|
|
154
|
+
*/
|
|
155
|
+
export declare function bindMethodsToServiceLocator(serviceLocator: ServiceLocator, target: {}): {
|
|
156
|
+
run: <T>(fn: () => T) => T;
|
|
157
|
+
enterScope: () => void;
|
|
158
|
+
exitScope: () => void;
|
|
159
|
+
};
|
|
160
|
+
export declare const serviceLocator: ServiceLocatorInterface;
|
|
161
|
+
export {};
|
|
162
|
+
//# sourceMappingURL=service_locator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"service_locator.d.ts","sourceRoot":"","sources":["../src/service_locator.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE9C,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC9E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,UAAU,uBAAuB;IAC7B;;;OAGG;IACH,gBAAgB,IAAI,aAAa,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAErD;;;OAGG;IACH,eAAe,IAAI,YAAY,CAAC;IAEhC;;;;;OAKG;IACH,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAElD;;;OAGG;IACH,gBAAgB,IAAI,aAAa,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAErD;;;OAGG;IACH,SAAS,IAAI,aAAa,CAAC;IAE3B;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;IAEvC;;;OAGG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC;IAE3C,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,cAAc,GAAG,SAAS,CAAC;IAElF,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;IAE5F;;;OAGG;IACH,wBAAwB,IAAI,IAAI,CAAC;IAEjC;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,cAAe,YAAW,uBAAuB;IAC1D,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,MAAM,CAAC,CAAgB;IAE/B;;;OAGG;IACH,OAAO,CAAC,eAAe,CAAoD;IAE3E;;;;;;;OAOG;gBAEC,aAAa,CAAC,EAAE,aAAa,EAC7B,YAAY,CAAC,EAAE,YAAY,EAC3B,aAAa,CAAC,EAAE,aAAa,EAC7B,MAAM,CAAC,EAAE,aAAa;IAQ1B,gBAAgB,IAAI,aAAa;IAQjC,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI;IAcpD,eAAe,IAAI,YAAY;IAc/B,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI;IAcjD,gBAAgB,IAAI,aAAa;IAkBjC,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI;IAcpD,SAAS,IAAI,aAAa;IAO1B,SAAS,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAYtC,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa;IAI1C,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,cAAc,GAAG,SAAS;IAIjF,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,cAAc,GAAG,IAAI;IAY3F,wBAAwB,IAAI,IAAI;IAchC,KAAK,IAAI,IAAI;CAOhB;AASD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,2BAA2B,CACvC,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,EAAE,GACX;IAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAAC,UAAU,EAAE,MAAM,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,IAAI,CAAA;CAAE,CA4C/E;AAED,eAAO,MAAM,cAAc,yBAczB,CAAC"}
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
import { MemoryStorage } from '@crawlee/memory-storage';
|
|
3
|
+
import log from '@apify/log';
|
|
4
|
+
import { Configuration } from './configuration.js';
|
|
5
|
+
import { ServiceConflictError } from './errors.js';
|
|
6
|
+
import { LocalEventManager } from './events/local_event_manager.js';
|
|
7
|
+
import { ApifyLogAdapter } from './log.js';
|
|
8
|
+
/**
|
|
9
|
+
* Service locator for managing the services used by Crawlee.
|
|
10
|
+
*
|
|
11
|
+
* All services are initialized to their default value lazily.
|
|
12
|
+
*
|
|
13
|
+
* There are two primary usage patterns:
|
|
14
|
+
*
|
|
15
|
+
* **1. Global service locator (for default services):**
|
|
16
|
+
* ```typescript
|
|
17
|
+
* import { serviceLocator, BasicCrawler } from 'crawlee';
|
|
18
|
+
*
|
|
19
|
+
* // Optionally configure global services before creating crawlers
|
|
20
|
+
* serviceLocator.setStorageClient(myCustomClient);
|
|
21
|
+
*
|
|
22
|
+
* // Crawler uses global services
|
|
23
|
+
* const crawler = new BasicCrawler({ ... });
|
|
24
|
+
* ```
|
|
25
|
+
*
|
|
26
|
+
* **2. Per-crawler services (recommended for isolation):**
|
|
27
|
+
* ```typescript
|
|
28
|
+
* import { BasicCrawler, Configuration, LocalEventManager } from 'crawlee';
|
|
29
|
+
* import { MemoryStorage } from '@crawlee/memory-storage';
|
|
30
|
+
*
|
|
31
|
+
* const crawler = new BasicCrawler({
|
|
32
|
+
* requestHandler: async ({ request }) => { ... },
|
|
33
|
+
* configuration: new Configuration({ ... }), // custom config
|
|
34
|
+
* storageClient: new MemoryStorage(), // custom storage
|
|
35
|
+
* eventManager: LocalEventManager.fromConfig(), // custom events
|
|
36
|
+
* });
|
|
37
|
+
* // Crawler has its own isolated ServiceLocator instance
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export class ServiceLocator {
|
|
41
|
+
configuration;
|
|
42
|
+
eventManager;
|
|
43
|
+
storageClient;
|
|
44
|
+
logger;
|
|
45
|
+
/**
|
|
46
|
+
* Storage managers for Dataset, KeyValueStore, and RequestQueue.
|
|
47
|
+
* Manages caching and lifecycle of storage instances.
|
|
48
|
+
*/
|
|
49
|
+
storageManagers = new Map();
|
|
50
|
+
/**
|
|
51
|
+
* Creates a new ServiceLocator instance.
|
|
52
|
+
*
|
|
53
|
+
* @param configuration Optional configuration instance to use
|
|
54
|
+
* @param eventManager Optional event manager instance to use
|
|
55
|
+
* @param storageClient Optional storage client instance to use
|
|
56
|
+
* @param logger Optional logger instance to use
|
|
57
|
+
*/
|
|
58
|
+
constructor(configuration, eventManager, storageClient, logger) {
|
|
59
|
+
this.configuration = configuration;
|
|
60
|
+
this.eventManager = eventManager;
|
|
61
|
+
this.storageClient = storageClient;
|
|
62
|
+
this.logger = logger;
|
|
63
|
+
}
|
|
64
|
+
getConfiguration() {
|
|
65
|
+
if (!this.configuration) {
|
|
66
|
+
this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
|
|
67
|
+
this.configuration = new Configuration();
|
|
68
|
+
}
|
|
69
|
+
return this.configuration;
|
|
70
|
+
}
|
|
71
|
+
setConfiguration(configuration) {
|
|
72
|
+
// Same instance, no need to do anything
|
|
73
|
+
if (this.configuration === configuration) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
// Already have a different configuration that was retrieved
|
|
77
|
+
if (this.configuration) {
|
|
78
|
+
throw new ServiceConflictError('Configuration', configuration, this.configuration);
|
|
79
|
+
}
|
|
80
|
+
this.configuration = configuration;
|
|
81
|
+
}
|
|
82
|
+
getEventManager() {
|
|
83
|
+
if (!this.eventManager) {
|
|
84
|
+
this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
|
|
85
|
+
if (!this.configuration) {
|
|
86
|
+
this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
|
|
87
|
+
'It is advised to explicitly first set the configuration instead.');
|
|
88
|
+
}
|
|
89
|
+
this.eventManager = LocalEventManager.fromConfig(this.getConfiguration());
|
|
90
|
+
}
|
|
91
|
+
return this.eventManager;
|
|
92
|
+
}
|
|
93
|
+
setEventManager(eventManager) {
|
|
94
|
+
// Same instance, no need to do anything
|
|
95
|
+
if (this.eventManager === eventManager) {
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
// Already have a different event manager that was retrieved
|
|
99
|
+
if (this.eventManager) {
|
|
100
|
+
throw new ServiceConflictError('EventManager', eventManager, this.eventManager);
|
|
101
|
+
}
|
|
102
|
+
this.eventManager = eventManager;
|
|
103
|
+
}
|
|
104
|
+
getStorageClient() {
|
|
105
|
+
if (!this.storageClient) {
|
|
106
|
+
this.getLogger().debug('No storage client set, implicitly creating and using default MemoryStorage.');
|
|
107
|
+
if (!this.configuration) {
|
|
108
|
+
this.getLogger().warning('Implicit creation of storage client will implicitly set configuration as side effect. ' +
|
|
109
|
+
'It is advised to explicitly first set the configuration instead.');
|
|
110
|
+
}
|
|
111
|
+
const config = this.getConfiguration();
|
|
112
|
+
this.storageClient = new MemoryStorage({
|
|
113
|
+
persistStorage: config.get('persistStorage'),
|
|
114
|
+
logger: this.getLogger().child({ prefix: 'MemoryStorage' }),
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
return this.storageClient;
|
|
118
|
+
}
|
|
119
|
+
setStorageClient(storageClient) {
|
|
120
|
+
// Same instance, no need to do anything
|
|
121
|
+
if (this.storageClient === storageClient) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
// Already have a different storage client that was retrieved
|
|
125
|
+
if (this.storageClient) {
|
|
126
|
+
throw new ServiceConflictError('StorageClient', storageClient, this.storageClient);
|
|
127
|
+
}
|
|
128
|
+
this.storageClient = storageClient;
|
|
129
|
+
}
|
|
130
|
+
getLogger() {
|
|
131
|
+
if (!this.logger) {
|
|
132
|
+
this.logger = new ApifyLogAdapter(log);
|
|
133
|
+
}
|
|
134
|
+
return this.logger;
|
|
135
|
+
}
|
|
136
|
+
setLogger(logger) {
|
|
137
|
+
if (this.logger === logger) {
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
if (this.logger) {
|
|
141
|
+
throw new ServiceConflictError('Logger', logger, this.logger);
|
|
142
|
+
}
|
|
143
|
+
this.logger = logger;
|
|
144
|
+
}
|
|
145
|
+
getChildLog(prefix) {
|
|
146
|
+
return this.getLogger().child({ prefix });
|
|
147
|
+
}
|
|
148
|
+
getStorageManager(constructor) {
|
|
149
|
+
return this.storageManagers.get(constructor);
|
|
150
|
+
}
|
|
151
|
+
setStorageManager(constructor, storageManager) {
|
|
152
|
+
if (this.storageManagers.has(constructor)) {
|
|
153
|
+
throw new ServiceConflictError(`StorageManager(${constructor.name})`, storageManager, this.storageManagers.get(constructor));
|
|
154
|
+
}
|
|
155
|
+
this.storageManagers.set(constructor, storageManager);
|
|
156
|
+
}
|
|
157
|
+
clearStorageManagerCache() {
|
|
158
|
+
this.storageManagers.forEach((manager) => {
|
|
159
|
+
// KeyValueStore has a clearCache method on its instances
|
|
160
|
+
// TODO this uses fragile string matching and `any` casts into private fields - remove as part of
|
|
161
|
+
// https://github.com/apify/crawlee/issues/3075 (Storage instance management will be reworked significantly)
|
|
162
|
+
if (manager.name === 'KeyValueStore') {
|
|
163
|
+
manager.cache?.forEach((item) => {
|
|
164
|
+
item.clearCache?.();
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
this.storageManagers.clear();
|
|
169
|
+
}
|
|
170
|
+
reset() {
|
|
171
|
+
this.configuration = undefined;
|
|
172
|
+
this.eventManager = undefined;
|
|
173
|
+
this.storageClient = undefined;
|
|
174
|
+
this.logger = undefined;
|
|
175
|
+
this.clearStorageManagerCache();
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Used as the default service provider when crawlers don't specify custom services.
|
|
180
|
+
*/
|
|
181
|
+
const globalServiceLocator = new ServiceLocator();
|
|
182
|
+
const serviceLocatorStorage = new AsyncLocalStorage();
|
|
183
|
+
/**
|
|
184
|
+
* Wraps all methods on `target` so that any code they invoke will see the given
|
|
185
|
+
* `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
|
|
186
|
+
*
|
|
187
|
+
* Walks the prototype chain and replaces each method on the *instance* (not the prototype)
|
|
188
|
+
* with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
|
|
189
|
+
*
|
|
190
|
+
* The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
|
|
191
|
+
* wrapped method — including `super` calls, since the prototype methods execute within the
|
|
192
|
+
* context established by the instance-level wrapper.
|
|
193
|
+
*
|
|
194
|
+
* @internal
|
|
195
|
+
* @returns Scope control functions: `run` executes a callback within the scoped context,
|
|
196
|
+
* `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
|
|
197
|
+
*/
|
|
198
|
+
export function bindMethodsToServiceLocator(serviceLocator, target) {
|
|
199
|
+
let proto = Object.getPrototypeOf(target);
|
|
200
|
+
while (proto !== null && proto !== Object.prototype) {
|
|
201
|
+
const propertyKeys = [...Object.getOwnPropertyNames(proto), ...Object.getOwnPropertySymbols(proto)];
|
|
202
|
+
for (const propertyKey of propertyKeys) {
|
|
203
|
+
const descriptor = Object.getOwnPropertyDescriptor(proto, propertyKey);
|
|
204
|
+
// We use property descriptors rather than accessing target[propertyKey] directly,
|
|
205
|
+
// because that would trigger getters and cause unwanted side effects.
|
|
206
|
+
// Skip getters, setters, and constructors — only wrap regular methods.
|
|
207
|
+
if (propertyKey === 'constructor' ||
|
|
208
|
+
!descriptor ||
|
|
209
|
+
descriptor.get ||
|
|
210
|
+
descriptor.set ||
|
|
211
|
+
typeof descriptor.value !== 'function')
|
|
212
|
+
continue;
|
|
213
|
+
const original = descriptor.value;
|
|
214
|
+
target[propertyKey] = (...args) => {
|
|
215
|
+
return serviceLocatorStorage.run(serviceLocator, () => {
|
|
216
|
+
return original.apply(target, args);
|
|
217
|
+
});
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
proto = Object.getPrototypeOf(proto);
|
|
221
|
+
}
|
|
222
|
+
let previousStore;
|
|
223
|
+
return {
|
|
224
|
+
run: (fn) => serviceLocatorStorage.run(serviceLocator, fn),
|
|
225
|
+
enterScope: () => {
|
|
226
|
+
previousStore = serviceLocatorStorage.getStore();
|
|
227
|
+
serviceLocatorStorage.enterWith(serviceLocator);
|
|
228
|
+
},
|
|
229
|
+
exitScope: () => {
|
|
230
|
+
serviceLocatorStorage.enterWith(previousStore); // casting to any so that `undefined` is accepted - this "unsets" the AsyncLocalStorage
|
|
231
|
+
},
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
export const serviceLocator = new Proxy({}, {
|
|
235
|
+
get(_target, prop) {
|
|
236
|
+
const active = serviceLocatorStorage.getStore() ?? globalServiceLocator;
|
|
237
|
+
const value = Reflect.get(active, prop, active);
|
|
238
|
+
if (typeof value === 'function') {
|
|
239
|
+
return value.bind(active);
|
|
240
|
+
}
|
|
241
|
+
return value;
|
|
242
|
+
},
|
|
243
|
+
set(_target, prop) {
|
|
244
|
+
throw new TypeError(`Cannot set property '${String(prop)}' on serviceLocator directly. Use the setter methods (e.g. setConfiguration(), setStorageClient()) instead.`);
|
|
245
|
+
},
|
|
246
|
+
});
|
|
247
|
+
//# sourceMappingURL=service_locator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"service_locator.js","sourceRoot":"","sources":["../src/service_locator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAqF3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,OAAO,cAAc;IACf,aAAa,CAAiB;IAC9B,YAAY,CAAgB;IAC5B,aAAa,CAAiB;IAC9B,MAAM,CAAiB;IAE/B;;;OAGG;IACK,eAAe,GAAG,IAAI,GAAG,EAAyC,CAAC;IAE3E;;;;;;;OAOG;IACH,YACI,aAA6B,EAC7B,YAA2B,EAC3B,aAA6B,EAC7B,MAAsB;QAEtB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,gBAAgB;QACZ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,4EAA4E,CAAC,CAAC;YACrG,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,gBAAgB,CAAC,aAA4B;QACzC,wCAAwC;QACxC,IAAI,IAAI,CAAC,aAAa,KAAK,aAAa,EAAE,CAAC;YACvC,OAAO;QACX,CAAC;QAED,4DAA4D;QAC5D,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,MAAM,IAAI,oBAAoB,CAAC,eAAe,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED,eAAe;QACX,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACrB,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,gFAAgF,CAAC,CAAC;YACzG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;gBACtB,IAAI,CAAC,SAAS,EAAE,CAAC,OAAO,CACpB,uFAAuF;oBACnF,kEAAkE,CACzE,CAAC;YACN,CAAC;YACD,IAAI,CAAC,YAAY,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,eAAe,CAAC,YAA0B;QACtC,wCAAwC;QACxC,IAAI,IAAI,CAAC,YAAY,KAAK,YAAY,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,4DAA4D;QAC5D,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,MAAM,IAAI,oBAAoB,CAAC,cAAc,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACpF,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACrC,CAAC;IAED,gBAAgB;QACZ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,6EAA6E,CAAC,CAAC;YACtG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;gBACtB,IAAI,CAAC,SAAS,EAAE,CAAC,OAAO,CACpB,wFAAwF;oBACpF,kEAAkE,CACzE,CAAC;YACN,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvC,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC;gBACnC,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC;gBAC5C,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;aAC9D,CAAC,CAAC;QACP,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,gBAAgB,CAAC,aAA4B;QACzC,wCAAwC;QACxC,IAAI,IAAI,CAAC,aAAa,KAAK,aAAa,EAAE,CAAC;YACvC,OAAO;QACX,CAAC;QAED,6DAA6D;QAC7D,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,MAAM,IAAI,oBAAoB,CAAC,eAAe,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED,SAAS;QACL,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED,SAAS,CAAC,MAAqB;QAC3B,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YACzB,OAAO;QACX,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,MAAM,IAAI,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,WAAW,CAAC,MAAc;QACtB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,iBAAiB,CAAC,WAAkC;QAChD,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACjD,CAAC;IAED,iBAAiB,CAAC,WAAkC,EAAE,cAA8B;QAChF,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,oBAAoB,CAC1B,kBAAkB,WAAW,CAAC,IAAI,GAAG,EACrC,cAAc,EACd,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,CACxC,CAAC;QACN,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC1D,CAAC;IAED,wBAAwB;QACpB,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACrC,yDAAyD;YACzD,iGAAiG;YACjG,6GAA6G;YAC7G,IAAK,OAAe,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBAC3C,OAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,IAAS,EAAE,EAAE;oBAC1C,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;gBACxB,CAAC,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAED,KAAK;QACD,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC;QAC9B,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;QACxB,IAAI,CAAC,wBAAwB,EAAE,CAAC;IACpC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,oBAAoB,GAAG,IAAI,cAAc,EAAE,CAAC;AAElD,MAAM,qBAAqB,GAAG,IAAI,iBAAiB,EAA2B,CAAC;AAE/E;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,2BAA2B,CACvC,cAA8B,EAC9B,MAAU;IAEV,IAAI,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAE1C,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC;QAClD,MAAM,YAAY,GAAG,CAAC,GAAG,MAAM,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC;QAEpG,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACrC,MAAM,UAAU,GAAG,MAAM,CAAC,wBAAwB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAEvE,kFAAkF;YAClF,sEAAsE;YACtE,uEAAuE;YACvE,IACI,WAAW,KAAK,aAAa;gBAC7B,CAAC,UAAU;gBACX,UAAU,CAAC,GAAG;gBACd,UAAU,CAAC,GAAG;gBACd,OAAO,UAAU,CAAC,KAAK,KAAK,UAAU;gBAEtC,SAAS;YAEb,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC;YACjC,MAA2C,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,IAAW,EAAE,EAAE;gBAC3E,OAAO,qBAAqB,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE;oBAClD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACxC,CAAC,CAAC,CAAC;YACP,CAAC,CAAC;QACN,CAAC;QAED,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,aAAkD,CAAC;IAEvD,OAAO;QACH,GAAG,EAAE,CAAI,EAAW,EAAK,EAAE,CAAC,qBAAqB,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,CAAC;QACzE,UAAU,EAAE,GAAG,EAAE;YACb,aAAa,GAAG,qBAAqB,CAAC,QAAQ,EAAE,CAAC;YACjD,qBAAqB,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACpD,CAAC;QACD,SAAS,EAAE,GAAG,EAAE;YACZ,qBAAqB,CAAC,SAAS,CAAC,aAAoB,CAAC,CAAC,CAAC,uFAAuF;QAClJ,CAAC;KACJ,CAAC;AACN,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,KAAK,CAAC,EAA6B,EAAE;IACnE,GAAG,CAAC,OAAO,EAAE,IAAI;QACb,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACxE,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAChD,IAAI,OAAO,KAAK,KAAK,UAAU,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,GAAG,CAAC,OAAO,EAAE,IAAI;QACb,MAAM,IAAI,SAAS,CACf,wBAAwB,MAAM,CAAC,IAAI,CAAC,6GAA6G,CACpJ,CAAC;IACN,CAAC;CACJ,CAAC,CAAC"}
|