@crawlee/core 4.0.0-beta.6 → 4.0.0-beta.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/autoscaling/autoscaled_pool.d.ts +3 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -9
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -13
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +18 -29
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +85 -227
- package/configuration.d.ts.map +1 -1
- package/configuration.js +159 -223
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +4 -2
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +18 -12
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +71 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +123 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +19 -28
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +12 -20
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/crawler_utils.d.ts +2 -2
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -1
- package/crawlers/crawler_utils.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +15 -15
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +21 -18
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +32 -18
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +45 -24
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +25 -8
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +69 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +33 -3
- package/errors.d.ts.map +1 -1
- package/errors.js +48 -4
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +8 -5
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +7 -9
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +33 -39
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +3 -2
- package/index.d.ts.map +1 -1
- package/index.js +2 -1
- package/index.js.map +1 -1
- package/log.d.ts +82 -2
- package/log.d.ts.map +1 -1
- package/log.js +102 -0
- package/log.js.map +1 -1
- package/package.json +9 -10
- package/proxy_configuration.d.ts +14 -148
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +19 -167
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +121 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +142 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +74 -10
- package/request.d.ts.map +1 -1
- package/request.js +85 -23
- package/request.js.map +1 -1
- package/router.d.ts.map +1 -1
- package/router.js.map +1 -1
- package/serialization.js +1 -1
- package/serialization.js.map +1 -1
- package/service_locator.d.ts +157 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +234 -0
- package/service_locator.js.map +1 -0
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +0 -1
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +26 -72
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +36 -98
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +65 -71
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +102 -95
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +64 -19
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +92 -25
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +3 -1
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +3 -1
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +80 -10
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +110 -26
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +9 -9
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +13 -8
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +87 -22
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +126 -76
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +1 -3
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +2 -4
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +3 -3
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +4 -5
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +5 -5
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +10 -7
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_instance_manager.d.ts +91 -0
- package/storages/storage_instance_manager.d.ts.map +1 -0
- package/storages/storage_instance_manager.js +236 -0
- package/storages/storage_instance_manager.js.map +1 -0
- package/storages/utils.d.ts +25 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +6 -5
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/validators.d.ts +4 -0
- package/validators.d.ts.map +1 -1
- package/validators.js +4 -0
- package/validators.js.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import type { BaseHttpClient } from '@crawlee/types';
|
|
1
2
|
import { type ParseSitemapOptions } from '@crawlee/utils';
|
|
2
|
-
import { Configuration } from '../configuration.js';
|
|
3
3
|
import type { GlobInput, RegExpInput } from '../enqueue_links/shared.js';
|
|
4
4
|
import { Request } from '../request.js';
|
|
5
5
|
import type { IRequestList } from './request_list.js';
|
|
@@ -81,9 +81,9 @@ export interface SitemapRequestListOptions extends UrlConstraints {
|
|
|
81
81
|
*/
|
|
82
82
|
parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
|
|
83
83
|
/**
|
|
84
|
-
*
|
|
84
|
+
* Custom HTTP client to be used for sitemap loading.
|
|
85
85
|
*/
|
|
86
|
-
|
|
86
|
+
httpClient?: BaseHttpClient;
|
|
87
87
|
}
|
|
88
88
|
/**
|
|
89
89
|
* A list of URLs to crawl parsed from a sitemap.
|
|
@@ -132,7 +132,7 @@ export declare class SitemapRequestList implements IRequestList {
|
|
|
132
132
|
/**
|
|
133
133
|
* Proxy URL to be used for sitemap loading.
|
|
134
134
|
*/
|
|
135
|
-
private proxyUrl
|
|
135
|
+
private proxyUrl?;
|
|
136
136
|
/**
|
|
137
137
|
* Logger instance.
|
|
138
138
|
*/
|
|
@@ -224,7 +224,7 @@ export declare class SitemapRequestList implements IRequestList {
|
|
|
224
224
|
* @inheritDoc
|
|
225
225
|
*/
|
|
226
226
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
227
|
-
[Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/
|
|
227
|
+
[Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/types").Dictionary>, void, unknown>;
|
|
228
228
|
/**
|
|
229
229
|
* @inheritDoc
|
|
230
230
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap_request_list.d.ts","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAEA,OAAO,
|
|
1
|
+
{"version":3,"file":"sitemap_request_list.d.ts","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAgB,KAAK,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAKxE,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAoB,MAAM,4BAA4B,CAAC;AAI3F,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAMtD,UAAU,cAAc;IACpB;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC;IAE1C;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,yBAA0B,SAAQ,cAAc;IAC7D;;OAEG;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,CAAC,EAAE;QACjB;;;WAGG;QACH,MAAM,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IACF;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,mBAAmB,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,GAAG,UAAU,CAAC,CAAC;IACnF;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B;AAiBD;;;;GAIG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACnD;;;OAGG;IACH,UAAU,cAAqB;IAE/B,2DAA2D;IAC3D,OAAO,CAAC,SAAS,CAAqB;IAEtC;;;;OAIG;IACH,OAAO,CAAC,WAAW,CAA8B;IAEjD;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAa5B;IAEF;;;;;OAKG;IACH,OAAO,CAAC,cAAc,CAAY;IAElC;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY,CAAS;IAE7B,iDAAiD;IACjD,OAAO,CAAC,eAAe,CAAK;IAE5B,OAAO,CAAC,eAAe,CAAC,CAAS;IAEjC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAE9B,OAAO,CAAC,MAAM,CAAS;IAEvB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,CAAS;IAE1B;;OAEG;IACH,OAAO,CAAC,GAAG,CAAgB;IAE3B,OAAO,CAAC,wBAAwB,CAA0B;IAC1D,OAAO,CAAC,iBAAiB,CAA0B;IAEnD,8CAA8C;IAC9C,OAAO,CAAC,MAAM,CAAe;IAE7B,OAAO,CAAC,kBAAkB,CAAgE;IAE1F,gBAAgB;IAChB,OAAO;IAwDP;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAOvB;;;;OAIG;IACH,OAAO,CAAC,UAAU;IAWlB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAO7B;;;;OAIG;YACW,WAAW;IAkBzB;;;;;OAKG;YACW,WAAW;IAqBzB;;;;OAIG;IACH,oBAAoB,IAAI,OAAO;IAO/B;;;;OAIG;YACW,IAAI;IAuClB;;;;;;OAMG;WACU,IAAI,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA6BlF;;OAEG;IACH,MAAM,IAAI,MAAM;IAIhB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAIjC;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YA6CrB,YAAY;IA+B1B;;OAEG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAkBjD;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAS7B;;OAEG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAMrD;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAOzD,OAAO,CAAC,+BAA+B;CAQ1C"}
|
|
@@ -2,10 +2,9 @@ import { Transform } from 'node:stream';
|
|
|
2
2
|
import { parseSitemap } from '@crawlee/utils';
|
|
3
3
|
import { minimatch } from 'minimatch';
|
|
4
4
|
import ow from 'ow';
|
|
5
|
-
import defaultLog from '@apify/log';
|
|
6
|
-
import { Configuration } from '../configuration.js';
|
|
7
5
|
import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps } from '../enqueue_links/shared.js';
|
|
8
6
|
import { Request } from '../request.js';
|
|
7
|
+
import { serviceLocator } from '../service_locator.js';
|
|
9
8
|
import { KeyValueStore } from './key_value_store.js';
|
|
10
9
|
import { purgeDefaultStorages } from './utils.js';
|
|
11
10
|
/** @internal */
|
|
@@ -74,7 +73,7 @@ export class SitemapRequestList {
|
|
|
74
73
|
/**
|
|
75
74
|
* Logger instance.
|
|
76
75
|
*/
|
|
77
|
-
log
|
|
76
|
+
log;
|
|
78
77
|
urlExcludePatternObjects = [];
|
|
79
78
|
urlPatternObjects = [];
|
|
80
79
|
/** EventManager used to handle persistence */
|
|
@@ -96,7 +95,8 @@ export class SitemapRequestList {
|
|
|
96
95
|
config: ow.optional.object,
|
|
97
96
|
persistenceOptions: ow.optional.object,
|
|
98
97
|
}));
|
|
99
|
-
const { globs, exclude, regexps
|
|
98
|
+
const { globs, exclude, regexps } = options;
|
|
99
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestList' });
|
|
100
100
|
if (exclude?.length) {
|
|
101
101
|
for (const excl of exclude) {
|
|
102
102
|
if (typeof excl === 'string' || 'glob' in excl) {
|
|
@@ -118,7 +118,7 @@ export class SitemapRequestList {
|
|
|
118
118
|
this.proxyUrl = options.proxyUrl;
|
|
119
119
|
this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
|
|
120
120
|
this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
|
|
121
|
-
this.events =
|
|
121
|
+
this.events = serviceLocator.getEventManager();
|
|
122
122
|
this.persistState = this.persistState.bind(this);
|
|
123
123
|
}
|
|
124
124
|
/**
|
|
@@ -253,12 +253,15 @@ export class SitemapRequestList {
|
|
|
253
253
|
* Track the loading progress using the `isSitemapFullyLoaded` property.
|
|
254
254
|
*/
|
|
255
255
|
static async open(options) {
|
|
256
|
+
const { httpClient, ...restOptions } = options;
|
|
256
257
|
const requestList = new SitemapRequestList({
|
|
257
|
-
...
|
|
258
|
+
...restOptions,
|
|
258
259
|
persistStateKey: options.persistStateKey ?? STATE_PERSISTENCE_KEY,
|
|
259
260
|
});
|
|
260
261
|
await requestList.restoreState();
|
|
261
|
-
void requestList.load({
|
|
262
|
+
void requestList.load({
|
|
263
|
+
parseSitemapOptions: { logger: serviceLocator.getLogger(), ...options.parseSitemapOptions, httpClient },
|
|
264
|
+
});
|
|
262
265
|
if (requestList.persistenceOptions.enable) {
|
|
263
266
|
requestList.events.on("persistState" /* EventType.PERSIST_STATE */, requestList.persistState);
|
|
264
267
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap_request_list.js","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EAAE,YAAY,EAA4B,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,UAAU,MAAM,YAAY,CAAC;AAEpC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,6BAA6B,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAE9G,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,qBAAqB,GAAG,4BAA4B,CAAC;AAuG3D;;;;GAIG;AACH,MAAM,OAAO,kBAAkB;IAC3B;;;OAGG;IACH,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,2DAA2D;IACnD,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC;;;;OAIG;IACK,WAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEjD;;OAEG;IACK,sBAAsB,GAA2B;QACrD;;WAEG;QACH,oBAAoB,EAAE,IAAI;QAC1B;;WAEG;QACH,iBAAiB,EAAE,IAAI,GAAG,EAAU;QACpC;;WAEG;QACH,kBAAkB,EAAE,IAAI,GAAG,EAAU;KACxC,CAAC;IAEF;;;;;OAKG;IACK,cAAc,CAAY;IAElC;;;;;;;OAOG;IACK,YAAY,GAAG,KAAK,CAAC;IAE7B,iDAAiD;IACzC,eAAe,GAAG,CAAC,CAAC;IAEpB,eAAe,CAAU;IAEzB,KAAK,CAAiB;IAEtB,MAAM,GAAG,KAAK,CAAC;IAEvB;;OAEG;IACK,QAAQ,CAAqB;IAErC;;OAEG;IACK,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;IAEzD,wBAAwB,GAAuB,EAAE,CAAC;IAClD,iBAAiB,GAAuB,EAAE,CAAC;IAEnD,8CAA8C;IACtC,MAAM,CAAe;IAErB,kBAAkB,CAAgE;IAE1F,gBAAgB;IAChB,YAAoB,OAAkC;QAClD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACvC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE;YACzB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;YACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE,EAAE,GAAG,OAAO,CAAC;QAEtF,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBAC7C,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjF,CAAC;qBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACpD,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACrF,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/E,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAE1E,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC,CAAC;QAEzE,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAEvC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,aAAqB;QACzC,OAAO,IAAI,SAAS,CAAC;YACjB,UAAU,EAAE,IAAI;YAChB,aAAa;SAChB,CAAC,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAW;QAC1B,OAAO,CAAC,aAAa,EAAE,EAAE;YACrB,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;YAEvC,MAAM,YAAY,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC;YAC5D,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC;YAE9E,OAAO,OAAO,CAAC,YAAY,IAAI,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,GAAW;QACrC,OAAO,CACH,CAAC,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACzD,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAC7F,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,GAAkB;QACxC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC3D,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,+EAA+E;gBAC/E,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC3C,OAAO,CAAC,OAAO,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,oBAAoB;QAChB,OAAO,CACH,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,KAAK,IAAI;YACzD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,IAAI,KAAK,CAAC,CAC5D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,IAAI,CAAC,EACf,mBAAmB,GACsD;QACzE,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACxD,MAAM,UAAU,GACZ,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBAChD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC;YAE1E,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACrF,GAAG,mBAAmB;oBACtB,QAAQ,EAAE,CAAC;oBACX,kBAAkB,EAAE,IAAI;iBAC3B,CAAC,EAAE,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACzB,2BAA2B;wBAC3B,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC7D,SAAS;oBACb,CAAC;oBAED,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/D,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAClE,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YACtD,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,GAAG,IAAI,CAAC;QAC5D,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAkC;QAChD,MAAM,WAAW,GAAG,IAAI,kBAAkB,CAAC;YACvC,GAAG,OAAO;YACV,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,qBAAqB;SACpE,CAAC,CAAC;QACH,MAAM,WAAW,CAAC,YAAY,EAAE,CAAC;QACjC,KAAK,WAAW,CAAC,IAAI,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAE5E,IAAI,WAAW,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACxC,WAAW,CAAC,MAAM,CAAC,EAAE,+CAA0B,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5C,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,UAAU,CAAC,GAAG,EAAE;gBACZ,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM;QACF,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAClH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,OAAO,CACH,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,CAC7G,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,KAAK,CAAC,CAAC;IACjF,CAAC;IAED;;OAEG;IACH,YAAY;QACR,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1C,MAAM,QAAQ,GAAG,EAAE,CAAC;QAEpB,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACf,MAAM;YACV,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,0EAA0E;QAC1E,2FAA2F;QAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAAC,CAAC;QAElF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YACzB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YACpC,SAAS,CAAC,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAEhC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE;YAC5C,sBAAsB,EAAE;gBACpB,kBAAkB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;gBAC9E,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBACtE,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;aAC/E;YACD,QAAQ;YACR,SAAS,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,+EAA+E;YACnI,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACnD,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;SACY,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,YAAY;QACtB,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpD,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEvF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,sBAAsB,GAAG;YAC1B,kBAAkB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;YAC5E,oBAAoB,EAAE,KAAK,CAAC,sBAAsB,CAAC,oBAAoB;YACvE,iBAAiB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;SAC7E,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAEpD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QAClB,0CAA0C;QAC1C,IAAI,OAAO,GAA8B,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAC9E,IAAI,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACJ,0CAA0C;YAC1C,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEpB,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB;QACjC,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,sDAAsD;IAChG,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAEO,+BAA+B,CAAC,GAAW;QAC/C,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,4CAA4C,GAAG,GAAG,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,GAAG,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"sitemap_request_list.js","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,EAAE,YAAY,EAA4B,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,6BAA6B,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAG9G,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,qBAAqB,GAAG,4BAA4B,CAAC;AAuG3D;;;;GAIG;AACH,MAAM,OAAO,kBAAkB;IAC3B;;;OAGG;IACH,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,2DAA2D;IACnD,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC;;;;OAIG;IACK,WAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEjD;;OAEG;IACK,sBAAsB,GAA2B;QACrD;;WAEG;QACH,oBAAoB,EAAE,IAAI;QAC1B;;WAEG;QACH,iBAAiB,EAAE,IAAI,GAAG,EAAU;QACpC;;WAEG;QACH,kBAAkB,EAAE,IAAI,GAAG,EAAU;KACxC,CAAC;IAEF;;;;;OAKG;IACK,cAAc,CAAY;IAElC;;;;;;;OAOG;IACK,YAAY,GAAG,KAAK,CAAC;IAE7B,iDAAiD;IACzC,eAAe,GAAG,CAAC,CAAC;IAEpB,eAAe,CAAU;IAEzB,KAAK,CAAiB;IAEtB,MAAM,GAAG,KAAK,CAAC;IAEvB;;OAEG;IACK,QAAQ,CAAU;IAE1B;;OAEG;IACK,GAAG,CAAgB;IAEnB,wBAAwB,GAAuB,EAAE,CAAC;IAClD,iBAAiB,GAAuB,EAAE,CAAC;IAEnD,8CAA8C;IACtC,MAAM,CAAe;IAErB,kBAAkB,CAAgE;IAE1F,gBAAgB;IAChB,YAAoB,OAAkC;QAClD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACvC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE;YACzB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;YACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAE5C,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;QAE9E,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBAC7C,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjF,CAAC;qBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACpD,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACrF,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/E,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAE1E,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC,CAAC;QAEzE,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAE/C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,aAAqB;QACzC,OAAO,IAAI,SAAS,CAAC;YACjB,UAAU,EAAE,IAAI;YAChB,aAAa;SAChB,CAAC,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAW;QAC1B,OAAO,CAAC,aAAa,EAAE,EAAE;YACrB,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;YAEvC,MAAM,YAAY,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC;YAC5D,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC;YAE9E,OAAO,OAAO,CAAC,YAAY,IAAI,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,GAAW;QACrC,OAAO,CACH,CAAC,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACzD,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAC7F,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,GAAkB;QACxC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC3D,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,+EAA+E;gBAC/E,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC3C,OAAO,CAAC,OAAO,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,oBAAoB;QAChB,OAAO,CACH,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,KAAK,IAAI;YACzD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,IAAI,KAAK,CAAC,CAC5D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,IAAI,CAAC,EACf,mBAAmB,GAGtB;QACG,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACxD,MAAM,UAAU,GACZ,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBAChD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC;YAE1E,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACrF,GAAG,mBAAmB;oBACtB,QAAQ,EAAE,CAAC;oBACX,kBAAkB,EAAE,IAAI;iBAC3B,CAAC,EAAE,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACzB,2BAA2B;wBAC3B,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC7D,SAAS;oBACb,CAAC;oBAED,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/D,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAClE,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YACtD,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,GAAG,IAAI,CAAC;QAC5D,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAkC;QAChD,MAAM,EAAE,UAAU,EAAE,GAAG,WAAW,EAAE,GAAG,OAAO,CAAC;QAE/C,MAAM,WAAW,GAAG,IAAI,kBAAkB,CAAC;YACvC,GAAG,WAAW;YACd,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,qBAAqB;SACpE,CAAC,CAAC;QACH,MAAM,WAAW,CAAC,YAAY,EAAE,CAAC;QACjC,KAAK,WAAW,CAAC,IAAI,CAAC;YAClB,mBAAmB,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,UAAU,EAAE;SAC1G,CAAC,CAAC;QAEH,IAAI,WAAW,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACxC,WAAW,CAAC,MAAM,CAAC,EAAE,+CAA0B,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5C,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,UAAU,CAAC,GAAG,EAAE;gBACZ,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM;QACF,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAClH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,OAAO,CACH,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,CAC7G,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,KAAK,CAAC,CAAC;IACjF,CAAC;IAED;;OAEG;IACH,YAAY;QACR,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1C,MAAM,QAAQ,GAAG,EAAE,CAAC;QAEpB,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACf,MAAM;YACV,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,0EAA0E;QAC1E,2FAA2F;QAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAAC,CAAC;QAElF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YACzB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YACpC,SAAS,CAAC,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAEhC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE;YAC5C,sBAAsB,EAAE;gBACpB,kBAAkB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;gBAC9E,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBACtE,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;aAC/E;YACD,QAAQ;YACR,SAAS,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,+EAA+E;YACnI,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACnD,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;SACY,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,YAAY;QACtB,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpD,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEvF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,sBAAsB,GAAG;YAC1B,kBAAkB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;YAC5E,oBAAoB,EAAE,KAAK,CAAC,sBAAsB,CAAC,oBAAoB;YACvE,iBAAiB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;SAC7E,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAEpD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QAClB,0CAA0C;QAC1C,IAAI,OAAO,GAA8B,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAC9E,IAAI,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACJ,0CAA0C;YAC1C,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEpB,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB;QACjC,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,sDAAsD;IAChG,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAEO,+BAA+B,CAAC,GAAW;QAC/C,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,4CAA4C,GAAG,GAAG,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,GAAG,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import type { DatasetClient, KeyValueStoreClient, RequestQueueClient, StorageClient, StorageIdentifier } from '@crawlee/types';
|
|
2
|
+
import type { Constructor } from '../typedefs.js';
|
|
3
|
+
export type { StorageIdentifier } from '@crawlee/types';
|
|
4
|
+
/**
|
|
5
|
+
* Matches an `IStorage` – a storage "frontend" (Dataset, KeyValueStore, RequestQueue).
|
|
6
|
+
*/
|
|
7
|
+
export interface IStorage {
|
|
8
|
+
id: string;
|
|
9
|
+
name?: string;
|
|
10
|
+
}
|
|
11
|
+
type Hashable = string;
|
|
12
|
+
/**
|
|
13
|
+
* Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
|
|
14
|
+
*
|
|
15
|
+
* A single instance manages all storage types. Instances are cached by
|
|
16
|
+
* `(storageClass, id/name/alias, clientCacheKey)` so the same storage is never opened twice.
|
|
17
|
+
*
|
|
18
|
+
* The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
|
|
19
|
+
* and a pre-bound `clientOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
|
|
20
|
+
* assigns a reserved default alias.
|
|
21
|
+
*
|
|
22
|
+
* @ignore
|
|
23
|
+
*/
|
|
24
|
+
export declare class StorageInstanceManager {
|
|
25
|
+
private readonly cache;
|
|
26
|
+
private readonly openerLocks;
|
|
27
|
+
/**
|
|
28
|
+
* Open (or retrieve from cache) a storage instance.
|
|
29
|
+
*
|
|
30
|
+
* @param cls The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
|
|
31
|
+
* @param id Storage ID (mutually exclusive with `name` and `alias`).
|
|
32
|
+
* @param name Storage name (mutually exclusive with `id` and `alias`).
|
|
33
|
+
* @param alias Run-scoped alias (mutually exclusive with `id` and `name`).
|
|
34
|
+
* Automatically assigned when no identifier is provided.
|
|
35
|
+
* @param clientOpener A **lazy** factory that creates the sub-client.
|
|
36
|
+
* Only called on a cache miss.
|
|
37
|
+
* @param clientCacheKey Opaque key identifying the storage backend, so that the same logical
|
|
38
|
+
* storage opened through different clients is cached separately.
|
|
39
|
+
*/
|
|
40
|
+
openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, { id, name, alias, clientOpener, clientCacheKey, }: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
|
|
41
|
+
clientOpener: () => Promise<DatasetClient | KeyValueStoreClient | RequestQueueClient>;
|
|
42
|
+
clientCacheKey: Hashable;
|
|
43
|
+
}): Promise<TStorage>;
|
|
44
|
+
/**
|
|
45
|
+
* Remove a storage instance from the cache (called from `storage.drop()`).
|
|
46
|
+
*/
|
|
47
|
+
removeFromCache(instance: IStorage): void;
|
|
48
|
+
/**
|
|
49
|
+
* Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
|
|
50
|
+
* instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
|
|
51
|
+
* Called during service locator reset.
|
|
52
|
+
*/
|
|
53
|
+
clearCache(): void;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* A storage identifier where exactly one of `id`, `name`, or `alias` is specified.
|
|
57
|
+
* Produced by {@link resolveStorageIdentifier} from ambiguous user input.
|
|
58
|
+
*/
|
|
59
|
+
export type ExplicitStorageIdentifier = {
|
|
60
|
+
id: string;
|
|
61
|
+
name?: never;
|
|
62
|
+
alias?: never;
|
|
63
|
+
} | {
|
|
64
|
+
id?: never;
|
|
65
|
+
name: string;
|
|
66
|
+
alias?: never;
|
|
67
|
+
} | {
|
|
68
|
+
id?: never;
|
|
69
|
+
name?: never;
|
|
70
|
+
alias: string;
|
|
71
|
+
};
|
|
72
|
+
/**
|
|
73
|
+
* Represents the case where no identifier was provided — the caller wants the default storage.
|
|
74
|
+
*/
|
|
75
|
+
export interface DefaultStorageIdentifier {
|
|
76
|
+
id?: never;
|
|
77
|
+
name?: never;
|
|
78
|
+
alias?: never;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
|
|
82
|
+
* `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
|
|
83
|
+
* the `StorageInstanceManager` and `StorageClient.create*Client` expect.
|
|
84
|
+
*
|
|
85
|
+
* - `null` / `undefined` / `{}` → default storage alias
|
|
86
|
+
* - `string` → resolved via `storageExists` (ID-first, then name)
|
|
87
|
+
* - `{ id }` → `{ id }`
|
|
88
|
+
* - `{ name }` → `{ name }`
|
|
89
|
+
*/
|
|
90
|
+
export declare function resolveStorageIdentifier(identifier: string | StorageIdentifier | null | undefined, client: StorageClient, storageType: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<ExplicitStorageIdentifier>;
|
|
91
|
+
//# sourceMappingURL=storage_instance_manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage_instance_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_instance_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,aAAa,EACb,mBAAmB,EACnB,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EACpB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,KAAK,QAAQ,GAAG,MAAM,CAAC;AA0HvB;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAiC;IAE7D;;;;;;;;;;;;OAYG;IACG,WAAW,CAAC,QAAQ,SAAS,QAAQ,EACvC,GAAG,EAAE,WAAW,CAAC,QAAQ,CAAC,EAC1B,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,YAAY,EACZ,cAAc,GACjB,EAAE,CAAC,yBAAyB,GAAG,wBAAwB,CAAC,GAAG;QACxD,YAAY,EAAE,MAAM,OAAO,CAAC,aAAa,GAAG,mBAAmB,GAAG,kBAAkB,CAAC,CAAC;QACtF,cAAc,EAAE,QAAQ,CAAC;KAC5B,GACF,OAAO,CAAC,QAAQ,CAAC;IAmEpB;;OAEG;IACH,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAIzC;;;;OAIG;IACH,UAAU,IAAI,IAAI;CASrB;AAED;;;GAGG;AACH,MAAM,MAAM,yBAAyB,GAC/B;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC3C;IAAE,EAAE,CAAC,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC3C;IAAE,EAAE,CAAC,EAAE,KAAK,CAAC;IAAC,IAAI,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAElD;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACrC,EAAE,CAAC,EAAE,KAAK,CAAC;IACX,IAAI,CAAC,EAAE,KAAK,CAAC;IACb,KAAK,CAAC,EAAE,KAAK,CAAC;CACjB;AAED;;;;;;;;;GASG;AACH,wBAAsB,wBAAwB,CAC1C,UAAU,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,GAAG,SAAS,EACzD,MAAM,EAAE,aAAa,EACrB,WAAW,EAAE,SAAS,GAAG,eAAe,GAAG,cAAc,GAC1D,OAAO,CAAC,yBAAyB,CAAC,CAsBpC"}
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
import { AsyncQueue } from '@sapphire/async-queue';
|
|
2
|
+
/** Reserved alias for the default (unnamed) storage. */
|
|
3
|
+
const DEFAULT_STORAGE_ALIAS = '__default__';
|
|
4
|
+
/**
|
|
5
|
+
* Three-tier cache for storage instances, modelled after crawlee-python's `_StorageCache`.
|
|
6
|
+
*
|
|
7
|
+
* Each tier maps `[storageClass][key][clientCacheKey] → instance`:
|
|
8
|
+
* - `byId` — keyed by the backend-assigned storage id
|
|
9
|
+
* - `byName` — keyed by the persistent storage name
|
|
10
|
+
* - `byAlias` — keyed by a run-scoped alias (e.g. `'__default__'` for unnamed storages)
|
|
11
|
+
*/
|
|
12
|
+
class StorageCache {
|
|
13
|
+
byId = new Map();
|
|
14
|
+
byName = new Map();
|
|
15
|
+
byAlias = new Map();
|
|
16
|
+
get(cls, { id, name, alias, clientCacheKey, }) {
|
|
17
|
+
for (const [tier, key] of [
|
|
18
|
+
[this.byId, id],
|
|
19
|
+
[this.byName, name],
|
|
20
|
+
[this.byAlias, alias],
|
|
21
|
+
]) {
|
|
22
|
+
if (key === undefined)
|
|
23
|
+
continue;
|
|
24
|
+
const cached = tier.get(cls)?.get(key)?.get(clientCacheKey);
|
|
25
|
+
if (cached) {
|
|
26
|
+
if (cached instanceof cls) {
|
|
27
|
+
return cached;
|
|
28
|
+
}
|
|
29
|
+
throw new Error('Cached storage instance type mismatch.');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
/** Write a single entry into a given tier. */
|
|
35
|
+
setInMap(tier, cls, key, instance, clientCacheKey) {
|
|
36
|
+
if (!tier.has(cls))
|
|
37
|
+
tier.set(cls, new Map());
|
|
38
|
+
const keyMap = tier.get(cls);
|
|
39
|
+
if (!keyMap.has(key))
|
|
40
|
+
keyMap.set(key, new Map());
|
|
41
|
+
keyMap.get(key).set(clientCacheKey, instance);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Cache an instance under its actual id, name, and an optional alias.
|
|
45
|
+
*/
|
|
46
|
+
set(cls, instance, clientCacheKey, alias) {
|
|
47
|
+
// Always cache by id.
|
|
48
|
+
this.setInMap(this.byId, cls, instance.id, instance, clientCacheKey);
|
|
49
|
+
// Cache by name — only for named storages.
|
|
50
|
+
if (instance.name) {
|
|
51
|
+
this.setInMap(this.byName, cls, instance.name, instance, clientCacheKey);
|
|
52
|
+
}
|
|
53
|
+
// Cache by alias — only for unnamed storages opened via alias.
|
|
54
|
+
if (alias !== undefined) {
|
|
55
|
+
this.setInMap(this.byAlias, cls, alias, instance, clientCacheKey);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
removeFromCache(instance) {
|
|
59
|
+
const storageType = instance.constructor;
|
|
60
|
+
for (const tier of [this.byId, this.byName, this.byAlias]) {
|
|
61
|
+
const classMap = tier.get(storageType);
|
|
62
|
+
if (!classMap)
|
|
63
|
+
continue;
|
|
64
|
+
for (const keyMap of classMap.values()) {
|
|
65
|
+
for (const [cacheKey, cached] of keyMap) {
|
|
66
|
+
if (cached === instance) {
|
|
67
|
+
keyMap.delete(cacheKey);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/** Iterate all cached instances across all storage types. */
|
|
74
|
+
*allValues() {
|
|
75
|
+
const seen = new Set();
|
|
76
|
+
for (const classMap of this.byId.values()) {
|
|
77
|
+
for (const keyMap of classMap.values()) {
|
|
78
|
+
for (const instance of keyMap.values()) {
|
|
79
|
+
if (!seen.has(instance)) {
|
|
80
|
+
seen.add(instance);
|
|
81
|
+
yield instance;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
clear() {
|
|
88
|
+
this.byId.clear();
|
|
89
|
+
this.byName.clear();
|
|
90
|
+
this.byAlias.clear();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
|
|
95
|
+
*
|
|
96
|
+
* A single instance manages all storage types. Instances are cached by
|
|
97
|
+
* `(storageClass, id/name/alias, clientCacheKey)` so the same storage is never opened twice.
|
|
98
|
+
*
|
|
99
|
+
* The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
|
|
100
|
+
* and a pre-bound `clientOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
|
|
101
|
+
* assigns a reserved default alias.
|
|
102
|
+
*
|
|
103
|
+
* @ignore
|
|
104
|
+
*/
|
|
105
|
+
export class StorageInstanceManager {
|
|
106
|
+
cache = new StorageCache();
|
|
107
|
+
openerLocks = new Map();
|
|
108
|
+
/**
|
|
109
|
+
* Open (or retrieve from cache) a storage instance.
|
|
110
|
+
*
|
|
111
|
+
* @param cls The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
|
|
112
|
+
* @param id Storage ID (mutually exclusive with `name` and `alias`).
|
|
113
|
+
* @param name Storage name (mutually exclusive with `id` and `alias`).
|
|
114
|
+
* @param alias Run-scoped alias (mutually exclusive with `id` and `name`).
|
|
115
|
+
* Automatically assigned when no identifier is provided.
|
|
116
|
+
* @param clientOpener A **lazy** factory that creates the sub-client.
|
|
117
|
+
* Only called on a cache miss.
|
|
118
|
+
* @param clientCacheKey Opaque key identifying the storage backend, so that the same logical
|
|
119
|
+
* storage opened through different clients is cached separately.
|
|
120
|
+
*/
|
|
121
|
+
async openStorage(cls, { id, name, alias, clientOpener, clientCacheKey, }) {
|
|
122
|
+
// Auto-set alias='__default__' when no parameters are specified (mirrors crawlee-python).
|
|
123
|
+
if (!id && !name && !alias) {
|
|
124
|
+
alias = DEFAULT_STORAGE_ALIAS;
|
|
125
|
+
}
|
|
126
|
+
// Fast-path cache check (no lock).
|
|
127
|
+
if (alias !== undefined) {
|
|
128
|
+
const cached = this.cache.get(cls, { alias, clientCacheKey });
|
|
129
|
+
if (cached)
|
|
130
|
+
return cached;
|
|
131
|
+
}
|
|
132
|
+
else if (id) {
|
|
133
|
+
const cached = this.cache.get(cls, { id, clientCacheKey });
|
|
134
|
+
if (cached)
|
|
135
|
+
return cached;
|
|
136
|
+
}
|
|
137
|
+
else if (name) {
|
|
138
|
+
const cached = this.cache.get(cls, { name, clientCacheKey });
|
|
139
|
+
if (cached)
|
|
140
|
+
return cached;
|
|
141
|
+
}
|
|
142
|
+
const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
|
|
143
|
+
const lockKey = `${cls.name}:${identifierKey}:${clientCacheKey}`;
|
|
144
|
+
if (!this.openerLocks.has(lockKey)) {
|
|
145
|
+
this.openerLocks.set(lockKey, new AsyncQueue());
|
|
146
|
+
}
|
|
147
|
+
const queue = this.openerLocks.get(lockKey);
|
|
148
|
+
await queue.wait();
|
|
149
|
+
try {
|
|
150
|
+
// Double-check cache under lock (another caller may have filled it while we waited).
|
|
151
|
+
if (alias !== undefined) {
|
|
152
|
+
const cached = this.cache.get(cls, { alias, clientCacheKey });
|
|
153
|
+
if (cached)
|
|
154
|
+
return cached;
|
|
155
|
+
}
|
|
156
|
+
else if (id) {
|
|
157
|
+
const cached = this.cache.get(cls, { id, clientCacheKey });
|
|
158
|
+
if (cached)
|
|
159
|
+
return cached;
|
|
160
|
+
}
|
|
161
|
+
else if (name) {
|
|
162
|
+
const cached = this.cache.get(cls, { name, clientCacheKey });
|
|
163
|
+
if (cached)
|
|
164
|
+
return cached;
|
|
165
|
+
}
|
|
166
|
+
// Cache miss — create the sub-client and storage instance.
|
|
167
|
+
const subClient = await clientOpener();
|
|
168
|
+
const storageInfo = await subClient.getMetadata();
|
|
169
|
+
const instance = new cls({
|
|
170
|
+
id: storageInfo.id,
|
|
171
|
+
name: storageInfo.name,
|
|
172
|
+
client: subClient,
|
|
173
|
+
});
|
|
174
|
+
// Atomic cache writes (no awaits between these).
|
|
175
|
+
this.cache.set(cls, instance, clientCacheKey, alias);
|
|
176
|
+
return instance;
|
|
177
|
+
}
|
|
178
|
+
finally {
|
|
179
|
+
queue.shift();
|
|
180
|
+
// Clean up idle locks so the map doesn't grow unboundedly
|
|
181
|
+
// (mirrors crawlee-python's WeakValueDictionary behaviour).
|
|
182
|
+
if (queue.remaining === 0) {
|
|
183
|
+
this.openerLocks.delete(lockKey);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Remove a storage instance from the cache (called from `storage.drop()`).
|
|
189
|
+
*/
|
|
190
|
+
removeFromCache(instance) {
|
|
191
|
+
this.cache.removeFromCache(instance);
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
|
|
195
|
+
* instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
|
|
196
|
+
* Called during service locator reset.
|
|
197
|
+
*/
|
|
198
|
+
clearCache() {
|
|
199
|
+
for (const instance of this.cache.allValues()) {
|
|
200
|
+
if ('clearCache' in instance && typeof instance.clearCache === 'function') {
|
|
201
|
+
instance.clearCache();
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
this.cache.clear();
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
|
|
209
|
+
* `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
|
|
210
|
+
* the `StorageInstanceManager` and `StorageClient.create*Client` expect.
|
|
211
|
+
*
|
|
212
|
+
* - `null` / `undefined` / `{}` → default storage alias
|
|
213
|
+
* - `string` → resolved via `storageExists` (ID-first, then name)
|
|
214
|
+
* - `{ id }` → `{ id }`
|
|
215
|
+
* - `{ name }` → `{ name }`
|
|
216
|
+
*/
|
|
217
|
+
export async function resolveStorageIdentifier(identifier, client, storageType) {
|
|
218
|
+
if (identifier === null || identifier === undefined) {
|
|
219
|
+
return { alias: DEFAULT_STORAGE_ALIAS };
|
|
220
|
+
}
|
|
221
|
+
if (typeof identifier === 'string') {
|
|
222
|
+
if (client.storageExists && (await client.storageExists(identifier, storageType))) {
|
|
223
|
+
return { id: identifier };
|
|
224
|
+
}
|
|
225
|
+
return { name: identifier };
|
|
226
|
+
}
|
|
227
|
+
if (identifier.id) {
|
|
228
|
+
return { id: identifier.id };
|
|
229
|
+
}
|
|
230
|
+
if (identifier.name) {
|
|
231
|
+
return { name: identifier.name };
|
|
232
|
+
}
|
|
233
|
+
// Empty object — treated as default storage.
|
|
234
|
+
return { alias: DEFAULT_STORAGE_ALIAS };
|
|
235
|
+
}
|
|
236
|
+
//# sourceMappingURL=storage_instance_manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage_instance_manager.js","sourceRoot":"","sources":["../../src/storages/storage_instance_manager.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAgBnD,wDAAwD;AACxD,MAAM,qBAAqB,GAAG,aAAa,CAAC;AAI5C;;;;;;;GAOG;AACH,MAAM,YAAY;IACL,IAAI,GAAc,IAAI,GAAG,EAAE,CAAC;IAC5B,MAAM,GAAc,IAAI,GAAG,EAAE,CAAC;IAC9B,OAAO,GAAc,IAAI,GAAG,EAAE,CAAC;IAExC,GAAG,CACC,GAAmB,EACnB,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,cAAc,GAKc;QAEhC,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI;YACtB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACf,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;YACnB,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;SACa,EAAE,CAAC;YACrC,IAAI,GAAG,KAAK,SAAS;gBAAE,SAAS;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,cAAc,CAAC,CAAC;YAC5D,IAAI,MAAM,EAAE,CAAC;gBACT,IAAI,MAAM,YAAa,GAAuD,EAAE,CAAC;oBAC7E,OAAO,MAAW,CAAC;gBACvB,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC9D,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,8CAA8C;IACtC,QAAQ,CACZ,IAAe,EACf,GAAmB,EACnB,GAAW,EACX,QAAW,EACX,cAAwB;QAExB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,GAAG,CAAqB,GAAmB,EAAE,QAAW,EAAE,cAAwB,EAAE,KAAc;QAC9F,sBAAsB;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QAErE,2CAA2C;QAC3C,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QAC7E,CAAC;QAED,+DAA+D;QAC/D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QACtE,CAAC;IACL,CAAC;IAED,eAAe,CAAC,QAAkB;QAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAoC,CAAC;QAElE,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACvC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;oBACtC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;wBACtB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAC5B,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,6DAA6D;IAC7D,CAAC,SAAS;QACN,MAAM,IAAI,GAAG,IAAI,GAAG,EAAY,CAAC;QACjC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACtB,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBACnB,MAAM,QAAQ,CAAC;oBACnB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,KAAK;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACJ;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,sBAAsB;IACd,KAAK,GAAG,IAAI,YAAY,EAAE,CAAC;IAC3B,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE7D;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,WAAW,CACb,GAA0B,EAC1B,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,YAAY,EACZ,cAAc,GAIjB;QAED,0FAA0F;QAC1F,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACzB,KAAK,GAAG,qBAAqB,CAAC;QAClC,CAAC;QAED,mCAAmC;QACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;YAC9D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,EAAE,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;YAC3D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,IAAI,EAAE,CAAC;YACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;YAC7D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;QAED,MAAM,aAAa,GAAG,EAAE,IAAI,IAAI,IAAI,KAAK,IAAI,qBAAqB,CAAC;QACnE,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,aAAa,IAAI,cAAc,EAAE,CAAC;QAEjE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QAE7C,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,IAAI,CAAC;YACD,qFAAqF;YACrF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC9D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;iBAAM,IAAI,EAAE,EAAE,CAAC;gBACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC3D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;iBAAM,IAAI,IAAI,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC7D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;YAED,2DAA2D;YAC3D,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,MAChB,SACH,CAAC,WAAW,EAAE,CAAC;YAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;gBACrB,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,SAAS;aACpB,CAAa,CAAC;YAEf,iDAAiD;YACjD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC;YAErD,OAAO,QAAQ,CAAC;QACpB,CAAC;gBAAS,CAAC;YACP,KAAK,CAAC,KAAK,EAAE,CAAC;YAEd,0DAA0D;YAC1D,4DAA4D;YAC5D,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;gBACxB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,QAAkB;QAC9B,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,UAAU;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;YAC5C,IAAI,YAAY,IAAI,QAAQ,IAAI,OAAQ,QAAgB,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBAChF,QAAgB,CAAC,UAAU,EAAE,CAAC;YACnC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACJ;AAoBD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,UAAyD,EACzD,MAAqB,EACrB,WAAyD;IAEzD,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAClD,OAAO,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC5C,CAAC;IAED,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACjC,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;YAChF,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAChC,CAAC;IAED,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;IACrC,CAAC;IAED,6CAA6C;IAC7C,OAAO,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;AAC5C,CAAC"}
|
package/storages/utils.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type { Dictionary, StorageClient } from '@crawlee/types';
|
|
1
|
+
import type { BaseHttpClient, Dictionary, StorageClient } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
|
+
import type { ProxyConfiguration } from '../proxy_configuration.js';
|
|
3
4
|
/**
|
|
4
5
|
* Options for purging default storage.
|
|
5
6
|
*/
|
|
@@ -89,5 +90,28 @@ export declare const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10000;
|
|
|
89
90
|
* @internal
|
|
90
91
|
*/
|
|
91
92
|
export declare const MAX_QUERIES_FOR_CONSISTENCY = 6;
|
|
93
|
+
/**
|
|
94
|
+
* Options for the static `open()` method on storage classes ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
|
|
95
|
+
*/
|
|
96
|
+
export interface StorageOpenOptions {
|
|
97
|
+
/**
|
|
98
|
+
* SDK configuration instance, defaults to the static register.
|
|
99
|
+
*/
|
|
100
|
+
config?: Configuration;
|
|
101
|
+
/**
|
|
102
|
+
* Optional storage client that should be used to open storages.
|
|
103
|
+
*/
|
|
104
|
+
storageClient?: StorageClient;
|
|
105
|
+
/**
|
|
106
|
+
* Used to pass the proxy configuration for the `requestsFromUrl` objects.
|
|
107
|
+
* Takes advantage of the internal address rotation and authentication process.
|
|
108
|
+
* If undefined, the `requestsFromUrl` requests will be made without proxy.
|
|
109
|
+
*/
|
|
110
|
+
proxyConfiguration?: ProxyConfiguration;
|
|
111
|
+
/**
|
|
112
|
+
* HTTP client to be used to download the list of URLs in `RequestQueue`.
|
|
113
|
+
*/
|
|
114
|
+
httpClient?: BaseHttpClient;
|
|
115
|
+
}
|
|
92
116
|
export {};
|
|
93
117
|
//# sourceMappingURL=utils.d.ts.map
|
package/storages/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEhF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAIpE;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwB1G,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,GAAS,KAAK,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B"}
|
package/storages/utils.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import crypto from 'node:crypto';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
|
+
import { serviceLocator } from '../service_locator.js';
|
|
3
4
|
import { KeyValueStore } from './key_value_store.js';
|
|
4
5
|
export async function purgeDefaultStorages(configOrOptions, client) {
|
|
5
6
|
const options = configOrOptions instanceof Configuration
|
|
@@ -8,11 +9,11 @@ export async function purgeDefaultStorages(configOrOptions, client) {
|
|
|
8
9
|
config: configOrOptions,
|
|
9
10
|
}
|
|
10
11
|
: (configOrOptions ?? {});
|
|
11
|
-
const { config =
|
|
12
|
-
({ client =
|
|
12
|
+
const { config = serviceLocator.getConfiguration(), onlyPurgeOnce = false } = options;
|
|
13
|
+
({ client = serviceLocator.getStorageClient() } = options);
|
|
13
14
|
const casted = client;
|
|
14
15
|
// if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
|
|
15
|
-
if (!onlyPurgeOnce || (config.
|
|
16
|
+
if (!onlyPurgeOnce || (config.purgeOnStart && !casted.__purged)) {
|
|
16
17
|
casted.__purged = true;
|
|
17
18
|
await casted.purge?.();
|
|
18
19
|
}
|
|
@@ -27,8 +28,8 @@ export async function purgeDefaultStorages(configOrOptions, client) {
|
|
|
27
28
|
* @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
|
|
28
29
|
*/
|
|
29
30
|
export async function useState(name, defaultValue = {}, options) {
|
|
30
|
-
const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
|
|
31
|
-
config: options?.config ||
|
|
31
|
+
const kvStore = await KeyValueStore.open(options?.keyValueStoreName ? { name: options.keyValueStoreName } : null, {
|
|
32
|
+
config: options?.config || serviceLocator.getConfiguration(),
|
|
32
33
|
});
|
|
33
34
|
return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
|
|
34
35
|
}
|
package/storages/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACtF,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9G,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE;KAC/D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AAChB,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,MAAM,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}
|
package/typedefs.d.ts
CHANGED
|
@@ -6,5 +6,5 @@ export type Awaitable<T> = T | PromiseLike<T>;
|
|
|
6
6
|
export declare function entries<T extends {}>(obj: T): [keyof T, T[keyof T]][];
|
|
7
7
|
/** @ignore */
|
|
8
8
|
export declare function keys<T extends {}>(obj: T): (keyof T)[];
|
|
9
|
-
export
|
|
9
|
+
export type { AllowedHttpMethods } from '@crawlee/types';
|
|
10
10
|
//# sourceMappingURL=typedefs.d.ts.map
|
package/typedefs.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../src/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;AAEjE,cAAc;AACd,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;AAE9C,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACxD;AAED,cAAc;AACd,wBAAgB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,CAAC,EAAE,CACzC;AAED,
|
|
1
|
+
{"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../src/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;AAEjE,cAAc;AACd,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;AAE9C,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACxD;AAED,cAAc;AACd,wBAAgB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,CAAC,EAAE,CACzC;AAED,YAAY,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}
|
package/validators.d.ts
CHANGED
|
@@ -17,5 +17,9 @@ export declare const validators: {
|
|
|
17
17
|
validator: boolean;
|
|
18
18
|
message: (label: string) => string;
|
|
19
19
|
};
|
|
20
|
+
sessionPool: (value: Dictionary) => {
|
|
21
|
+
validator: boolean;
|
|
22
|
+
message: (label: string) => string;
|
|
23
|
+
};
|
|
20
24
|
};
|
|
21
25
|
//# sourceMappingURL=validators.d.ts.map
|
package/validators.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,gBAAgB;AAChB,eAAO,MAAM,UAAU;yBAEE,UAAU;;yBAEV,MAAM;;gCAEC,UAAU;;yBAEjB,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;0BAEL,UAAU;;yBAEX,MAAM;;
|
|
1
|
+
{"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,gBAAgB;AAChB,eAAO,MAAM,UAAU;yBAEE,UAAU;;yBAEV,MAAM;;gCAEC,UAAU;;yBAEjB,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;0BAEL,UAAU;;yBAEX,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;CAG9B,CAAC"}
|