@crawlee/core 4.0.0-beta.6 → 4.0.0-beta.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/autoscaling/autoscaled_pool.d.ts +3 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -9
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -13
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +18 -29
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +85 -227
- package/configuration.d.ts.map +1 -1
- package/configuration.js +159 -223
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +4 -2
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +18 -12
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +71 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +123 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +19 -28
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +12 -20
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/crawler_utils.d.ts +2 -2
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -1
- package/crawlers/crawler_utils.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +15 -15
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +21 -18
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +32 -18
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +45 -24
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +25 -8
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +69 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +33 -3
- package/errors.d.ts.map +1 -1
- package/errors.js +48 -4
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +8 -5
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +7 -9
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +33 -39
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +3 -2
- package/index.d.ts.map +1 -1
- package/index.js +2 -1
- package/index.js.map +1 -1
- package/log.d.ts +82 -2
- package/log.d.ts.map +1 -1
- package/log.js +102 -0
- package/log.js.map +1 -1
- package/package.json +9 -10
- package/proxy_configuration.d.ts +14 -148
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +19 -167
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +121 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +142 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +74 -10
- package/request.d.ts.map +1 -1
- package/request.js +85 -23
- package/request.js.map +1 -1
- package/router.d.ts.map +1 -1
- package/router.js.map +1 -1
- package/serialization.js +1 -1
- package/serialization.js.map +1 -1
- package/service_locator.d.ts +157 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +234 -0
- package/service_locator.js.map +1 -0
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +0 -1
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +26 -72
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +36 -98
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +65 -71
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +102 -95
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +64 -19
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +92 -25
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +3 -1
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +3 -1
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +80 -10
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +110 -26
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +9 -9
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +13 -8
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +87 -22
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +126 -76
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +1 -3
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +2 -4
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +3 -3
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +4 -5
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +5 -5
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +10 -7
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_instance_manager.d.ts +91 -0
- package/storages/storage_instance_manager.d.ts.map +1 -0
- package/storages/storage_instance_manager.js +236 -0
- package/storages/storage_instance_manager.js.map +1 -0
- package/storages/utils.d.ts +25 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +6 -5
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/validators.d.ts +4 -0
- package/validators.d.ts.map +1 -1
- package/validators.js +4 -0
- package/validators.js.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/storages/dataset.js
CHANGED
|
@@ -2,10 +2,10 @@ import { stringify } from 'csv-stringify/sync';
|
|
|
2
2
|
import ow from 'ow';
|
|
3
3
|
import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
|
|
4
4
|
import { Configuration } from '../configuration.js';
|
|
5
|
-
import {
|
|
5
|
+
import { serviceLocator } from '../service_locator.js';
|
|
6
6
|
import { checkStorageAccess } from './access_checking.js';
|
|
7
7
|
import { KeyValueStore } from './key_value_store.js';
|
|
8
|
-
import {
|
|
8
|
+
import { resolveStorageIdentifier } from './storage_instance_manager.js';
|
|
9
9
|
import { purgeDefaultStorages } from './utils.js';
|
|
10
10
|
/** @internal */
|
|
11
11
|
export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
@@ -132,7 +132,7 @@ export class Dataset {
|
|
|
132
132
|
id;
|
|
133
133
|
name;
|
|
134
134
|
client;
|
|
135
|
-
log
|
|
135
|
+
log;
|
|
136
136
|
/**
|
|
137
137
|
* @internal
|
|
138
138
|
*/
|
|
@@ -140,7 +140,8 @@ export class Dataset {
|
|
|
140
140
|
this.config = config;
|
|
141
141
|
this.id = options.id;
|
|
142
142
|
this.name = options.name;
|
|
143
|
-
this.client = options.client
|
|
143
|
+
this.client = options.client;
|
|
144
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'Dataset' });
|
|
144
145
|
}
|
|
145
146
|
/**
|
|
146
147
|
* Stores an object or an array of objects to the dataset.
|
|
@@ -233,7 +234,14 @@ export class Dataset {
|
|
|
233
234
|
const kvStore = await KeyValueStore.open(options?.toKVS ?? null, { config: this.config });
|
|
234
235
|
const items = await this.export(options);
|
|
235
236
|
if (contentType === 'text/csv') {
|
|
236
|
-
|
|
237
|
+
// To handle empty dataset exports gracefully.
|
|
238
|
+
if (items.length === 0) {
|
|
239
|
+
await kvStore.setValue(key, '', { contentType });
|
|
240
|
+
return items;
|
|
241
|
+
}
|
|
242
|
+
const keys = options?.collectAllKeys
|
|
243
|
+
? Array.from(new Set(items.flatMap(Object.keys)))
|
|
244
|
+
: Object.keys(items[0]);
|
|
237
245
|
const value = stringify([
|
|
238
246
|
keys,
|
|
239
247
|
...items.map((item) => {
|
|
@@ -248,7 +256,6 @@ export class Dataset {
|
|
|
248
256
|
return items;
|
|
249
257
|
}
|
|
250
258
|
throw new Error(`Unsupported content type: ${contentType}`);
|
|
251
|
-
return items;
|
|
252
259
|
}
|
|
253
260
|
/**
|
|
254
261
|
* Save entire default dataset's contents into one JSON file within a key-value store.
|
|
@@ -293,28 +300,23 @@ export class Dataset {
|
|
|
293
300
|
/**
|
|
294
301
|
* Returns an object containing general information about the dataset.
|
|
295
302
|
*
|
|
296
|
-
* The function returns the same object as the Apify API Client's
|
|
297
|
-
* [getDataset](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-datasets-getDataset)
|
|
298
|
-
* function, which in turn calls the
|
|
299
|
-
* [Get dataset](https://apify.com/docs/api/v2#/reference/datasets/dataset/get-dataset)
|
|
300
|
-
* API endpoint.
|
|
301
|
-
*
|
|
302
303
|
* **Example:**
|
|
303
304
|
* ```
|
|
304
305
|
* {
|
|
305
306
|
* id: "WkzbQMuFYuamGv3YF",
|
|
306
307
|
* name: "my-dataset",
|
|
307
|
-
* userId: "wRsJZtadYvn4mBZmm",
|
|
308
308
|
* createdAt: new Date("2015-12-12T07:34:14.202Z"),
|
|
309
309
|
* modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
|
|
310
310
|
* accessedAt: new Date("2015-12-14T08:36:13.202Z"),
|
|
311
311
|
* itemCount: 14,
|
|
312
312
|
* }
|
|
313
313
|
* ```
|
|
314
|
+
*
|
|
315
|
+
* @throws If the underlying storage no longer exists (e.g. it was deleted externally).
|
|
314
316
|
*/
|
|
315
317
|
async getInfo() {
|
|
316
318
|
checkStorageAccess();
|
|
317
|
-
return this.client.
|
|
319
|
+
return this.client.getMetadata();
|
|
318
320
|
}
|
|
319
321
|
/**
|
|
320
322
|
* Iterates over dataset items, yielding each in turn to an `iteratee` function.
|
|
@@ -388,6 +390,68 @@ export class Dataset {
|
|
|
388
390
|
await this.forEach(wrappedFunc, options);
|
|
389
391
|
return currentMemo;
|
|
390
392
|
}
|
|
393
|
+
/**
|
|
394
|
+
* Iterates over dataset items using an async generator,
|
|
395
|
+
* allowing the use of `for await...of` syntax.
|
|
396
|
+
*
|
|
397
|
+
* **Example usage:**
|
|
398
|
+
* ```javascript
|
|
399
|
+
* const dataset = await Dataset.open('my-results');
|
|
400
|
+
* for await (const item of dataset.values()) {
|
|
401
|
+
* console.log(item);
|
|
402
|
+
* }
|
|
403
|
+
* ```
|
|
404
|
+
*
|
|
405
|
+
* @param options Options for the iteration.
|
|
406
|
+
*/
|
|
407
|
+
values(options = {}) {
|
|
408
|
+
checkStorageAccess();
|
|
409
|
+
const result = this.client.listItems(options);
|
|
410
|
+
if (!(Symbol.asyncIterator in result)) {
|
|
411
|
+
Object.defineProperty(result, Symbol.asyncIterator, {
|
|
412
|
+
get() {
|
|
413
|
+
throw new Error('Resource client "listItems" method does not return an async iterable.');
|
|
414
|
+
},
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
return result;
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Iterates over dataset entries (index-value pairs) using an async generator,
|
|
421
|
+
* allowing the use of `for await...of` syntax.
|
|
422
|
+
*
|
|
423
|
+
* **Example usage:**
|
|
424
|
+
* ```javascript
|
|
425
|
+
* const dataset = await Dataset.open('my-results');
|
|
426
|
+
* for await (const [index, item] of dataset.entries()) {
|
|
427
|
+
* console.log(`Item at ${index}: ${JSON.stringify(item)}`);
|
|
428
|
+
* }
|
|
429
|
+
* ```
|
|
430
|
+
*
|
|
431
|
+
* @param options Options for the iteration.
|
|
432
|
+
*/
|
|
433
|
+
entries(options = {}) {
|
|
434
|
+
checkStorageAccess();
|
|
435
|
+
if (!this.client.listEntries) {
|
|
436
|
+
throw new Error('Resource client is missing the "listEntries" method.');
|
|
437
|
+
}
|
|
438
|
+
return this.client.listEntries(options);
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Default async iterator for the dataset, iterating over items.
|
|
442
|
+
* Allows using the dataset directly in a `for await...of` loop.
|
|
443
|
+
*
|
|
444
|
+
* **Example usage:**
|
|
445
|
+
* ```javascript
|
|
446
|
+
* const dataset = await Dataset.open('my-results');
|
|
447
|
+
* for await (const item of dataset) {
|
|
448
|
+
* console.log(item);
|
|
449
|
+
* }
|
|
450
|
+
* ```
|
|
451
|
+
*/
|
|
452
|
+
async *[Symbol.asyncIterator]() {
|
|
453
|
+
yield* this.values();
|
|
454
|
+
}
|
|
391
455
|
/**
|
|
392
456
|
* Removes the dataset either from the Apify cloud storage or from the local directory,
|
|
393
457
|
* depending on the mode of operation.
|
|
@@ -395,8 +459,7 @@ export class Dataset {
|
|
|
395
459
|
async drop() {
|
|
396
460
|
checkStorageAccess();
|
|
397
461
|
await this.client.delete();
|
|
398
|
-
|
|
399
|
-
manager.closeStorage(this);
|
|
462
|
+
serviceLocator.getStorageInstanceManager().removeFromCache(this);
|
|
400
463
|
}
|
|
401
464
|
/**
|
|
402
465
|
* Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
|
|
@@ -407,23 +470,27 @@ export class Dataset {
|
|
|
407
470
|
*
|
|
408
471
|
* For more details and code examples, see the {@link Dataset} class.
|
|
409
472
|
*
|
|
410
|
-
* @param [
|
|
411
|
-
* ID or name of the dataset to be opened. If
|
|
412
|
-
*
|
|
473
|
+
* @param [identifier]
|
|
474
|
+
* ID or name of the dataset to be opened. If a string is provided, it will first be
|
|
475
|
+
* looked up as an ID; if no such storage exists, it will be treated as a name.
|
|
476
|
+
* If `null` or `undefined`, the function returns the default dataset associated with the crawler run.
|
|
413
477
|
* @param [options] Storage manager options.
|
|
414
478
|
*/
|
|
415
|
-
static async open(
|
|
479
|
+
static async open(identifier, options = {}) {
|
|
416
480
|
checkStorageAccess();
|
|
417
|
-
ow(datasetIdOrName, ow.optional.string);
|
|
418
481
|
ow(options, ow.object.exactShape({
|
|
419
482
|
config: ow.optional.object.instanceOf(Configuration),
|
|
420
483
|
storageClient: ow.optional.object,
|
|
421
484
|
}));
|
|
422
485
|
options.config ??= Configuration.getGlobalConfig();
|
|
423
|
-
options.storageClient
|
|
424
|
-
await purgeDefaultStorages({ onlyPurgeOnce: true, client
|
|
425
|
-
const
|
|
426
|
-
return
|
|
486
|
+
const client = options.storageClient ?? serviceLocator.getStorageClient();
|
|
487
|
+
await purgeDefaultStorages({ onlyPurgeOnce: true, client, config: options.config });
|
|
488
|
+
const resolved = await resolveStorageIdentifier(identifier, client, 'Dataset');
|
|
489
|
+
return serviceLocator.getStorageInstanceManager().openStorage(this, {
|
|
490
|
+
...resolved,
|
|
491
|
+
clientOpener: () => client.createDatasetClient(resolved),
|
|
492
|
+
clientCacheKey: client.getStorageClientCacheKey?.() ?? client.constructor.name,
|
|
493
|
+
});
|
|
427
494
|
}
|
|
428
495
|
/**
|
|
429
496
|
* Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.
|
package/storages/dataset.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAY,GAAG,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;IACpG,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC;YAChD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;QACjE,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;QAC9D,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AAqFD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAM,OAAO,OAAO;IAWH;IAVb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,MAAM,CAAsB;IAC5B,GAAG,GAAQ,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAE5C;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAC;IACzE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO;QACX,CAAC;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAG,KAAK,EAAE,MAAM,GAAG,CAAC,EAAiB,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;YAEzE,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAE3B,IAAI,KAAK,CAAC,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrC,MAAM,cAAc,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,cAAc,EAAE,CAAC;QAEvB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,SAAS,CAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;QAE5D,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,+BAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,eAA+B,EAC/B,UAAiC,EAAE;QAEnC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,OAAO,CAAC,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAE5D,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAgB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAE/E,OAAO,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAGrD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;IACpG,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC;YAChD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;QACjE,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;QAC9D,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AA6FD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAM,OAAO,OAAO;IAWH;IAVb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,MAAM,CAAsB;IAC5B,GAAG,CAAgB;IAEnB;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO;QACX,CAAC;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAG,KAAK,EAAE,MAAM,GAAG,CAAC,EAAiB,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;YAEzE,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAE3B,IAAI,KAAK,CAAC,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrC,MAAM,cAAc,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,cAAc,EAAE,CAAC;QAEvB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,8CAA8C;YAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBACjD,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,EAAE,cAAc;gBAChC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAE5B,MAAM,KAAK,GAAG,SAAS,CAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;IAChE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,+BAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,UAAkC,EAAE;QACvC,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAuD,CAAC;QAEpG,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa,EAAE;gBAChD,GAAG;oBACC,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;gBAC7F,CAAC;aACJ,CAAC,CAAC;QACP,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,OAAO,CACH,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IACzB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,cAAc,CAAC,yBAAyB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,UAA8C,EAC9C,UAA8B,EAAE;QAEhC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QAEnD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;QAE1E,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAEpF,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAE/E,OAAO,cAAc,CAAC,yBAAyB,EAAE,CAAC,WAAW,CAAgB,IAAI,EAAE;YAC/E,GAAG,QAAQ;YACX,YAAY,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,QAAQ,CAAC;YACxD,cAAc,EAAE,MAAM,CAAC,wBAAwB,EAAE,EAAE,IAAI,MAAM,CAAC,WAAW,CAAC,IAAI;SACjF,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ"}
|
package/storages/index.d.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
export * from './dataset.js';
|
|
2
2
|
export * from './key_value_store.js';
|
|
3
3
|
export * from './request_list.js';
|
|
4
|
+
export * from './request_list_adapter.js';
|
|
4
5
|
export * from './request_provider.js';
|
|
5
6
|
export { RequestQueueV1 } from './request_queue.js';
|
|
6
7
|
export { RequestQueue } from './request_queue_v2.js';
|
|
7
8
|
export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
|
|
8
|
-
export * from './
|
|
9
|
+
export * from './storage_instance_manager.js';
|
|
9
10
|
export * from './utils.js';
|
|
10
11
|
export * from './access_checking.js';
|
|
11
12
|
export * from './sitemap_request_list.js';
|
|
13
|
+
export * from './request_manager_tandem.js';
|
|
12
14
|
//# sourceMappingURL=index.d.ts.map
|
package/storages/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,+BAA+B,CAAC;AAC9C,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,6BAA6B,CAAC"}
|
package/storages/index.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
export * from './dataset.js';
|
|
2
2
|
export * from './key_value_store.js';
|
|
3
3
|
export * from './request_list.js';
|
|
4
|
+
export * from './request_list_adapter.js';
|
|
4
5
|
export * from './request_provider.js';
|
|
5
6
|
export { RequestQueueV1 } from './request_queue.js';
|
|
6
7
|
export { RequestQueue } from './request_queue_v2.js';
|
|
7
8
|
export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
|
|
8
|
-
export * from './
|
|
9
|
+
export * from './storage_instance_manager.js';
|
|
9
10
|
export * from './utils.js';
|
|
10
11
|
export * from './access_checking.js';
|
|
11
12
|
export * from './sitemap_request_list.js';
|
|
13
|
+
export * from './request_manager_tandem.js';
|
|
12
14
|
//# sourceMappingURL=index.js.map
|
package/storages/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,+BAA+B,CAAC;AAC9C,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,6BAA6B,CAAC"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type { Dictionary,
|
|
1
|
+
import type { Dictionary, KeyValueStoreClient } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
3
|
import type { Awaitable } from '../typedefs.js';
|
|
4
|
-
import type {
|
|
4
|
+
import type { StorageIdentifier } from './storage_instance_manager.js';
|
|
5
|
+
import type { StorageOpenOptions } from './utils.js';
|
|
5
6
|
/**
|
|
6
7
|
* Helper function to possibly stringify value if options.contentType is not set.
|
|
7
8
|
*
|
|
@@ -70,7 +71,6 @@ export declare class KeyValueStore {
|
|
|
70
71
|
readonly config: Configuration;
|
|
71
72
|
readonly id: string;
|
|
72
73
|
readonly name?: string;
|
|
73
|
-
readonly storageObject?: Record<string, unknown>;
|
|
74
74
|
private readonly client;
|
|
75
75
|
private persistStateEventStarted;
|
|
76
76
|
/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
|
|
@@ -226,10 +226,72 @@ export declare class KeyValueStore {
|
|
|
226
226
|
*/
|
|
227
227
|
forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
|
|
228
228
|
private _forEachKey;
|
|
229
|
+
/**
|
|
230
|
+
* Iterates over key-value store keys using an async generator,
|
|
231
|
+
* allowing the use of `for await...of` syntax.
|
|
232
|
+
*
|
|
233
|
+
* **Example usage:**
|
|
234
|
+
* ```javascript
|
|
235
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
236
|
+
* for await (const key of keyValueStore.keys()) {
|
|
237
|
+
* console.log(key);
|
|
238
|
+
* }
|
|
239
|
+
* ```
|
|
240
|
+
*
|
|
241
|
+
* @param options Options for the iteration.
|
|
242
|
+
*/
|
|
243
|
+
keys(options?: KeyValueStoreIteratorOptions): AsyncGenerator<string, void, undefined>;
|
|
244
|
+
/**
|
|
245
|
+
* Iterates over key-value store values using an async generator,
|
|
246
|
+
* allowing the use of `for await...of` syntax.
|
|
247
|
+
*
|
|
248
|
+
* **Example usage:**
|
|
249
|
+
* ```javascript
|
|
250
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
251
|
+
* for await (const value of keyValueStore.values()) {
|
|
252
|
+
* console.log(value);
|
|
253
|
+
* }
|
|
254
|
+
* ```
|
|
255
|
+
*
|
|
256
|
+
* @param options Options for the iteration.
|
|
257
|
+
*/
|
|
258
|
+
values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
|
|
259
|
+
/**
|
|
260
|
+
* Iterates over key-value store entries (key-value pairs) using an async generator,
|
|
261
|
+
* allowing the use of `for await...of` syntax.
|
|
262
|
+
*
|
|
263
|
+
* **Example usage:**
|
|
264
|
+
* ```javascript
|
|
265
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
266
|
+
* for await (const [key, value] of keyValueStore.entries()) {
|
|
267
|
+
* console.log(`${key}: ${value}`);
|
|
268
|
+
* }
|
|
269
|
+
* ```
|
|
270
|
+
*
|
|
271
|
+
* @param options Options for the iteration.
|
|
272
|
+
*/
|
|
273
|
+
entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
|
|
274
|
+
/**
|
|
275
|
+
* Default async iterator for the key-value store, iterating over entries (key-value pairs).
|
|
276
|
+
* Allows using the store directly in a `for await...of` loop.
|
|
277
|
+
*
|
|
278
|
+
* **Example usage:**
|
|
279
|
+
* ```javascript
|
|
280
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
281
|
+
* for await (const [key, value] of keyValueStore) {
|
|
282
|
+
* console.log(`${key}: ${value}`);
|
|
283
|
+
* }
|
|
284
|
+
* ```
|
|
285
|
+
*/
|
|
286
|
+
[Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
|
|
229
287
|
/**
|
|
230
288
|
* Returns a file URL for the given key.
|
|
289
|
+
*
|
|
290
|
+
* If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
|
|
291
|
+
*
|
|
292
|
+
* @param key The key of the record to generate the public URL for.
|
|
231
293
|
*/
|
|
232
|
-
getPublicUrl(key: string): string
|
|
294
|
+
getPublicUrl(key: string): Promise<string | undefined>;
|
|
233
295
|
/**
|
|
234
296
|
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
235
297
|
*
|
|
@@ -239,12 +301,13 @@ export declare class KeyValueStore {
|
|
|
239
301
|
*
|
|
240
302
|
* For more details and code examples, see the {@link KeyValueStore} class.
|
|
241
303
|
*
|
|
242
|
-
* @param [
|
|
243
|
-
* ID or name of the key-value store to be opened. If
|
|
244
|
-
*
|
|
304
|
+
* @param [identifier]
|
|
305
|
+
* ID or name of the key-value store to be opened. If a string is provided, it will first be
|
|
306
|
+
* looked up as an ID; if no such storage exists, it will be treated as a name.
|
|
307
|
+
* If `null` or `undefined`, the function returns the default key-value store associated with the crawler run.
|
|
245
308
|
* @param [options] Storage manager options.
|
|
246
309
|
*/
|
|
247
|
-
static open(
|
|
310
|
+
static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
|
|
248
311
|
/**
|
|
249
312
|
* Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
250
313
|
*
|
|
@@ -380,8 +443,7 @@ export interface KeyConsumer {
|
|
|
380
443
|
export interface KeyValueStoreOptions {
|
|
381
444
|
id: string;
|
|
382
445
|
name?: string;
|
|
383
|
-
client:
|
|
384
|
-
storageObject?: Record<string, unknown>;
|
|
446
|
+
client: KeyValueStoreClient;
|
|
385
447
|
}
|
|
386
448
|
export interface RecordOptions {
|
|
387
449
|
/**
|
|
@@ -402,5 +464,13 @@ export interface KeyValueStoreIteratorOptions {
|
|
|
402
464
|
* All keys up to this one (including) are skipped from the result.
|
|
403
465
|
*/
|
|
404
466
|
exclusiveStartKey?: string;
|
|
467
|
+
/**
|
|
468
|
+
* If set, only keys that start with this prefix are returned.
|
|
469
|
+
*/
|
|
470
|
+
prefix?: string;
|
|
471
|
+
/**
|
|
472
|
+
* Collection name to use for listing keys.
|
|
473
|
+
*/
|
|
474
|
+
collection?: string;
|
|
405
475
|
}
|
|
406
476
|
//# sourceMappingURL=key_value_store.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,
|
|
1
|
+
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOtE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAIrD;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,CAAC,EAAE,OAAO,CAAC,EAAE,SAAS;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MA0B5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAclB,QAAQ,CAAC,MAAM;IAbnB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IAEvD;;OAEG;gBAEC,OAAO,EAAE,oBAAoB,EACpB,MAAM,gBAAkC;IAOrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAC3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IA0CrE;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuB3G,OAAO,CAAC,uBAAuB;IA6B/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsE3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAMlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAMpF,WAAW;IA0BzB;;;;;;;;;;;;;OAaG;IACI,IAAI,CAAC,OAAO,GAAE,4BAAiC,GAAG,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC;IAUhG;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAUhG;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,CAAC,GAAG,OAAO,EACf,OAAO,GAAE,4BAAiC,GAC3C,aAAa,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC;IAUtD;;;;;;;;;;;OAWG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,cAAc,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC;IAI1F;;;;;;OAMG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAI5D;;;;;;;;;;;;;;OAcG;WACU,IAAI,CACb,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,aAAa,CAAC;IAyBzB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAClE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAmC5E;;;;OAIG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAK3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;OAkBG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CA6B/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,mBAAmB,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB"}
|