@crawlee/core 4.0.0-beta.6 → 4.0.0-beta.60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/README.md +9 -5
  2. package/autoscaling/autoscaled_pool.d.ts +3 -5
  3. package/autoscaling/autoscaled_pool.d.ts.map +1 -1
  4. package/autoscaling/autoscaled_pool.js +3 -9
  5. package/autoscaling/autoscaled_pool.js.map +1 -1
  6. package/autoscaling/snapshotter.d.ts +3 -13
  7. package/autoscaling/snapshotter.d.ts.map +1 -1
  8. package/autoscaling/snapshotter.js +18 -29
  9. package/autoscaling/snapshotter.js.map +1 -1
  10. package/autoscaling/system_status.d.ts +0 -3
  11. package/autoscaling/system_status.d.ts.map +1 -1
  12. package/autoscaling/system_status.js +2 -3
  13. package/autoscaling/system_status.js.map +1 -1
  14. package/configuration.d.ts +85 -227
  15. package/configuration.d.ts.map +1 -1
  16. package/configuration.js +159 -223
  17. package/configuration.js.map +1 -1
  18. package/cookie_utils.d.ts +4 -2
  19. package/cookie_utils.d.ts.map +1 -1
  20. package/cookie_utils.js +18 -12
  21. package/cookie_utils.js.map +1 -1
  22. package/crawlers/context_pipeline.d.ts +71 -0
  23. package/crawlers/context_pipeline.d.ts.map +1 -0
  24. package/crawlers/context_pipeline.js +123 -0
  25. package/crawlers/context_pipeline.js.map +1 -0
  26. package/crawlers/crawler_commons.d.ts +19 -28
  27. package/crawlers/crawler_commons.d.ts.map +1 -1
  28. package/crawlers/crawler_commons.js +12 -20
  29. package/crawlers/crawler_commons.js.map +1 -1
  30. package/crawlers/crawler_utils.d.ts +2 -2
  31. package/crawlers/crawler_utils.d.ts.map +1 -1
  32. package/crawlers/crawler_utils.js +1 -1
  33. package/crawlers/crawler_utils.js.map +1 -1
  34. package/crawlers/error_snapshotter.d.ts +3 -2
  35. package/crawlers/error_snapshotter.d.ts.map +1 -1
  36. package/crawlers/error_snapshotter.js +2 -2
  37. package/crawlers/error_snapshotter.js.map +1 -1
  38. package/crawlers/error_tracker.d.ts +2 -1
  39. package/crawlers/error_tracker.d.ts.map +1 -1
  40. package/crawlers/error_tracker.js.map +1 -1
  41. package/crawlers/index.d.ts +1 -1
  42. package/crawlers/index.d.ts.map +1 -1
  43. package/crawlers/index.js +1 -1
  44. package/crawlers/index.js.map +1 -1
  45. package/crawlers/internals/types.d.ts +8 -0
  46. package/crawlers/internals/types.d.ts.map +1 -0
  47. package/crawlers/internals/types.js +2 -0
  48. package/crawlers/internals/types.js.map +1 -0
  49. package/crawlers/statistics.d.ts +15 -15
  50. package/crawlers/statistics.d.ts.map +1 -1
  51. package/crawlers/statistics.js +21 -18
  52. package/crawlers/statistics.js.map +1 -1
  53. package/enqueue_links/enqueue_links.d.ts +32 -18
  54. package/enqueue_links/enqueue_links.d.ts.map +1 -1
  55. package/enqueue_links/enqueue_links.js +45 -24
  56. package/enqueue_links/enqueue_links.js.map +1 -1
  57. package/enqueue_links/shared.d.ts +25 -8
  58. package/enqueue_links/shared.d.ts.map +1 -1
  59. package/enqueue_links/shared.js +69 -37
  60. package/enqueue_links/shared.js.map +1 -1
  61. package/errors.d.ts +33 -3
  62. package/errors.d.ts.map +1 -1
  63. package/errors.js +48 -4
  64. package/errors.js.map +1 -1
  65. package/events/event_manager.d.ts +8 -5
  66. package/events/event_manager.d.ts.map +1 -1
  67. package/events/event_manager.js +7 -9
  68. package/events/event_manager.js.map +1 -1
  69. package/events/local_event_manager.d.ts +14 -4
  70. package/events/local_event_manager.d.ts.map +1 -1
  71. package/events/local_event_manager.js +33 -39
  72. package/events/local_event_manager.js.map +1 -1
  73. package/index.d.ts +3 -2
  74. package/index.d.ts.map +1 -1
  75. package/index.js +2 -1
  76. package/index.js.map +1 -1
  77. package/log.d.ts +82 -2
  78. package/log.d.ts.map +1 -1
  79. package/log.js +102 -0
  80. package/log.js.map +1 -1
  81. package/package.json +9 -10
  82. package/proxy_configuration.d.ts +14 -148
  83. package/proxy_configuration.d.ts.map +1 -1
  84. package/proxy_configuration.js +19 -167
  85. package/proxy_configuration.js.map +1 -1
  86. package/recoverable_state.d.ts +121 -0
  87. package/recoverable_state.d.ts.map +1 -0
  88. package/recoverable_state.js +142 -0
  89. package/recoverable_state.js.map +1 -0
  90. package/request.d.ts +74 -10
  91. package/request.d.ts.map +1 -1
  92. package/request.js +85 -23
  93. package/request.js.map +1 -1
  94. package/router.d.ts.map +1 -1
  95. package/router.js.map +1 -1
  96. package/serialization.js +1 -1
  97. package/serialization.js.map +1 -1
  98. package/service_locator.d.ts +157 -0
  99. package/service_locator.d.ts.map +1 -0
  100. package/service_locator.js +234 -0
  101. package/service_locator.js.map +1 -0
  102. package/session_pool/index.d.ts +0 -1
  103. package/session_pool/index.d.ts.map +1 -1
  104. package/session_pool/index.js +0 -1
  105. package/session_pool/index.js.map +1 -1
  106. package/session_pool/session.d.ts +26 -72
  107. package/session_pool/session.d.ts.map +1 -1
  108. package/session_pool/session.js +36 -98
  109. package/session_pool/session.js.map +1 -1
  110. package/session_pool/session_pool.d.ts +65 -71
  111. package/session_pool/session_pool.d.ts.map +1 -1
  112. package/session_pool/session_pool.js +102 -95
  113. package/session_pool/session_pool.js.map +1 -1
  114. package/storages/dataset.d.ts +64 -19
  115. package/storages/dataset.d.ts.map +1 -1
  116. package/storages/dataset.js +92 -25
  117. package/storages/dataset.js.map +1 -1
  118. package/storages/index.d.ts +3 -1
  119. package/storages/index.d.ts.map +1 -1
  120. package/storages/index.js +3 -1
  121. package/storages/index.js.map +1 -1
  122. package/storages/key_value_store.d.ts +80 -10
  123. package/storages/key_value_store.d.ts.map +1 -1
  124. package/storages/key_value_store.js +110 -26
  125. package/storages/key_value_store.js.map +1 -1
  126. package/storages/request_list.d.ts +9 -9
  127. package/storages/request_list.d.ts.map +1 -1
  128. package/storages/request_list.js +13 -8
  129. package/storages/request_list.js.map +1 -1
  130. package/storages/request_list_adapter.d.ts +58 -0
  131. package/storages/request_list_adapter.d.ts.map +1 -0
  132. package/storages/request_list_adapter.js +81 -0
  133. package/storages/request_list_adapter.js.map +1 -0
  134. package/storages/request_manager_tandem.d.ts +68 -0
  135. package/storages/request_manager_tandem.d.ts.map +1 -0
  136. package/storages/request_manager_tandem.js +124 -0
  137. package/storages/request_manager_tandem.js.map +1 -0
  138. package/storages/request_provider.d.ts +87 -22
  139. package/storages/request_provider.d.ts.map +1 -1
  140. package/storages/request_provider.js +126 -76
  141. package/storages/request_provider.js.map +1 -1
  142. package/storages/request_queue.d.ts +1 -3
  143. package/storages/request_queue.d.ts.map +1 -1
  144. package/storages/request_queue.js +2 -4
  145. package/storages/request_queue.js.map +1 -1
  146. package/storages/request_queue_v2.d.ts +3 -3
  147. package/storages/request_queue_v2.d.ts.map +1 -1
  148. package/storages/request_queue_v2.js +4 -5
  149. package/storages/request_queue_v2.js.map +1 -1
  150. package/storages/sitemap_request_list.d.ts +5 -5
  151. package/storages/sitemap_request_list.d.ts.map +1 -1
  152. package/storages/sitemap_request_list.js +10 -7
  153. package/storages/sitemap_request_list.js.map +1 -1
  154. package/storages/storage_instance_manager.d.ts +91 -0
  155. package/storages/storage_instance_manager.d.ts.map +1 -0
  156. package/storages/storage_instance_manager.js +236 -0
  157. package/storages/storage_instance_manager.js.map +1 -0
  158. package/storages/utils.d.ts +25 -1
  159. package/storages/utils.d.ts.map +1 -1
  160. package/storages/utils.js +6 -5
  161. package/storages/utils.js.map +1 -1
  162. package/typedefs.d.ts +1 -1
  163. package/typedefs.d.ts.map +1 -1
  164. package/validators.d.ts +4 -0
  165. package/validators.d.ts.map +1 -1
  166. package/validators.js +4 -0
  167. package/validators.js.map +1 -1
  168. package/crawlers/crawler_extension.d.ts +0 -12
  169. package/crawlers/crawler_extension.d.ts.map +0 -1
  170. package/crawlers/crawler_extension.js +0 -14
  171. package/crawlers/crawler_extension.js.map +0 -1
  172. package/http_clients/base-http-client.d.ts +0 -134
  173. package/http_clients/base-http-client.d.ts.map +0 -1
  174. package/http_clients/base-http-client.js +0 -33
  175. package/http_clients/base-http-client.js.map +0 -1
  176. package/http_clients/form-data-like.d.ts +0 -67
  177. package/http_clients/form-data-like.d.ts.map +0 -1
  178. package/http_clients/form-data-like.js +0 -5
  179. package/http_clients/form-data-like.js.map +0 -1
  180. package/http_clients/got-scraping-http-client.d.ts +0 -15
  181. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  182. package/http_clients/got-scraping-http-client.js +0 -69
  183. package/http_clients/got-scraping-http-client.js.map +0 -1
  184. package/http_clients/index.d.ts +0 -3
  185. package/http_clients/index.d.ts.map +0 -1
  186. package/http_clients/index.js +0 -3
  187. package/http_clients/index.js.map +0 -1
  188. package/session_pool/events.d.ts +0 -3
  189. package/session_pool/events.d.ts.map +0 -1
  190. package/session_pool/events.js +0 -3
  191. package/session_pool/events.js.map +0 -1
  192. package/storages/storage_manager.d.ts +0 -58
  193. package/storages/storage_manager.d.ts.map +0 -1
  194. package/storages/storage_manager.js +0 -105
  195. package/storages/storage_manager.js.map +0 -1
  196. package/tsconfig.build.tsbuildinfo +0 -1
@@ -2,10 +2,10 @@ import { stringify } from 'csv-stringify/sync';
2
2
  import ow from 'ow';
3
3
  import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
4
4
  import { Configuration } from '../configuration.js';
5
- import { log } from '../log.js';
5
+ import { serviceLocator } from '../service_locator.js';
6
6
  import { checkStorageAccess } from './access_checking.js';
7
7
  import { KeyValueStore } from './key_value_store.js';
8
- import { StorageManager } from './storage_manager.js';
8
+ import { resolveStorageIdentifier } from './storage_instance_manager.js';
9
9
  import { purgeDefaultStorages } from './utils.js';
10
10
  /** @internal */
11
11
  export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
@@ -132,7 +132,7 @@ export class Dataset {
132
132
  id;
133
133
  name;
134
134
  client;
135
- log = log.child({ prefix: 'Dataset' });
135
+ log;
136
136
  /**
137
137
  * @internal
138
138
  */
@@ -140,7 +140,8 @@ export class Dataset {
140
140
  this.config = config;
141
141
  this.id = options.id;
142
142
  this.name = options.name;
143
- this.client = options.client.dataset(this.id);
143
+ this.client = options.client;
144
+ this.log = serviceLocator.getLogger().child({ prefix: 'Dataset' });
144
145
  }
145
146
  /**
146
147
  * Stores an object or an array of objects to the dataset.
@@ -233,7 +234,14 @@ export class Dataset {
233
234
  const kvStore = await KeyValueStore.open(options?.toKVS ?? null, { config: this.config });
234
235
  const items = await this.export(options);
235
236
  if (contentType === 'text/csv') {
236
- const keys = Object.keys(items[0]);
237
+ // To handle empty dataset exports gracefully.
238
+ if (items.length === 0) {
239
+ await kvStore.setValue(key, '', { contentType });
240
+ return items;
241
+ }
242
+ const keys = options?.collectAllKeys
243
+ ? Array.from(new Set(items.flatMap(Object.keys)))
244
+ : Object.keys(items[0]);
237
245
  const value = stringify([
238
246
  keys,
239
247
  ...items.map((item) => {
@@ -248,7 +256,6 @@ export class Dataset {
248
256
  return items;
249
257
  }
250
258
  throw new Error(`Unsupported content type: ${contentType}`);
251
- return items;
252
259
  }
253
260
  /**
254
261
  * Save entire default dataset's contents into one JSON file within a key-value store.
@@ -293,28 +300,23 @@ export class Dataset {
293
300
  /**
294
301
  * Returns an object containing general information about the dataset.
295
302
  *
296
- * The function returns the same object as the Apify API Client's
297
- * [getDataset](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-datasets-getDataset)
298
- * function, which in turn calls the
299
- * [Get dataset](https://apify.com/docs/api/v2#/reference/datasets/dataset/get-dataset)
300
- * API endpoint.
301
- *
302
303
  * **Example:**
303
304
  * ```
304
305
  * {
305
306
  * id: "WkzbQMuFYuamGv3YF",
306
307
  * name: "my-dataset",
307
- * userId: "wRsJZtadYvn4mBZmm",
308
308
  * createdAt: new Date("2015-12-12T07:34:14.202Z"),
309
309
  * modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
310
310
  * accessedAt: new Date("2015-12-14T08:36:13.202Z"),
311
311
  * itemCount: 14,
312
312
  * }
313
313
  * ```
314
+ *
315
+ * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
314
316
  */
315
317
  async getInfo() {
316
318
  checkStorageAccess();
317
- return this.client.get();
319
+ return this.client.getMetadata();
318
320
  }
319
321
  /**
320
322
  * Iterates over dataset items, yielding each in turn to an `iteratee` function.
@@ -388,6 +390,68 @@ export class Dataset {
388
390
  await this.forEach(wrappedFunc, options);
389
391
  return currentMemo;
390
392
  }
393
+ /**
394
+ * Iterates over dataset items using an async generator,
395
+ * allowing the use of `for await...of` syntax.
396
+ *
397
+ * **Example usage:**
398
+ * ```javascript
399
+ * const dataset = await Dataset.open('my-results');
400
+ * for await (const item of dataset.values()) {
401
+ * console.log(item);
402
+ * }
403
+ * ```
404
+ *
405
+ * @param options Options for the iteration.
406
+ */
407
+ values(options = {}) {
408
+ checkStorageAccess();
409
+ const result = this.client.listItems(options);
410
+ if (!(Symbol.asyncIterator in result)) {
411
+ Object.defineProperty(result, Symbol.asyncIterator, {
412
+ get() {
413
+ throw new Error('Resource client "listItems" method does not return an async iterable.');
414
+ },
415
+ });
416
+ }
417
+ return result;
418
+ }
419
+ /**
420
+ * Iterates over dataset entries (index-value pairs) using an async generator,
421
+ * allowing the use of `for await...of` syntax.
422
+ *
423
+ * **Example usage:**
424
+ * ```javascript
425
+ * const dataset = await Dataset.open('my-results');
426
+ * for await (const [index, item] of dataset.entries()) {
427
+ * console.log(`Item at ${index}: ${JSON.stringify(item)}`);
428
+ * }
429
+ * ```
430
+ *
431
+ * @param options Options for the iteration.
432
+ */
433
+ entries(options = {}) {
434
+ checkStorageAccess();
435
+ if (!this.client.listEntries) {
436
+ throw new Error('Resource client is missing the "listEntries" method.');
437
+ }
438
+ return this.client.listEntries(options);
439
+ }
440
+ /**
441
+ * Default async iterator for the dataset, iterating over items.
442
+ * Allows using the dataset directly in a `for await...of` loop.
443
+ *
444
+ * **Example usage:**
445
+ * ```javascript
446
+ * const dataset = await Dataset.open('my-results');
447
+ * for await (const item of dataset) {
448
+ * console.log(item);
449
+ * }
450
+ * ```
451
+ */
452
+ async *[Symbol.asyncIterator]() {
453
+ yield* this.values();
454
+ }
391
455
  /**
392
456
  * Removes the dataset either from the Apify cloud storage or from the local directory,
393
457
  * depending on the mode of operation.
@@ -395,8 +459,7 @@ export class Dataset {
395
459
  async drop() {
396
460
  checkStorageAccess();
397
461
  await this.client.delete();
398
- const manager = StorageManager.getManager(Dataset, this.config);
399
- manager.closeStorage(this);
462
+ serviceLocator.getStorageInstanceManager().removeFromCache(this);
400
463
  }
401
464
  /**
402
465
  * Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
@@ -407,23 +470,27 @@ export class Dataset {
407
470
  *
408
471
  * For more details and code examples, see the {@link Dataset} class.
409
472
  *
410
- * @param [datasetIdOrName]
411
- * ID or name of the dataset to be opened. If `null` or `undefined`,
412
- * the function returns the default dataset associated with the crawler run.
473
+ * @param [identifier]
474
+ * ID or name of the dataset to be opened. If a string is provided, it will first be
475
+ * looked up as an ID; if no such storage exists, it will be treated as a name.
476
+ * If `null` or `undefined`, the function returns the default dataset associated with the crawler run.
413
477
  * @param [options] Storage manager options.
414
478
  */
415
- static async open(datasetIdOrName, options = {}) {
479
+ static async open(identifier, options = {}) {
416
480
  checkStorageAccess();
417
- ow(datasetIdOrName, ow.optional.string);
418
481
  ow(options, ow.object.exactShape({
419
482
  config: ow.optional.object.instanceOf(Configuration),
420
483
  storageClient: ow.optional.object,
421
484
  }));
422
485
  options.config ??= Configuration.getGlobalConfig();
423
- options.storageClient ??= options.config.getStorageClient();
424
- await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
425
- const manager = StorageManager.getManager(this, options.config);
426
- return manager.openStorage(datasetIdOrName, options.storageClient);
486
+ const client = options.storageClient ?? serviceLocator.getStorageClient();
487
+ await purgeDefaultStorages({ onlyPurgeOnce: true, client, config: options.config });
488
+ const resolved = await resolveStorageIdentifier(identifier, client, 'Dataset');
489
+ return serviceLocator.getStorageInstanceManager().openStorage(this, {
490
+ ...resolved,
491
+ clientOpener: () => client.createDatasetClient(resolved),
492
+ clientCacheKey: client.getStorageClientCacheKey?.() ?? client.constructor.name,
493
+ });
427
494
  }
428
495
  /**
429
496
  * Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.
@@ -1 +1 @@
1
- {"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAY,GAAG,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;IACpG,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC;YAChD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;QACjE,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;QAC9D,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AAqFD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAM,OAAO,OAAO;IAWH;IAVb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,MAAM,CAAsB;IAC5B,GAAG,GAAQ,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAE5C;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAC;IACzE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO;QACX,CAAC;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAG,KAAK,EAAE,MAAM,GAAG,CAAC,EAAiB,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;YAEzE,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAE3B,IAAI,KAAK,CAAC,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrC,MAAM,cAAc,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,cAAc,EAAE,CAAC;QAEvB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,SAAS,CAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;QAE5D,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,+BAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,eAA+B,EAC/B,UAAiC,EAAE;QAEnC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,OAAO,CAAC,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAE5D,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAgB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAE/E,OAAO,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ"}
1
+ {"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAGrD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;IACpG,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC;YAChD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;QACjE,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;QAC9D,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AA6FD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAM,OAAO,OAAO;IAWH;IAVb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,MAAM,CAAsB;IAC5B,GAAG,CAAgB;IAEnB;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO;QACX,CAAC;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAG,KAAK,EAAE,MAAM,GAAG,CAAC,EAAiB,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;YAEzE,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAE3B,IAAI,KAAK,CAAC,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrC,MAAM,cAAc,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,cAAc,EAAE,CAAC;QAEvB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,8CAA8C;YAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBACjD,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,EAAE,cAAc;gBAChC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAE5B,MAAM,KAAK,GAAG,SAAS,CAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;IAChE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,+BAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,UAAkC,EAAE;QACvC,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAuD,CAAC;QAEpG,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,aAAa,EAAE;gBAChD,GAAG;oBACC,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;gBAC7F,CAAC;aACJ,CAAC,CAAC;QACP,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,OAAO,CACH,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IACzB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,cAAc,CAAC,yBAAyB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,UAA8C,EAC9C,UAA8B,EAAE;QAEhC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QAEnD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;QAE1E,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAEpF,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAE/E,OAAO,cAAc,CAAC,yBAAyB,EAAE,CAAC,WAAW,CAAgB,IAAI,EAAE;YAC/E,GAAG,QAAQ;YACX,YAAY,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,QAAQ,CAAC;YACxD,cAAc,EAAE,MAAM,CAAC,wBAAwB,EAAE,EAAE,IAAI,MAAM,CAAC,WAAW,CAAC,IAAI;SACjF,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ"}
@@ -1,12 +1,14 @@
1
1
  export * from './dataset.js';
2
2
  export * from './key_value_store.js';
3
3
  export * from './request_list.js';
4
+ export * from './request_list_adapter.js';
4
5
  export * from './request_provider.js';
5
6
  export { RequestQueueV1 } from './request_queue.js';
6
7
  export { RequestQueue } from './request_queue_v2.js';
7
8
  export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
8
- export * from './storage_manager.js';
9
+ export * from './storage_instance_manager.js';
9
10
  export * from './utils.js';
10
11
  export * from './access_checking.js';
11
12
  export * from './sitemap_request_list.js';
13
+ export * from './request_manager_tandem.js';
12
14
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,+BAA+B,CAAC;AAC9C,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,6BAA6B,CAAC"}
package/storages/index.js CHANGED
@@ -1,12 +1,14 @@
1
1
  export * from './dataset.js';
2
2
  export * from './key_value_store.js';
3
3
  export * from './request_list.js';
4
+ export * from './request_list_adapter.js';
4
5
  export * from './request_provider.js';
5
6
  export { RequestQueueV1 } from './request_queue.js';
6
7
  export { RequestQueue } from './request_queue_v2.js';
7
8
  export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
8
- export * from './storage_manager.js';
9
+ export * from './storage_instance_manager.js';
9
10
  export * from './utils.js';
10
11
  export * from './access_checking.js';
11
12
  export * from './sitemap_request_list.js';
13
+ export * from './request_manager_tandem.js';
12
14
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,+BAA+B,CAAC;AAC9C,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,6BAA6B,CAAC"}
@@ -1,7 +1,8 @@
1
- import type { Dictionary, StorageClient } from '@crawlee/types';
1
+ import type { Dictionary, KeyValueStoreClient } from '@crawlee/types';
2
2
  import { Configuration } from '../configuration.js';
3
3
  import type { Awaitable } from '../typedefs.js';
4
- import type { StorageManagerOptions } from './storage_manager.js';
4
+ import type { StorageIdentifier } from './storage_instance_manager.js';
5
+ import type { StorageOpenOptions } from './utils.js';
5
6
  /**
6
7
  * Helper function to possibly stringify value if options.contentType is not set.
7
8
  *
@@ -70,7 +71,6 @@ export declare class KeyValueStore {
70
71
  readonly config: Configuration;
71
72
  readonly id: string;
72
73
  readonly name?: string;
73
- readonly storageObject?: Record<string, unknown>;
74
74
  private readonly client;
75
75
  private persistStateEventStarted;
76
76
  /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
@@ -226,10 +226,72 @@ export declare class KeyValueStore {
226
226
  */
227
227
  forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
228
228
  private _forEachKey;
229
+ /**
230
+ * Iterates over key-value store keys using an async generator,
231
+ * allowing the use of `for await...of` syntax.
232
+ *
233
+ * **Example usage:**
234
+ * ```javascript
235
+ * const keyValueStore = await KeyValueStore.open();
236
+ * for await (const key of keyValueStore.keys()) {
237
+ * console.log(key);
238
+ * }
239
+ * ```
240
+ *
241
+ * @param options Options for the iteration.
242
+ */
243
+ keys(options?: KeyValueStoreIteratorOptions): AsyncGenerator<string, void, undefined>;
244
+ /**
245
+ * Iterates over key-value store values using an async generator,
246
+ * allowing the use of `for await...of` syntax.
247
+ *
248
+ * **Example usage:**
249
+ * ```javascript
250
+ * const keyValueStore = await KeyValueStore.open();
251
+ * for await (const value of keyValueStore.values()) {
252
+ * console.log(value);
253
+ * }
254
+ * ```
255
+ *
256
+ * @param options Options for the iteration.
257
+ */
258
+ values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
259
+ /**
260
+ * Iterates over key-value store entries (key-value pairs) using an async generator,
261
+ * allowing the use of `for await...of` syntax.
262
+ *
263
+ * **Example usage:**
264
+ * ```javascript
265
+ * const keyValueStore = await KeyValueStore.open();
266
+ * for await (const [key, value] of keyValueStore.entries()) {
267
+ * console.log(`${key}: ${value}`);
268
+ * }
269
+ * ```
270
+ *
271
+ * @param options Options for the iteration.
272
+ */
273
+ entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
274
+ /**
275
+ * Default async iterator for the key-value store, iterating over entries (key-value pairs).
276
+ * Allows using the store directly in a `for await...of` loop.
277
+ *
278
+ * **Example usage:**
279
+ * ```javascript
280
+ * const keyValueStore = await KeyValueStore.open();
281
+ * for await (const [key, value] of keyValueStore) {
282
+ * console.log(`${key}: ${value}`);
283
+ * }
284
+ * ```
285
+ */
286
+ [Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
229
287
  /**
230
288
  * Returns a file URL for the given key.
289
+ *
290
+ * If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
291
+ *
292
+ * @param key The key of the record to generate the public URL for.
231
293
  */
232
- getPublicUrl(key: string): string;
294
+ getPublicUrl(key: string): Promise<string | undefined>;
233
295
  /**
234
296
  * Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
235
297
  *
@@ -239,12 +301,13 @@ export declare class KeyValueStore {
239
301
  *
240
302
  * For more details and code examples, see the {@link KeyValueStore} class.
241
303
  *
242
- * @param [storeIdOrName]
243
- * ID or name of the key-value store to be opened. If `null` or `undefined`,
244
- * the function returns the default key-value store associated with the crawler run.
304
+ * @param [identifier]
305
+ * ID or name of the key-value store to be opened. If a string is provided, it will first be
306
+ * looked up as an ID; if no such storage exists, it will be treated as a name.
307
+ * If `null` or `undefined`, the function returns the default key-value store associated with the crawler run.
245
308
  * @param [options] Storage manager options.
246
309
  */
247
- static open(storeIdOrName?: string | null, options?: StorageManagerOptions): Promise<KeyValueStore>;
310
+ static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
248
311
  /**
249
312
  * Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
250
313
  *
@@ -380,8 +443,7 @@ export interface KeyConsumer {
380
443
  export interface KeyValueStoreOptions {
381
444
  id: string;
382
445
  name?: string;
383
- client: StorageClient;
384
- storageObject?: Record<string, unknown>;
446
+ client: KeyValueStoreClient;
385
447
  }
386
448
  export interface RecordOptions {
387
449
  /**
@@ -402,5 +464,13 @@ export interface KeyValueStoreIteratorOptions {
402
464
  * All keys up to this one (including) are skipped from the result.
403
465
  */
404
466
  exclusiveStartKey?: string;
467
+ /**
468
+ * If set, only keys that start with this prefix are returned.
469
+ */
470
+ prefix?: string;
471
+ /**
472
+ * Collection name to use for listing keys.
473
+ */
474
+ collection?: string;
405
475
  }
406
476
  //# sourceMappingURL=key_value_store.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAQrF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAIlE;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,CAAC,EAAE,OAAO,CAAC,EAAE,SAAS;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MA0B5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAelB,QAAQ,CAAC,MAAM;IAdnB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IAEvD;;OAEG;gBAEC,OAAO,EAAE,oBAAoB,EACpB,MAAM,gBAAkC;IAQrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAC3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IA0CrE;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuB3G,OAAO,CAAC,uBAAuB;IA2B/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsE3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAMlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAMpF,WAAW;IAwBzB;;OAEG;IACH,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAKjC;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;IAsB7G;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAClE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAmC5E;;;;OAIG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAK3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;OAkBG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CA6B/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B"}
1
+ {"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOtE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAIrD;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,CAAC,EAAE,OAAO,CAAC,EAAE,SAAS;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MA0B5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAclB,QAAQ,CAAC,MAAM;IAbnB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IAEvD;;OAEG;gBAEC,OAAO,EAAE,oBAAoB,EACpB,MAAM,gBAAkC;IAOrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAC3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IA0CrE;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuB3G,OAAO,CAAC,uBAAuB;IA6B/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsE3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAMlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAMpF,WAAW;IA0BzB;;;;;;;;;;;;;OAaG;IACI,IAAI,CAAC,OAAO,GAAE,4BAAiC,GAAG,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC;IAUhG;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAUhG;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,CAAC,GAAG,OAAO,EACf,OAAO,GAAE,4BAAiC,GAC3C,aAAa,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC;IAUtD;;;;;;;;;;;OAWG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,cAAc,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC;IAI1F;;;;;;OAMG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAI5D;;;;;;;;;;;;;;OAcG;WACU,IAAI,CACb,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,aAAa,CAAC;IAyBzB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAClE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAmC5E;;;;OAIG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAK3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;OAkBG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CA6B/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,mBAAmB,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB"}