@crawlee/core 4.0.0-beta.6 → 4.0.0-beta.60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/README.md +9 -5
  2. package/autoscaling/autoscaled_pool.d.ts +3 -5
  3. package/autoscaling/autoscaled_pool.d.ts.map +1 -1
  4. package/autoscaling/autoscaled_pool.js +3 -9
  5. package/autoscaling/autoscaled_pool.js.map +1 -1
  6. package/autoscaling/snapshotter.d.ts +3 -13
  7. package/autoscaling/snapshotter.d.ts.map +1 -1
  8. package/autoscaling/snapshotter.js +18 -29
  9. package/autoscaling/snapshotter.js.map +1 -1
  10. package/autoscaling/system_status.d.ts +0 -3
  11. package/autoscaling/system_status.d.ts.map +1 -1
  12. package/autoscaling/system_status.js +2 -3
  13. package/autoscaling/system_status.js.map +1 -1
  14. package/configuration.d.ts +85 -227
  15. package/configuration.d.ts.map +1 -1
  16. package/configuration.js +159 -223
  17. package/configuration.js.map +1 -1
  18. package/cookie_utils.d.ts +4 -2
  19. package/cookie_utils.d.ts.map +1 -1
  20. package/cookie_utils.js +18 -12
  21. package/cookie_utils.js.map +1 -1
  22. package/crawlers/context_pipeline.d.ts +71 -0
  23. package/crawlers/context_pipeline.d.ts.map +1 -0
  24. package/crawlers/context_pipeline.js +123 -0
  25. package/crawlers/context_pipeline.js.map +1 -0
  26. package/crawlers/crawler_commons.d.ts +19 -28
  27. package/crawlers/crawler_commons.d.ts.map +1 -1
  28. package/crawlers/crawler_commons.js +12 -20
  29. package/crawlers/crawler_commons.js.map +1 -1
  30. package/crawlers/crawler_utils.d.ts +2 -2
  31. package/crawlers/crawler_utils.d.ts.map +1 -1
  32. package/crawlers/crawler_utils.js +1 -1
  33. package/crawlers/crawler_utils.js.map +1 -1
  34. package/crawlers/error_snapshotter.d.ts +3 -2
  35. package/crawlers/error_snapshotter.d.ts.map +1 -1
  36. package/crawlers/error_snapshotter.js +2 -2
  37. package/crawlers/error_snapshotter.js.map +1 -1
  38. package/crawlers/error_tracker.d.ts +2 -1
  39. package/crawlers/error_tracker.d.ts.map +1 -1
  40. package/crawlers/error_tracker.js.map +1 -1
  41. package/crawlers/index.d.ts +1 -1
  42. package/crawlers/index.d.ts.map +1 -1
  43. package/crawlers/index.js +1 -1
  44. package/crawlers/index.js.map +1 -1
  45. package/crawlers/internals/types.d.ts +8 -0
  46. package/crawlers/internals/types.d.ts.map +1 -0
  47. package/crawlers/internals/types.js +2 -0
  48. package/crawlers/internals/types.js.map +1 -0
  49. package/crawlers/statistics.d.ts +15 -15
  50. package/crawlers/statistics.d.ts.map +1 -1
  51. package/crawlers/statistics.js +21 -18
  52. package/crawlers/statistics.js.map +1 -1
  53. package/enqueue_links/enqueue_links.d.ts +32 -18
  54. package/enqueue_links/enqueue_links.d.ts.map +1 -1
  55. package/enqueue_links/enqueue_links.js +45 -24
  56. package/enqueue_links/enqueue_links.js.map +1 -1
  57. package/enqueue_links/shared.d.ts +25 -8
  58. package/enqueue_links/shared.d.ts.map +1 -1
  59. package/enqueue_links/shared.js +69 -37
  60. package/enqueue_links/shared.js.map +1 -1
  61. package/errors.d.ts +33 -3
  62. package/errors.d.ts.map +1 -1
  63. package/errors.js +48 -4
  64. package/errors.js.map +1 -1
  65. package/events/event_manager.d.ts +8 -5
  66. package/events/event_manager.d.ts.map +1 -1
  67. package/events/event_manager.js +7 -9
  68. package/events/event_manager.js.map +1 -1
  69. package/events/local_event_manager.d.ts +14 -4
  70. package/events/local_event_manager.d.ts.map +1 -1
  71. package/events/local_event_manager.js +33 -39
  72. package/events/local_event_manager.js.map +1 -1
  73. package/index.d.ts +3 -2
  74. package/index.d.ts.map +1 -1
  75. package/index.js +2 -1
  76. package/index.js.map +1 -1
  77. package/log.d.ts +82 -2
  78. package/log.d.ts.map +1 -1
  79. package/log.js +102 -0
  80. package/log.js.map +1 -1
  81. package/package.json +9 -10
  82. package/proxy_configuration.d.ts +14 -148
  83. package/proxy_configuration.d.ts.map +1 -1
  84. package/proxy_configuration.js +19 -167
  85. package/proxy_configuration.js.map +1 -1
  86. package/recoverable_state.d.ts +121 -0
  87. package/recoverable_state.d.ts.map +1 -0
  88. package/recoverable_state.js +142 -0
  89. package/recoverable_state.js.map +1 -0
  90. package/request.d.ts +74 -10
  91. package/request.d.ts.map +1 -1
  92. package/request.js +85 -23
  93. package/request.js.map +1 -1
  94. package/router.d.ts.map +1 -1
  95. package/router.js.map +1 -1
  96. package/serialization.js +1 -1
  97. package/serialization.js.map +1 -1
  98. package/service_locator.d.ts +157 -0
  99. package/service_locator.d.ts.map +1 -0
  100. package/service_locator.js +234 -0
  101. package/service_locator.js.map +1 -0
  102. package/session_pool/index.d.ts +0 -1
  103. package/session_pool/index.d.ts.map +1 -1
  104. package/session_pool/index.js +0 -1
  105. package/session_pool/index.js.map +1 -1
  106. package/session_pool/session.d.ts +26 -72
  107. package/session_pool/session.d.ts.map +1 -1
  108. package/session_pool/session.js +36 -98
  109. package/session_pool/session.js.map +1 -1
  110. package/session_pool/session_pool.d.ts +65 -71
  111. package/session_pool/session_pool.d.ts.map +1 -1
  112. package/session_pool/session_pool.js +102 -95
  113. package/session_pool/session_pool.js.map +1 -1
  114. package/storages/dataset.d.ts +64 -19
  115. package/storages/dataset.d.ts.map +1 -1
  116. package/storages/dataset.js +92 -25
  117. package/storages/dataset.js.map +1 -1
  118. package/storages/index.d.ts +3 -1
  119. package/storages/index.d.ts.map +1 -1
  120. package/storages/index.js +3 -1
  121. package/storages/index.js.map +1 -1
  122. package/storages/key_value_store.d.ts +80 -10
  123. package/storages/key_value_store.d.ts.map +1 -1
  124. package/storages/key_value_store.js +110 -26
  125. package/storages/key_value_store.js.map +1 -1
  126. package/storages/request_list.d.ts +9 -9
  127. package/storages/request_list.d.ts.map +1 -1
  128. package/storages/request_list.js +13 -8
  129. package/storages/request_list.js.map +1 -1
  130. package/storages/request_list_adapter.d.ts +58 -0
  131. package/storages/request_list_adapter.d.ts.map +1 -0
  132. package/storages/request_list_adapter.js +81 -0
  133. package/storages/request_list_adapter.js.map +1 -0
  134. package/storages/request_manager_tandem.d.ts +68 -0
  135. package/storages/request_manager_tandem.d.ts.map +1 -0
  136. package/storages/request_manager_tandem.js +124 -0
  137. package/storages/request_manager_tandem.js.map +1 -0
  138. package/storages/request_provider.d.ts +87 -22
  139. package/storages/request_provider.d.ts.map +1 -1
  140. package/storages/request_provider.js +126 -76
  141. package/storages/request_provider.js.map +1 -1
  142. package/storages/request_queue.d.ts +1 -3
  143. package/storages/request_queue.d.ts.map +1 -1
  144. package/storages/request_queue.js +2 -4
  145. package/storages/request_queue.js.map +1 -1
  146. package/storages/request_queue_v2.d.ts +3 -3
  147. package/storages/request_queue_v2.d.ts.map +1 -1
  148. package/storages/request_queue_v2.js +4 -5
  149. package/storages/request_queue_v2.js.map +1 -1
  150. package/storages/sitemap_request_list.d.ts +5 -5
  151. package/storages/sitemap_request_list.d.ts.map +1 -1
  152. package/storages/sitemap_request_list.js +10 -7
  153. package/storages/sitemap_request_list.js.map +1 -1
  154. package/storages/storage_instance_manager.d.ts +91 -0
  155. package/storages/storage_instance_manager.d.ts.map +1 -0
  156. package/storages/storage_instance_manager.js +236 -0
  157. package/storages/storage_instance_manager.js.map +1 -0
  158. package/storages/utils.d.ts +25 -1
  159. package/storages/utils.d.ts.map +1 -1
  160. package/storages/utils.js +6 -5
  161. package/storages/utils.js.map +1 -1
  162. package/typedefs.d.ts +1 -1
  163. package/typedefs.d.ts.map +1 -1
  164. package/validators.d.ts +4 -0
  165. package/validators.d.ts.map +1 -1
  166. package/validators.js +4 -0
  167. package/validators.js.map +1 -1
  168. package/crawlers/crawler_extension.d.ts +0 -12
  169. package/crawlers/crawler_extension.d.ts.map +0 -1
  170. package/crawlers/crawler_extension.js +0 -14
  171. package/crawlers/crawler_extension.js.map +0 -1
  172. package/http_clients/base-http-client.d.ts +0 -134
  173. package/http_clients/base-http-client.d.ts.map +0 -1
  174. package/http_clients/base-http-client.js +0 -33
  175. package/http_clients/base-http-client.js.map +0 -1
  176. package/http_clients/form-data-like.d.ts +0 -67
  177. package/http_clients/form-data-like.d.ts.map +0 -1
  178. package/http_clients/form-data-like.js +0 -5
  179. package/http_clients/form-data-like.js.map +0 -1
  180. package/http_clients/got-scraping-http-client.d.ts +0 -15
  181. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  182. package/http_clients/got-scraping-http-client.js +0 -69
  183. package/http_clients/got-scraping-http-client.js.map +0 -1
  184. package/http_clients/index.d.ts +0 -3
  185. package/http_clients/index.d.ts.map +0 -1
  186. package/http_clients/index.js +0 -3
  187. package/http_clients/index.js.map +0 -1
  188. package/session_pool/events.d.ts +0 -3
  189. package/session_pool/events.d.ts.map +0 -1
  190. package/session_pool/events.js +0 -3
  191. package/session_pool/events.js.map +0 -1
  192. package/storages/storage_manager.d.ts +0 -58
  193. package/storages/storage_manager.d.ts.map +0 -1
  194. package/storages/storage_manager.js +0 -105
  195. package/storages/storage_manager.js.map +0 -1
  196. package/tsconfig.build.tsbuildinfo +0 -1
@@ -1,5 +1,5 @@
1
+ import type { BaseHttpClient } from '@crawlee/types';
1
2
  import { type ParseSitemapOptions } from '@crawlee/utils';
2
- import { Configuration } from '../configuration.js';
3
3
  import type { GlobInput, RegExpInput } from '../enqueue_links/shared.js';
4
4
  import { Request } from '../request.js';
5
5
  import type { IRequestList } from './request_list.js';
@@ -81,9 +81,9 @@ export interface SitemapRequestListOptions extends UrlConstraints {
81
81
  */
82
82
  parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
83
83
  /**
84
- * Crawlee configuration
84
+ * Custom HTTP client to be used for sitemap loading.
85
85
  */
86
- config?: Configuration;
86
+ httpClient?: BaseHttpClient;
87
87
  }
88
88
  /**
89
89
  * A list of URLs to crawl parsed from a sitemap.
@@ -132,7 +132,7 @@ export declare class SitemapRequestList implements IRequestList {
132
132
  /**
133
133
  * Proxy URL to be used for sitemap loading.
134
134
  */
135
- private proxyUrl;
135
+ private proxyUrl?;
136
136
  /**
137
137
  * Logger instance.
138
138
  */
@@ -224,7 +224,7 @@ export declare class SitemapRequestList implements IRequestList {
224
224
  * @inheritDoc
225
225
  */
226
226
  // @ts-ignore optional peer dependency or compatibility with es2022
227
- [Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/utils").Dictionary>, void, unknown>;
227
+ [Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/types").Dictionary>, void, unknown>;
228
228
  /**
229
229
  * @inheritDoc
230
230
  */
@@ -1 +1 @@
1
- {"version":3,"file":"sitemap_request_list.d.ts","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgB,KAAK,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOxE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAoB,MAAM,4BAA4B,CAAC;AAG3F,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAMtD,UAAU,cAAc;IACpB;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC;IAE1C;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,yBAA0B,SAAQ,cAAc;IAC7D;;OAEG;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,CAAC,EAAE;QACjB;;;WAGG;QACH,MAAM,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IACF;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,mBAAmB,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,GAAG,UAAU,CAAC,CAAC;IACnF;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAiBD;;;;GAIG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACnD;;;OAGG;IACH,UAAU,cAAqB;IAE/B,2DAA2D;IAC3D,OAAO,CAAC,SAAS,CAAqB;IAEtC;;;;OAIG;IACH,OAAO,CAAC,WAAW,CAA8B;IAEjD;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAa5B;IAEF;;;;;OAKG;IACH,OAAO,CAAC,cAAc,CAAY;IAElC;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY,CAAS;IAE7B,iDAAiD;IACjD,OAAO,CAAC,eAAe,CAAK;IAE5B,OAAO,CAAC,eAAe,CAAC,CAAS;IAEjC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAE9B,OAAO,CAAC,MAAM,CAAS;IAEvB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAqB;IAErC;;OAEG;IACH,OAAO,CAAC,GAAG,CAAsD;IAEjE,OAAO,CAAC,wBAAwB,CAA0B;IAC1D,OAAO,CAAC,iBAAiB,CAA0B;IAEnD,8CAA8C;IAC9C,OAAO,CAAC,MAAM,CAAe;IAE7B,OAAO,CAAC,kBAAkB,CAAgE;IAE1F,gBAAgB;IAChB,OAAO;IAsDP;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAOvB;;;;OAIG;IACH,OAAO,CAAC,UAAU;IAWlB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAO7B;;;;OAIG;YACW,WAAW;IAkBzB;;;;;OAKG;YACW,WAAW;IAqBzB;;;;OAIG;IACH,oBAAoB,IAAI,OAAO;IAO/B;;;;OAIG;YACW,IAAI;IAqClB;;;;;;OAMG;WACU,IAAI,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAyBlF;;OAEG;IACH,MAAM,IAAI,MAAM;IAIhB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAIjC;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YA6CrB,YAAY;IA+B1B;;OAEG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAkBjD;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAS7B;;OAEG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAMrD;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAOzD,OAAO,CAAC,+BAA+B;CAQ1C"}
1
+ {"version":3,"file":"sitemap_request_list.d.ts","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAgB,KAAK,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAKxE,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAoB,MAAM,4BAA4B,CAAC;AAI3F,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAMtD,UAAU,cAAc;IACpB;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC;IAE1C;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,yBAA0B,SAAQ,cAAc;IAC7D;;OAEG;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,CAAC,EAAE;QACjB;;;WAGG;QACH,MAAM,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IACF;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,mBAAmB,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,GAAG,UAAU,CAAC,CAAC;IACnF;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B;AAiBD;;;;GAIG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACnD;;;OAGG;IACH,UAAU,cAAqB;IAE/B,2DAA2D;IAC3D,OAAO,CAAC,SAAS,CAAqB;IAEtC;;;;OAIG;IACH,OAAO,CAAC,WAAW,CAA8B;IAEjD;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAa5B;IAEF;;;;;OAKG;IACH,OAAO,CAAC,cAAc,CAAY;IAElC;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY,CAAS;IAE7B,iDAAiD;IACjD,OAAO,CAAC,eAAe,CAAK;IAE5B,OAAO,CAAC,eAAe,CAAC,CAAS;IAEjC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAE9B,OAAO,CAAC,MAAM,CAAS;IAEvB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,CAAS;IAE1B;;OAEG;IACH,OAAO,CAAC,GAAG,CAAgB;IAE3B,OAAO,CAAC,wBAAwB,CAA0B;IAC1D,OAAO,CAAC,iBAAiB,CAA0B;IAEnD,8CAA8C;IAC9C,OAAO,CAAC,MAAM,CAAe;IAE7B,OAAO,CAAC,kBAAkB,CAAgE;IAE1F,gBAAgB;IAChB,OAAO;IAwDP;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAOvB;;;;OAIG;IACH,OAAO,CAAC,UAAU;IAWlB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAO7B;;;;OAIG;YACW,WAAW;IAkBzB;;;;;OAKG;YACW,WAAW;IAqBzB;;;;OAIG;IACH,oBAAoB,IAAI,OAAO;IAO/B;;;;OAIG;YACW,IAAI;IAuClB;;;;;;OAMG;WACU,IAAI,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA6BlF;;OAEG;IACH,MAAM,IAAI,MAAM;IAIhB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAIjC;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YA6CrB,YAAY;IA+B1B;;OAEG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAkBjD;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAS7B;;OAEG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAMrD;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAOzD,OAAO,CAAC,+BAA+B;CAQ1C"}
@@ -2,10 +2,9 @@ import { Transform } from 'node:stream';
2
2
  import { parseSitemap } from '@crawlee/utils';
3
3
  import { minimatch } from 'minimatch';
4
4
  import ow from 'ow';
5
- import defaultLog from '@apify/log';
6
- import { Configuration } from '../configuration.js';
7
5
  import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps } from '../enqueue_links/shared.js';
8
6
  import { Request } from '../request.js';
7
+ import { serviceLocator } from '../service_locator.js';
9
8
  import { KeyValueStore } from './key_value_store.js';
10
9
  import { purgeDefaultStorages } from './utils.js';
11
10
  /** @internal */
@@ -74,7 +73,7 @@ export class SitemapRequestList {
74
73
  /**
75
74
  * Logger instance.
76
75
  */
77
- log = defaultLog.child({ prefix: 'SitemapRequestList' });
76
+ log;
78
77
  urlExcludePatternObjects = [];
79
78
  urlPatternObjects = [];
80
79
  /** EventManager used to handle persistence */
@@ -96,7 +95,8 @@ export class SitemapRequestList {
96
95
  config: ow.optional.object,
97
96
  persistenceOptions: ow.optional.object,
98
97
  }));
99
- const { globs, exclude, regexps, config = Configuration.getGlobalConfig() } = options;
98
+ const { globs, exclude, regexps } = options;
99
+ this.log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestList' });
100
100
  if (exclude?.length) {
101
101
  for (const excl of exclude) {
102
102
  if (typeof excl === 'string' || 'glob' in excl) {
@@ -118,7 +118,7 @@ export class SitemapRequestList {
118
118
  this.proxyUrl = options.proxyUrl;
119
119
  this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
120
120
  this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
121
- this.events = config.getEventManager();
121
+ this.events = serviceLocator.getEventManager();
122
122
  this.persistState = this.persistState.bind(this);
123
123
  }
124
124
  /**
@@ -253,12 +253,15 @@ export class SitemapRequestList {
253
253
  * Track the loading progress using the `isSitemapFullyLoaded` property.
254
254
  */
255
255
  static async open(options) {
256
+ const { httpClient, ...restOptions } = options;
256
257
  const requestList = new SitemapRequestList({
257
- ...options,
258
+ ...restOptions,
258
259
  persistStateKey: options.persistStateKey ?? STATE_PERSISTENCE_KEY,
259
260
  });
260
261
  await requestList.restoreState();
261
- void requestList.load({ parseSitemapOptions: options.parseSitemapOptions });
262
+ void requestList.load({
263
+ parseSitemapOptions: { logger: serviceLocator.getLogger(), ...options.parseSitemapOptions, httpClient },
264
+ });
262
265
  if (requestList.persistenceOptions.enable) {
263
266
  requestList.events.on("persistState" /* EventType.PERSIST_STATE */, requestList.persistState);
264
267
  }
@@ -1 +1 @@
1
- {"version":3,"file":"sitemap_request_list.js","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EAAE,YAAY,EAA4B,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,UAAU,MAAM,YAAY,CAAC;AAEpC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,6BAA6B,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAE9G,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,qBAAqB,GAAG,4BAA4B,CAAC;AAuG3D;;;;GAIG;AACH,MAAM,OAAO,kBAAkB;IAC3B;;;OAGG;IACH,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,2DAA2D;IACnD,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC;;;;OAIG;IACK,WAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEjD;;OAEG;IACK,sBAAsB,GAA2B;QACrD;;WAEG;QACH,oBAAoB,EAAE,IAAI;QAC1B;;WAEG;QACH,iBAAiB,EAAE,IAAI,GAAG,EAAU;QACpC;;WAEG;QACH,kBAAkB,EAAE,IAAI,GAAG,EAAU;KACxC,CAAC;IAEF;;;;;OAKG;IACK,cAAc,CAAY;IAElC;;;;;;;OAOG;IACK,YAAY,GAAG,KAAK,CAAC;IAE7B,iDAAiD;IACzC,eAAe,GAAG,CAAC,CAAC;IAEpB,eAAe,CAAU;IAEzB,KAAK,CAAiB;IAEtB,MAAM,GAAG,KAAK,CAAC;IAEvB;;OAEG;IACK,QAAQ,CAAqB;IAErC;;OAEG;IACK,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;IAEzD,wBAAwB,GAAuB,EAAE,CAAC;IAClD,iBAAiB,GAAuB,EAAE,CAAC;IAEnD,8CAA8C;IACtC,MAAM,CAAe;IAErB,kBAAkB,CAAgE;IAE1F,gBAAgB;IAChB,YAAoB,OAAkC;QAClD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACvC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE;YACzB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;YACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE,EAAE,GAAG,OAAO,CAAC;QAEtF,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBAC7C,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjF,CAAC;qBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACpD,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACrF,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/E,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAE1E,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC,CAAC;QAEzE,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAEvC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,aAAqB;QACzC,OAAO,IAAI,SAAS,CAAC;YACjB,UAAU,EAAE,IAAI;YAChB,aAAa;SAChB,CAAC,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAW;QAC1B,OAAO,CAAC,aAAa,EAAE,EAAE;YACrB,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;YAEvC,MAAM,YAAY,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC;YAC5D,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC;YAE9E,OAAO,OAAO,CAAC,YAAY,IAAI,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,GAAW;QACrC,OAAO,CACH,CAAC,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACzD,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAC7F,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,GAAkB;QACxC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC3D,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,+EAA+E;gBAC/E,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC3C,OAAO,CAAC,OAAO,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,oBAAoB;QAChB,OAAO,CACH,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,KAAK,IAAI;YACzD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,IAAI,KAAK,CAAC,CAC5D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,IAAI,CAAC,EACf,mBAAmB,GACsD;QACzE,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACxD,MAAM,UAAU,GACZ,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBAChD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC;YAE1E,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACrF,GAAG,mBAAmB;oBACtB,QAAQ,EAAE,CAAC;oBACX,kBAAkB,EAAE,IAAI;iBAC3B,CAAC,EAAE,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACzB,2BAA2B;wBAC3B,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC7D,SAAS;oBACb,CAAC;oBAED,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/D,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAClE,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YACtD,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,GAAG,IAAI,CAAC;QAC5D,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAkC;QAChD,MAAM,WAAW,GAAG,IAAI,kBAAkB,CAAC;YACvC,GAAG,OAAO;YACV,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,qBAAqB;SACpE,CAAC,CAAC;QACH,MAAM,WAAW,CAAC,YAAY,EAAE,CAAC;QACjC,KAAK,WAAW,CAAC,IAAI,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAE5E,IAAI,WAAW,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACxC,WAAW,CAAC,MAAM,CAAC,EAAE,+CAA0B,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5C,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,UAAU,CAAC,GAAG,EAAE;gBACZ,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM;QACF,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAClH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,OAAO,CACH,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,CAC7G,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,KAAK,CAAC,CAAC;IACjF,CAAC;IAED;;OAEG;IACH,YAAY;QACR,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1C,MAAM,QAAQ,GAAG,EAAE,CAAC;QAEpB,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACf,MAAM;YACV,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,0EAA0E;QAC1E,2FAA2F;QAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAAC,CAAC;QAElF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YACzB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YACpC,SAAS,CAAC,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAEhC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE;YAC5C,sBAAsB,EAAE;gBACpB,kBAAkB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;gBAC9E,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBACtE,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;aAC/E;YACD,QAAQ;YACR,SAAS,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,+EAA+E;YACnI,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACnD,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;SACY,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,YAAY;QACtB,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpD,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEvF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,sBAAsB,GAAG;YAC1B,kBAAkB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;YAC5E,oBAAoB,EAAE,KAAK,CAAC,sBAAsB,CAAC,oBAAoB;YACvE,iBAAiB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;SAC7E,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAEpD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QAClB,0CAA0C;QAC1C,IAAI,OAAO,GAA8B,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAC9E,IAAI,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACJ,0CAA0C;YAC1C,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEpB,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB;QACjC,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,sDAAsD;IAChG,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAEO,+BAA+B,CAAC,GAAW;QAC/C,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,4CAA4C,GAAG,GAAG,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,GAAG,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;CACJ"}
1
+ {"version":3,"file":"sitemap_request_list.js","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,EAAE,YAAY,EAA4B,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,6BAA6B,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAG9G,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,qBAAqB,GAAG,4BAA4B,CAAC;AAuG3D;;;;GAIG;AACH,MAAM,OAAO,kBAAkB;IAC3B;;;OAGG;IACH,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,2DAA2D;IACnD,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC;;;;OAIG;IACK,WAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEjD;;OAEG;IACK,sBAAsB,GAA2B;QACrD;;WAEG;QACH,oBAAoB,EAAE,IAAI;QAC1B;;WAEG;QACH,iBAAiB,EAAE,IAAI,GAAG,EAAU;QACpC;;WAEG;QACH,kBAAkB,EAAE,IAAI,GAAG,EAAU;KACxC,CAAC;IAEF;;;;;OAKG;IACK,cAAc,CAAY;IAElC;;;;;;;OAOG;IACK,YAAY,GAAG,KAAK,CAAC;IAE7B,iDAAiD;IACzC,eAAe,GAAG,CAAC,CAAC;IAEpB,eAAe,CAAU;IAEzB,KAAK,CAAiB;IAEtB,MAAM,GAAG,KAAK,CAAC;IAEvB;;OAEG;IACK,QAAQ,CAAU;IAE1B;;OAEG;IACK,GAAG,CAAgB;IAEnB,wBAAwB,GAAuB,EAAE,CAAC;IAClD,iBAAiB,GAAuB,EAAE,CAAC;IAEnD,8CAA8C;IACtC,MAAM,CAAe;IAErB,kBAAkB,CAAgE;IAE1F,gBAAgB;IAChB,YAAoB,OAAkC;QAClD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACvC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE;YACzB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;YACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAE5C,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;QAE9E,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBAC7C,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjF,CAAC;qBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACpD,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACrF,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/E,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAE1E,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC,CAAC;QAEzE,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAE/C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,aAAqB;QACzC,OAAO,IAAI,SAAS,CAAC;YACjB,UAAU,EAAE,IAAI;YAChB,aAAa;SAChB,CAAC,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAW;QAC1B,OAAO,CAAC,aAAa,EAAE,EAAE;YACrB,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;YAEvC,MAAM,YAAY,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC;YAC5D,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC;YAE9E,OAAO,OAAO,CAAC,YAAY,IAAI,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,GAAW;QACrC,OAAO,CACH,CAAC,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACzD,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAC7F,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,GAAkB;QACxC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC3D,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,+EAA+E;gBAC/E,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC3C,OAAO,CAAC,OAAO,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,oBAAoB;QAChB,OAAO,CACH,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,KAAK,IAAI;YACzD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,IAAI,KAAK,CAAC,CAC5D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,IAAI,CAAC,EACf,mBAAmB,GAGtB;QACG,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACxD,MAAM,UAAU,GACZ,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBAChD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC;YAE1E,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACrF,GAAG,mBAAmB;oBACtB,QAAQ,EAAE,CAAC;oBACX,kBAAkB,EAAE,IAAI;iBAC3B,CAAC,EAAE,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACzB,2BAA2B;wBAC3B,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC7D,SAAS;oBACb,CAAC;oBAED,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/D,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAClE,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YACtD,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,GAAG,IAAI,CAAC;QAC5D,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAkC;QAChD,MAAM,EAAE,UAAU,EAAE,GAAG,WAAW,EAAE,GAAG,OAAO,CAAC;QAE/C,MAAM,WAAW,GAAG,IAAI,kBAAkB,CAAC;YACvC,GAAG,WAAW;YACd,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,qBAAqB;SACpE,CAAC,CAAC;QACH,MAAM,WAAW,CAAC,YAAY,EAAE,CAAC;QACjC,KAAK,WAAW,CAAC,IAAI,CAAC;YAClB,mBAAmB,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,UAAU,EAAE;SAC1G,CAAC,CAAC;QAEH,IAAI,WAAW,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACxC,WAAW,CAAC,MAAM,CAAC,EAAE,+CAA0B,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5C,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,UAAU,CAAC,GAAG,EAAE;gBACZ,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM;QACF,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAClH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,OAAO,CACH,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,CAC7G,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,KAAK,CAAC,CAAC;IACjF,CAAC;IAED;;OAEG;IACH,YAAY;QACR,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1C,MAAM,QAAQ,GAAG,EAAE,CAAC;QAEpB,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACf,MAAM;YACV,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,0EAA0E;QAC1E,2FAA2F;QAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAAC,CAAC;QAElF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YACzB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YACpC,SAAS,CAAC,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAEhC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE;YAC5C,sBAAsB,EAAE;gBACpB,kBAAkB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;gBAC9E,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBACtE,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;aAC/E;YACD,QAAQ;YACR,SAAS,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,+EAA+E;YACnI,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACnD,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;SACY,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,YAAY;QACtB,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpD,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEvF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,sBAAsB,GAAG;YAC1B,kBAAkB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;YAC5E,oBAAoB,EAAE,KAAK,CAAC,sBAAsB,CAAC,oBAAoB;YACvE,iBAAiB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;SAC7E,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAEpD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QAClB,0CAA0C;QAC1C,IAAI,OAAO,GAA8B,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAC9E,IAAI,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACJ,0CAA0C;YAC1C,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEpB,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB;QACjC,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,sDAAsD;IAChG,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAEO,+BAA+B,CAAC,GAAW;QAC/C,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,4CAA4C,GAAG,GAAG,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,GAAG,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;CACJ"}
@@ -0,0 +1,91 @@
1
+ import type { DatasetClient, KeyValueStoreClient, RequestQueueClient, StorageClient, StorageIdentifier } from '@crawlee/types';
2
+ import type { Constructor } from '../typedefs.js';
3
+ export type { StorageIdentifier } from '@crawlee/types';
4
+ /**
5
+ * Matches an `IStorage` – a storage "frontend" (Dataset, KeyValueStore, RequestQueue).
6
+ */
7
+ export interface IStorage {
8
+ id: string;
9
+ name?: string;
10
+ }
11
+ type Hashable = string;
12
+ /**
13
+ * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
14
+ *
15
+ * A single instance manages all storage types. Instances are cached by
16
+ * `(storageClass, id/name/alias, clientCacheKey)` so the same storage is never opened twice.
17
+ *
18
+ * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
19
+ * and a pre-bound `clientOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
20
+ * assigns a reserved default alias.
21
+ *
22
+ * @ignore
23
+ */
24
+ export declare class StorageInstanceManager {
25
+ private readonly cache;
26
+ private readonly openerLocks;
27
+ /**
28
+ * Open (or retrieve from cache) a storage instance.
29
+ *
30
+ * @param cls The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
31
+ * @param id Storage ID (mutually exclusive with `name` and `alias`).
32
+ * @param name Storage name (mutually exclusive with `id` and `alias`).
33
+ * @param alias Run-scoped alias (mutually exclusive with `id` and `name`).
34
+ * Automatically assigned when no identifier is provided.
35
+ * @param clientOpener A **lazy** factory that creates the sub-client.
36
+ * Only called on a cache miss.
37
+ * @param clientCacheKey Opaque key identifying the storage backend, so that the same logical
38
+ * storage opened through different clients is cached separately.
39
+ */
40
+ openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, { id, name, alias, clientOpener, clientCacheKey, }: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
41
+ clientOpener: () => Promise<DatasetClient | KeyValueStoreClient | RequestQueueClient>;
42
+ clientCacheKey: Hashable;
43
+ }): Promise<TStorage>;
44
+ /**
45
+ * Remove a storage instance from the cache (called from `storage.drop()`).
46
+ */
47
+ removeFromCache(instance: IStorage): void;
48
+ /**
49
+ * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
50
+ * instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
51
+ * Called during service locator reset.
52
+ */
53
+ clearCache(): void;
54
+ }
55
+ /**
56
+ * A storage identifier where exactly one of `id`, `name`, or `alias` is specified.
57
+ * Produced by {@link resolveStorageIdentifier} from ambiguous user input.
58
+ */
59
+ export type ExplicitStorageIdentifier = {
60
+ id: string;
61
+ name?: never;
62
+ alias?: never;
63
+ } | {
64
+ id?: never;
65
+ name: string;
66
+ alias?: never;
67
+ } | {
68
+ id?: never;
69
+ name?: never;
70
+ alias: string;
71
+ };
72
+ /**
73
+ * Represents the case where no identifier was provided — the caller wants the default storage.
74
+ */
75
+ export interface DefaultStorageIdentifier {
76
+ id?: never;
77
+ name?: never;
78
+ alias?: never;
79
+ }
80
+ /**
81
+ * Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
82
+ * `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
83
+ * the `StorageInstanceManager` and `StorageClient.create*Client` expect.
84
+ *
85
+ * - `null` / `undefined` / `{}` → default storage alias
86
+ * - `string` → resolved via `storageExists` (ID-first, then name)
87
+ * - `{ id }` → `{ id }`
88
+ * - `{ name }` → `{ name }`
89
+ */
90
+ export declare function resolveStorageIdentifier(identifier: string | StorageIdentifier | null | undefined, client: StorageClient, storageType: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<ExplicitStorageIdentifier>;
91
+ //# sourceMappingURL=storage_instance_manager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"storage_instance_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_instance_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,aAAa,EACb,mBAAmB,EACnB,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EACpB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,KAAK,QAAQ,GAAG,MAAM,CAAC;AA0HvB;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAiC;IAE7D;;;;;;;;;;;;OAYG;IACG,WAAW,CAAC,QAAQ,SAAS,QAAQ,EACvC,GAAG,EAAE,WAAW,CAAC,QAAQ,CAAC,EAC1B,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,YAAY,EACZ,cAAc,GACjB,EAAE,CAAC,yBAAyB,GAAG,wBAAwB,CAAC,GAAG;QACxD,YAAY,EAAE,MAAM,OAAO,CAAC,aAAa,GAAG,mBAAmB,GAAG,kBAAkB,CAAC,CAAC;QACtF,cAAc,EAAE,QAAQ,CAAC;KAC5B,GACF,OAAO,CAAC,QAAQ,CAAC;IAmEpB;;OAEG;IACH,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAIzC;;;;OAIG;IACH,UAAU,IAAI,IAAI;CASrB;AAED;;;GAGG;AACH,MAAM,MAAM,yBAAyB,GAC/B;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC3C;IAAE,EAAE,CAAC,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC3C;IAAE,EAAE,CAAC,EAAE,KAAK,CAAC;IAAC,IAAI,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAElD;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACrC,EAAE,CAAC,EAAE,KAAK,CAAC;IACX,IAAI,CAAC,EAAE,KAAK,CAAC;IACb,KAAK,CAAC,EAAE,KAAK,CAAC;CACjB;AAED;;;;;;;;;GASG;AACH,wBAAsB,wBAAwB,CAC1C,UAAU,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,GAAG,SAAS,EACzD,MAAM,EAAE,aAAa,EACrB,WAAW,EAAE,SAAS,GAAG,eAAe,GAAG,cAAc,GAC1D,OAAO,CAAC,yBAAyB,CAAC,CAsBpC"}
@@ -0,0 +1,236 @@
1
+ import { AsyncQueue } from '@sapphire/async-queue';
2
+ /** Reserved alias for the default (unnamed) storage. */
3
+ const DEFAULT_STORAGE_ALIAS = '__default__';
4
+ /**
5
+ * Three-tier cache for storage instances, modelled after crawlee-python's `_StorageCache`.
6
+ *
7
+ * Each tier maps `[storageClass][key][clientCacheKey] → instance`:
8
+ * - `byId` — keyed by the backend-assigned storage id
9
+ * - `byName` — keyed by the persistent storage name
10
+ * - `byAlias` — keyed by a run-scoped alias (e.g. `'__default__'` for unnamed storages)
11
+ */
12
+ class StorageCache {
13
+ byId = new Map();
14
+ byName = new Map();
15
+ byAlias = new Map();
16
+ get(cls, { id, name, alias, clientCacheKey, }) {
17
+ for (const [tier, key] of [
18
+ [this.byId, id],
19
+ [this.byName, name],
20
+ [this.byAlias, alias],
21
+ ]) {
22
+ if (key === undefined)
23
+ continue;
24
+ const cached = tier.get(cls)?.get(key)?.get(clientCacheKey);
25
+ if (cached) {
26
+ if (cached instanceof cls) {
27
+ return cached;
28
+ }
29
+ throw new Error('Cached storage instance type mismatch.');
30
+ }
31
+ }
32
+ return undefined;
33
+ }
34
+ /** Write a single entry into a given tier. */
35
+ setInMap(tier, cls, key, instance, clientCacheKey) {
36
+ if (!tier.has(cls))
37
+ tier.set(cls, new Map());
38
+ const keyMap = tier.get(cls);
39
+ if (!keyMap.has(key))
40
+ keyMap.set(key, new Map());
41
+ keyMap.get(key).set(clientCacheKey, instance);
42
+ }
43
+ /**
44
+ * Cache an instance under its actual id, name, and an optional alias.
45
+ */
46
+ set(cls, instance, clientCacheKey, alias) {
47
+ // Always cache by id.
48
+ this.setInMap(this.byId, cls, instance.id, instance, clientCacheKey);
49
+ // Cache by name — only for named storages.
50
+ if (instance.name) {
51
+ this.setInMap(this.byName, cls, instance.name, instance, clientCacheKey);
52
+ }
53
+ // Cache by alias — only for unnamed storages opened via alias.
54
+ if (alias !== undefined) {
55
+ this.setInMap(this.byAlias, cls, alias, instance, clientCacheKey);
56
+ }
57
+ }
58
+ removeFromCache(instance) {
59
+ const storageType = instance.constructor;
60
+ for (const tier of [this.byId, this.byName, this.byAlias]) {
61
+ const classMap = tier.get(storageType);
62
+ if (!classMap)
63
+ continue;
64
+ for (const keyMap of classMap.values()) {
65
+ for (const [cacheKey, cached] of keyMap) {
66
+ if (cached === instance) {
67
+ keyMap.delete(cacheKey);
68
+ }
69
+ }
70
+ }
71
+ }
72
+ }
73
+ /** Iterate all cached instances across all storage types. */
74
+ *allValues() {
75
+ const seen = new Set();
76
+ for (const classMap of this.byId.values()) {
77
+ for (const keyMap of classMap.values()) {
78
+ for (const instance of keyMap.values()) {
79
+ if (!seen.has(instance)) {
80
+ seen.add(instance);
81
+ yield instance;
82
+ }
83
+ }
84
+ }
85
+ }
86
+ }
87
+ clear() {
88
+ this.byId.clear();
89
+ this.byName.clear();
90
+ this.byAlias.clear();
91
+ }
92
+ }
93
+ /**
94
+ * Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
95
+ *
96
+ * A single instance manages all storage types. Instances are cached by
97
+ * `(storageClass, id/name/alias, clientCacheKey)` so the same storage is never opened twice.
98
+ *
99
+ * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
100
+ * and a pre-bound `clientOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
101
+ * assigns a reserved default alias.
102
+ *
103
+ * @ignore
104
+ */
105
+ export class StorageInstanceManager {
106
+ cache = new StorageCache();
107
+ openerLocks = new Map();
108
+ /**
109
+ * Open (or retrieve from cache) a storage instance.
110
+ *
111
+ * @param cls The storage class constructor (e.g. `Dataset`, `KeyValueStore`, `RequestQueue`).
112
+ * @param id Storage ID (mutually exclusive with `name` and `alias`).
113
+ * @param name Storage name (mutually exclusive with `id` and `alias`).
114
+ * @param alias Run-scoped alias (mutually exclusive with `id` and `name`).
115
+ * Automatically assigned when no identifier is provided.
116
+ * @param clientOpener A **lazy** factory that creates the sub-client.
117
+ * Only called on a cache miss.
118
+ * @param clientCacheKey Opaque key identifying the storage backend, so that the same logical
119
+ * storage opened through different clients is cached separately.
120
+ */
121
+ async openStorage(cls, { id, name, alias, clientOpener, clientCacheKey, }) {
122
+ // Auto-set alias='__default__' when no parameters are specified (mirrors crawlee-python).
123
+ if (!id && !name && !alias) {
124
+ alias = DEFAULT_STORAGE_ALIAS;
125
+ }
126
+ // Fast-path cache check (no lock).
127
+ if (alias !== undefined) {
128
+ const cached = this.cache.get(cls, { alias, clientCacheKey });
129
+ if (cached)
130
+ return cached;
131
+ }
132
+ else if (id) {
133
+ const cached = this.cache.get(cls, { id, clientCacheKey });
134
+ if (cached)
135
+ return cached;
136
+ }
137
+ else if (name) {
138
+ const cached = this.cache.get(cls, { name, clientCacheKey });
139
+ if (cached)
140
+ return cached;
141
+ }
142
+ const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
143
+ const lockKey = `${cls.name}:${identifierKey}:${clientCacheKey}`;
144
+ if (!this.openerLocks.has(lockKey)) {
145
+ this.openerLocks.set(lockKey, new AsyncQueue());
146
+ }
147
+ const queue = this.openerLocks.get(lockKey);
148
+ await queue.wait();
149
+ try {
150
+ // Double-check cache under lock (another caller may have filled it while we waited).
151
+ if (alias !== undefined) {
152
+ const cached = this.cache.get(cls, { alias, clientCacheKey });
153
+ if (cached)
154
+ return cached;
155
+ }
156
+ else if (id) {
157
+ const cached = this.cache.get(cls, { id, clientCacheKey });
158
+ if (cached)
159
+ return cached;
160
+ }
161
+ else if (name) {
162
+ const cached = this.cache.get(cls, { name, clientCacheKey });
163
+ if (cached)
164
+ return cached;
165
+ }
166
+ // Cache miss — create the sub-client and storage instance.
167
+ const subClient = await clientOpener();
168
+ const storageInfo = await subClient.getMetadata();
169
+ const instance = new cls({
170
+ id: storageInfo.id,
171
+ name: storageInfo.name,
172
+ client: subClient,
173
+ });
174
+ // Atomic cache writes (no awaits between these).
175
+ this.cache.set(cls, instance, clientCacheKey, alias);
176
+ return instance;
177
+ }
178
+ finally {
179
+ queue.shift();
180
+ // Clean up idle locks so the map doesn't grow unboundedly
181
+ // (mirrors crawlee-python's WeakValueDictionary behaviour).
182
+ if (queue.remaining === 0) {
183
+ this.openerLocks.delete(lockKey);
184
+ }
185
+ }
186
+ }
187
+ /**
188
+ * Remove a storage instance from the cache (called from `storage.drop()`).
189
+ */
190
+ removeFromCache(instance) {
191
+ this.cache.removeFromCache(instance);
192
+ }
193
+ /**
194
+ * Clear the entire cache. Also calls `clearCache()` on any cached KeyValueStore
195
+ * instances (duck-typed to avoid importing KeyValueStore and circular dependencies).
196
+ * Called during service locator reset.
197
+ */
198
+ clearCache() {
199
+ for (const instance of this.cache.allValues()) {
200
+ if ('clearCache' in instance && typeof instance.clearCache === 'function') {
201
+ instance.clearCache();
202
+ }
203
+ }
204
+ this.cache.clear();
205
+ }
206
+ }
207
+ /**
208
+ * Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
209
+ * `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
210
+ * the `StorageInstanceManager` and `StorageClient.create*Client` expect.
211
+ *
212
+ * - `null` / `undefined` / `{}` → default storage alias
213
+ * - `string` → resolved via `storageExists` (ID-first, then name)
214
+ * - `{ id }` → `{ id }`
215
+ * - `{ name }` → `{ name }`
216
+ */
217
+ export async function resolveStorageIdentifier(identifier, client, storageType) {
218
+ if (identifier === null || identifier === undefined) {
219
+ return { alias: DEFAULT_STORAGE_ALIAS };
220
+ }
221
+ if (typeof identifier === 'string') {
222
+ if (client.storageExists && (await client.storageExists(identifier, storageType))) {
223
+ return { id: identifier };
224
+ }
225
+ return { name: identifier };
226
+ }
227
+ if (identifier.id) {
228
+ return { id: identifier.id };
229
+ }
230
+ if (identifier.name) {
231
+ return { name: identifier.name };
232
+ }
233
+ // Empty object — treated as default storage.
234
+ return { alias: DEFAULT_STORAGE_ALIAS };
235
+ }
236
+ //# sourceMappingURL=storage_instance_manager.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"storage_instance_manager.js","sourceRoot":"","sources":["../../src/storages/storage_instance_manager.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAgBnD,wDAAwD;AACxD,MAAM,qBAAqB,GAAG,aAAa,CAAC;AAI5C;;;;;;;GAOG;AACH,MAAM,YAAY;IACL,IAAI,GAAc,IAAI,GAAG,EAAE,CAAC;IAC5B,MAAM,GAAc,IAAI,GAAG,EAAE,CAAC;IAC9B,OAAO,GAAc,IAAI,GAAG,EAAE,CAAC;IAExC,GAAG,CACC,GAAmB,EACnB,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,cAAc,GAKc;QAEhC,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI;YACtB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACf,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;YACnB,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;SACa,EAAE,CAAC;YACrC,IAAI,GAAG,KAAK,SAAS;gBAAE,SAAS;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,cAAc,CAAC,CAAC;YAC5D,IAAI,MAAM,EAAE,CAAC;gBACT,IAAI,MAAM,YAAa,GAAuD,EAAE,CAAC;oBAC7E,OAAO,MAAW,CAAC;gBACvB,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC9D,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,8CAA8C;IACtC,QAAQ,CACZ,IAAe,EACf,GAAmB,EACnB,GAAW,EACX,QAAW,EACX,cAAwB;QAExB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,GAAG,CAAqB,GAAmB,EAAE,QAAW,EAAE,cAAwB,EAAE,KAAc;QAC9F,sBAAsB;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QAErE,2CAA2C;QAC3C,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QAC7E,CAAC;QAED,+DAA+D;QAC/D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QACtE,CAAC;IACL,CAAC;IAED,eAAe,CAAC,QAAkB;QAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAoC,CAAC;QAElE,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACvC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;oBACtC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;wBACtB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAC5B,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,6DAA6D;IAC7D,CAAC,SAAS;QACN,MAAM,IAAI,GAAG,IAAI,GAAG,EAAY,CAAC;QACjC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACtB,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBACnB,MAAM,QAAQ,CAAC;oBACnB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,KAAK;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACJ;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,sBAAsB;IACd,KAAK,GAAG,IAAI,YAAY,EAAE,CAAC;IAC3B,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE7D;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,WAAW,CACb,GAA0B,EAC1B,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,YAAY,EACZ,cAAc,GAIjB;QAED,0FAA0F;QAC1F,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACzB,KAAK,GAAG,qBAAqB,CAAC;QAClC,CAAC;QAED,mCAAmC;QACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;YAC9D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,EAAE,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;YAC3D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,IAAI,EAAE,CAAC;YACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;YAC7D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;QAED,MAAM,aAAa,GAAG,EAAE,IAAI,IAAI,IAAI,KAAK,IAAI,qBAAqB,CAAC;QACnE,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,aAAa,IAAI,cAAc,EAAE,CAAC;QAEjE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QAE7C,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,IAAI,CAAC;YACD,qFAAqF;YACrF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC9D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;iBAAM,IAAI,EAAE,EAAE,CAAC;gBACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC3D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;iBAAM,IAAI,IAAI,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC7D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;YAED,2DAA2D;YAC3D,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,MAChB,SACH,CAAC,WAAW,EAAE,CAAC;YAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;gBACrB,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,SAAS;aACpB,CAAa,CAAC;YAEf,iDAAiD;YACjD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC;YAErD,OAAO,QAAQ,CAAC;QACpB,CAAC;gBAAS,CAAC;YACP,KAAK,CAAC,KAAK,EAAE,CAAC;YAEd,0DAA0D;YAC1D,4DAA4D;YAC5D,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;gBACxB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,QAAkB;QAC9B,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,UAAU;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;YAC5C,IAAI,YAAY,IAAI,QAAQ,IAAI,OAAQ,QAAgB,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBAChF,QAAgB,CAAC,UAAU,EAAE,CAAC;YACnC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACJ;AAoBD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,UAAyD,EACzD,MAAqB,EACrB,WAAyD;IAEzD,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAClD,OAAO,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC5C,CAAC;IAED,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACjC,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;YAChF,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAChC,CAAC;IAED,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;IACrC,CAAC;IAED,6CAA6C;IAC7C,OAAO,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;AAC5C,CAAC"}
@@ -1,5 +1,6 @@
1
- import type { Dictionary, StorageClient } from '@crawlee/types';
1
+ import type { BaseHttpClient, Dictionary, StorageClient } from '@crawlee/types';
2
2
  import { Configuration } from '../configuration.js';
3
+ import type { ProxyConfiguration } from '../proxy_configuration.js';
3
4
  /**
4
5
  * Options for purging default storage.
5
6
  */
@@ -89,5 +90,28 @@ export declare const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10000;
89
90
  * @internal
90
91
  */
91
92
  export declare const MAX_QUERIES_FOR_CONSISTENCY = 6;
93
+ /**
94
+ * Options for the static `open()` method on storage classes ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
95
+ */
96
+ export interface StorageOpenOptions {
97
+ /**
98
+ * SDK configuration instance, defaults to the static register.
99
+ */
100
+ config?: Configuration;
101
+ /**
102
+ * Optional storage client that should be used to open storages.
103
+ */
104
+ storageClient?: StorageClient;
105
+ /**
106
+ * Used to pass the proxy configuration for the `requestsFromUrl` objects.
107
+ * Takes advantage of the internal address rotation and authentication process.
108
+ * If undefined, the `requestsFromUrl` requests will be made without proxy.
109
+ */
110
+ proxyConfiguration?: ProxyConfiguration;
111
+ /**
112
+ * HTTP client to be used to download the list of URLs in `RequestQueue`.
113
+ */
114
+ httpClient?: BaseHttpClient;
115
+ }
92
116
  export {};
93
117
  //# sourceMappingURL=utils.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwB1G,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,GAAS,KAAK,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC"}
1
+ {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEhF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAIpE;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwB1G,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,GAAS,KAAK,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B"}
package/storages/utils.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import crypto from 'node:crypto';
2
2
  import { Configuration } from '../configuration.js';
3
+ import { serviceLocator } from '../service_locator.js';
3
4
  import { KeyValueStore } from './key_value_store.js';
4
5
  export async function purgeDefaultStorages(configOrOptions, client) {
5
6
  const options = configOrOptions instanceof Configuration
@@ -8,11 +9,11 @@ export async function purgeDefaultStorages(configOrOptions, client) {
8
9
  config: configOrOptions,
9
10
  }
10
11
  : (configOrOptions ?? {});
11
- const { config = Configuration.getGlobalConfig(), onlyPurgeOnce = false } = options;
12
- ({ client = config.getStorageClient() } = options);
12
+ const { config = serviceLocator.getConfiguration(), onlyPurgeOnce = false } = options;
13
+ ({ client = serviceLocator.getStorageClient() } = options);
13
14
  const casted = client;
14
15
  // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
15
- if (!onlyPurgeOnce || (config.get('purgeOnStart') && !casted.__purged)) {
16
+ if (!onlyPurgeOnce || (config.purgeOnStart && !casted.__purged)) {
16
17
  casted.__purged = true;
17
18
  await casted.purge?.();
18
19
  }
@@ -27,8 +28,8 @@ export async function purgeDefaultStorages(configOrOptions, client) {
27
28
  * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
28
29
  */
29
30
  export async function useState(name, defaultValue = {}, options) {
30
- const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
31
- config: options?.config || Configuration.getGlobalConfig(),
31
+ const kvStore = await KeyValueStore.open(options?.keyValueStoreName ? { name: options.keyValueStoreName } : null, {
32
+ config: options?.config || serviceLocator.getConfiguration(),
32
33
  });
33
34
  return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
34
35
  }
@@ -1 +1 @@
1
- {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACpF,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACjE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE;KAC7D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AAChB,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,MAAM,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}
1
+ {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAIjC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,aAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACtF,CAAC,EAAE,MAAM,GAAG,cAAc,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9G,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE;KAC/D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AAChB,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,MAAM,CAAC;AAE1D;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}
package/typedefs.d.ts CHANGED
@@ -6,5 +6,5 @@ export type Awaitable<T> = T | PromiseLike<T>;
6
6
  export declare function entries<T extends {}>(obj: T): [keyof T, T[keyof T]][];
7
7
  /** @ignore */
8
8
  export declare function keys<T extends {}>(obj: T): (keyof T)[];
9
- export declare type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH';
9
+ export type { AllowedHttpMethods } from '@crawlee/types';
10
10
  //# sourceMappingURL=typedefs.d.ts.map
package/typedefs.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../src/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;AAEjE,cAAc;AACd,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;AAE9C,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACxD;AAED,cAAc;AACd,wBAAgB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,CAAC,EAAE,CACzC;AAED,MAAM,CAAC,OAAO,MAAM,kBAAkB,GAChC,KAAK,GACL,MAAM,GACN,MAAM,GACN,KAAK,GACL,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,OAAO,CAAC"}
1
+ {"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../src/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;AAEjE,cAAc;AACd,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;AAE9C,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACxD;AAED,cAAc;AACd,wBAAgB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,CAAC,EAAE,CACzC;AAED,YAAY,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}
package/validators.d.ts CHANGED
@@ -17,5 +17,9 @@ export declare const validators: {
17
17
  validator: boolean;
18
18
  message: (label: string) => string;
19
19
  };
20
+ sessionPool: (value: Dictionary) => {
21
+ validator: boolean;
22
+ message: (label: string) => string;
23
+ };
20
24
  };
21
25
  //# sourceMappingURL=validators.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,gBAAgB;AAChB,eAAO,MAAM,UAAU;yBAEE,UAAU;;yBAEV,MAAM;;gCAEC,UAAU;;yBAEjB,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;0BAEL,UAAU;;yBAEX,MAAM;;CAE9B,CAAC"}
1
+ {"version":3,"file":"validators.d.ts","sourceRoot":"","sources":["../src/validators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,gBAAgB;AAChB,eAAO,MAAM,UAAU;yBAEE,UAAU;;yBAEV,MAAM;;gCAEC,UAAU;;yBAEjB,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;0BAEL,UAAU;;yBAEX,MAAM;;yBAEN,UAAU;;yBAEV,MAAM;;CAG9B,CAAC"}