@crawlee/core 4.0.0-beta.7 → 4.0.0-beta.70

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/README.md +17 -13
  2. package/autoscaling/autoscaled_pool.d.ts +3 -5
  3. package/autoscaling/autoscaled_pool.d.ts.map +1 -1
  4. package/autoscaling/autoscaled_pool.js +3 -9
  5. package/autoscaling/autoscaled_pool.js.map +1 -1
  6. package/autoscaling/snapshotter.d.ts +3 -13
  7. package/autoscaling/snapshotter.d.ts.map +1 -1
  8. package/autoscaling/snapshotter.js +18 -29
  9. package/autoscaling/snapshotter.js.map +1 -1
  10. package/autoscaling/system_status.d.ts +0 -3
  11. package/autoscaling/system_status.d.ts.map +1 -1
  12. package/autoscaling/system_status.js +2 -3
  13. package/autoscaling/system_status.js.map +1 -1
  14. package/configuration.d.ts +88 -227
  15. package/configuration.d.ts.map +1 -1
  16. package/configuration.js +162 -223
  17. package/configuration.js.map +1 -1
  18. package/cookie_utils.d.ts +4 -2
  19. package/cookie_utils.d.ts.map +1 -1
  20. package/cookie_utils.js +18 -12
  21. package/cookie_utils.js.map +1 -1
  22. package/crawlers/context_pipeline.d.ts +71 -0
  23. package/crawlers/context_pipeline.d.ts.map +1 -0
  24. package/crawlers/context_pipeline.js +123 -0
  25. package/crawlers/context_pipeline.js.map +1 -0
  26. package/crawlers/crawler_commons.d.ts +20 -29
  27. package/crawlers/crawler_commons.d.ts.map +1 -1
  28. package/crawlers/crawler_commons.js +12 -20
  29. package/crawlers/crawler_commons.js.map +1 -1
  30. package/crawlers/crawler_utils.d.ts +2 -2
  31. package/crawlers/crawler_utils.d.ts.map +1 -1
  32. package/crawlers/crawler_utils.js +1 -1
  33. package/crawlers/crawler_utils.js.map +1 -1
  34. package/crawlers/error_snapshotter.d.ts +3 -2
  35. package/crawlers/error_snapshotter.d.ts.map +1 -1
  36. package/crawlers/error_snapshotter.js +2 -2
  37. package/crawlers/error_snapshotter.js.map +1 -1
  38. package/crawlers/error_tracker.d.ts +2 -1
  39. package/crawlers/error_tracker.d.ts.map +1 -1
  40. package/crawlers/error_tracker.js.map +1 -1
  41. package/crawlers/index.d.ts +1 -1
  42. package/crawlers/index.d.ts.map +1 -1
  43. package/crawlers/index.js +1 -1
  44. package/crawlers/index.js.map +1 -1
  45. package/crawlers/internals/types.d.ts +8 -0
  46. package/crawlers/internals/types.d.ts.map +1 -0
  47. package/crawlers/internals/types.js +2 -0
  48. package/crawlers/internals/types.js.map +1 -0
  49. package/crawlers/statistics.d.ts +16 -16
  50. package/crawlers/statistics.d.ts.map +1 -1
  51. package/crawlers/statistics.js +22 -25
  52. package/crawlers/statistics.js.map +1 -1
  53. package/enqueue_links/enqueue_links.d.ts +36 -21
  54. package/enqueue_links/enqueue_links.d.ts.map +1 -1
  55. package/enqueue_links/enqueue_links.js +47 -26
  56. package/enqueue_links/enqueue_links.js.map +1 -1
  57. package/enqueue_links/shared.d.ts +25 -8
  58. package/enqueue_links/shared.d.ts.map +1 -1
  59. package/enqueue_links/shared.js +69 -37
  60. package/enqueue_links/shared.js.map +1 -1
  61. package/errors.d.ts +33 -3
  62. package/errors.d.ts.map +1 -1
  63. package/errors.js +48 -4
  64. package/errors.js.map +1 -1
  65. package/events/event_manager.d.ts +34 -7
  66. package/events/event_manager.d.ts.map +1 -1
  67. package/events/event_manager.js +8 -9
  68. package/events/event_manager.js.map +1 -1
  69. package/events/local_event_manager.d.ts +14 -4
  70. package/events/local_event_manager.d.ts.map +1 -1
  71. package/events/local_event_manager.js +33 -39
  72. package/events/local_event_manager.js.map +1 -1
  73. package/index.d.ts +4 -2
  74. package/index.d.ts.map +1 -1
  75. package/index.js +3 -1
  76. package/index.js.map +1 -1
  77. package/log.d.ts +82 -2
  78. package/log.d.ts.map +1 -1
  79. package/log.js +102 -0
  80. package/log.js.map +1 -1
  81. package/memory-storage/consts.d.ts +5 -0
  82. package/memory-storage/consts.d.ts.map +1 -0
  83. package/memory-storage/consts.js +5 -0
  84. package/memory-storage/consts.js.map +1 -0
  85. package/memory-storage/index.d.ts +2 -0
  86. package/memory-storage/index.d.ts.map +1 -0
  87. package/memory-storage/index.js +2 -0
  88. package/memory-storage/index.js.map +1 -0
  89. package/memory-storage/memory-storage.d.ts +47 -0
  90. package/memory-storage/memory-storage.d.ts.map +1 -0
  91. package/memory-storage/memory-storage.js +137 -0
  92. package/memory-storage/memory-storage.js.map +1 -0
  93. package/memory-storage/resource-clients/common/base-client.d.ts +5 -0
  94. package/memory-storage/resource-clients/common/base-client.d.ts.map +1 -0
  95. package/memory-storage/resource-clients/common/base-client.js +7 -0
  96. package/memory-storage/resource-clients/common/base-client.js.map +1 -0
  97. package/memory-storage/resource-clients/dataset.d.ts +41 -0
  98. package/memory-storage/resource-clients/dataset.d.ts.map +1 -0
  99. package/memory-storage/resource-clients/dataset.js +114 -0
  100. package/memory-storage/resource-clients/dataset.js.map +1 -0
  101. package/memory-storage/resource-clients/key-value-store.d.ts +64 -0
  102. package/memory-storage/resource-clients/key-value-store.d.ts.map +1 -0
  103. package/memory-storage/resource-clients/key-value-store.js +204 -0
  104. package/memory-storage/resource-clients/key-value-store.js.map +1 -0
  105. package/memory-storage/resource-clients/request-queue.d.ts +97 -0
  106. package/memory-storage/resource-clients/request-queue.d.ts.map +1 -0
  107. package/memory-storage/resource-clients/request-queue.js +422 -0
  108. package/memory-storage/resource-clients/request-queue.js.map +1 -0
  109. package/memory-storage/utils.d.ts +12 -0
  110. package/memory-storage/utils.d.ts.map +1 -0
  111. package/memory-storage/utils.js +28 -0
  112. package/memory-storage/utils.js.map +1 -0
  113. package/package.json +12 -10
  114. package/proxy_configuration.d.ts +14 -148
  115. package/proxy_configuration.d.ts.map +1 -1
  116. package/proxy_configuration.js +19 -167
  117. package/proxy_configuration.js.map +1 -1
  118. package/recoverable_state.d.ts +121 -0
  119. package/recoverable_state.d.ts.map +1 -0
  120. package/recoverable_state.js +142 -0
  121. package/recoverable_state.js.map +1 -0
  122. package/request.d.ts +84 -12
  123. package/request.d.ts.map +1 -1
  124. package/request.js +107 -27
  125. package/request.js.map +1 -1
  126. package/router.d.ts.map +1 -1
  127. package/router.js.map +1 -1
  128. package/serialization.js +1 -1
  129. package/serialization.js.map +1 -1
  130. package/service_locator.d.ts +157 -0
  131. package/service_locator.d.ts.map +1 -0
  132. package/service_locator.js +239 -0
  133. package/service_locator.js.map +1 -0
  134. package/session_pool/consts.d.ts +1 -1
  135. package/session_pool/consts.d.ts.map +1 -1
  136. package/session_pool/consts.js +1 -1
  137. package/session_pool/consts.js.map +1 -1
  138. package/session_pool/fingerprint.d.ts +10 -0
  139. package/session_pool/fingerprint.d.ts.map +1 -0
  140. package/session_pool/fingerprint.js +31 -0
  141. package/session_pool/fingerprint.js.map +1 -0
  142. package/session_pool/index.d.ts +0 -1
  143. package/session_pool/index.d.ts.map +1 -1
  144. package/session_pool/index.js +0 -1
  145. package/session_pool/index.js.map +1 -1
  146. package/session_pool/session.d.ts +35 -72
  147. package/session_pool/session.d.ts.map +1 -1
  148. package/session_pool/session.js +46 -98
  149. package/session_pool/session.js.map +1 -1
  150. package/session_pool/session_pool.d.ts +71 -71
  151. package/session_pool/session_pool.d.ts.map +1 -1
  152. package/session_pool/session_pool.js +111 -99
  153. package/session_pool/session_pool.js.map +1 -1
  154. package/storages/access_checking.d.ts.map +1 -1
  155. package/storages/access_checking.js +5 -1
  156. package/storages/access_checking.js.map +1 -1
  157. package/storages/dataset.d.ts +98 -47
  158. package/storages/dataset.d.ts.map +1 -1
  159. package/storages/dataset.js +166 -123
  160. package/storages/dataset.js.map +1 -1
  161. package/storages/index.d.ts +8 -6
  162. package/storages/index.d.ts.map +1 -1
  163. package/storages/index.js +6 -6
  164. package/storages/index.js.map +1 -1
  165. package/storages/key_value_store.d.ts +163 -33
  166. package/storages/key_value_store.d.ts.map +1 -1
  167. package/storages/key_value_store.js +267 -119
  168. package/storages/key_value_store.js.map +1 -1
  169. package/storages/key_value_store_codec.d.ts +33 -0
  170. package/storages/key_value_store_codec.d.ts.map +1 -0
  171. package/storages/key_value_store_codec.js +114 -0
  172. package/storages/key_value_store_codec.js.map +1 -0
  173. package/storages/request_list.d.ts +32 -81
  174. package/storages/request_list.d.ts.map +1 -1
  175. package/storages/request_list.js +49 -39
  176. package/storages/request_list.js.map +1 -1
  177. package/storages/request_loader.d.ts +97 -0
  178. package/storages/request_loader.d.ts.map +1 -0
  179. package/storages/request_loader.js +2 -0
  180. package/storages/request_loader.js.map +1 -0
  181. package/storages/request_manager.d.ts +34 -0
  182. package/storages/request_manager.d.ts.map +1 -0
  183. package/storages/request_manager.js +2 -0
  184. package/storages/request_manager.js.map +1 -0
  185. package/storages/request_manager_tandem.d.ts +107 -0
  186. package/storages/request_manager_tandem.d.ts.map +1 -0
  187. package/storages/request_manager_tandem.js +198 -0
  188. package/storages/request_manager_tandem.js.map +1 -0
  189. package/storages/request_queue.d.ts +282 -45
  190. package/storages/request_queue.d.ts.map +1 -1
  191. package/storages/request_queue.js +592 -213
  192. package/storages/request_queue.js.map +1 -1
  193. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +29 -24
  194. package/storages/sitemap_request_loader.d.ts.map +1 -0
  195. package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +50 -46
  196. package/storages/sitemap_request_loader.js.map +1 -0
  197. package/storages/storage_instance_manager.d.ts +92 -0
  198. package/storages/storage_instance_manager.d.ts.map +1 -0
  199. package/storages/storage_instance_manager.js +261 -0
  200. package/storages/storage_instance_manager.js.map +1 -0
  201. package/storages/storage_stats.d.ts +49 -0
  202. package/storages/storage_stats.d.ts.map +1 -0
  203. package/storages/storage_stats.js +30 -0
  204. package/storages/storage_stats.js.map +1 -0
  205. package/storages/utils.d.ts +51 -5
  206. package/storages/utils.d.ts.map +1 -1
  207. package/storages/utils.js +60 -8
  208. package/storages/utils.js.map +1 -1
  209. package/typedefs.d.ts +1 -1
  210. package/typedefs.d.ts.map +1 -1
  211. package/validators.d.ts +8 -0
  212. package/validators.d.ts.map +1 -1
  213. package/validators.js +8 -0
  214. package/validators.js.map +1 -1
  215. package/crawlers/crawler_extension.d.ts +0 -12
  216. package/crawlers/crawler_extension.d.ts.map +0 -1
  217. package/crawlers/crawler_extension.js +0 -14
  218. package/crawlers/crawler_extension.js.map +0 -1
  219. package/http_clients/base-http-client.d.ts +0 -134
  220. package/http_clients/base-http-client.d.ts.map +0 -1
  221. package/http_clients/base-http-client.js +0 -33
  222. package/http_clients/base-http-client.js.map +0 -1
  223. package/http_clients/form-data-like.d.ts +0 -67
  224. package/http_clients/form-data-like.d.ts.map +0 -1
  225. package/http_clients/form-data-like.js +0 -5
  226. package/http_clients/form-data-like.js.map +0 -1
  227. package/http_clients/got-scraping-http-client.d.ts +0 -15
  228. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  229. package/http_clients/got-scraping-http-client.js +0 -69
  230. package/http_clients/got-scraping-http-client.js.map +0 -1
  231. package/http_clients/index.d.ts +0 -3
  232. package/http_clients/index.d.ts.map +0 -1
  233. package/http_clients/index.js +0 -3
  234. package/http_clients/index.js.map +0 -1
  235. package/session_pool/events.d.ts +0 -3
  236. package/session_pool/events.d.ts.map +0 -1
  237. package/session_pool/events.js +0 -3
  238. package/session_pool/events.js.map +0 -1
  239. package/storages/request_provider.d.ts +0 -307
  240. package/storages/request_provider.d.ts.map +0 -1
  241. package/storages/request_provider.js +0 -555
  242. package/storages/request_provider.js.map +0 -1
  243. package/storages/request_queue_v2.d.ts +0 -87
  244. package/storages/request_queue_v2.d.ts.map +0 -1
  245. package/storages/request_queue_v2.js +0 -438
  246. package/storages/request_queue_v2.js.map +0 -1
  247. package/storages/sitemap_request_list.d.ts.map +0 -1
  248. package/storages/sitemap_request_list.js.map +0 -1
  249. package/storages/storage_manager.d.ts +0 -58
  250. package/storages/storage_manager.d.ts.map +0 -1
  251. package/storages/storage_manager.js +0 -105
  252. package/storages/storage_manager.js.map +0 -1
  253. package/tsconfig.build.tsbuildinfo +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAY,GAAG,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;IACpG,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC;YAChD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;QACjE,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;QAC9D,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AAqFD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAM,OAAO,OAAO;IAWH;IAVb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,MAAM,CAAsB;IAC5B,GAAG,GAAQ,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAE5C;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAC;IACzE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO;QACX,CAAC;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAG,KAAK,EAAE,MAAM,GAAG,CAAC,EAAiB,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;YAEzE,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAE3B,IAAI,KAAK,CAAC,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrC,MAAM,cAAc,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,cAAc,EAAE,CAAC;QAEvB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,SAAS,CAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;QAE5D,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,+BAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,eAA+B,EAC/B,UAAiC,EAAE;QAEnC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,OAAO,CAAC,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAE5D,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAgB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAE/E,OAAO,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ"}
1
+ {"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGzD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAEtE,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CAAI,IAAO,EAAE,KAAc;IAC7D,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;AACL,CAAC;AA6FD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAM,OAAO,OAAO;IAgBH;IAfb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,OAAO,CAAuB;IAC9B,GAAG,CAAgB;IAEF,YAAY,GAAG,IAAI,mBAAmB,CAAe;QAClE,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;KAChB,CAAC,CAAC;IAEH;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IACvE,CAAC;IAED;;;OAGG;IACH,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IACrC,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAE5B,4CAA4C;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC;YACD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACnC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,8CAA8C;YAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBACjD,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,EAAE,cAAc;gBAChC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAE5B,MAAM,KAAK,GAAG,SAAS,CAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;IAChE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,+BAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,CAAC,eAAe,CAAC,OAA+B;QAC1D,IAAI,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAChC,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,MAAM;gBACF,GAAG,IAAI;gBACP,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,IAAI,CAAmB,CAAC;aACrE,CAAC;QACN,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,CAAC,UAAU,CACrB,OAA+B,EAC/B,QAAQ,GAAG,+BAA+B;QAE1C,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC;QACjC,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,UAAU,GAAG,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClG,IAAI,UAAU,IAAI,CAAC;gBAAE,MAAM;YAE3B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACnC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;YACnF,MAAM,IAAI,CAAC;YAEX,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YAC7B,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,UAAU,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK;gBAAE,MAAM;YACtF,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAChC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,UAAkC,EAAE;QACvC,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAC3C,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK;SACrC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,UAAkC,EAAE;QACxC,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC;YAChD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK;SACrC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IACzB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,cAAc,CAAC,yBAAyB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,UAA8C,EAC9C,UAA8B,EAAE;QAEhC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACrC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QAEnD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,cAAc,CAAC,iBAAiB,EAAE,CAAC;QAEpF,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE5F,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC;QAEvF,OAAO,cAAc,CAAC,yBAAyB,EAAE,CAAC,WAAW,CAAgB,IAAI,EAAE;YAC/E,GAAG,QAAQ;YACX,aAAa,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,oBAAoB,CAAC,QAAQ,CAAC;YAClE,eAAe,EAAE,cAAc,CAAC,yBAAyB,EAAE,EAAE,IAAI,cAAc,CAAC,WAAW,CAAC,IAAI;SACnG,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ"}
@@ -1,12 +1,14 @@
1
1
  export * from './dataset.js';
2
2
  export * from './key_value_store.js';
3
+ export * from './key_value_store_codec.js';
3
4
  export * from './request_list.js';
4
- export * from './request_provider.js';
5
- export { RequestQueueV1 } from './request_queue.js';
6
- export { RequestQueue } from './request_queue_v2.js';
7
- export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
8
- export * from './storage_manager.js';
5
+ export type * from './request_loader.js';
6
+ export type * from './request_manager.js';
7
+ export * from './request_queue.js';
8
+ export * from './storage_instance_manager.js';
9
+ export * from './storage_stats.js';
9
10
  export * from './utils.js';
10
11
  export * from './access_checking.js';
11
- export * from './sitemap_request_list.js';
12
+ export * from './sitemap_request_loader.js';
13
+ export * from './request_manager_tandem.js';
12
14
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,mBAAmB,CAAC;AAClC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,sBAAsB,CAAC;AAC1C,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC"}
package/storages/index.js CHANGED
@@ -1,12 +1,12 @@
1
1
  export * from './dataset.js';
2
2
  export * from './key_value_store.js';
3
+ export * from './key_value_store_codec.js';
3
4
  export * from './request_list.js';
4
- export * from './request_provider.js';
5
- export { RequestQueueV1 } from './request_queue.js';
6
- export { RequestQueue } from './request_queue_v2.js';
7
- export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
8
- export * from './storage_manager.js';
5
+ export * from './request_queue.js';
6
+ export * from './storage_instance_manager.js';
7
+ export * from './storage_stats.js';
9
8
  export * from './utils.js';
10
9
  export * from './access_checking.js';
11
- export * from './sitemap_request_list.js';
10
+ export * from './sitemap_request_loader.js';
11
+ export * from './request_manager_tandem.js';
12
12
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,mBAAmB,CAAC;AAGlC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC"}
@@ -1,15 +1,9 @@
1
- import type { Dictionary, StorageClient } from '@crawlee/types';
1
+ import type { Dictionary, KeyValueStoreBackend } from '@crawlee/types';
2
2
  import { Configuration } from '../configuration.js';
3
3
  import type { Awaitable } from '../typedefs.js';
4
- import type { StorageManagerOptions } from './storage_manager.js';
5
- /**
6
- * Helper function to possibly stringify value if options.contentType is not set.
7
- *
8
- * @ignore
9
- */
10
- export declare const maybeStringify: <T>(value: T, options: {
11
- contentType?: string;
12
- }) => T;
4
+ import type { KeyValueStoreStats } from './storage_stats.js';
5
+ import type { StorageIdentifier } from './storage_instance_manager.js';
6
+ import type { StorageOpenOptions } from './utils.js';
13
7
  /**
14
8
  * The `KeyValueStore` class represents a key-value store, a simple data storage that is used
15
9
  * for saving and reading data records or files. Each data record is
@@ -70,15 +64,20 @@ export declare class KeyValueStore {
70
64
  readonly config: Configuration;
71
65
  readonly id: string;
72
66
  readonly name?: string;
73
- readonly storageObject?: Record<string, unknown>;
74
- private readonly client;
67
+ private readonly backend;
75
68
  private persistStateEventStarted;
76
69
  /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
77
70
  private readonly cache;
71
+ private readonly statsTracker;
78
72
  /**
79
73
  * @internal
80
74
  */
81
75
  constructor(options: KeyValueStoreOptions, config?: Configuration);
76
+ /**
77
+ * Backend-independent usage counters tracked for this key-value store (read / write / delete /
78
+ * list operations issued to the underlying storage backend). Counted per backend call.
79
+ */
80
+ get stats(): KeyValueStoreStats;
82
81
  /**
83
82
  * Gets a value from the key-value store.
84
83
  *
@@ -143,6 +142,33 @@ export declare class KeyValueStore {
143
142
  * on the MIME content type of the record, or the default value if the key is missing from the store.
144
143
  */
145
144
  getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
145
+ /**
146
+ * Reads a record from the key-value store without parsing the value.
147
+ *
148
+ * Use this when you need the raw bytes and the content type — for example, to run your own
149
+ * parser (`simdjson`, a custom XML library, etc.) or to forward the bytes verbatim.
150
+ *
151
+ * There is no symmetric `setRecord` method, because {@link KeyValueStore.setValue} already
152
+ * passes a `Buffer` (or `string` / `Stream`) through unchanged when an explicit `contentType`
153
+ * is provided. To write pre-serialized bytes, call
154
+ * `setValue(key, buffer, { contentType: 'application/json; charset=utf-8' })`.
155
+ *
156
+ * Returns `null` if the record does not exist.
157
+ *
158
+ * **Example usage:**
159
+ * ```javascript
160
+ * const store = await KeyValueStore.open();
161
+ * const record = await store.getRecord('huge.json');
162
+ * if (record) {
163
+ * const data = simdjson.parse(record.value);
164
+ * }
165
+ * ```
166
+ *
167
+ * @param key
168
+ * Unique key of the record. It can be at most 256 characters long and only consist
169
+ * of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
170
+ */
171
+ getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
146
172
  /**
147
173
  * Tests whether a record with the given key exists in the key-value store without retrieving its value.
148
174
  *
@@ -152,6 +178,8 @@ export declare class KeyValueStore {
152
178
  recordExists(key: string): Promise<boolean>;
153
179
  getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
154
180
  private ensurePersistStateEvent;
181
+ private fetchKeyValuePages;
182
+ private fetchKeyPages;
155
183
  /**
156
184
  * Saves or deletes a record in the key-value store.
157
185
  * The function returns a promise that resolves once the record has been saved or deleted.
@@ -207,7 +235,7 @@ export declare class KeyValueStore {
207
235
  * Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
208
236
  * Each invocation of `iteratee` is called with three arguments: `(key, index, info)`, where `key`
209
237
  * is the record key, `index` is a zero-based index of the key in the current iteration
210
- * (regardless of `options.exclusiveStartKey`) and `info` is an object that contains a single property `size`
238
+ * and `info` is an object that contains a single property `size`
211
239
  * indicating size of the record in bytes.
212
240
  *
213
241
  * If the `iteratee` function returns a Promise then it is awaited before the next call.
@@ -225,11 +253,99 @@ export declare class KeyValueStore {
225
253
  * @param [options] All `forEachKey()` parameters.
226
254
  */
227
255
  forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
228
- private _forEachKey;
256
+ /**
257
+ * Returns key-value store keys.
258
+ *
259
+ * When awaited (`await store.keys()`), returns all keys as a flat `string[]` array.
260
+ * When used as an async iterable (`for await...of`), iterates over all keys across pages
261
+ * without loading everything into memory at once.
262
+ *
263
+ * **Example usage:**
264
+ * ```javascript
265
+ * const keyValueStore = await KeyValueStore.open();
266
+ *
267
+ * // Iterate over all keys (memory-efficient for large stores)
268
+ * for await (const key of keyValueStore.keys()) {
269
+ * console.log(key);
270
+ * }
271
+ *
272
+ * // Or fetch all keys at once
273
+ * const allKeys = await keyValueStore.keys();
274
+ * console.log(allKeys);
275
+ * ```
276
+ *
277
+ * @param options Options for the iteration.
278
+ */
279
+ keys(options?: KeyValueStoreIteratorOptions): AsyncIterable<string> & Promise<string[]>;
280
+ /**
281
+ * Returns key-value store values.
282
+ *
283
+ * When awaited (`await store.values()`), returns all values as a flat `T[]` array.
284
+ * When used as an async iterable (`for await...of`), iterates over all values across pages
285
+ * without loading everything into memory at once.
286
+ *
287
+ * **Example usage:**
288
+ * ```javascript
289
+ * const keyValueStore = await KeyValueStore.open();
290
+ *
291
+ * // Iterate over all values (memory-efficient for large stores)
292
+ * for await (const value of keyValueStore.values()) {
293
+ * console.log(value);
294
+ * }
295
+ *
296
+ * // Or fetch all values at once
297
+ * const allValues = await keyValueStore.values();
298
+ * console.log(allValues);
299
+ * ```
300
+ *
301
+ * @param options Options for the iteration.
302
+ */
303
+ values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
304
+ /**
305
+ * Returns key-value store entries (key-value pairs).
306
+ *
307
+ * When awaited (`await store.entries()`), returns all entries as a flat `[key, value][]` array.
308
+ * When used as an async iterable (`for await...of`), iterates over all entries across pages
309
+ * without loading everything into memory at once.
310
+ *
311
+ * **Example usage:**
312
+ * ```javascript
313
+ * const keyValueStore = await KeyValueStore.open();
314
+ *
315
+ * // Iterate over all entries (memory-efficient for large stores)
316
+ * for await (const [key, value] of keyValueStore.entries()) {
317
+ * console.log(`${key}: ${value}`);
318
+ * }
319
+ *
320
+ * // Or fetch all entries at once
321
+ * const allEntries = await keyValueStore.entries();
322
+ * console.log(allEntries);
323
+ * ```
324
+ *
325
+ * @param options Options for the iteration.
326
+ */
327
+ entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
328
+ /**
329
+ * Default async iterator for the key-value store, iterating over entries (key-value pairs).
330
+ * Allows using the store directly in a `for await...of` loop.
331
+ *
332
+ * **Example usage:**
333
+ * ```javascript
334
+ * const keyValueStore = await KeyValueStore.open();
335
+ * for await (const [key, value] of keyValueStore) {
336
+ * console.log(`${key}: ${value}`);
337
+ * }
338
+ * ```
339
+ */
340
+ [Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
229
341
  /**
230
342
  * Returns a file URL for the given key.
343
+ *
344
+ * If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
345
+ *
346
+ * @param key The key of the record to generate the public URL for.
231
347
  */
232
- getPublicUrl(key: string): string;
348
+ getPublicUrl(key: string): Promise<string | undefined>;
233
349
  /**
234
350
  * Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
235
351
  *
@@ -239,12 +355,13 @@ export declare class KeyValueStore {
239
355
  *
240
356
  * For more details and code examples, see the {@link KeyValueStore} class.
241
357
  *
242
- * @param [storeIdOrName]
243
- * ID or name of the key-value store to be opened. If `null` or `undefined`,
244
- * the function returns the default key-value store associated with the crawler run.
358
+ * @param [identifier]
359
+ * ID or name of the key-value store to be opened. If a string is provided, it will first be
360
+ * looked up as an ID; if no such storage exists, it will be treated as a name.
361
+ * If `null` or `undefined`, the function returns the default key-value store associated with the crawler run.
245
362
  * @param [options] Storage manager options.
246
363
  */
247
- static open(storeIdOrName?: string | null, options?: StorageManagerOptions): Promise<KeyValueStore>;
364
+ static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
248
365
  /**
249
366
  * Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
250
367
  *
@@ -303,6 +420,19 @@ export declare class KeyValueStore {
303
420
  * @ignore
304
421
  */
305
422
  static getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
423
+ /**
424
+ * Reads a record from the default {@link KeyValueStore} associated with the current crawler run
425
+ * without parsing the value.
426
+ *
427
+ * This is just a convenient shortcut for {@link KeyValueStore.getRecord}. Returns `null` if the
428
+ * record does not exist.
429
+ *
430
+ * @param key
431
+ * Unique key of the record. It can be at most 256 characters long and only consist
432
+ * of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
433
+ * @ignore
434
+ */
435
+ static getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
306
436
  /**
307
437
  * Tests whether a record with the given key exists in the default {@link KeyValueStore} associated with the current crawler run.
308
438
  * @param key The queried record key.
@@ -344,8 +474,9 @@ export declare class KeyValueStore {
344
474
  static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
345
475
  /**
346
476
  * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
347
- * By default, it will try to find root input files (either extension-less, `.json` or `.txt`),
348
- * or alternatively read the input from the default {@link KeyValueStore}.
477
+ *
478
+ * The input is read from the default {@link KeyValueStore} under the configured input key
479
+ * (`CRAWLEE_INPUT_KEY`, default `INPUT`).
349
480
  *
350
481
  * Note that the `getInput()` function does not cache the value read from the key-value store.
351
482
  * If you need to use the input multiple times in your crawler,
@@ -380,27 +511,26 @@ export interface KeyConsumer {
380
511
  export interface KeyValueStoreOptions {
381
512
  id: string;
382
513
  name?: string;
383
- client: StorageClient;
384
- storageObject?: Record<string, unknown>;
514
+ backend: KeyValueStoreBackend;
515
+ }
516
+ /**
517
+ * A raw, unparsed key-value store record as returned by {@link KeyValueStore.getRecord}: the
518
+ * verbatim bytes plus the content type, with parsing left to the caller.
519
+ */
520
+ export interface KeyValueStoreRawRecord {
521
+ value: Buffer | ArrayBuffer;
522
+ contentType: string | null;
385
523
  }
386
524
  export interface RecordOptions {
387
525
  /**
388
526
  * Specifies a custom MIME content type of the record.
389
527
  */
390
528
  contentType?: string;
391
- /**
392
- * Specifies a custom timeout for the `set-record` API call, in seconds.
393
- */
394
- timeoutSecs?: number;
395
- /**
396
- * If set to `true`, the `set-record` API call will not be retried if it times out.
397
- */
398
- doNotRetryTimeouts?: boolean;
399
529
  }
400
530
  export interface KeyValueStoreIteratorOptions {
401
531
  /**
402
- * All keys up to this one (including) are skipped from the result.
532
+ * If set, only keys that start with this prefix are returned.
403
533
  */
404
- exclusiveStartKey?: string;
534
+ prefix?: string;
405
535
  }
406
536
  //# sourceMappingURL=key_value_store.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAQrF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAIlE;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,CAAC,EAAE,OAAO,CAAC,EAAE,SAAS;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MA0B5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAelB,QAAQ,CAAC,MAAM;IAdnB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IAEvD;;OAEG;gBAEC,OAAO,EAAE,oBAAoB,EACpB,MAAM,gBAAkC;IAQrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAC3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IA0CrE;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuB3G,OAAO,CAAC,uBAAuB;IA2B/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsE3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAMlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAMpF,WAAW;IAwBzB;;OAEG;IACH,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAKjC;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;IAsB7G;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAClE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAmC5E;;;;OAIG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAK3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;OAkBG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CA6B/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B"}
1
+ {"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,oBAAoB,EAAyB,MAAM,gBAAgB,CAAC;AAK9F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE7D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAQrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAqBlB,QAAQ,CAAC,MAAM;IApBnB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAC/C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IAEvD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAK1B;IAEH;;OAEG;gBAEC,OAAO,EAAE,oBAAoB,EACpB,MAAM,gBAAkC;IAOrD;;;OAGG;IACH,IAAI,KAAK,IAAI,kBAAkB,CAE9B;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAC3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAkDrE;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAcpE;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuB3G,OAAO,CAAC,uBAAuB;YAsBhB,kBAAkB;YAkBlB,aAAa;IAmB5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA4D3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAMlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBlG;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,IAAI,CAAC,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAS3F;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAShG;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,CAAC,GAAG,OAAO,EACf,OAAO,GAAE,4BAAiC,GAC3C,aAAa,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC;IAStD;;;;;;;;;;;OAWG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,cAAc,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC;IAI1F;;;;;;OAMG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAI5D;;;;;;;;;;;;;;OAcG;WACU,IAAI,CACb,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,aAAa,CAAC;IAyBzB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAClE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAmC5E;;;;;;;;;;;OAWG;WACU,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAK3E;;;;OAIG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAK3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;;OAmBG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CAI/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,oBAAoB,CAAC;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,KAAK,EAAE,MAAM,GAAG,WAAW,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB"}