@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/autoscaling/autoscaled_pool.d.ts +296 -0
  2. package/autoscaling/autoscaled_pool.d.ts.map +1 -0
  3. package/autoscaling/autoscaled_pool.js +673 -0
  4. package/autoscaling/autoscaled_pool.js.map +1 -0
  5. package/autoscaling/index.d.ts +4 -0
  6. package/autoscaling/index.d.ts.map +1 -0
  7. package/autoscaling/index.js +7 -0
  8. package/autoscaling/index.js.map +1 -0
  9. package/autoscaling/snapshotter.d.ts +193 -0
  10. package/autoscaling/snapshotter.d.ts.map +1 -0
  11. package/autoscaling/snapshotter.js +377 -0
  12. package/autoscaling/snapshotter.js.map +1 -0
  13. package/autoscaling/system_status.d.ts +181 -0
  14. package/autoscaling/system_status.d.ts.map +1 -0
  15. package/autoscaling/system_status.js +194 -0
  16. package/autoscaling/system_status.js.map +1 -0
  17. package/cache_container.d.ts +18 -0
  18. package/cache_container.d.ts.map +1 -0
  19. package/cache_container.js +42 -0
  20. package/cache_container.js.map +1 -0
  21. package/configuration.d.ts +150 -0
  22. package/configuration.d.ts.map +1 -0
  23. package/configuration.js +284 -0
  24. package/configuration.js.map +1 -0
  25. package/constants.d.ts +3 -0
  26. package/constants.d.ts.map +1 -0
  27. package/constants.js +6 -0
  28. package/constants.js.map +1 -0
  29. package/crawlers/crawler_commons.d.ts +25 -0
  30. package/crawlers/crawler_commons.d.ts.map +1 -0
  31. package/crawlers/crawler_commons.js +3 -0
  32. package/crawlers/crawler_commons.js.map +1 -0
  33. package/crawlers/crawler_extension.d.ts +11 -0
  34. package/crawlers/crawler_extension.d.ts.map +1 -0
  35. package/crawlers/crawler_extension.js +30 -0
  36. package/crawlers/crawler_extension.js.map +1 -0
  37. package/crawlers/crawler_utils.d.ts +20 -0
  38. package/crawlers/crawler_utils.d.ts.map +1 -0
  39. package/crawlers/crawler_utils.js +61 -0
  40. package/crawlers/crawler_utils.js.map +1 -0
  41. package/crawlers/index.d.ts +5 -0
  42. package/crawlers/index.d.ts.map +1 -0
  43. package/crawlers/index.js +8 -0
  44. package/crawlers/index.js.map +1 -0
  45. package/crawlers/statistics.d.ts +147 -0
  46. package/crawlers/statistics.d.ts.map +1 -0
  47. package/crawlers/statistics.js +360 -0
  48. package/crawlers/statistics.js.map +1 -0
  49. package/enqueue_links/enqueue_links.d.ts +151 -0
  50. package/enqueue_links/enqueue_links.d.ts.map +1 -0
  51. package/enqueue_links/enqueue_links.js +152 -0
  52. package/enqueue_links/enqueue_links.js.map +1 -0
  53. package/enqueue_links/index.d.ts +3 -0
  54. package/enqueue_links/index.d.ts.map +1 -0
  55. package/enqueue_links/index.js +6 -0
  56. package/enqueue_links/index.js.map +1 -0
  57. package/enqueue_links/shared.d.ts +60 -0
  58. package/enqueue_links/shared.d.ts.map +1 -0
  59. package/enqueue_links/shared.js +148 -0
  60. package/enqueue_links/shared.js.map +1 -0
  61. package/errors.d.ts +8 -0
  62. package/errors.d.ts.map +1 -0
  63. package/errors.js +12 -0
  64. package/errors.js.map +1 -0
  65. package/events/event_manager.d.ts +41 -0
  66. package/events/event_manager.d.ts.map +1 -0
  67. package/events/event_manager.js +103 -0
  68. package/events/event_manager.js.map +1 -0
  69. package/events/index.d.ts +3 -0
  70. package/events/index.d.ts.map +1 -0
  71. package/events/index.js +6 -0
  72. package/events/index.js.map +1 -0
  73. package/events/local_event_manager.d.ts +32 -0
  74. package/events/local_event_manager.d.ts.map +1 -0
  75. package/events/local_event_manager.js +114 -0
  76. package/events/local_event_manager.js.map +1 -0
  77. package/index.d.ts +14 -0
  78. package/index.d.ts.map +1 -0
  79. package/index.js +17 -0
  80. package/index.js.map +1 -0
  81. package/index.mjs +61 -0
  82. package/log.d.ts +3 -0
  83. package/log.d.ts.map +1 -0
  84. package/log.js +12 -0
  85. package/log.js.map +1 -0
  86. package/package.json +4 -8
  87. package/proxy_configuration.d.ts +181 -0
  88. package/proxy_configuration.d.ts.map +1 -0
  89. package/proxy_configuration.js +209 -0
  90. package/proxy_configuration.js.map +1 -0
  91. package/request.d.ts +185 -0
  92. package/request.d.ts.map +1 -0
  93. package/request.js +330 -0
  94. package/request.js.map +1 -0
  95. package/serialization.d.ts +33 -0
  96. package/serialization.d.ts.map +1 -0
  97. package/serialization.js +146 -0
  98. package/serialization.js.map +1 -0
  99. package/session_pool/errors.d.ts +8 -0
  100. package/session_pool/errors.d.ts.map +1 -0
  101. package/session_pool/errors.js +20 -0
  102. package/session_pool/errors.js.map +1 -0
  103. package/session_pool/events.d.ts +3 -0
  104. package/session_pool/events.d.ts.map +1 -0
  105. package/session_pool/events.js +6 -0
  106. package/session_pool/events.js.map +1 -0
  107. package/session_pool/index.d.ts +6 -0
  108. package/session_pool/index.d.ts.map +1 -0
  109. package/session_pool/index.js +9 -0
  110. package/session_pool/index.js.map +1 -0
  111. package/session_pool/session.d.ts +212 -0
  112. package/session_pool/session.d.ts.map +1 -0
  113. package/session_pool/session.js +360 -0
  114. package/session_pool/session.js.map +1 -0
  115. package/session_pool/session_pool.d.ts +222 -0
  116. package/session_pool/session_pool.d.ts.map +1 -0
  117. package/session_pool/session_pool.js +389 -0
  118. package/session_pool/session_pool.js.map +1 -0
  119. package/session_pool/session_utils.d.ts +9 -0
  120. package/session_pool/session_utils.d.ts.map +1 -0
  121. package/session_pool/session_utils.js +19 -0
  122. package/session_pool/session_utils.js.map +1 -0
  123. package/storages/dataset.d.ts +317 -0
  124. package/storages/dataset.d.ts.map +1 -0
  125. package/storages/dataset.js +367 -0
  126. package/storages/dataset.js.map +1 -0
  127. package/storages/index.d.ts +7 -0
  128. package/storages/index.d.ts.map +1 -0
  129. package/storages/index.js +11 -0
  130. package/storages/index.js.map +1 -0
  131. package/storages/key_value_store.d.ts +242 -0
  132. package/storages/key_value_store.d.ts.map +1 -0
  133. package/storages/key_value_store.js +320 -0
  134. package/storages/key_value_store.js.map +1 -0
  135. package/storages/request_list.d.ts +469 -0
  136. package/storages/request_list.d.ts.map +1 -0
  137. package/storages/request_list.js +724 -0
  138. package/storages/request_list.js.map +1 -0
  139. package/storages/request_queue.d.ts +315 -0
  140. package/storages/request_queue.d.ts.map +1 -0
  141. package/storages/request_queue.js +730 -0
  142. package/storages/request_queue.js.map +1 -0
  143. package/storages/storage.d.ts +260 -0
  144. package/storages/storage.d.ts.map +1 -0
  145. package/storages/storage.js +3 -0
  146. package/storages/storage.js.map +1 -0
  147. package/storages/storage_manager.d.ts +40 -0
  148. package/storages/storage_manager.d.ts.map +1 -0
  149. package/storages/storage_manager.js +119 -0
  150. package/storages/storage_manager.js.map +1 -0
  151. package/tsconfig.build.tsbuildinfo +1 -0
  152. package/typedefs.d.ts +12 -0
  153. package/typedefs.d.ts.map +1 -0
  154. package/typedefs.js +15 -0
  155. package/typedefs.js.map +1 -0
  156. package/validators.d.ts +21 -0
  157. package/validators.d.ts.map +1 -0
  158. package/validators.js +26 -0
  159. package/validators.js.map +1 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":";;;;AAAA,0CAA6D;AAC7D,0DAAiE;AACjE,gDAAwD;AACxD,4DAA4B;AAC5B,mDAA2D;AAC3D,oDAAoB;AACpB,uDAAmD;AACnD,gCAA6B;AAC7B,wCAAqD;AAGrD,MAAM,mBAAmB,GAAG,OAAS,CAAC;AAEtC;;;GAGG;AACU,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAEzC,gBAAgB;AACH,QAAA,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACU,QAAA,mCAAmC,GAAG,KAAM,CAAC;AAE1D;;;GAGG;AACU,QAAA,2BAA2B,GAAG,CAAC,CAAC;AAE7C;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC;;;;GAIG;AACU,QAAA,gCAAgC,GAAG,IAAI,CAAC;AAErD;;;;;;;;GAQG;AACH,SAAgB,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,gBAAM;SACb,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAE3B,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7B,CAAC;AARD,oCAQC;AAiCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,MAAa,YAAY;IAgDrB;;OAEG;IACH,YAAY,OAA4B;QAlDxC;;;;mBAAM,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;WAAC;QAC5C;;;;;WAAW;QACX;;;;;WAAc;QACd;;;;mBAAc,EAAE;WAAC;QACjB;;;;mBAAY,IAAA,gCAAoB,GAAE;WAAC;QACnC;;;;;WAA2B;QAE3B;;;;WAIG;QACH;;;;mBAAwB,IAAI,+BAAc,EAAU;WAAC;QACrD;;;;mBAMY,IAAI;WAAC;QAEjB,6DAA6D;QAC7D,8EAA8E;QAC9E;;;;mBAAa,IAAI,GAAG,EAAE;WAAC;QAEvB,iEAAiE;QACjE,yFAAyF;QACzF,kDAAkD;QAClD;;;;mBAAe,IAAI,IAAI,EAAE;WAAC;QAC1B;;;;mBAAwB,CAAC,GAAG,IAAI;WAAC,CAAC,4DAA4D;QAE9F,oFAAoF;QACpF,uDAAuD;QACvD,yCAAyC;QACzC;;;;mBAAkB,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,2BAA2B,EAAE,CAAC;WAAC;QAE3E,mFAAmF;QACnF,mFAAmF;QACnF;;;;mBAAoB,CAAC;WAAC;QACtB;;;;mBAAsB,CAAC;WAAC;QAExB,0DAA0D;QAC1D,uEAAuE;QACvE;;;;mBAAgB,IAAI,yBAAQ,CAE1B,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC;WAAC;QAMlC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE;YAC/C,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;SAChC,CAAuB,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,eAAe;QACX,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,UAAU,CAAC,WAAqC,EAAE,UAAwC,EAAE;QAC9F,IAAA,YAAE,EAAC,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnC,GAAG,EAAE,YAAE,CAAC,MAAM;YACd,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC;QACJ,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,MAAM,OAAO,GAAG,WAAW,YAAY,iBAAO;YAC1C,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,IAAI,iBAAO,CAAC,WAAW,CAAC,CAAC;QAE/B,mDAAmD;QACnD,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjD,sCAAsC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEpD,IAAI,UAAU,EAAE;YACZ,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACH,iBAAiB,EAAE,IAAI;gBACvB,mFAAmF;gBACnF,0FAA0F;gBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;gBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;aAClC,CAAC;SACL;QAED,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAuB,CAAC;QACtG,kBAAkB,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QAEjD,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,kBAAkB,CAAC;QAC5D,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAEjD,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YAC/F,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAEzB,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;SAC1D;QAED,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,WAAW,CACb,YAA0C,EAC1C,UAAwC,EAAE;QAE1C,IAAA,YAAE,EAAC,YAAY,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACpD,GAAG,EAAE,YAAE,CAAC,MAAM;YACd,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC,CAAC;QACL,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtD,MAAM,kBAAkB,GAAG,CAAC,SAAiB,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAElD,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;YAE1B,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YAC5C,mBAAmB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAEhD,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC;QAEF,MAAM,OAAO,GAA2B;YACpC,iBAAiB,EAAE,EAAE;YACrB,mBAAmB,EAAE,EAAE;SAC1B,CAAC;QAEF,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;YAC9C,OAAO,WAAW,YAAY,iBAAO;gBACjC,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,IAAI,iBAAO,CAAC,WAAW,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,IAAI,GAAG,EAAmB,CAAC;QAEjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;YAC5B,mDAAmD;YACnD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvD,sCAAsC;YACtC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEpD,IAAI,UAAU,EAAE;gBACZ,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAC3B,iBAAiB,EAAE,IAAI;oBACvB,mFAAmF;oBACnF,0FAA0F;oBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;oBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;oBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;iBAClC,CAAC,CAAC;aACN;iBAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;gBAC9C,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;aACjD;SACJ;QAED,yDAAyD;QACzD,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;YACrB,OAAO,OAAO,CAAC;SAClB;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAElG,8BAA8B;QAC9B,OAAO,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,CAAC;QAE7D,yCAAyC;QACzC,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,iBAAiB,EAAE;YACnD,4CAA4C;YAC5C,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE3C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAE1D,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC;YACpD,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEzC,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;gBAC/F,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAEzB,qEAAqE;gBACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;aAC1D;SACJ;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,EAAU;QACvB,IAAA,YAAE,EAAC,EAAE,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAElB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,OAAO,IAAI,iBAAO,CAAC,cAA2C,CAAC,CAAC;IACpE,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAEvD,oCAAoC;QACpC,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC;QAEhC,mCAAmC;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE;YAC/E,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6DAA6D,EAAE;gBAC5E,aAAa;gBACb,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC9C,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC;aAC7D,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;SACf;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,IAAI,OAAO,CAAC;QACZ,IAAI;YACA,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,qFAAqF;YACrF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACtC,MAAM,CAAC,CAAC;SACX;QAED,6HAA6H;QAE7H,yIAAyI;QACzI,6EAA6E;QAC7E,uGAAuG;QACvG,2FAA2F;QAC3F,0FAA0F;QAC1F,IAAI,CAAC,OAAO,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0EAA0E,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YAC9G,UAAU,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAC1C,CAAC,EAAE,wCAAgC,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC;SACf;QAED,8FAA8F;QAC9F,wGAAwG;QACxG,4GAA4G;QAC5G,sDAAsD;QACtD,IAAI,OAAO,CAAC,SAAS,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iEAAiE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YACrG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC9C,OAAO,IAAI,CAAC;SACf;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAC/B,EAAE,EAAE,YAAE,CAAC,MAAM;YACb,SAAS,EAAE,YAAE,CAAC,MAAM;YACpB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAChC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;YAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uBAAuB,OAAO,CAAC,EAAE,6CAA6C,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1H,OAAO,IAAI,CAAC;SACf;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAChE,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,CAAuB,CAAC;QAC5G,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAC9B,kBAAkB,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QAEjD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACnC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAE3C,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,EAAE;YACvC,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC9B;QAED,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,UAAwC,EAAE;QAC7E,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAC/B,EAAE,EAAE,YAAE,CAAC,MAAM;YACb,SAAS,EAAE,YAAE,CAAC,MAAM;SACvB,CAAC,CAAC,CAAC;QACJ,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;YAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,OAAO,CAAC,EAAE,kCAAkC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClH,OAAO,IAAI,CAAC;SACf;QAED,oEAAoE;QACpE,wEAAwE;QACxE,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAuB,CAAC;QACzG,kBAAkB,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACjD,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,yHAAyH;QACzH,2HAA2H;QAC3H,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;gBAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3G,OAAO;aACV;YAED,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAEnC,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAC5D,CAAC,EAAE,wCAAgC,CAAC,CAAC;QAErC,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,qBAAqB,EAAE;YAC9F,MAAM,OAAO,GAAG,2CAA2C,IAAI,CAAC,qBAAqB,GAAG,GAAG,8BAA8B,CAAC;YAC1H,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAEhF,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAChE,OAAO,gBAAgB,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACjG,CAAC;IAEO,MAAM;QACV,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAClC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,QAAgB,EAAE,kBAAsC;QAC5E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC7B,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;YAC/C,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,iBAAiB,EAAE,kBAAkB,CAAC,iBAAiB;SAC1D,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;OAUG;IACO,KAAK,CAAC,qBAAqB,CACjC,iBAAiB,GAAG,KAAK,EACzB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,yBAAiB,EAAE,6BAAqB,CAAC,EACnF,SAAS,GAAG,CAAC;QAEb,sCAAsC;QACtC,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAEjD,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE;YAC7B,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC;YAElC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,MAAM;iBACnC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC;iBACnB,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,kBAAkB,EAAE,EAAE,EAAE;gBACrD,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE;oBAC3C,uFAAuF;oBACvF,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAU,CAAC;wBAAE,OAAO;oBAE/G,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;oBACpD,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE;wBACxC,SAAS;wBACT,iBAAiB,EAAE,KAAK;wBACxB,iBAAiB,EAAE,IAAI;wBACvB,SAAS;qBACZ,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;gBAEH,mGAAmG;gBACnG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;gBAElC,OAAO;oBACH,eAAe,EAAE,KAAK,CAAC,MAAM,IAAI,KAAK;oBACtC,SAAS,EAAE,KAAK;oBAChB,eAAe,EAAE,IAAI,IAAI,CAAC,eAAe,CAAC;oBAC1C,cAAc;oBACd,kBAAkB;iBACrB,CAAC;YACN,CAAC,CAAC,CAAC;SACV;QAED,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;QAE7H,4DAA4D;QAE5D,2DAA2D;QAC3D,wFAAwF;QACxF,kEAAkE;QAClE,0DAA0D;QAC1D,kDAAkD;QAElD,uFAAuF;QACvF,IAAI,SAAS,IAAI,qCAA4B,EAAE;YAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uDAAuD,qCAA4B,GAAG,CAAC,CAAC;SAC5G;QACD,MAAM,2BAA2B,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC;eAC9D,eAAe;eACf,SAAS,GAAG,qCAA4B,CAAC;QAEhD,6DAA6D;QAC7D,iGAAiG;QACjG,kFAAkF;QAClF,MAAM,oBAAoB,GAAG,CAAC,cAAc,GAAG,CAAC,eAAe,IAAI,2CAAmC,CAAC;QACvG,MAAM,mBAAmB,GAAG,CAAC,kBAAkB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,mBAAmB,CAAC;QACtG,uFAAuF;QACvF,MAAM,0BAA0B,GAAG,iBAAiB,IAAI,CAAC,oBAAoB,IAAI,CAAC,mBAAmB,CAAC;QAEtG,6DAA6D;QAC7D,IAAI,CAAC,2BAA2B,IAAI,CAAC,0BAA0B;YAAE,OAAO,IAAI,CAAC;QAE7E,yGAAyG;QACzG,0GAA0G;QAC1G,IAAI,CAAC,2BAA2B,IAAI,SAAS,GAAG,mCAA2B;YAAE,OAAO,KAAK,CAAC;QAE1F,MAAM,SAAS,GAAG,2BAA2B;YACzC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC;QAEhB,+DAA+D;QAC/D,IAAI,0BAA0B,EAAE;YAC5B,MAAM,WAAW,GAAG,2CAAmC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC1F,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,WAAW,oFAAoF,CAAC,CAAC;YAC9H,MAAM,IAAA,qBAAK,EAAC,WAAW,CAAC,CAAC;SAC5B;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,SAAS,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACnF,CAAC;IAED;;OAEG;IACK,2BAA2B,CAAC,SAAiB,EAAE,SAAkB;QACrE,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;SACtD;aAAM,IAAI,IAAI,CAAC,iBAAiB,GAAG,6BAAqB,EAAE;YACvD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;SACvD;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,YAAY,CAAC,CAAC;QACjD,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY;QACd,gFAAgF;QAChF,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3D,OAAO,mBAAmB,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B;QAC3C,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,YAAY,CAAC,CAAC;QACjD,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;CACJ;AAjoBD,oCAioBC"}
@@ -0,0 +1,260 @@
1
+ /// <reference types="node" />
2
+ import { RequestOptions as RequestModel } from '../request';
3
+ import { AllowedHttpMethods, Dictionary } from '../typedefs';
4
+ /**
5
+ * A helper class that is used to report results from various
6
+ * {@link RequestQueue} functions as well as {@link enqueueLinks}.
7
+ */
8
+ export interface QueueOperationInfo {
9
+ /** Indicates if request was already present in the queue. */
10
+ wasAlreadyPresent: boolean;
11
+ /** Indicates if request was already marked as handled. */
12
+ wasAlreadyHandled: boolean;
13
+ /** The ID of the added request */
14
+ requestId: string;
15
+ }
16
+ export interface DatasetCollectionClientOptions {
17
+ storageDir: string;
18
+ }
19
+ export interface DatasetCollectionData {
20
+ id: string;
21
+ name?: string;
22
+ createdAt: Date;
23
+ modifiedAt: Date;
24
+ accessedAt: Date;
25
+ }
26
+ /**
27
+ * Dataset collection client.
28
+ */
29
+ export interface DatasetCollectionClient {
30
+ list(): Promise<PaginatedList<Dataset>>;
31
+ getOrCreate(name: string): Promise<DatasetCollectionData>;
32
+ }
33
+ export interface Dataset extends DatasetCollectionData {
34
+ itemCount: number;
35
+ }
36
+ export interface DatasetClientUpdateOptions {
37
+ name?: string;
38
+ }
39
+ export interface DatasetClientListOptions {
40
+ desc?: boolean;
41
+ limit?: number;
42
+ offset?: number;
43
+ }
44
+ export interface PaginatedList<Data> {
45
+ /** Total count of entries in the dataset. */
46
+ total: number;
47
+ /** Count of dataset entries returned in this set. */
48
+ count: number;
49
+ /** Position of the first returned entry in the dataset. */
50
+ offset: number;
51
+ /** Maximum number of dataset entries requested. */
52
+ limit: number;
53
+ /** Should the results be in descending order. */
54
+ desc: boolean;
55
+ /** Dataset entries based on chosen format parameter. */
56
+ items: Data[];
57
+ }
58
+ export interface DatasetInfo {
59
+ id: string;
60
+ name?: string;
61
+ userId?: string;
62
+ createdAt: Date;
63
+ modifiedAt: Date;
64
+ accessedAt: Date;
65
+ itemCount: number;
66
+ cleanItemCount?: number;
67
+ actId?: string;
68
+ actRunId?: string;
69
+ stats?: DatasetStats;
70
+ fields?: string[];
71
+ }
72
+ export interface DatasetStats {
73
+ readCount?: number;
74
+ writeCount?: number;
75
+ deleteCount?: number;
76
+ storageBytes?: number;
77
+ }
78
+ export interface DatasetClient<Data extends Dictionary = Dictionary> {
79
+ get(): Promise<DatasetInfo | undefined>;
80
+ update(newFields: DatasetClientUpdateOptions): Promise<DatasetInfo>;
81
+ delete(): Promise<void>;
82
+ downloadItems(...args: unknown[]): Promise<Buffer>;
83
+ listItems(options?: DatasetClientListOptions): Promise<PaginatedList<Data>>;
84
+ pushItems(items: Data | Data[] | string | string[]): Promise<void>;
85
+ }
86
+ export interface KeyValueStoreInfo {
87
+ id: string;
88
+ name?: string;
89
+ userId: string;
90
+ createdAt: Date;
91
+ modifiedAt: Date;
92
+ accessedAt: Date;
93
+ actId?: string;
94
+ actRunId?: string;
95
+ stats?: KeyValueStoreStats;
96
+ }
97
+ export interface KeyValueStoreStats {
98
+ readCount?: number;
99
+ writeCount?: number;
100
+ deleteCount?: number;
101
+ listCount?: number;
102
+ storageBytes?: number;
103
+ }
104
+ /**
105
+ * Key-value store collection client.
106
+ */
107
+ export interface KeyValueStoreCollectionClient {
108
+ list(): Promise<PaginatedList<KeyValueStoreInfo>>;
109
+ getOrCreate(name: string): Promise<KeyValueStoreInfo>;
110
+ }
111
+ export interface KeyValueStoreRecord {
112
+ key: string;
113
+ value: any;
114
+ contentType?: string;
115
+ }
116
+ export interface KeyValueStoreClientUpdateOptions {
117
+ name?: string;
118
+ }
119
+ export interface KeyValueStoreClientListOptions {
120
+ limit?: number;
121
+ exclusiveStartKey?: string;
122
+ }
123
+ export interface KeyValueStoreItemData {
124
+ key: string;
125
+ size: number;
126
+ }
127
+ export interface KeyValueStoreClientListData {
128
+ count: number;
129
+ limit: number;
130
+ exclusiveStartKey?: string;
131
+ isTruncated: boolean;
132
+ nextExclusiveStartKey?: string;
133
+ items: KeyValueStoreItemData[];
134
+ }
135
+ export interface KeyValueStoreClientGetRecordOptions {
136
+ buffer?: boolean;
137
+ stream?: boolean;
138
+ }
139
+ /**
140
+ * Key-value Store client.
141
+ */
142
+ export interface KeyValueStoreClient {
143
+ get(): Promise<KeyValueStoreInfo | undefined>;
144
+ update(newFields: KeyValueStoreClientUpdateOptions): Promise<KeyValueStoreInfo>;
145
+ delete(): Promise<void>;
146
+ listKeys(options?: KeyValueStoreClientListOptions): Promise<KeyValueStoreClientListData>;
147
+ getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise<KeyValueStoreRecord | undefined>;
148
+ setRecord(record: KeyValueStoreRecord): Promise<void>;
149
+ deleteRecord(key: string): Promise<void>;
150
+ }
151
+ /**
152
+ * Request queue collection client.
153
+ */
154
+ export interface RequestQueueCollectionClient {
155
+ list(): Promise<PaginatedList<RequestQueueInfo>>;
156
+ getOrCreate(name: string): Promise<RequestQueueInfo>;
157
+ }
158
+ export interface QueueHead {
159
+ limit: number;
160
+ queueModifiedAt: Date;
161
+ hadMultipleClients: boolean;
162
+ items: RequestModel[];
163
+ }
164
+ export interface RequestQueueInfo {
165
+ id: string;
166
+ name?: string;
167
+ userId: string;
168
+ createdAt: Date;
169
+ modifiedAt: Date;
170
+ accessedAt: Date;
171
+ expireAt?: string;
172
+ totalRequestCount: number;
173
+ handledRequestCount: number;
174
+ pendingRequestCount: number;
175
+ actId?: string;
176
+ actRunId?: string;
177
+ hadMultipleClients: boolean;
178
+ stats: RequestQueueStats;
179
+ }
180
+ export interface RequestQueueStats {
181
+ readCount?: number;
182
+ writeCount?: number;
183
+ deleteCount?: number;
184
+ headItemReadCount?: number;
185
+ storageBytes?: number;
186
+ }
187
+ export interface RequestQueueClientOptions {
188
+ name: string;
189
+ storageDir: string;
190
+ }
191
+ export interface ListOptions {
192
+ /**
193
+ * @default 100
194
+ */
195
+ limit?: number;
196
+ }
197
+ export interface RequestOptions {
198
+ forefront?: boolean;
199
+ [k: string]: unknown;
200
+ }
201
+ export interface RequestSchema {
202
+ uniqueKey: string;
203
+ url: string;
204
+ method?: AllowedHttpMethods;
205
+ payload?: string;
206
+ retryCount?: number;
207
+ errorMessages?: string[];
208
+ headers?: Dictionary<string>;
209
+ userData?: Dictionary;
210
+ handledAt?: string;
211
+ noRetry?: boolean;
212
+ loadedUrl?: string;
213
+ }
214
+ export interface ProcessedRequest {
215
+ uniqueKey: string;
216
+ requestId: string;
217
+ wasAlreadyPresent: boolean;
218
+ wasAlreadyHandled: boolean;
219
+ }
220
+ export interface UnprocessedRequest {
221
+ uniqueKey: string;
222
+ url: string;
223
+ method?: AllowedHttpMethods;
224
+ }
225
+ export interface BatchAddRequestsResult {
226
+ processedRequests: ProcessedRequest[];
227
+ unprocessedRequests: UnprocessedRequest[];
228
+ }
229
+ export interface RequestQueueClient {
230
+ get(): Promise<RequestQueueInfo | undefined>;
231
+ update(newFields: {
232
+ name?: string;
233
+ }): Promise<RequestQueueInfo | undefined>;
234
+ delete(): Promise<void>;
235
+ listHead(options?: ListOptions): Promise<QueueHead>;
236
+ addRequest(request: RequestModel, options?: RequestOptions): Promise<QueueOperationInfo>;
237
+ batchAddRequests(requests: RequestSchema[], options?: RequestOptions): Promise<BatchAddRequestsResult>;
238
+ getRequest(id: string): Promise<RequestOptions | undefined>;
239
+ updateRequest(request: RequestModel, options?: RequestOptions): Promise<QueueOperationInfo>;
240
+ deleteRequest(_id: string): Promise<unknown>;
241
+ }
242
+ export interface RequestQueueOptions {
243
+ clientKey?: string;
244
+ timeoutSecs?: number;
245
+ }
246
+ /**
247
+ * Represents a storage capable of working with datasets, KV stores and request queues.
248
+ */
249
+ export interface StorageClient {
250
+ datasets(): DatasetCollectionClient;
251
+ dataset(id: string): DatasetClient;
252
+ keyValueStores(): KeyValueStoreCollectionClient;
253
+ keyValueStore(id: string): KeyValueStoreClient;
254
+ requestQueues(): RequestQueueCollectionClient;
255
+ requestQueue(id: string, options?: RequestQueueOptions): RequestQueueClient;
256
+ stats?: {
257
+ rateLimitErrors: number[];
258
+ };
259
+ }
260
+ //# sourceMappingURL=storage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../src/storages/storage.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,cAAc,IAAI,YAAY,EAAE,MAAM,YAAY,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE7D;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAE/B,6DAA6D;IAC7D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,0DAA0D;IAC1D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;CAErB;AAED,MAAM,WAAW,8BAA8B;IAC3C,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACpC,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IACxC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;CAC7D;AAED,MAAM,WAAW,OAAQ,SAAQ,qBAAqB;IAClD,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,0BAA0B;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB;IACrC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa,CAAC,IAAI;IAC/B,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,IAAI,EAAE,OAAO,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AACD,MAAM,WAAW,YAAY;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,aAAa,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAC/D,GAAG,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC;IACxC,MAAM,CAAC,SAAS,EAAE,0BAA0B,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACpE,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,aAAa,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,SAAS,CAAC,OAAO,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5E,SAAS,CAAC,KAAK,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,iBAAiB;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC1C,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAClD,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,mBAAmB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,GAAG,CAAC;IACX,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,gCAAgC;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,8BAA8B;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,qBAAqB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,2BAA2B;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,OAAO,CAAC;IACrB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,KAAK,EAAE,qBAAqB,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,mCAAmC;IAChD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAChC,GAAG,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAAC;IAC9C,MAAM,CAAC,SAAS,EAAE,gCAAgC,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAChF,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,8BAA8B,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;IACzF,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mCAAmC,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAAC;IAChH,SAAS,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IACzC,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACjD,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,SAAS;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,IAAI,CAAC;IACtB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,KAAK,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,KAAK,EAAE,iBAAiB,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IACxB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC3B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AAED,MAAM,WAAW,sBAAsB;IACnC,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;IACtC,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;CAC7C;AAED,MAAM,WAAW,kBAAkB;IAC/B,GAAG,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC;IAC7C,MAAM,CAAC,SAAS,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC;IAC5E,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACpD,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACzF,gBAAgB,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACvG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IAC5D,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5F,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,mBAAmB;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B,QAAQ,IAAI,uBAAuB,CAAC;IACpC,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,aAAa,CAAC;IACnC,cAAc,IAAI,6BAA6B,CAAC;IAChD,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC/C,aAAa,IAAI,4BAA4B,CAAC;IAC9C,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,kBAAkB,CAAC;IAC5E,KAAK,CAAC,EAAE;QAAE,eAAe,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CACzC"}
@@ -0,0 +1,3 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ //# sourceMappingURL=storage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"storage.js","sourceRoot":"","sources":["../../src/storages/storage.ts"],"names":[],"mappings":""}
@@ -0,0 +1,40 @@
1
+ import { Configuration } from '../configuration';
2
+ import { Constructor } from '../typedefs';
3
+ import { StorageClient } from './storage';
4
+ export interface IStorage {
5
+ id: string;
6
+ name?: string;
7
+ }
8
+ /**
9
+ * StorageManager takes care of opening remote or local storages.
10
+ * @ignore
11
+ */
12
+ export declare class StorageManager<T extends IStorage = IStorage> {
13
+ private readonly config;
14
+ private static readonly MAX_OPENED_STORAGES;
15
+ private readonly name;
16
+ private readonly StorageConstructor;
17
+ private readonly cache;
18
+ constructor(StorageConstructor: Constructor<T>, config?: Configuration);
19
+ openStorage(idOrName?: string, client?: StorageClient): Promise<T>;
20
+ closeStorage(storage: {
21
+ id: string;
22
+ name?: string;
23
+ }): void;
24
+ /**
25
+ * Helper function that first requests storage by ID and if storage doesn't exist then gets it by name.
26
+ */
27
+ protected _getOrCreateStorage(storageIdOrName: string, storageConstructorName: string, apiClient: StorageClient): Promise<import("./storage").DatasetCollectionData>;
28
+ protected _getStorageClientFactories(client: StorageClient, storageConstructorName: string): {
29
+ createStorageClient: ((id: string) => import("./storage").DatasetClient<import("../typedefs").Dictionary<unknown>>) | ((id: string) => import("./storage").KeyValueStoreClient) | ((id: string, options?: import("./storage").RequestQueueOptions | undefined) => import("./storage").RequestQueueClient);
30
+ createStorageCollectionClient: (() => import("./storage").DatasetCollectionClient) | (() => import("./storage").KeyValueStoreCollectionClient) | (() => import("./storage").RequestQueueCollectionClient);
31
+ };
32
+ protected _addStorageToCache(storage: T): void;
33
+ }
34
+ export interface StorageManagerOptions {
35
+ /**
36
+ * SDK configuration instance, defaults to the static register.
37
+ */
38
+ config?: Configuration;
39
+ }
40
+ //# sourceMappingURL=storage_manager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"storage_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAc1C,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,qBAAa,cAAc,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAQjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAP3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IACnD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA+C;IACpE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoC;IACvE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;gBAGhC,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,EACjB,MAAM,gBAAkC;IAOvD,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC;IAyBxE,YAAY,CAAC,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAU1D;;OAEG;cACa,mBAAmB,CAAC,eAAe,EAAE,MAAM,EAAE,sBAAsB,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa;IAcrH,SAAS,CAAC,0BAA0B,CAAC,MAAM,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM;;;;IAY1F,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;CASjD;AAKD,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B"}
@@ -0,0 +1,119 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.StorageManager = void 0;
4
+ const consts_1 = require("@apify/consts");
5
+ const datastructures_1 = require("@apify/datastructures");
6
+ const configuration_1 = require("../configuration");
7
+ const DEFAULT_ID_ENV_VAR_NAMES = {
8
+ Dataset: consts_1.ENV_VARS.DEFAULT_DATASET_ID,
9
+ KeyValueStore: consts_1.ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID,
10
+ RequestQueue: consts_1.ENV_VARS.DEFAULT_REQUEST_QUEUE_ID,
11
+ };
12
+ const DEFAULT_ID_CONFIG_KEYS = {
13
+ Dataset: 'defaultDatasetId',
14
+ KeyValueStore: 'defaultKeyValueStoreId',
15
+ RequestQueue: 'defaultRequestQueueId',
16
+ };
17
+ /**
18
+ * StorageManager takes care of opening remote or local storages.
19
+ * @ignore
20
+ */
21
+ class StorageManager {
22
+ constructor(StorageConstructor, config = configuration_1.Configuration.getGlobalConfig()) {
23
+ Object.defineProperty(this, "config", {
24
+ enumerable: true,
25
+ configurable: true,
26
+ writable: true,
27
+ value: config
28
+ });
29
+ Object.defineProperty(this, "name", {
30
+ enumerable: true,
31
+ configurable: true,
32
+ writable: true,
33
+ value: void 0
34
+ });
35
+ Object.defineProperty(this, "StorageConstructor", {
36
+ enumerable: true,
37
+ configurable: true,
38
+ writable: true,
39
+ value: void 0
40
+ });
41
+ Object.defineProperty(this, "cache", {
42
+ enumerable: true,
43
+ configurable: true,
44
+ writable: true,
45
+ value: void 0
46
+ });
47
+ this.StorageConstructor = StorageConstructor;
48
+ this.name = this.StorageConstructor.name;
49
+ this.cache = new datastructures_1.LruCache({ maxLength: StorageManager.MAX_OPENED_STORAGES });
50
+ }
51
+ async openStorage(idOrName, client) {
52
+ if (!idOrName) {
53
+ const defaultIdEnvVarName = DEFAULT_ID_ENV_VAR_NAMES[this.name];
54
+ const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
55
+ idOrName = this.config.get(defaultIdConfigKey);
56
+ if (!idOrName)
57
+ throw new Error(`The '${defaultIdEnvVarName}' environment variable is not defined.`);
58
+ }
59
+ const cacheKey = idOrName;
60
+ let storage = this.cache.get(cacheKey);
61
+ if (!storage) {
62
+ client ?? (client = this.config.getStorageClient());
63
+ const storageObject = await this._getOrCreateStorage(idOrName, this.name, client);
64
+ storage = new this.StorageConstructor({
65
+ id: storageObject.id,
66
+ name: storageObject.name,
67
+ client,
68
+ });
69
+ this._addStorageToCache(storage);
70
+ }
71
+ return storage;
72
+ }
73
+ closeStorage(storage) {
74
+ const idKey = storage.id;
75
+ this.cache.remove(idKey);
76
+ if (storage.name) {
77
+ const nameKey = storage.name;
78
+ this.cache.remove(nameKey);
79
+ }
80
+ }
81
+ /**
82
+ * Helper function that first requests storage by ID and if storage doesn't exist then gets it by name.
83
+ */
84
+ async _getOrCreateStorage(storageIdOrName, storageConstructorName, apiClient) {
85
+ const { createStorageClient, createStorageCollectionClient, } = this._getStorageClientFactories(apiClient, storageConstructorName);
86
+ const storageClient = createStorageClient(storageIdOrName);
87
+ const existingStorage = await storageClient.get();
88
+ if (existingStorage)
89
+ return existingStorage;
90
+ const storageCollectionClient = createStorageCollectionClient();
91
+ return storageCollectionClient.getOrCreate(storageIdOrName);
92
+ }
93
+ _getStorageClientFactories(client, storageConstructorName) {
94
+ // Dataset => dataset
95
+ const clientName = storageConstructorName[0].toLowerCase() + storageConstructorName.slice(1);
96
+ // dataset => datasets
97
+ const collectionClientName = `${clientName}s`;
98
+ return {
99
+ createStorageClient: client[clientName].bind(client),
100
+ createStorageCollectionClient: client[collectionClientName].bind(client),
101
+ };
102
+ }
103
+ _addStorageToCache(storage) {
104
+ const idKey = storage.id;
105
+ this.cache.add(idKey, storage);
106
+ if (storage.name) {
107
+ const nameKey = storage.name;
108
+ this.cache.add(nameKey, storage);
109
+ }
110
+ }
111
+ }
112
+ exports.StorageManager = StorageManager;
113
+ Object.defineProperty(StorageManager, "MAX_OPENED_STORAGES", {
114
+ enumerable: true,
115
+ configurable: true,
116
+ writable: true,
117
+ value: 1000
118
+ });
119
+ //# sourceMappingURL=storage_manager.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":";;;AAAA,0CAAyC;AACzC,0DAAiD;AACjD,oDAAiD;AAIjD,MAAM,wBAAwB,GAAG;IAC7B,OAAO,EAAE,iBAAQ,CAAC,kBAAkB;IACpC,aAAa,EAAE,iBAAQ,CAAC,0BAA0B;IAClD,YAAY,EAAE,iBAAQ,CAAC,wBAAwB;CACzC,CAAC;AAEX,MAAM,sBAAsB,GAAG;IAC3B,OAAO,EAAE,kBAAkB;IAC3B,aAAa,EAAE,wBAAwB;IACvC,YAAY,EAAE,uBAAuB;CAC/B,CAAC;AAOX;;;GAGG;AACH,MAAa,cAAc;IAMvB,YACI,kBAAkC,EACjB,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QANrB;;;;;WAAoE;QACpE;;;;;WAAuE;QACvE;;;;;WAAoC;QAMhC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAoD,CAAC;QACzF,IAAI,CAAC,KAAK,GAAG,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAiB,EAAE,MAAsB;QACvD,IAAI,CAAC,QAAQ,EAAE;YACX,MAAM,mBAAmB,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChE,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7D,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAW,CAAC;YACzD,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,mBAAmB,wCAAwC,CAAC,CAAC;SACvG;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC;QAC1B,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,OAAO,EAAE;YACV,MAAM,KAAN,MAAM,GAAK,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,EAAC;YAC1C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClF,OAAO,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC;gBAClC,EAAE,EAAE,aAAa,CAAC,EAAE;gBACpB,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,MAAM;aACT,CAAC,CAAC;YACH,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;SACpC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,YAAY,CAAC,OAAsC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzB,IAAI,OAAO,CAAC,IAAI,EAAE;YACd,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SAC9B;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,eAAuB,EAAE,sBAA8B,EAAE,SAAwB;QACjH,MAAM,EACF,mBAAmB,EACnB,6BAA6B,GAChC,GAAG,IAAI,CAAC,0BAA0B,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC;QAEvE,MAAM,aAAa,GAAG,mBAAmB,CAAC,eAAe,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,CAAC;QAClD,IAAI,eAAe;YAAE,OAAO,eAAe,CAAC;QAE5C,MAAM,uBAAuB,GAAG,6BAA6B,EAAE,CAAC;QAChE,OAAO,uBAAuB,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAES,0BAA0B,CAAC,MAAqB,EAAE,sBAA8B;QACtF,qBAAqB;QACrB,MAAM,UAAU,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAgB,CAAC;QAC5G,sBAAsB;QACtB,MAAM,oBAAoB,GAAG,GAAG,UAAU,GAA4B,CAAC;QAEvE,OAAO;YACH,mBAAmB,EAAE,MAAM,CAAC,UAAW,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YACrD,6BAA6B,EAAE,MAAM,CAAC,oBAAqB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SAC5E,CAAC;IACN,CAAC;IAES,kBAAkB,CAAC,OAAU;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,OAAO,CAAC,IAAI,EAAE;YACd,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;SACpC;IACL,CAAC;;AAvFL,wCAwFC;AAvFG;;;;WAA8C,IAAI;GAAC"}