@crawlee/core 3.0.0-beta.9 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +75 -133
  2. package/autoscaling/autoscaled_pool.d.ts +5 -5
  3. package/autoscaling/autoscaled_pool.d.ts.map +1 -1
  4. package/autoscaling/autoscaled_pool.js +16 -16
  5. package/autoscaling/autoscaled_pool.js.map +1 -1
  6. package/autoscaling/snapshotter.d.ts +6 -6
  7. package/autoscaling/snapshotter.d.ts.map +1 -1
  8. package/autoscaling/snapshotter.js +12 -8
  9. package/autoscaling/snapshotter.js.map +1 -1
  10. package/autoscaling/system_status.d.ts +1 -1
  11. package/autoscaling/system_status.d.ts.map +1 -1
  12. package/configuration.d.ts +77 -62
  13. package/configuration.d.ts.map +1 -1
  14. package/configuration.js +85 -87
  15. package/configuration.js.map +1 -1
  16. package/crawlers/crawler_commons.d.ts +7 -12
  17. package/crawlers/crawler_commons.d.ts.map +1 -1
  18. package/crawlers/crawler_utils.d.ts +1 -1
  19. package/crawlers/crawler_utils.d.ts.map +1 -1
  20. package/crawlers/statistics.d.ts.map +1 -1
  21. package/crawlers/statistics.js +1 -1
  22. package/crawlers/statistics.js.map +1 -1
  23. package/enqueue_links/enqueue_links.d.ts +25 -23
  24. package/enqueue_links/enqueue_links.d.ts.map +1 -1
  25. package/enqueue_links/enqueue_links.js +23 -23
  26. package/enqueue_links/enqueue_links.js.map +1 -1
  27. package/enqueue_links/shared.d.ts +11 -6
  28. package/enqueue_links/shared.d.ts.map +1 -1
  29. package/enqueue_links/shared.js +7 -2
  30. package/enqueue_links/shared.js.map +1 -1
  31. package/errors.d.ts +16 -0
  32. package/errors.d.ts.map +1 -0
  33. package/errors.js +22 -0
  34. package/errors.js.map +1 -0
  35. package/events/event_manager.d.ts +12 -8
  36. package/events/event_manager.d.ts.map +1 -1
  37. package/events/event_manager.js +17 -6
  38. package/events/event_manager.js.map +1 -1
  39. package/events/local_event_manager.d.ts +3 -5
  40. package/events/local_event_manager.d.ts.map +1 -1
  41. package/events/local_event_manager.js +3 -5
  42. package/events/local_event_manager.js.map +1 -1
  43. package/index.d.ts +4 -0
  44. package/index.d.ts.map +1 -1
  45. package/index.js +5 -0
  46. package/index.js.map +1 -1
  47. package/index.mjs +6 -0
  48. package/package.json +22 -27
  49. package/proxy_configuration.d.ts +7 -14
  50. package/proxy_configuration.d.ts.map +1 -1
  51. package/proxy_configuration.js +6 -15
  52. package/proxy_configuration.js.map +1 -1
  53. package/request.d.ts +11 -6
  54. package/request.d.ts.map +1 -1
  55. package/request.js +5 -1
  56. package/request.js.map +1 -1
  57. package/router.d.ts +93 -0
  58. package/router.d.ts.map +1 -0
  59. package/router.js +137 -0
  60. package/router.js.map +1 -0
  61. package/serialization.d.ts.map +1 -1
  62. package/serialization.js +1 -2
  63. package/serialization.js.map +1 -1
  64. package/session_pool/session.d.ts +10 -15
  65. package/session_pool/session.d.ts.map +1 -1
  66. package/session_pool/session.js +18 -18
  67. package/session_pool/session.js.map +1 -1
  68. package/session_pool/session_pool.d.ts +3 -2
  69. package/session_pool/session_pool.d.ts.map +1 -1
  70. package/session_pool/session_pool.js +4 -5
  71. package/session_pool/session_pool.js.map +1 -1
  72. package/session_pool/session_utils.d.ts +6 -5
  73. package/session_pool/session_utils.d.ts.map +1 -1
  74. package/session_pool/session_utils.js +3 -0
  75. package/session_pool/session_utils.js.map +1 -1
  76. package/storages/dataset.d.ts +37 -15
  77. package/storages/dataset.d.ts.map +1 -1
  78. package/storages/dataset.js +44 -12
  79. package/storages/dataset.js.map +1 -1
  80. package/storages/index.d.ts +1 -0
  81. package/storages/index.d.ts.map +1 -1
  82. package/storages/index.js +1 -0
  83. package/storages/index.js.map +1 -1
  84. package/storages/key_value_store.d.ts +121 -23
  85. package/storages/key_value_store.d.ts.map +1 -1
  86. package/storages/key_value_store.js +178 -24
  87. package/storages/key_value_store.js.map +1 -1
  88. package/storages/request_list.d.ts +32 -35
  89. package/storages/request_list.d.ts.map +1 -1
  90. package/storages/request_list.js +32 -24
  91. package/storages/request_list.js.map +1 -1
  92. package/storages/request_queue.d.ts +12 -8
  93. package/storages/request_queue.d.ts.map +1 -1
  94. package/storages/request_queue.js +18 -10
  95. package/storages/request_queue.js.map +1 -1
  96. package/storages/storage_manager.d.ts +9 -5
  97. package/storages/storage_manager.d.ts.map +1 -1
  98. package/storages/storage_manager.js +28 -18
  99. package/storages/storage_manager.js.map +1 -1
  100. package/storages/utils.d.ts +16 -0
  101. package/storages/utils.d.ts.map +1 -0
  102. package/storages/utils.js +26 -0
  103. package/storages/utils.js.map +1 -0
  104. package/tsconfig.build.tsbuildinfo +1 -1
  105. package/typedefs.d.ts +0 -2
  106. package/typedefs.d.ts.map +1 -1
  107. package/typedefs.js.map +1 -1
  108. package/validators.d.ts +5 -5
  109. package/validators.d.ts.map +1 -1
  110. package/validators.js.map +1 -1
@@ -1 +1 @@
1
- {"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AACA,gDAA4F;AAC5F,0CAA+C;AAC/C,oDAAoB;AAEpB,oDAAiD;AACjD,gCAA2C;AAI3C,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAC9C,MAAM,mCAAmC,GAAG,KAAK,CAAC;AA2DlD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAa,WAAW;IAuBpB;;OAEG;IACH,YAAY,UAA8B,EAAE;QAzB5C;;;;;WAAS;QACT;;;;;WAAsB;QACtB;;;;;WAAsB;QACtB;;;;;WAAqB;QACrB;;;;;WAAwC;QACxC;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;;WAAyB;QACzB;;;;;WAA2B;QAC3B;;;;;WAAwB;QACxB;;;;;WAAwB;QAExB;;;;mBAA8B,EAAE;WAAC;QACjC;;;;mBAA0C,EAAE;WAAC;QAC7C;;;;mBAAoC,EAAE;WAAC;QACvC;;;;mBAAoC,EAAE;WAAC;QAEvC;;;;mBAAsC,IAAK;WAAC;QAC5C;;;;mBAAmC,IAAK;WAAC;QAEzC;;;;mBAAkD,IAAI;WAAC;QAMnD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,SAAU,EAChB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,GACrC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAE5C,IAAI,CAAC,+BAA+B,GAAG,6BAA6B,GAAG,IAAI,CAAC;QAC5E,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,mFAAmF;QACnF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACnD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAE,CAAC,CAAC;QACvF,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,0EAA0E,CAAC,CAAC,CAAC,8BAA8B;QAE5M,iCAAiC;QACjC,4BAA4B;QAE5B,sBAAsB;QACtB,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACrH,IAAI,CAAC,cAAc,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5G,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACN,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7D,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;QACtD,6BAA6B;IACjC,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,oBAA6B;QAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,oBAA6B;QACtC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;OAEG;IACO,UAAU,CAAgC,SAAc,EAAE,oBAA6B;QAC7F,IAAI,CAAC,oBAAoB;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,GAAG;YAAE,OAAO,MAAM,CAAC;QAExB,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAChD,OAAO,GAAG,EAAE,EAAE;YACV,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,oBAAoB,EAAE;gBAC3D,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;aAC5B;iBAAM;gBACH,MAAM;aACT;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,UAAsB;QAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,QAAQ,GAAmB;YAC7B,SAAS;YACT,YAAY,EAAE,eAAgB,GAAG,IAAI,CAAC,cAAe,GAAG,IAAI,CAAC,kBAAkB;YAC/E,SAAS,EAAE,eAAe;SAC7B,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,UAAsB;QACnD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,IAAI,CAAC,kCAAkC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,kCAAkC,GAAG,mCAAmC;YAAE,OAAO;QAEnJ,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAe,CAAC;QAC7E,MAAM,aAAa,GAAG,IAAI,CAAC,cAAe,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,oBAAoB,CAAC;QAClG,MAAM,qBAAqB,GAAG,qBAAqB,GAAG,aAAa,CAAC;QACpE,MAAM,kBAAkB,GAAG,eAAgB,GAAG,qBAAqB,CAAC;QAEpE,IAAI,kBAAkB,EAAE;YACpB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAgB,GAAG,IAAI,CAAC,cAAe,CAAC,GAAG,GAAG,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;YAChE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,mCAAmC;kBAC9C,SAAS,IAAI,CAAC,eAAgB,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,cAAe,CAAC,QAAQ,cAAc,2CAA2C,CAAC,CAAC;YAC5I,IAAI,CAAC,kCAAkC,GAAG,SAAS,CAAC;SACvD;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,gBAA+B;QACxD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,CAAC;SACpB,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrF,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC;YAEhF,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;YAChE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;SACxE;QAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,UAAsB;QACzC,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACnB,SAAS;YACT,YAAY,EAAE,eAAgB;YAC9B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,eAAgB,GAAG,GAAG,CAAC;SAC/C,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CAAC,gBAA+B;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC,CAAC,wCAAwC;QACzG,MAAM,eAAe,GAAG,cAAc,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC;QAEjF,+BAA+B;QAC/B,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,mBAAmB,EAAE,eAAe;SACvC,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/E,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CAAC;YACjD,MAAM,KAAK,GAAG,eAAe,GAAG,mBAAmB,CAAC;YACpD,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;SAClE;QAED,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,SAAoF,EAAE,GAAS;QACrH,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,qBAAqB;gBAAE,QAAQ,EAAE,CAAC;;gBACtF,MAAM;SACd;QACD,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,OAAO,IAAA,qBAAa,GAAE,CAAC;IAC3B,CAAC;CACJ;AA/RD,kCA+RC"}
1
+ {"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AAEA,gDAA0E;AAC1E,0CAA+C;AAC/C,oDAAoB;AAEpB,oDAAiD;AACjD,gCAA2C;AAK3C,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAC9C,MAAM,mCAAmC,GAAG,KAAK,CAAC;AA2DlD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAa,WAAW;IAuBpB;;OAEG;IACH,YAAY,UAA8B,EAAE;QAzB5C;;;;;WAAS;QACT;;;;;WAAsB;QACtB;;;;;WAAsB;QACtB;;;;;WAAqB;QACrB;;;;;WAAwC;QACxC;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;;WAAyB;QACzB;;;;;WAA2B;QAC3B;;;;;WAAwB;QACxB;;;;;WAAwB;QAExB;;;;mBAA8B,EAAE;WAAC;QACjC;;;;mBAA0C,EAAE;WAAC;QAC7C;;;;mBAAoC,EAAE;WAAC;QACvC;;;;mBAAoC,EAAE;WAAC;QAEvC;;;;mBAAsC,IAAK;WAAC;QAC5C;;;;mBAAmC,IAAK;WAAC;QAEzC;;;;mBAAkD,IAAI;WAAC;QAMnD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,SAAU,EAChB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,GACrC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAE5C,IAAI,CAAC,+BAA+B,GAAG,6BAA6B,GAAG,IAAI,CAAC;QAC5E,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,mFAAmF;QACnF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAExD,IAAI,YAAY,GAAG,CAAC,EAAE;YAClB,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC;SACpD;aAAM;YACH,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YACnD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAE,CAAC,CAAC;YACvF,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,OAAO;kBAClG,sGAAsG,CAAC,CAAC;SACjH;QAED,sBAAsB;QACtB,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACrH,IAAI,CAAC,cAAc,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5G,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACN,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7D,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,oBAA6B;QAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,oBAA6B;QACtC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;OAEG;IACO,UAAU,CAAgC,SAAc,EAAE,oBAA6B;QAC7F,IAAI,CAAC,oBAAoB;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,GAAG;YAAE,OAAO,MAAM,CAAC;QAExB,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAChD,OAAO,GAAG,EAAE,EAAE;YACV,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,oBAAoB,EAAE;gBAC3D,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;aAC5B;iBAAM;gBACH,MAAM;aACT;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,UAAsB;QAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,QAAQ,GAAmB;YAC7B,SAAS;YACT,YAAY,EAAE,eAAgB,GAAG,IAAI,CAAC,cAAe,GAAG,IAAI,CAAC,kBAAkB;YAC/E,SAAS,EAAE,eAAe;SAC7B,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,UAAsB;QACnD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,IAAI,CAAC,kCAAkC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,kCAAkC,GAAG,mCAAmC;YAAE,OAAO;QAEnJ,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAe,CAAC;QAC7E,MAAM,aAAa,GAAG,IAAI,CAAC,cAAe,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,oBAAoB,CAAC;QAClG,MAAM,qBAAqB,GAAG,qBAAqB,GAAG,aAAa,CAAC;QACpE,MAAM,kBAAkB,GAAG,eAAgB,GAAG,qBAAqB,CAAC;QAEpE,IAAI,kBAAkB,EAAE;YACpB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAgB,GAAG,IAAI,CAAC,cAAe,CAAC,GAAG,GAAG,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;YAChE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,mCAAmC;kBAC9C,SAAS,IAAI,CAAC,eAAgB,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,cAAe,CAAC,QAAQ,cAAc,2CAA2C,CAAC,CAAC;YAC5I,IAAI,CAAC,kCAAkC,GAAG,SAAS,CAAC;SACvD;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,gBAA+B;QACxD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,CAAC;SACpB,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrF,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC;YAEhF,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;YAChE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;SACxE;QAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,UAAsB;QACzC,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACnB,SAAS;YACT,YAAY,EAAE,eAAgB;YAC9B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,eAAgB,GAAG,GAAG,CAAC;SAC/C,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CAAC,gBAA+B;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC,CAAC,wCAAwC;QACzG,MAAM,eAAe,GAAG,cAAc,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC;QAEjF,+BAA+B;QAC/B,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,mBAAmB,EAAE,eAAe;SACvC,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/E,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CAAC;YACjD,MAAM,KAAK,GAAG,eAAe,GAAG,mBAAmB,CAAC;YACpD,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;SAClE;QAED,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,SAAoF,EAAE,GAAS;QACrH,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,qBAAqB;gBAAE,QAAQ,EAAE,CAAC;;gBACtF,MAAM;SACd;QACD,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,OAAO,IAAA,qBAAa,GAAE,CAAC;IAC3B,CAAC;CACJ;AAlSD,kCAkSC"}
@@ -1,5 +1,5 @@
1
1
  import { Snapshotter } from './snapshotter';
2
- import { Configuration } from '../configuration';
2
+ import type { Configuration } from '../configuration';
3
3
  /**
4
4
  * Represents the current status of the system.
5
5
  */
@@ -1 +1 @@
1
- {"version":3,"file":"system_status.d.ts","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,2CAA2C;IAC3C,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,UAAU,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,SAAS,CAAC,EAAE,IAAI,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;OAIG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;OAEG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,yBAAyB,EAAE,MAAM,CAAC;IAClC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAS;IACrD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;gBAE9B,OAAO,GAAE,mBAAwB;IA6B7C;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,IAAI,UAAU;IAI9B;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB,IAAI,UAAU;IAIjC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;IAclE;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAK3D;;;OAGG;IACH,SAAS,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAK9D;;;OAGG;IACH,SAAS,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAKxD;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;IAKxE;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAC;QAAC,YAAY,EAAE,OAAO,CAAA;KAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,UAAU;CAoB9H"}
1
+ {"version":3,"file":"system_status.d.ts","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,2CAA2C;IAC3C,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,UAAU,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,SAAS,CAAC,EAAE,IAAI,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;OAIG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;OAEG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,yBAAyB,EAAE,MAAM,CAAC;IAClC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAS;IACrD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;gBAE9B,OAAO,GAAE,mBAAwB;IA6B7C;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,IAAI,UAAU;IAI9B;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB,IAAI,UAAU;IAIjC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;IAclE;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAK3D;;;OAGG;IACH,SAAS,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAK9D;;;OAGG;IACH,SAAS,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAKxD;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;IAKxE;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAC;QAAC,YAAY,EAAE,OAAO,CAAA;KAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,UAAU;CAoB9H"}
@@ -1,98 +1,114 @@
1
1
  /// <reference types="node" />
2
- import { MemoryStorage, MemoryStorageOptions } from '@crawlee/memory-storage';
2
+ import type { MemoryStorageOptions } from '@crawlee/memory-storage';
3
+ import { MemoryStorage } from '@crawlee/memory-storage';
3
4
  import { AsyncLocalStorage } from 'node:async_hooks';
4
- import { StorageClient } from '@crawlee/types';
5
- import { Dictionary } from './typedefs';
6
- import { EventManager } from './events';
5
+ import type { Dictionary, StorageClient } from '@crawlee/types';
6
+ import { LogLevel } from '@apify/log';
7
+ import type { EventManager } from './events';
7
8
  export interface ConfigurationOptions {
8
9
  storageClient?: StorageClient;
9
10
  eventManager?: EventManager;
10
11
  storageClientOptions?: Dictionary;
11
12
  defaultDatasetId?: string;
13
+ purgeOnStart?: boolean;
12
14
  defaultKeyValueStoreId?: string;
13
15
  defaultRequestQueueId?: string;
14
16
  maxUsedCpuRatio?: number;
15
17
  availableMemoryRatio?: number;
18
+ memoryMbytes?: number;
16
19
  persistStateIntervalMillis?: number;
17
20
  systemInfoIntervalMillis?: number;
18
- metamorphAfterSleepMillis?: number;
19
21
  inputKey?: string;
20
- actorId?: string;
21
- actorRunId?: string;
22
- actorTaskId?: string;
23
- containerPort?: number;
24
- containerUrl?: string;
25
- proxyHostname?: string;
26
- proxyPassword?: string;
27
- proxyPort?: number;
22
+ headless?: boolean;
23
+ xvfb?: boolean;
24
+ chromeExecutablePath?: string;
25
+ defaultBrowserPath?: string;
26
+ disableBrowserSandbox?: boolean;
27
+ logLevel?: LogLevel | LogLevel[keyof LogLevel];
28
28
  }
29
29
  /**
30
- * `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
30
+ * `Configuration` is a value object holding Crawlee configuration. By default, there is a
31
+ * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
32
+ * Places that depend on a configurable behaviour depend on this class as have the global
33
+ * instance as the default value.
31
34
  *
32
- * 1. When using `Apify` class, we can get the instance configuration via `sdk.config`
33
- * ```js
34
- * const { Apify } = require('apify');
35
+ * ```js
36
+ * import { BasicCrawler, Configuration } from 'crawlee';
35
37
  *
36
- * const sdk = new Apify({ token: '123' });
37
- * console.log(sdk.config.get('token')); // '123'
38
- * ```
39
- * 2. To get the global configuration (singleton instance). It will respect the environment variables.
40
- * ```js
41
- * // returns the token from APIFY_TOKEN env var
42
- * console.log(Configuration.getGlobalConfig().get('token'));
43
- * ```
38
+ * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
39
+ * const crawler = new BasicCrawler({ ... }, config);
40
+ * ```
41
+ *
42
+ * The configuration provided via environment variables always takes precedence. We can also
43
+ * define the `crawlee.json` file in the project root directory which will serve as a baseline,
44
+ * so the options provided in constructor will override those.
45
+ *
46
+ * > In other words, the precedence is: crawlee.json < constructor options < environment variables.
44
47
  *
45
48
  * ## Supported Configuration Options
46
49
  *
47
50
  * Key | Environment Variable | Default Value
48
51
  * ---|---|---
49
- * `defaultDatasetId` | `APIFY_DEFAULT_DATASET_ID` | `'default'`
50
- * `defaultKeyValueStoreId` | `APIFY_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
51
- * `defaultRequestQueueId` | `APIFY_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
52
- * `persistStateIntervalMillis` | `APIFY_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
52
+ * `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
53
+ * `logLevel` | `CRAWLEE_LOG_LEVEL` | -
54
+ * `headless` | `CRAWLEE_HEADLESS` | -
55
+ * `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
56
+ * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
57
+ * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
58
+ * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
53
59
  *
54
60
  * ## Advanced Configuration Options
55
61
  *
56
62
  * Key | Environment Variable | Default Value
57
63
  * ---|---|---
58
- * `actorId` | `APIFY_ACTOR_ID` | -
59
- * `actorRunId` | `APIFY_ACTOR_RUN_ID` | -
60
- * `actorTaskId` | `APIFY_ACTOR_TASK_ID` | -
61
- * `containerPort` | `APIFY_CONTAINER_PORT` | `4321`
62
- * `containerUrl` | `APIFY_CONTAINER_URL` | `'http://localhost:4321'`
63
- * `inputKey` | `APIFY_INPUT_KEY` | `'INPUT'`
64
- * `metamorphAfterSleepMillis` | `APIFY_METAMORPH_AFTER_SLEEP_MILLIS` | `300e3`
65
- * `proxyHostname` | `APIFY_PROXY_HOSTNAME` | `'proxy.apify.com'`
66
- * `proxyPassword` | `APIFY_PROXY_PASSWORD` | -
67
- * `proxyPort` | `APIFY_PROXY_PORT` | `8000`
68
- *
69
- * ## Not Supported environment variables
70
- *
71
- * - `MEMORY_MBYTES`
72
- * - `HEADLESS`
73
- * - `XVFB`
74
- * - `CHROME_EXECUTABLE_PATH`
64
+ * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
65
+ * `xvfb` | `CRAWLEE_XVFB` | -
66
+ * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
67
+ * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
75
68
  */
76
69
  export declare class Configuration {
77
70
  /**
78
- * Maps environment variables to config keys (e.g. `APIFY_PROXY_PORT` to `proxyPort`)
71
+ * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
79
72
  */
80
- private static ENV_MAP;
81
- /**
82
- * Maps config keys to environment variables (e.g. `proxyPort` to `APIFY_PROXY_PORT`).
83
- */
84
- private static ENV_MAP_REVERSED;
85
- private static BOOLEAN_VARS;
86
- private static INTEGER_VARS;
87
- private static DEFAULTS;
73
+ protected static ENV_MAP: {
74
+ CRAWLEE_AVAILABLE_MEMORY_RATIO: string;
75
+ CRAWLEE_PURGE_ON_START: string;
76
+ CRAWLEE_MEMORY_MBYTES: string;
77
+ CRAWLEE_DEFAULT_DATASET_ID: string;
78
+ CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: string;
79
+ CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: string;
80
+ CRAWLEE_INPUT_KEY: string;
81
+ CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: string;
82
+ CRAWLEE_HEADLESS: string;
83
+ CRAWLEE_XVFB: string;
84
+ CRAWLEE_CHROME_EXECUTABLE_PATH: string;
85
+ CRAWLEE_DEFAULT_BROWSER_PATH: string;
86
+ CRAWLEE_DISABLE_BROWSER_SANDBOX: string;
87
+ CRAWLEE_LOG_LEVEL: string;
88
+ };
89
+ protected static BOOLEAN_VARS: string[];
90
+ protected static INTEGER_VARS: string[];
91
+ protected static DEFAULTS: {
92
+ defaultKeyValueStoreId: string;
93
+ defaultDatasetId: string;
94
+ defaultRequestQueueId: string;
95
+ inputKey: string;
96
+ maxUsedCpuRatio: number;
97
+ availableMemoryRatio: number;
98
+ storageClientOptions: {};
99
+ purgeOnStart: boolean;
100
+ persistStateIntervalMillis: number;
101
+ systemInfoIntervalMillis: number;
102
+ };
88
103
  /**
89
104
  * Provides access to the current-instance-scoped Configuration without passing it around in parameters.
90
105
  * @internal
91
106
  */
92
107
  static storage: AsyncLocalStorage<Configuration>;
93
- private options;
94
- private services;
95
- private static globalConfig?;
108
+ protected options: Map<keyof ConfigurationOptions, ConfigurationOptions[keyof ConfigurationOptions]>;
109
+ protected services: Map<string, unknown>;
110
+ /** @internal */
111
+ static globalConfig?: Configuration;
96
112
  /**
97
113
  * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
98
114
  */
@@ -103,7 +119,7 @@ export declare class Configuration {
103
119
  * in the above section.
104
120
  */
105
121
  get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U;
106
- private _castEnvValue;
122
+ protected _castEnvValue(key: keyof ConfigurationOptions, value: number | string | boolean): string | number | boolean;
107
123
  /**
108
124
  * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
109
125
  * To reset a value, we can omit the `value` argument or pass `undefined` there.
@@ -114,7 +130,7 @@ export declare class Configuration {
114
130
  * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
115
131
  * return the same client instance.
116
132
  *
117
- * Caching works based on the API URL and token, so calling this method with different options will return
133
+ * Caching works based on the `storageClientOptions`, so calling this method with different options will return
118
134
  * multiple instances, one for each variant of the options.
119
135
  * @internal
120
136
  */
@@ -129,8 +145,6 @@ export declare class Configuration {
129
145
  useEventManager(events: EventManager): void;
130
146
  /**
131
147
  * Returns the global configuration instance. It will respect the environment variables.
132
- * When used inside SDK instance (`sdk.main()`), the return value is equal to `sdk.config`.
133
- *
134
148
  */
135
149
  static getGlobalConfig(): Configuration;
136
150
  /**
@@ -146,5 +160,6 @@ export declare class Configuration {
146
160
  * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
147
161
  */
148
162
  static resetGlobalState(): void;
163
+ protected buildOptions(options: ConfigurationOptions): void;
149
164
  }
150
165
  //# sourceMappingURL=configuration.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAW,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,YAAY,EAAqB,MAAM,UAAU,CAAC;AAG3D,MAAM,WAAW,oBAAoB;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,OAAO,CAyBpB;IAEF;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAGE;IAEjC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAgB;IAE3C,OAAO,CAAC,MAAM,CAAC,YAAY,CAAkD;IAE7E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAerB;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,OAAO,CAAC,OAAO,CAAoF;IACnG,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAgB;IAE5C;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAO9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IAkBzG,OAAO,CAAC,aAAa;IAarB;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAatE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;;;OAIG;IACH,MAAM,CAAC,eAAe,IAAK,aAAa;IASxC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;CAGlC"}
1
+ {"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG7C,MAAM,WAAW,oBAAoB;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;CAClD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,OAAO;;;;;;;;;;;;;;;MAetB;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,CAAiE;IAExG,SAAS,CAAC,MAAM,CAAC,YAAY,WAA8E;IAE3G,SAAS,CAAC,MAAM,CAAC,QAAQ;;;;;;;;;;;MAWvB;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,SAAS,CAAC,OAAO,EAAG,GAAG,CAAC,MAAM,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;IACtG,SAAS,CAAC,QAAQ,uBAA8B;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAC,EAAE,aAAa,CAAC;IAEpC;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAe9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IA2BzG,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IAazF;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAatE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,aAAa;IASvC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;IAI/B,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,oBAAoB;CAgBvD"}
package/configuration.js CHANGED
@@ -1,58 +1,54 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.Configuration = void 0;
4
- const consts_1 = require("@apify/consts");
4
+ const tslib_1 = require("tslib");
5
5
  const memory_storage_1 = require("@crawlee/memory-storage");
6
+ const fs_extra_1 = require("fs-extra");
7
+ const node_path_1 = require("node:path");
6
8
  const node_async_hooks_1 = require("node:async_hooks");
7
9
  const node_events_1 = require("node:events");
10
+ const log_1 = tslib_1.__importStar(require("@apify/log"));
8
11
  const typedefs_1 = require("./typedefs");
9
12
  const events_1 = require("./events");
10
13
  /**
11
- * `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
14
+ * `Configuration` is a value object holding Crawlee configuration. By default, there is a
15
+ * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
16
+ * Places that depend on a configurable behaviour depend on this class as have the global
17
+ * instance as the default value.
12
18
  *
13
- * 1. When using `Apify` class, we can get the instance configuration via `sdk.config`
14
- * ```js
15
- * const { Apify } = require('apify');
19
+ * ```js
20
+ * import { BasicCrawler, Configuration } from 'crawlee';
16
21
  *
17
- * const sdk = new Apify({ token: '123' });
18
- * console.log(sdk.config.get('token')); // '123'
19
- * ```
20
- * 2. To get the global configuration (singleton instance). It will respect the environment variables.
21
- * ```js
22
- * // returns the token from APIFY_TOKEN env var
23
- * console.log(Configuration.getGlobalConfig().get('token'));
24
- * ```
22
+ * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
23
+ * const crawler = new BasicCrawler({ ... }, config);
24
+ * ```
25
+ *
26
+ * The configuration provided via environment variables always takes precedence. We can also
27
+ * define the `crawlee.json` file in the project root directory which will serve as a baseline,
28
+ * so the options provided in constructor will override those.
29
+ *
30
+ * > In other words, the precedence is: crawlee.json < constructor options < environment variables.
25
31
  *
26
32
  * ## Supported Configuration Options
27
33
  *
28
34
  * Key | Environment Variable | Default Value
29
35
  * ---|---|---
30
- * `defaultDatasetId` | `APIFY_DEFAULT_DATASET_ID` | `'default'`
31
- * `defaultKeyValueStoreId` | `APIFY_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
32
- * `defaultRequestQueueId` | `APIFY_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
33
- * `persistStateIntervalMillis` | `APIFY_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
36
+ * `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
37
+ * `logLevel` | `CRAWLEE_LOG_LEVEL` | -
38
+ * `headless` | `CRAWLEE_HEADLESS` | -
39
+ * `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
40
+ * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
41
+ * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
42
+ * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
34
43
  *
35
44
  * ## Advanced Configuration Options
36
45
  *
37
46
  * Key | Environment Variable | Default Value
38
47
  * ---|---|---
39
- * `actorId` | `APIFY_ACTOR_ID` | -
40
- * `actorRunId` | `APIFY_ACTOR_RUN_ID` | -
41
- * `actorTaskId` | `APIFY_ACTOR_TASK_ID` | -
42
- * `containerPort` | `APIFY_CONTAINER_PORT` | `4321`
43
- * `containerUrl` | `APIFY_CONTAINER_URL` | `'http://localhost:4321'`
44
- * `inputKey` | `APIFY_INPUT_KEY` | `'INPUT'`
45
- * `metamorphAfterSleepMillis` | `APIFY_METAMORPH_AFTER_SLEEP_MILLIS` | `300e3`
46
- * `proxyHostname` | `APIFY_PROXY_HOSTNAME` | `'proxy.apify.com'`
47
- * `proxyPassword` | `APIFY_PROXY_PASSWORD` | -
48
- * `proxyPort` | `APIFY_PROXY_PORT` | `8000`
49
- *
50
- * ## Not Supported environment variables
51
- *
52
- * - `MEMORY_MBYTES`
53
- * - `HEADLESS`
54
- * - `XVFB`
55
- * - `CHROME_EXECUTABLE_PATH`
48
+ * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
49
+ * `xvfb` | `CRAWLEE_XVFB` | -
50
+ * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
51
+ * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
56
52
  */
57
53
  class Configuration {
58
54
  /**
@@ -71,9 +67,15 @@ class Configuration {
71
67
  writable: true,
72
68
  value: new Map()
73
69
  });
74
- this.options = new Map((0, typedefs_1.entries)(options));
70
+ this.buildOptions(options);
75
71
  // Increase the global limit for event emitter memory leak warnings.
76
72
  node_events_1.EventEmitter.defaultMaxListeners = 50;
73
+ // set the log level to support CRAWLEE_ prefixed env var too
74
+ const logLevel = this.get('logLevel');
75
+ if (logLevel) {
76
+ const level = Number.isFinite(+logLevel) ? +logLevel : log_1.LogLevel[String(logLevel).toUpperCase()];
77
+ log_1.default.setLevel(level);
78
+ }
77
79
  }
78
80
  /**
79
81
  * Returns configured value. First checks the environment variables, then provided configuration,
@@ -81,9 +83,16 @@ class Configuration {
81
83
  * in the above section.
82
84
  */
83
85
  get(key, defaultValue) {
84
- // prefer env vars
85
- const envKey = Configuration.ENV_MAP_REVERSED[key] ?? key;
86
- const envValue = process.env[envKey];
86
+ // prefer env vars, always iterate through the whole map as there might be duplicate env vars for the same option
87
+ let envValue;
88
+ for (const [k, v] of (0, typedefs_1.entries)(Configuration.ENV_MAP)) {
89
+ if (key === v) {
90
+ envValue = process.env[k];
91
+ if (envValue) {
92
+ break;
93
+ }
94
+ }
95
+ }
87
96
  if (envValue != null) {
88
97
  return this._castEnvValue(key, envValue);
89
98
  }
@@ -116,7 +125,7 @@ class Configuration {
116
125
  * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
117
126
  * return the same client instance.
118
127
  *
119
- * Caching works based on the API URL and token, so calling this method with different options will return
128
+ * Caching works based on the `storageClientOptions`, so calling this method with different options will return
120
129
  * multiple instances, one for each variant of the options.
121
130
  * @internal
122
131
  */
@@ -159,8 +168,6 @@ class Configuration {
159
168
  }
160
169
  /**
161
170
  * Returns the global configuration instance. It will respect the environment variables.
162
- * When used inside SDK instance (`sdk.main()`), the return value is equal to `sdk.config`.
163
- *
164
171
  */
165
172
  static getGlobalConfig() {
166
173
  if (Configuration.storage.getStore()) {
@@ -188,81 +195,72 @@ class Configuration {
188
195
  static resetGlobalState() {
189
196
  delete this.globalConfig;
190
197
  }
198
+ buildOptions(options) {
199
+ // try to load configuration from crawlee.json as the baseline
200
+ const path = (0, node_path_1.join)(process.cwd(), 'crawlee.json');
201
+ if ((0, fs_extra_1.pathExistsSync)(path)) {
202
+ try {
203
+ const file = (0, fs_extra_1.readFileSync)(path);
204
+ const optionsFromFileConfig = JSON.parse(file.toString());
205
+ Object.assign(options, optionsFromFileConfig);
206
+ }
207
+ catch {
208
+ // ignore
209
+ }
210
+ }
211
+ this.options = new Map((0, typedefs_1.entries)(options));
212
+ }
191
213
  }
192
214
  exports.Configuration = Configuration;
193
215
  /**
194
- * Maps environment variables to config keys (e.g. `APIFY_PROXY_PORT` to `proxyPort`)
216
+ * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
195
217
  */
196
218
  Object.defineProperty(Configuration, "ENV_MAP", {
197
219
  enumerable: true,
198
220
  configurable: true,
199
221
  writable: true,
200
222
  value: {
201
- // TODO prefix once we have a package name
202
- AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
203
- APIFY_DEFAULT_DATASET_ID: 'defaultDatasetId',
204
- APIFY_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
205
- APIFY_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
206
- APIFY_METAMORPH_AFTER_SLEEP_MILLIS: 'metamorphAfterSleepMillis',
207
- APIFY_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
208
- APIFY_TEST_PERSIST_INTERVAL_MILLIS: 'persistStateIntervalMillis',
209
- APIFY_INPUT_KEY: 'inputKey',
210
- APIFY_ACTOR_ID: 'actorId',
211
- APIFY_ACTOR_RUN_ID: 'actorRunId',
212
- APIFY_ACTOR_TASK_ID: 'actorTaskId',
213
- APIFY_CONTAINER_PORT: 'containerPort',
214
- APIFY_CONTAINER_URL: 'containerUrl',
215
- APIFY_PROXY_HOSTNAME: 'proxyHostname',
216
- APIFY_PROXY_PASSWORD: 'proxyPassword',
217
- APIFY_PROXY_PORT: 'proxyPort',
218
- // not supported, use env vars directly:
219
- // APIFY_MEMORY_MBYTES: 'memoryMbytes',
220
- // APIFY_HEADLESS: 'headless',
221
- // APIFY_XVFB: 'xvfb',
222
- // APIFY_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
223
+ CRAWLEE_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
224
+ CRAWLEE_PURGE_ON_START: 'purgeOnStart',
225
+ CRAWLEE_MEMORY_MBYTES: 'memoryMbytes',
226
+ CRAWLEE_DEFAULT_DATASET_ID: 'defaultDatasetId',
227
+ CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
228
+ CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
229
+ CRAWLEE_INPUT_KEY: 'inputKey',
230
+ CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
231
+ CRAWLEE_HEADLESS: 'headless',
232
+ CRAWLEE_XVFB: 'xvfb',
233
+ CRAWLEE_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
234
+ CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
235
+ CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
236
+ CRAWLEE_LOG_LEVEL: 'logLevel',
223
237
  }
224
238
  });
225
- /**
226
- * Maps config keys to environment variables (e.g. `proxyPort` to `APIFY_PROXY_PORT`).
227
- */
228
- Object.defineProperty(Configuration, "ENV_MAP_REVERSED", {
229
- enumerable: true,
230
- configurable: true,
231
- writable: true,
232
- value: (0, typedefs_1.entries)(Configuration.ENV_MAP).reduce((obj, [key, value]) => {
233
- obj[value] = key;
234
- return obj;
235
- }, {})
236
- });
237
239
  Object.defineProperty(Configuration, "BOOLEAN_VARS", {
238
240
  enumerable: true,
239
241
  configurable: true,
240
242
  writable: true,
241
- value: []
243
+ value: ['purgeOnStart', 'headless', 'xvfb', 'disableBrowserSandbox']
242
244
  });
243
245
  Object.defineProperty(Configuration, "INTEGER_VARS", {
244
246
  enumerable: true,
245
247
  configurable: true,
246
248
  writable: true,
247
- value: ['proxyPort', 'memoryMbytes', 'containerPort']
249
+ value: ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis']
248
250
  });
249
251
  Object.defineProperty(Configuration, "DEFAULTS", {
250
252
  enumerable: true,
251
253
  configurable: true,
252
254
  writable: true,
253
255
  value: {
254
- defaultKeyValueStoreId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID],
255
- defaultDatasetId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_DATASET_ID],
256
- defaultRequestQueueId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_REQUEST_QUEUE_ID],
256
+ defaultKeyValueStoreId: 'default',
257
+ defaultDatasetId: 'default',
258
+ defaultRequestQueueId: 'default',
259
+ inputKey: 'INPUT',
257
260
  maxUsedCpuRatio: 0.95,
258
261
  availableMemoryRatio: 0.25,
259
262
  storageClientOptions: {},
260
- inputKey: 'INPUT',
261
- proxyHostname: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_HOSTNAME],
262
- proxyPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_PORT],
263
- containerPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_PORT],
264
- containerUrl: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_URL],
265
- metamorphAfterSleepMillis: 300000,
263
+ purgeOnStart: true,
266
264
  persistStateIntervalMillis: 60000,
267
265
  systemInfoIntervalMillis: 60000,
268
266
  }
@@ -1 +1 @@
1
- {"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;AAAA,0CAAyD;AACzD,4DAA8E;AAC9E,uDAAqD;AACrD,6CAA2C;AAE3C,yCAAiD;AACjD,qCAA2D;AA2B3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAa,aAAa;IAsEtB;;OAEG;IACH,YAAY,UAAgC,EAAE;QAP9C;;;;;WAAmG;QACnG;;;;mBAAmB,IAAI,GAAG,EAAmB;WAAC;QAO1C,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAA,kBAAO,EAAC,OAAO,CAAC,CAAC,CAAC;QAEzC,oEAAoE;QACpE,0BAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,kBAAkB;QAClB,MAAM,MAAM,GAAG,aAAa,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;QAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErC,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;SACjD;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACrC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAEO,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACnF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;SAC7D;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC3D;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC5D;QAED,MAAM,YAAY,GAAG,IAAI,0BAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;SACvD;QAED,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;SAC5C;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;;AAxNL,sCAyNC;AAxNG;;GAEG;AACH;;;;WAAyB;QACrB,0CAA0C;QAC1C,sBAAsB,EAAE,sBAAsB;QAE9C,wBAAwB,EAAE,kBAAkB;QAC5C,gCAAgC,EAAE,wBAAwB;QAC1D,8BAA8B,EAAE,uBAAuB;QACvD,kCAAkC,EAAE,2BAA2B;QAC/D,mCAAmC,EAAE,4BAA4B;QACjE,kCAAkC,EAAE,4BAA4B;QAChE,eAAe,EAAE,UAAU;QAC3B,cAAc,EAAE,SAAS;QACzB,kBAAkB,EAAE,YAAY;QAChC,mBAAmB,EAAE,aAAa;QAClC,oBAAoB,EAAE,eAAe;QACrC,mBAAmB,EAAE,cAAc;QACnC,oBAAoB,EAAE,eAAe;QACrC,oBAAoB,EAAE,eAAe;QACrC,gBAAgB,EAAE,WAAW;QAE7B,wCAAwC;QACxC,uCAAuC;QACvC,8BAA8B;QAC9B,sBAAsB;QACtB,wDAAwD;KAC3D;GAAC;AAEF;;GAEG;AACH;;;;WAAkC,IAAA,kBAAO,EAAC,aAAa,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC1F,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;QACjB,OAAO,GAAG,CAAC;IACf,CAAC,EAAE,EAA4B,CAAC;GAAC;AAEjC;;;;WAAwC,EAAE;GAAC;AAE3C;;;;WAA8B,CAAC,WAAW,EAAE,cAAc,EAAE,eAAe,CAAC;GAAC;AAE7E;;;;WAA0B;QACtB,sBAAsB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,0BAA0B,CAAC;QAC3E,gBAAgB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,kBAAkB,CAAC;QAC7D,qBAAqB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,wBAAwB,CAAC;QACxE,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,QAAQ,EAAE,OAAO;QACjB,aAAa,EAAE,uBAAc,CAAC,iBAAQ,CAAC,cAAc,CAAC;QACtD,SAAS,EAAE,CAAC,uBAAc,CAAC,iBAAQ,CAAC,UAAU,CAAC;QAC/C,aAAa,EAAE,CAAC,uBAAc,CAAC,iBAAQ,CAAC,cAAc,CAAC;QACvD,YAAY,EAAE,uBAAc,CAAC,iBAAQ,CAAC,aAAa,CAAC;QACpD,yBAAyB,EAAE,MAAO;QAClC,0BAA0B,EAAE,KAAM;QAClC,wBAAwB,EAAE,KAAM;KACnC;GAAC;AAEF;;;GAGG;AACH;;;;WAAiB,IAAI,oCAAiB,EAAiB;GAAC"}
1
+ {"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;;AACA,4DAAwD;AACxD,uCAAwD;AACxD,yCAAiC;AACjC,uDAAqD;AACrD,6CAA2C;AAE3C,0DAA2C;AAC3C,yCAAqC;AAErC,qCAA6C;AAwB7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAa,aAAa;IAkDtB;;OAEG;IACH,YAAY,UAAgC,EAAE;QAT9C;;;;;WAAsG;QACtG;;;;mBAAqB,IAAI,GAAG,EAAmB;WAAC;QAS5C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,0BAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,QAAQ,EAAE;YACV,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAyB,CAAC,CAAC;YACvH,aAAG,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;SACnC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAA,kBAAO,EAAC,aAAa,CAAC,OAAO,CAAC,EAAE;YACjD,IAAI,GAAG,KAAK,CAAC,EAAE;gBACX,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAE1B,IAAI,QAAQ,EAAE;oBACV,MAAM;iBACT;aACJ;SACJ;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;SACjD;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACrC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;SAC7D;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC3D;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC5D;QAED,MAAM,YAAY,GAAG,IAAI,0BAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;SACvD;QAED,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;SAC5C;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAA,gBAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,IAAA,yBAAc,EAAC,IAAI,CAAC,EAAE;YACtB,IAAI;gBACA,MAAM,IAAI,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;aACjD;YAAC,MAAM;gBACJ,SAAS;aACZ;SACJ;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAA,kBAAO,EAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC;;AApOL,sCAqOC;AApOG;;GAEG;AACH;;;;WAA2B;QACvB,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;QACxD,iBAAiB,EAAE,UAAU;KAChC;GAAC;AAEF;;;;WAA0C,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,uBAAuB,CAAC;GAAC;AAExG;;;;WAAgC,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC;GAAC;AAE3G;;;;WAA4B;QACxB,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,0BAA0B,EAAE,KAAM;QAClC,wBAAwB,EAAE,KAAM;KACnC;GAAC;AAEF;;;GAGG;AACH;;;;WAAiB,IAAI,oCAAiB,EAAiB;GAAC"}
@@ -1,13 +1,14 @@
1
- import { Log } from '../log';
2
- import { ProxyInfo } from '../proxy_configuration';
3
- import { Request } from '../request';
4
- import { Session } from '../session_pool/session';
5
- export interface CrawlingContext extends Record<PropertyKey, unknown> {
1
+ import type { Dictionary } from '@crawlee/types';
2
+ import type { Log } from '../log';
3
+ import type { ProxyInfo } from '../proxy_configuration';
4
+ import type { Request } from '../request';
5
+ import type { Session } from '../session_pool/session';
6
+ export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends Record<PropertyKey, unknown> {
6
7
  id: string;
7
8
  /**
8
9
  * The original {@link Request} object.
9
10
  */
10
- request: Request;
11
+ request: Request<UserData>;
11
12
  session?: Session;
12
13
  /**
13
14
  * An object with information about currently used proxy by the crawler
@@ -16,10 +17,4 @@ export interface CrawlingContext extends Record<PropertyKey, unknown> {
16
17
  proxyInfo?: ProxyInfo;
17
18
  log: Log;
18
19
  }
19
- export interface CrawlerHandleFailedRequestInput extends CrawlingContext {
20
- /**
21
- * The Error thrown by `requestHandler`.
22
- */
23
- error: Error;
24
- }
25
20
  //# sourceMappingURL=crawler_commons.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,MAAM,WAAW,eAAgB,SAAQ,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC;IACjE,EAAE,EAAE,MAAM,CAAC;IACX;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,+BAAgC,SAAQ,eAAe;IACpE;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC;CAChB"}
1
+ {"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEvD,MAAM,WAAW,eAAe,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC;IAC3G,EAAE,EAAE,MAAM,CAAC;IACX;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3B,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,GAAG,EAAE,GAAG,CAAC;CACZ"}
@@ -1,4 +1,4 @@
1
- import { Session } from '../session_pool/session';
1
+ import type { Session } from '../session_pool/session';
2
2
  /**
3
3
  * Handles timeout request
4
4
  * @internal
@@ -1 +1 @@
1
- {"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,QAK1G;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CAsBzE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,SAAK,EAAE,aAAa,SAAK,GAAG,MAAM,CAmBvF"}
1
+ {"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEvD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,QAK1G;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CAsBzE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,SAAK,EAAE,aAAa,SAAK,GAAG,MAAM,CAmBvF"}