@crawlee/core 3.10.1-beta.4 → 3.10.1-beta.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/autoscaling/autoscaled_pool.d.ts.map +1 -1
  2. package/autoscaling/autoscaled_pool.js +3 -3
  3. package/autoscaling/autoscaled_pool.js.map +1 -1
  4. package/autoscaling/snapshotter.d.ts.map +1 -1
  5. package/autoscaling/snapshotter.js +7 -6
  6. package/autoscaling/snapshotter.js.map +1 -1
  7. package/autoscaling/system_status.d.ts.map +1 -1
  8. package/autoscaling/system_status.js +4 -1
  9. package/autoscaling/system_status.js.map +1 -1
  10. package/configuration.d.ts.map +1 -1
  11. package/configuration.js +3 -1
  12. package/configuration.js.map +1 -1
  13. package/cookie_utils.d.ts.map +1 -1
  14. package/cookie_utils.js +4 -2
  15. package/cookie_utils.js.map +1 -1
  16. package/crawlers/crawler_commons.d.ts.map +1 -1
  17. package/crawlers/crawler_commons.js +13 -5
  18. package/crawlers/crawler_commons.js.map +1 -1
  19. package/crawlers/error_snapshotter.d.ts +7 -4
  20. package/crawlers/error_snapshotter.d.ts.map +1 -1
  21. package/crawlers/error_snapshotter.js +9 -2
  22. package/crawlers/error_snapshotter.js.map +1 -1
  23. package/crawlers/error_tracker.d.ts.map +1 -1
  24. package/crawlers/error_tracker.js +18 -10
  25. package/crawlers/error_tracker.js.map +1 -1
  26. package/crawlers/statistics.d.ts.map +1 -1
  27. package/crawlers/statistics.js +3 -1
  28. package/crawlers/statistics.js.map +1 -1
  29. package/enqueue_links/enqueue_links.d.ts.map +1 -1
  30. package/enqueue_links/enqueue_links.js +4 -3
  31. package/enqueue_links/enqueue_links.js.map +1 -1
  32. package/enqueue_links/shared.d.ts.map +1 -1
  33. package/enqueue_links/shared.js +6 -7
  34. package/enqueue_links/shared.js.map +1 -1
  35. package/events/local_event_manager.d.ts.map +1 -1
  36. package/events/local_event_manager.js +2 -2
  37. package/events/local_event_manager.js.map +1 -1
  38. package/log.d.ts +1 -1
  39. package/log.d.ts.map +1 -1
  40. package/log.js.map +1 -1
  41. package/package.json +5 -5
  42. package/proxy_configuration.d.ts.map +1 -1
  43. package/proxy_configuration.js +1 -1
  44. package/proxy_configuration.js.map +1 -1
  45. package/request.d.ts +1 -1
  46. package/request.d.ts.map +1 -1
  47. package/request.js +12 -16
  48. package/request.js.map +1 -1
  49. package/router.d.ts.map +1 -1
  50. package/router.js +3 -3
  51. package/router.js.map +1 -1
  52. package/serialization.d.ts.map +1 -1
  53. package/serialization.js.map +1 -1
  54. package/session_pool/session.d.ts.map +1 -1
  55. package/session_pool/session.js +3 -1
  56. package/session_pool/session.js.map +1 -1
  57. package/session_pool/session_pool.d.ts.map +1 -1
  58. package/session_pool/session_pool.js +1 -4
  59. package/session_pool/session_pool.js.map +1 -1
  60. package/storages/access_checking.d.ts.map +1 -1
  61. package/storages/access_checking.js.map +1 -1
  62. package/storages/dataset.d.ts.map +1 -1
  63. package/storages/dataset.js +4 -10
  64. package/storages/dataset.js.map +1 -1
  65. package/storages/key_value_store.d.ts +4 -0
  66. package/storages/key_value_store.d.ts.map +1 -1
  67. package/storages/key_value_store.js +12 -4
  68. package/storages/key_value_store.js.map +1 -1
  69. package/storages/request_list.d.ts +4 -4
  70. package/storages/request_list.d.ts.map +1 -1
  71. package/storages/request_list.js +10 -8
  72. package/storages/request_list.js.map +1 -1
  73. package/storages/request_provider.d.ts.map +1 -1
  74. package/storages/request_provider.js +31 -16
  75. package/storages/request_provider.js.map +1 -1
  76. package/storages/request_queue.d.ts.map +1 -1
  77. package/storages/request_queue.js +13 -10
  78. package/storages/request_queue.js.map +1 -1
  79. package/storages/request_queue_v2.d.ts.map +1 -1
  80. package/storages/request_queue_v2.js +3 -1
  81. package/storages/request_queue_v2.js.map +1 -1
  82. package/storages/storage_manager.d.ts.map +1 -1
  83. package/storages/storage_manager.js +2 -2
  84. package/storages/storage_manager.js.map +1 -1
  85. package/storages/utils.d.ts.map +1 -1
  86. package/storages/utils.js +11 -11
  87. package/storages/utils.js.map +1 -1
  88. package/tsconfig.build.tsbuildinfo +1 -1
  89. package/typedefs.d.ts.map +1 -1
package/storages/utils.js CHANGED
@@ -6,11 +6,13 @@ const node_crypto_1 = tslib_1.__importDefault(require("node:crypto"));
6
6
  const key_value_store_1 = require("./key_value_store");
7
7
  const configuration_1 = require("../configuration");
8
8
  async function purgeDefaultStorages(configOrOptions, client) {
9
- const options = configOrOptions instanceof configuration_1.Configuration ? {
10
- client,
11
- config: configOrOptions,
12
- } : configOrOptions ?? {};
13
- const { config = configuration_1.Configuration.getGlobalConfig(), onlyPurgeOnce = false, } = options;
9
+ const options = configOrOptions instanceof configuration_1.Configuration
10
+ ? {
11
+ client,
12
+ config: configOrOptions,
13
+ }
14
+ : configOrOptions ?? {};
15
+ const { config = configuration_1.Configuration.getGlobalConfig(), onlyPurgeOnce = false } = options;
14
16
  ({ client = config.getStorageClient() } = options);
15
17
  const casted = client;
16
18
  // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
@@ -30,7 +32,9 @@ exports.purgeDefaultStorages = purgeDefaultStorages;
30
32
  * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
31
33
  */
32
34
  async function useState(name, defaultValue = {}, options) {
33
- const kvStore = await key_value_store_1.KeyValueStore.open(options?.keyValueStoreName, { config: options?.config || configuration_1.Configuration.getGlobalConfig() });
35
+ const kvStore = await key_value_store_1.KeyValueStore.open(options?.keyValueStoreName, {
36
+ config: options?.config || configuration_1.Configuration.getGlobalConfig(),
37
+ });
34
38
  return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
35
39
  }
36
40
  exports.useState = useState;
@@ -44,11 +48,7 @@ exports.useState = useState;
44
48
  * @internal
45
49
  */
46
50
  function getRequestId(uniqueKey) {
47
- const str = node_crypto_1.default
48
- .createHash('sha256')
49
- .update(uniqueKey)
50
- .digest('base64')
51
- .replace(/[+/=]/g, '');
51
+ const str = node_crypto_1.default.createHash('sha256').update(uniqueKey).digest('base64').replace(/[+/=]/g, '');
52
52
  return str.slice(0, 15);
53
53
  }
54
54
  exports.getRequestId = getRequestId;
@@ -1 +1 @@
1
- {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":";;;;AAAA,sEAAiC;AAIjC,uDAAkD;AAClD,oDAAiD;AAwC1C,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GAA+B,eAAe,YAAY,6BAAa,CAAC,CAAC,CAAC;QACnF,MAAM;QACN,MAAM,EAAE,eAAe;KAC1B,CAAC,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC;IAC1B,MAAM,EACF,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,aAAa,GAAG,KAAK,GACxB,GAAG,OAAO,CAAC;IACZ,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AArBD,oDAqBC;AAWD;;;;;;;;GAQG;AACI,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,6BAAa,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IACrI,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AAPD,4BAOC;AAED;;;;;;;;GAQG;AACH,SAAgB,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,qBAAM;SACb,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAE3B,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AARD,oCAQC;AAED;;;GAGG;AACU,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACU,QAAA,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AACH,QAAA,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACU,QAAA,mCAAmC,GAAG,KAAM,CAAC;AAE1D;;;GAGG;AACU,QAAA,2BAA2B,GAAG,CAAC,CAAC"}
1
+ {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":";;;;AAAA,sEAAiC;AAIjC,uDAAkD;AAClD,oDAAiD;AAwC1C,KAAK,UAAU,oBAAoB,CACtC,eAA4D,EAC5D,MAAsB;IAEtB,MAAM,OAAO,GACT,eAAe,YAAY,6BAAa;QACpC,CAAC,CAAC;YACI,MAAM;YACN,MAAM,EAAE,eAAe;SAC1B;QACH,CAAC,CAAC,eAAe,IAAI,EAAE,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IACpF,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAgD,CAAC;IAEhE,oGAAoG;IACpG,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,CAAC;AACL,CAAC;AArBD,oDAqBC;AAWD;;;;;;;;GAQG;AACI,KAAK,UAAU,QAAQ,CAC1B,IAAa,EACb,eAAe,EAAW,EAC1B,OAAyB;IAEzB,MAAM,OAAO,GAAG,MAAM,+BAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE;QACjE,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,6BAAa,CAAC,eAAe,EAAE;KAC7D,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,iBAAiB,CAAQ,IAAI,IAAI,sBAAsB,EAAE,YAAY,CAAC,CAAC;AAC1F,CAAC;AATD,4BASC;AAED;;;;;;;;GAQG;AACH,SAAgB,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,qBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEjG,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAJD,oCAIC;AAED;;;GAGG;AACU,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;GAIG;AACU,QAAA,gCAAgC,GAAG,IAAI,CAAC;AAErD,gBAAgB;AACH,QAAA,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACU,QAAA,mCAAmC,GAAG,KAAM,CAAC;AAE1D;;;GAGG;AACU,QAAA,2BAA2B,GAAG,CAAC,CAAC"}