@crawlee/basic 4.0.0-beta.170 → 4.0.0-beta.171

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/index.d.ts +10 -0
  2. package/index.js +10 -0
  3. package/internals/autoscaling/autoscaled_pool.d.ts +196 -0
  4. package/internals/autoscaling/autoscaled_pool.js +385 -0
  5. package/internals/autoscaling/concurrency_system.d.ts +267 -0
  6. package/internals/autoscaling/concurrency_system.js +364 -0
  7. package/internals/autoscaling/cpu_load_signal.d.ts +43 -0
  8. package/internals/autoscaling/cpu_load_signal.js +46 -0
  9. package/internals/autoscaling/event_loop_load_signal.d.ts +51 -0
  10. package/internals/autoscaling/event_loop_load_signal.js +60 -0
  11. package/internals/autoscaling/index.d.ts +9 -0
  12. package/internals/autoscaling/index.js +9 -0
  13. package/internals/autoscaling/load_signal.d.ts +98 -0
  14. package/internals/autoscaling/load_signal.js +103 -0
  15. package/internals/autoscaling/memory_load_signal.d.ts +47 -0
  16. package/internals/autoscaling/memory_load_signal.js +104 -0
  17. package/internals/autoscaling/snapshotter.d.ts +84 -0
  18. package/internals/autoscaling/snapshotter.js +67 -0
  19. package/internals/autoscaling/storage_backend_load_signal.d.ts +56 -0
  20. package/internals/autoscaling/storage_backend_load_signal.js +73 -0
  21. package/internals/autoscaling/system_status.d.ts +122 -0
  22. package/internals/autoscaling/system_status.js +139 -0
  23. package/internals/autoscaling/weighted_avg.d.ts +5 -0
  24. package/internals/autoscaling/weighted_avg.js +14 -0
  25. package/internals/basic-crawler.d.ts +11 -2
  26. package/internals/basic-crawler.js +13 -1
  27. package/internals/cookie_utils.d.ts +44 -0
  28. package/internals/cookie_utils.js +122 -0
  29. package/internals/crawler-run.d.ts +3 -1
  30. package/internals/crawler-run.js +1 -1
  31. package/internals/crawlers/context_pipeline.d.ts +70 -0
  32. package/internals/crawlers/context_pipeline.js +122 -0
  33. package/internals/crawlers/crawler_commons.d.ts +193 -0
  34. package/internals/crawlers/crawler_commons.js +1 -0
  35. package/internals/crawlers/error_snapshotter.d.ts +57 -0
  36. package/internals/crawlers/error_snapshotter.js +117 -0
  37. package/internals/crawlers/error_tracker.d.ts +54 -0
  38. package/internals/crawlers/error_tracker.js +308 -0
  39. package/internals/crawlers/index.d.ts +5 -0
  40. package/internals/crawlers/index.js +4 -0
  41. package/internals/crawlers/internals/types.d.ts +7 -0
  42. package/internals/crawlers/internals/types.js +1 -0
  43. package/internals/crawlers/statistics.d.ts +318 -0
  44. package/internals/crawlers/statistics.js +525 -0
  45. package/internals/enqueue_links/enqueue_links.d.ts +152 -0
  46. package/internals/enqueue_links/enqueue_links.js +76 -0
  47. package/internals/enqueue_links/index.d.ts +2 -0
  48. package/internals/enqueue_links/index.js +2 -0
  49. package/internals/enqueue_links/shared.d.ts +98 -0
  50. package/internals/enqueue_links/shared.js +259 -0
  51. package/internals/errors.d.ts +58 -0
  52. package/internals/errors.js +72 -0
  53. package/internals/http.d.ts +9 -0
  54. package/internals/http.js +28 -0
  55. package/internals/router.d.ts +306 -0
  56. package/internals/router.js +310 -0
  57. package/internals/session_pool/consts.d.ts +3 -0
  58. package/internals/session_pool/consts.js +3 -0
  59. package/internals/session_pool/errors.d.ts +7 -0
  60. package/internals/session_pool/errors.js +11 -0
  61. package/internals/session_pool/fingerprint.d.ts +9 -0
  62. package/internals/session_pool/fingerprint.js +30 -0
  63. package/internals/session_pool/index.d.ts +4 -0
  64. package/internals/session_pool/index.js +4 -0
  65. package/internals/session_pool/session.d.ts +153 -0
  66. package/internals/session_pool/session.js +221 -0
  67. package/internals/session_pool/session_pool.d.ts +242 -0
  68. package/internals/session_pool/session_pool.js +429 -0
  69. package/internals/sitemap_request_loader.d.ts +196 -0
  70. package/internals/sitemap_request_loader.js +432 -0
  71. package/internals/throttling_request_manager.d.ts +217 -0
  72. package/internals/throttling_request_manager.js +829 -0
  73. package/package.json +11 -7
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.170",
3
+ "version": "4.0.0-beta.171",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -42,18 +42,22 @@
42
42
  "@apify/datastructures": "^3.0.1",
43
43
  "@apify/timeout": "^1.0.1",
44
44
  "@apify/utilities": "^3.0.1",
45
- "@crawlee/core": "4.0.0-beta.170",
46
- "@crawlee/http-client": "4.0.0-beta.170",
47
- "@crawlee/types": "4.0.0-beta.170",
48
- "@crawlee/utils": "4.0.0-beta.170",
45
+ "@crawlee/core": "4.0.0-beta.171",
46
+ "@crawlee/http-client": "4.0.0-beta.171",
47
+ "@crawlee/types": "4.0.0-beta.171",
48
+ "@crawlee/utils": "4.0.0-beta.171",
49
+ "@sapphire/async-queue": "^1.5.5",
50
+ "@standard-schema/spec": "^1.0.0",
49
51
  "csv-stringify": "^6.5.2",
52
+ "minimatch": "^10.0.1",
50
53
  "tldts": "^7.0.6",
54
+ "tough-cookie": "^6.0.0",
51
55
  "tslib": "^2.8.1",
52
56
  "type-fest": "^4.41.0",
53
57
  "zod": "^4.5.4"
54
58
  },
55
59
  "optionalDependencies": {
56
- "@crawlee/impit-client": "4.0.0-beta.170"
60
+ "@crawlee/impit-client": "4.0.0-beta.171"
57
61
  },
58
62
  "lerna": {
59
63
  "command": {
@@ -62,5 +66,5 @@
62
66
  }
63
67
  }
64
68
  },
65
- "gitHead": "86e1c891da983125e9e18f1a2327fbcf5e0f5fc2"
69
+ "gitHead": "e4367913045aa04189285719c9bd6912591d34bc"
66
70
  }