@crawlee/basic 4.0.0-beta.99 → 4.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +1 -1
  2. package/index.d.ts +11 -1
  3. package/index.js +11 -0
  4. package/internals/autoscaling/autoscaled_pool.d.ts +178 -0
  5. package/internals/autoscaling/autoscaled_pool.js +387 -0
  6. package/internals/autoscaling/concurrency_system.d.ts +235 -0
  7. package/internals/autoscaling/concurrency_system.js +357 -0
  8. package/internals/autoscaling/cpu_load_signal.d.ts +43 -0
  9. package/internals/autoscaling/cpu_load_signal.js +46 -0
  10. package/internals/autoscaling/event_loop_load_signal.d.ts +51 -0
  11. package/internals/autoscaling/event_loop_load_signal.js +60 -0
  12. package/internals/autoscaling/index.d.ts +9 -0
  13. package/internals/autoscaling/index.js +9 -0
  14. package/internals/autoscaling/load_signal.d.ts +98 -0
  15. package/internals/autoscaling/load_signal.js +103 -0
  16. package/internals/autoscaling/memory_load_signal.d.ts +45 -0
  17. package/internals/autoscaling/memory_load_signal.js +104 -0
  18. package/internals/autoscaling/snapshotter.d.ts +84 -0
  19. package/internals/autoscaling/snapshotter.js +67 -0
  20. package/internals/autoscaling/storage_backend_load_signal.d.ts +56 -0
  21. package/internals/autoscaling/storage_backend_load_signal.js +73 -0
  22. package/internals/autoscaling/system_status.d.ts +112 -0
  23. package/internals/autoscaling/system_status.js +139 -0
  24. package/internals/autoscaling/weighted_avg.d.ts +5 -0
  25. package/internals/autoscaling/weighted_avg.js +14 -0
  26. package/internals/basic-crawler.d.ts +185 -389
  27. package/internals/basic-crawler.js +982 -684
  28. package/internals/cookie_utils.d.ts +40 -0
  29. package/internals/cookie_utils.js +122 -0
  30. package/internals/crawler-run.d.ts +76 -0
  31. package/internals/crawler-run.js +142 -0
  32. package/internals/crawlers/context_pipeline.d.ts +75 -0
  33. package/internals/crawlers/context_pipeline.js +109 -0
  34. package/internals/crawlers/crawler_commons.d.ts +216 -0
  35. package/internals/crawlers/crawler_commons.js +1 -0
  36. package/internals/crawlers/error_snapshotter.d.ts +29 -0
  37. package/internals/crawlers/error_snapshotter.js +116 -0
  38. package/internals/crawlers/error_tracker.d.ts +51 -0
  39. package/internals/crawlers/error_tracker.js +316 -0
  40. package/internals/crawlers/index.d.ts +4 -0
  41. package/internals/crawlers/index.js +3 -0
  42. package/internals/crawlers/internals/types.d.ts +7 -0
  43. package/internals/crawlers/internals/types.js +1 -0
  44. package/internals/crawlers/statistics.d.ts +316 -0
  45. package/internals/crawlers/statistics.js +525 -0
  46. package/internals/crawling_request.d.ts +86 -0
  47. package/internals/crawling_request.js +127 -0
  48. package/internals/enqueue_links/enqueue_links.d.ts +152 -0
  49. package/internals/enqueue_links/enqueue_links.js +76 -0
  50. package/internals/enqueue_links/index.d.ts +3 -0
  51. package/internals/enqueue_links/index.js +3 -0
  52. package/internals/enqueue_links/shared.d.ts +106 -0
  53. package/internals/enqueue_links/shared.js +258 -0
  54. package/internals/errors.d.ts +68 -0
  55. package/internals/errors.js +83 -0
  56. package/internals/http.d.ts +9 -0
  57. package/internals/http.js +28 -0
  58. package/internals/request-timeout.d.ts +6 -0
  59. package/internals/request-timeout.js +27 -2
  60. package/internals/router.d.ts +299 -0
  61. package/internals/router.js +312 -0
  62. package/internals/send-request.d.ts +2 -1
  63. package/internals/send-request.js +1 -0
  64. package/internals/session_pool/consts.d.ts +1 -0
  65. package/internals/session_pool/consts.js +1 -0
  66. package/internals/session_pool/errors.d.ts +7 -0
  67. package/internals/session_pool/errors.js +11 -0
  68. package/internals/session_pool/fingerprint.d.ts +9 -0
  69. package/internals/session_pool/fingerprint.js +30 -0
  70. package/internals/session_pool/index.d.ts +4 -0
  71. package/internals/session_pool/index.js +4 -0
  72. package/internals/session_pool/session.d.ts +151 -0
  73. package/internals/session_pool/session.js +225 -0
  74. package/internals/session_pool/session_pool.d.ts +193 -0
  75. package/internals/session_pool/session_pool.js +453 -0
  76. package/internals/sitemap_request_loader.d.ts +158 -0
  77. package/internals/sitemap_request_loader.js +432 -0
  78. package/internals/throttling_request_manager.d.ts +229 -0
  79. package/internals/throttling_request_manager.js +843 -0
  80. package/package.json +17 -14
package/package.json CHANGED
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.99",
3
+ "version": "4.0.0-rc.1",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
- "node": ">=22.0.0"
6
+ "node": ">=22.13.0"
7
7
  },
8
8
  "type": "module",
9
9
  "exports": {
@@ -39,22 +39,25 @@
39
39
  "access": "public"
40
40
  },
41
41
  "dependencies": {
42
- "@apify/datastructures": "^2.0.0",
43
- "@apify/timeout": "^0.4.4",
44
- "@apify/utilities": "^2.15.5",
45
- "@crawlee/core": "4.0.0-beta.99",
46
- "@crawlee/http-client": "4.0.0-beta.99",
47
- "@crawlee/types": "4.0.0-beta.99",
48
- "@crawlee/utils": "4.0.0-beta.99",
42
+ "@apify/datastructures": "^3.0.1",
43
+ "@apify/timeout": "^1.0.1",
44
+ "@apify/utilities": "^3.0.1",
45
+ "@crawlee/core": "4.0.0-rc.1",
46
+ "@crawlee/http-client": "4.0.0-rc.1",
47
+ "@crawlee/types": "4.0.0-rc.1",
48
+ "@crawlee/utils": "4.0.0-rc.1",
49
+ "@sapphire/async-queue": "^1.5.5",
50
+ "@standard-schema/spec": "^1.0.0",
49
51
  "csv-stringify": "^6.5.2",
50
- "fs-extra": "^11.3.0",
51
- "ow": "^2.0.0",
52
+ "minimatch": "^10.2.6",
52
53
  "tldts": "^7.0.6",
54
+ "tough-cookie": "^6.0.0",
53
55
  "tslib": "^2.8.1",
54
- "type-fest": "^4.41.0"
56
+ "type-fest": "^4.41.0",
57
+ "zod": "^4.5.4"
55
58
  },
56
59
  "optionalDependencies": {
57
- "@crawlee/impit-client": "^4.0.0-beta.99"
60
+ "@crawlee/impit-client": "4.0.0-rc.1"
58
61
  },
59
62
  "lerna": {
60
63
  "command": {
@@ -63,5 +66,5 @@
63
66
  }
64
67
  }
65
68
  },
66
- "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
69
+ "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
67
70
  }