@crawlee/basic 4.0.0-rc.0 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +1 -1
  2. package/index.d.ts +11 -1
  3. package/index.js +11 -0
  4. package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
  5. package/internals/autoscaling/autoscaled-pool.js +389 -0
  6. package/internals/autoscaling/concurrency-system.d.ts +235 -0
  7. package/internals/autoscaling/concurrency-system.js +357 -0
  8. package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
  9. package/internals/autoscaling/cpu-load-signal.js +46 -0
  10. package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
  11. package/internals/autoscaling/event-loop-load-signal.js +60 -0
  12. package/internals/autoscaling/index.d.ts +9 -0
  13. package/internals/autoscaling/index.js +9 -0
  14. package/internals/autoscaling/load-signal.d.ts +98 -0
  15. package/internals/autoscaling/load-signal.js +103 -0
  16. package/internals/autoscaling/memory-load-signal.d.ts +45 -0
  17. package/internals/autoscaling/memory-load-signal.js +104 -0
  18. package/internals/autoscaling/snapshotter.d.ts +84 -0
  19. package/internals/autoscaling/snapshotter.js +67 -0
  20. package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
  21. package/internals/autoscaling/storage-backend-load-signal.js +73 -0
  22. package/internals/autoscaling/system-status.d.ts +112 -0
  23. package/internals/autoscaling/system-status.js +139 -0
  24. package/internals/autoscaling/weighted-avg.d.ts +5 -0
  25. package/internals/autoscaling/weighted-avg.js +14 -0
  26. package/internals/basic-crawler.d.ts +84 -312
  27. package/internals/basic-crawler.js +709 -598
  28. package/internals/cookie-utils.d.ts +40 -0
  29. package/internals/cookie-utils.js +122 -0
  30. package/internals/crawler-run.d.ts +76 -0
  31. package/internals/crawler-run.js +144 -0
  32. package/internals/crawlers/context-pipeline.d.ts +75 -0
  33. package/internals/crawlers/context-pipeline.js +109 -0
  34. package/internals/crawlers/crawler-commons.d.ts +216 -0
  35. package/internals/crawlers/crawler-commons.js +1 -0
  36. package/internals/crawlers/error-snapshotter.d.ts +29 -0
  37. package/internals/crawlers/error-snapshotter.js +116 -0
  38. package/internals/crawlers/error-tracker.d.ts +51 -0
  39. package/internals/crawlers/error-tracker.js +316 -0
  40. package/internals/crawlers/index.d.ts +4 -0
  41. package/internals/crawlers/index.js +3 -0
  42. package/internals/crawlers/internals/types.d.ts +7 -0
  43. package/internals/crawlers/internals/types.js +1 -0
  44. package/internals/crawlers/statistics.d.ts +326 -0
  45. package/internals/crawlers/statistics.js +559 -0
  46. package/internals/crawling-request.d.ts +86 -0
  47. package/internals/crawling-request.js +127 -0
  48. package/internals/enqueue-links/enqueue-links.d.ts +147 -0
  49. package/internals/enqueue-links/enqueue-links.js +35 -0
  50. package/internals/enqueue-links/index.d.ts +3 -0
  51. package/internals/enqueue-links/index.js +3 -0
  52. package/internals/enqueue-links/shared.d.ts +107 -0
  53. package/internals/enqueue-links/shared.js +228 -0
  54. package/internals/errors.d.ts +73 -0
  55. package/internals/errors.js +108 -0
  56. package/internals/http.d.ts +9 -0
  57. package/internals/http.js +28 -0
  58. package/internals/request-timeout.d.ts +6 -0
  59. package/internals/request-timeout.js +27 -2
  60. package/internals/router.d.ts +299 -0
  61. package/internals/router.js +312 -0
  62. package/internals/send-request.js +17 -8
  63. package/internals/session-pool/consts.d.ts +1 -0
  64. package/internals/session-pool/consts.js +1 -0
  65. package/internals/session-pool/errors.d.ts +7 -0
  66. package/internals/session-pool/errors.js +11 -0
  67. package/internals/session-pool/fingerprint.d.ts +9 -0
  68. package/internals/session-pool/fingerprint.js +30 -0
  69. package/internals/session-pool/index.d.ts +4 -0
  70. package/internals/session-pool/index.js +4 -0
  71. package/internals/session-pool/session-pool.d.ts +193 -0
  72. package/internals/session-pool/session-pool.js +453 -0
  73. package/internals/session-pool/session.d.ts +151 -0
  74. package/internals/session-pool/session.js +225 -0
  75. package/internals/sitemap-request-loader.d.ts +165 -0
  76. package/internals/sitemap-request-loader.js +492 -0
  77. package/internals/throttling-request-manager.d.ts +229 -0
  78. package/internals/throttling-request-manager.js +860 -0
  79. package/package.json +19 -15
package/package.json CHANGED
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-rc.0",
3
+ "version": "4.0.0-rc.2",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
- "node": ">=22.0.0"
6
+ "node": ">=22.13.0"
7
7
  },
8
8
  "type": "module",
9
9
  "exports": {
@@ -39,21 +39,25 @@
39
39
  "access": "public"
40
40
  },
41
41
  "dependencies": {
42
- "@apify/datastructures": "^2.0.0",
43
- "@apify/timeout": "^0.4.4",
44
- "@apify/utilities": "^2.15.5",
45
- "@crawlee/core": "4.0.0-rc.0",
46
- "@crawlee/http-client": "4.0.0-rc.0",
47
- "@crawlee/types": "4.0.0-rc.0",
48
- "@crawlee/utils": "4.0.0-rc.0",
49
- "csv-stringify": "^6.5.2",
50
- "tldts": "^7.0.6",
51
- "tslib": "^2.8.1",
42
+ "@apify/datastructures": "^3.0.1",
43
+ "@apify/timeout": "^1.0.1",
44
+ "@apify/utilities": "^3.1.0",
45
+ "@crawlee/core": "4.0.0-rc.2",
46
+ "@crawlee/http-client": "4.0.0-rc.2",
47
+ "@crawlee/types": "4.0.0-rc.2",
48
+ "@crawlee/utils": "4.0.0-rc.2",
49
+ "@sapphire/async-queue": "^1.5.5",
50
+ "@standard-schema/spec": "^1.1.0",
51
+ "csv-stringify": "^6.9.0",
52
+ "minimatch": "^11.0.5",
53
+ "tldts": "^7.4.19",
54
+ "tough-cookie": "^6.1.0",
55
+ "tslib": "^2.9.1",
52
56
  "type-fest": "^4.41.0",
53
- "zod": "^4.4.3"
57
+ "zod": "^4.6.7"
54
58
  },
55
59
  "optionalDependencies": {
56
- "@crawlee/impit-client": "^4.0.0-rc.0"
60
+ "@crawlee/impit-client": "4.0.0-rc.2"
57
61
  },
58
62
  "lerna": {
59
63
  "command": {
@@ -62,5 +66,5 @@
62
66
  }
63
67
  }
64
68
  },
65
- "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
69
+ "gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
66
70
  }