@crawlee/basic 4.0.0-rc.0 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.d.ts +11 -1
- package/index.js +11 -0
- package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
- package/internals/autoscaling/autoscaled-pool.js +389 -0
- package/internals/autoscaling/concurrency-system.d.ts +235 -0
- package/internals/autoscaling/concurrency-system.js +357 -0
- package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
- package/internals/autoscaling/cpu-load-signal.js +46 -0
- package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
- package/internals/autoscaling/event-loop-load-signal.js +60 -0
- package/internals/autoscaling/index.d.ts +9 -0
- package/internals/autoscaling/index.js +9 -0
- package/internals/autoscaling/load-signal.d.ts +98 -0
- package/internals/autoscaling/load-signal.js +103 -0
- package/internals/autoscaling/memory-load-signal.d.ts +45 -0
- package/internals/autoscaling/memory-load-signal.js +104 -0
- package/internals/autoscaling/snapshotter.d.ts +84 -0
- package/internals/autoscaling/snapshotter.js +67 -0
- package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
- package/internals/autoscaling/storage-backend-load-signal.js +73 -0
- package/internals/autoscaling/system-status.d.ts +112 -0
- package/internals/autoscaling/system-status.js +139 -0
- package/internals/autoscaling/weighted-avg.d.ts +5 -0
- package/internals/autoscaling/weighted-avg.js +14 -0
- package/internals/basic-crawler.d.ts +84 -312
- package/internals/basic-crawler.js +709 -598
- package/internals/cookie-utils.d.ts +40 -0
- package/internals/cookie-utils.js +122 -0
- package/internals/crawler-run.d.ts +76 -0
- package/internals/crawler-run.js +144 -0
- package/internals/crawlers/context-pipeline.d.ts +75 -0
- package/internals/crawlers/context-pipeline.js +109 -0
- package/internals/crawlers/crawler-commons.d.ts +216 -0
- package/internals/crawlers/crawler-commons.js +1 -0
- package/internals/crawlers/error-snapshotter.d.ts +29 -0
- package/internals/crawlers/error-snapshotter.js +116 -0
- package/internals/crawlers/error-tracker.d.ts +51 -0
- package/internals/crawlers/error-tracker.js +316 -0
- package/internals/crawlers/index.d.ts +4 -0
- package/internals/crawlers/index.js +3 -0
- package/internals/crawlers/internals/types.d.ts +7 -0
- package/internals/crawlers/internals/types.js +1 -0
- package/internals/crawlers/statistics.d.ts +326 -0
- package/internals/crawlers/statistics.js +559 -0
- package/internals/crawling-request.d.ts +86 -0
- package/internals/crawling-request.js +127 -0
- package/internals/enqueue-links/enqueue-links.d.ts +147 -0
- package/internals/enqueue-links/enqueue-links.js +35 -0
- package/internals/enqueue-links/index.d.ts +3 -0
- package/internals/enqueue-links/index.js +3 -0
- package/internals/enqueue-links/shared.d.ts +107 -0
- package/internals/enqueue-links/shared.js +228 -0
- package/internals/errors.d.ts +73 -0
- package/internals/errors.js +108 -0
- package/internals/http.d.ts +9 -0
- package/internals/http.js +28 -0
- package/internals/request-timeout.d.ts +6 -0
- package/internals/request-timeout.js +27 -2
- package/internals/router.d.ts +299 -0
- package/internals/router.js +312 -0
- package/internals/send-request.js +17 -8
- package/internals/session-pool/consts.d.ts +1 -0
- package/internals/session-pool/consts.js +1 -0
- package/internals/session-pool/errors.d.ts +7 -0
- package/internals/session-pool/errors.js +11 -0
- package/internals/session-pool/fingerprint.d.ts +9 -0
- package/internals/session-pool/fingerprint.js +30 -0
- package/internals/session-pool/index.d.ts +4 -0
- package/internals/session-pool/index.js +4 -0
- package/internals/session-pool/session-pool.d.ts +193 -0
- package/internals/session-pool/session-pool.js +453 -0
- package/internals/session-pool/session.d.ts +151 -0
- package/internals/session-pool/session.js +225 -0
- package/internals/sitemap-request-loader.d.ts +165 -0
- package/internals/sitemap-request-loader.js +492 -0
- package/internals/throttling-request-manager.d.ts +229 -0
- package/internals/throttling-request-manager.js +860 -0
- package/package.json +19 -15
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-rc.
|
|
3
|
+
"version": "4.0.0-rc.2",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=22.
|
|
6
|
+
"node": ">=22.13.0"
|
|
7
7
|
},
|
|
8
8
|
"type": "module",
|
|
9
9
|
"exports": {
|
|
@@ -39,21 +39,25 @@
|
|
|
39
39
|
"access": "public"
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"@apify/datastructures": "^
|
|
43
|
-
"@apify/timeout": "^0.
|
|
44
|
-
"@apify/utilities": "^
|
|
45
|
-
"@crawlee/core": "4.0.0-rc.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-rc.
|
|
47
|
-
"@crawlee/types": "4.0.0-rc.
|
|
48
|
-
"@crawlee/utils": "4.0.0-rc.
|
|
49
|
-
"
|
|
50
|
-
"
|
|
51
|
-
"
|
|
42
|
+
"@apify/datastructures": "^3.0.1",
|
|
43
|
+
"@apify/timeout": "^1.0.1",
|
|
44
|
+
"@apify/utilities": "^3.1.0",
|
|
45
|
+
"@crawlee/core": "4.0.0-rc.2",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-rc.2",
|
|
47
|
+
"@crawlee/types": "4.0.0-rc.2",
|
|
48
|
+
"@crawlee/utils": "4.0.0-rc.2",
|
|
49
|
+
"@sapphire/async-queue": "^1.5.5",
|
|
50
|
+
"@standard-schema/spec": "^1.1.0",
|
|
51
|
+
"csv-stringify": "^6.9.0",
|
|
52
|
+
"minimatch": "^11.0.5",
|
|
53
|
+
"tldts": "^7.4.19",
|
|
54
|
+
"tough-cookie": "^6.1.0",
|
|
55
|
+
"tslib": "^2.9.1",
|
|
52
56
|
"type-fest": "^4.41.0",
|
|
53
|
-
"zod": "^4.
|
|
57
|
+
"zod": "^4.6.7"
|
|
54
58
|
},
|
|
55
59
|
"optionalDependencies": {
|
|
56
|
-
"@crawlee/impit-client": "
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-rc.2"
|
|
57
61
|
},
|
|
58
62
|
"lerna": {
|
|
59
63
|
"command": {
|
|
@@ -62,5 +66,5 @@
|
|
|
62
66
|
}
|
|
63
67
|
}
|
|
64
68
|
},
|
|
65
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
|
|
66
70
|
}
|