@crawlee/basic 4.0.0-beta.237 → 4.0.0-beta.239
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1455,7 +1455,15 @@ export class BasicCrawler {
|
|
|
1455
1455
|
if (cachedRobotsTxtFile) {
|
|
1456
1456
|
return cachedRobotsTxtFile;
|
|
1457
1457
|
}
|
|
1458
|
-
|
|
1458
|
+
// Fetch robots.txt the way pages are fetched: through the crawler's HTTP client, with a session's browser
|
|
1459
|
+
// fingerprint and proxy. Sites with bot protection answer a bare request with 4xx or 5xx, and that status
|
|
1460
|
+
// alone would then decide whether everything is allowed or disallowed.
|
|
1461
|
+
const robotsTxtFile = await RobotsTxtFile.find(url, {
|
|
1462
|
+
httpClient: this.httpClient,
|
|
1463
|
+
session: await this.sessionPool.getSession(),
|
|
1464
|
+
timeoutMillis: 30_000,
|
|
1465
|
+
logger: this.log,
|
|
1466
|
+
});
|
|
1459
1467
|
this.#robotsTxtFileCache.add(origin, robotsTxtFile);
|
|
1460
1468
|
return robotsTxtFile;
|
|
1461
1469
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.239",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.0.1",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.239",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.239",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.239",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.239",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.0.0",
|
|
51
51
|
"csv-stringify": "^6.5.2",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.5.4"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.239"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "5fadaae0607c497eeffd0029fb17955f69d2ed46"
|
|
70
70
|
}
|