@crawlee/utils 4.0.0-beta.207 → 4.0.0-beta.209
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/robots.js +7 -5
- package/package.json +4 -4
package/internals/robots.js
CHANGED
|
@@ -62,13 +62,12 @@ export class RobotsTxtFile {
|
|
|
62
62
|
timeoutMillis: options?.timeoutMillis,
|
|
63
63
|
signal: options?.signal,
|
|
64
64
|
});
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
if (response.status === 404) {
|
|
65
|
+
// RFC 9309: an unavailable (4xx) robots.txt allows everything, an unreachable (5xx) one disallows everything.
|
|
66
|
+
if (response.status >= 400 && response.status < 600) {
|
|
67
|
+
const allowed = response.status < 500;
|
|
69
68
|
return new RobotsTxtFile(url, {
|
|
70
69
|
isAllowed() {
|
|
71
|
-
return
|
|
70
|
+
return allowed;
|
|
72
71
|
},
|
|
73
72
|
getSitemaps() {
|
|
74
73
|
return [];
|
|
@@ -78,6 +77,9 @@ export class RobotsTxtFile {
|
|
|
78
77
|
},
|
|
79
78
|
}, proxyUrl, logger);
|
|
80
79
|
}
|
|
80
|
+
if (response.status < 200 || response.status >= 300) {
|
|
81
|
+
throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
|
|
82
|
+
}
|
|
81
83
|
// @ts-ignore
|
|
82
84
|
return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
|
|
83
85
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.209",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -44,8 +44,8 @@
|
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@apify/ps-tree": "^1.2.0",
|
|
46
46
|
"@apify/validations": "^1.1.0",
|
|
47
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
48
|
-
"@crawlee/types": "4.0.0-beta.
|
|
47
|
+
"@crawlee/http-client": "4.0.0-beta.209",
|
|
48
|
+
"@crawlee/types": "4.0.0-beta.209",
|
|
49
49
|
"@types/sax": "^1.2.7",
|
|
50
50
|
"cheerio": "^1.0.0",
|
|
51
51
|
"domhandler": "^5.0.3",
|
|
@@ -64,5 +64,5 @@
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
},
|
|
67
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "ccf82c59e230033a34bddcbecb1f4a8e803eaa11"
|
|
68
68
|
}
|