@crawlee/utils 4.0.0-beta.208 → 4.0.0-beta.209

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/internals/robots.js +7 -5
  2. package/package.json +4 -4
package/internals/robots.js CHANGED
@@ -62,13 +62,12 @@ export class RobotsTxtFile {
62
62
  timeoutMillis: options?.timeoutMillis,
63
63
  signal: options?.signal,
64
64
  });
65
- if (response.status < 200 || response.status >= 300) {
66
- throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
67
- }
68
- if (response.status === 404) {
65
+ // RFC 9309: an unavailable (4xx) robots.txt allows everything, an unreachable (5xx) one disallows everything.
66
+ if (response.status >= 400 && response.status < 600) {
67
+ const allowed = response.status < 500;
69
68
  return new RobotsTxtFile(url, {
70
69
  isAllowed() {
71
- return true;
70
+ return allowed;
72
71
  },
73
72
  getSitemaps() {
74
73
  return [];
@@ -78,6 +77,9 @@ export class RobotsTxtFile {
78
77
  },
79
78
  }, proxyUrl, logger);
80
79
  }
80
+ if (response.status < 200 || response.status >= 300) {
81
+ throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
82
+ }
81
83
  // @ts-ignore
82
84
  return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
83
85
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.208",
3
+ "version": "4.0.0-beta.209",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -44,8 +44,8 @@
44
44
  "dependencies": {
45
45
  "@apify/ps-tree": "^1.2.0",
46
46
  "@apify/validations": "^1.1.0",
47
- "@crawlee/http-client": "4.0.0-beta.208",
48
- "@crawlee/types": "4.0.0-beta.208",
47
+ "@crawlee/http-client": "4.0.0-beta.209",
48
+ "@crawlee/types": "4.0.0-beta.209",
49
49
  "@types/sax": "^1.2.7",
50
50
  "cheerio": "^1.0.0",
51
51
  "domhandler": "^5.0.3",
@@ -64,5 +64,5 @@
64
64
  }
65
65
  }
66
66
  },
67
- "gitHead": "81d3fbc1abd10fd34be2694a98f4a36e2ebc2f64"
67
+ "gitHead": "ccf82c59e230033a34bddcbecb1f4a8e803eaa11"
68
68
  }