@crawlee/utils 4.0.0-beta.80 → 4.0.0-beta.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/robots.d.ts +1 -7
- package/internals/sitemap.d.ts +1 -1
- package/package.json +4 -4
package/internals/robots.d.ts
CHANGED
|
@@ -45,13 +45,7 @@ export declare class RobotsTxtFile {
|
|
|
45
45
|
* @param [proxyUrl] a proxy to be used for fetching the robots.txt file
|
|
46
46
|
*/
|
|
47
47
|
static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
|
|
48
|
-
|
|
49
|
-
signal?: AbortSignal;
|
|
50
|
-
timeoutMillis?: number;
|
|
51
|
-
proxyUrl?: string;
|
|
52
|
-
httpClient?: BaseHttpClient;
|
|
53
|
-
logger?: CrawleeLogger;
|
|
54
|
-
}): Promise<RobotsTxtFile>;
|
|
48
|
+
private static load;
|
|
55
49
|
/**
|
|
56
50
|
* Check if a URL should be crawled by robots.
|
|
57
51
|
* @param url the URL to check against the rules in robots.txt
|
package/internals/sitemap.d.ts
CHANGED
|
@@ -94,7 +94,7 @@ export declare class Sitemap {
|
|
|
94
94
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
95
95
|
*/
|
|
96
96
|
static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
|
97
|
-
|
|
97
|
+
private static parse;
|
|
98
98
|
}
|
|
99
99
|
/**
|
|
100
100
|
* Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.81",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -42,8 +42,8 @@
|
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
44
|
"@apify/ps-tree": "^1.2.0",
|
|
45
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
46
|
-
"@crawlee/types": "4.0.0-beta.
|
|
45
|
+
"@crawlee/http-client": "4.0.0-beta.81",
|
|
46
|
+
"@crawlee/types": "4.0.0-beta.81",
|
|
47
47
|
"@types/sax": "^1.2.7",
|
|
48
48
|
"cheerio": "^1.0.0",
|
|
49
49
|
"domhandler": "^5.0.3",
|
|
@@ -61,5 +61,5 @@
|
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
63
|
},
|
|
64
|
-
"gitHead": "
|
|
64
|
+
"gitHead": "80dc6b4fc82237e63a51a71153809ec8dfd0cc50"
|
|
65
65
|
}
|