@crawlee/utils 4.0.0-beta.237 → 4.0.0-beta.238
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/robots.d.ts +3 -1
- package/internals/robots.js +3 -1
- package/package.json +4 -4
package/internals/robots.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { BaseHttpClient } from '@crawlee/http-client';
|
|
2
|
-
import type { CrawleeLogger } from '@crawlee/types';
|
|
2
|
+
import type { CrawleeLogger, ISession } from '@crawlee/types';
|
|
3
3
|
import { Sitemap } from './sitemap.js';
|
|
4
4
|
import { type EnqueueStrategy } from './url.js';
|
|
5
5
|
export interface RobotsTxtFileSitemapsOptions {
|
|
@@ -38,12 +38,14 @@ export declare class RobotsTxtFile {
|
|
|
38
38
|
* @param [options.signal] an AbortSignal to cancel the request
|
|
39
39
|
* @param [options.timeoutMillis] timeout in milliseconds for the request
|
|
40
40
|
* @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
|
|
41
|
+
* @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
|
|
41
42
|
*/
|
|
42
43
|
static find(url: string, options?: {
|
|
43
44
|
signal?: AbortSignal;
|
|
44
45
|
timeoutMillis?: number;
|
|
45
46
|
proxyUrl?: string;
|
|
46
47
|
httpClient?: BaseHttpClient;
|
|
48
|
+
session?: ISession;
|
|
47
49
|
logger?: CrawleeLogger;
|
|
48
50
|
}): Promise<RobotsTxtFile>;
|
|
49
51
|
/**
|
package/internals/robots.js
CHANGED
|
@@ -38,6 +38,7 @@ export class RobotsTxtFile {
|
|
|
38
38
|
* @param [options.signal] an AbortSignal to cancel the request
|
|
39
39
|
* @param [options.timeoutMillis] timeout in milliseconds for the request
|
|
40
40
|
* @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
|
|
41
|
+
* @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
|
|
41
42
|
*/
|
|
42
43
|
static async find(url, options) {
|
|
43
44
|
const robotsTxtFileUrl = new URL(url);
|
|
@@ -56,8 +57,9 @@ export class RobotsTxtFile {
|
|
|
56
57
|
return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
|
|
57
58
|
}
|
|
58
59
|
static async #load(url, options) {
|
|
59
|
-
const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
|
|
60
|
+
const { proxyUrl, logger, session, httpClient = new FetchHttpClient() } = options || {};
|
|
60
61
|
const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
|
|
62
|
+
session,
|
|
61
63
|
proxyUrl,
|
|
62
64
|
timeoutMillis: options?.timeoutMillis,
|
|
63
65
|
signal: options?.signal,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.238",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -43,8 +43,8 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@apify/validations": "^1.1.0",
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.238",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.238",
|
|
48
48
|
"@types/sax": "^1.2.7",
|
|
49
49
|
"cheerio": "^1.0.0",
|
|
50
50
|
"domhandler": "^6.0.0",
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "8c6b2f3b5a811ca68672b6c68b6587a925baf2aa"
|
|
67
67
|
}
|