@crawlee/utils 4.0.0-beta.25 → 4.0.0-beta.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +10 -17
- package/index.js +8 -16
- package/internal.d.ts +11 -0
- package/internal.js +10 -0
- package/internals/blocked.d.ts +0 -1
- package/internals/blocked.js +0 -1
- package/internals/cheerio.d.ts +1 -3
- package/internals/cheerio.js +25 -16
- package/internals/extract-microdata.d.ts +24 -0
- package/internals/extract-microdata.js +118 -0
- package/internals/extract-urls.d.ts +1 -2
- package/internals/extract-urls.js +18 -17
- package/internals/general.d.ts +0 -25
- package/internals/general.js +2 -110
- package/internals/iterables.d.ts +0 -80
- package/internals/iterables.js +0 -126
- package/internals/mutation-journal.d.ts +29 -0
- package/internals/mutation-journal.js +173 -0
- package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
- package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
- package/internals/robots.d.ts +35 -15
- package/internals/robots.js +66 -30
- package/internals/schemas.d.ts +114 -0
- package/internals/schemas.js +115 -0
- package/internals/sitemap.d.ts +44 -3
- package/internals/sitemap.js +171 -85
- package/internals/social.d.ts +2 -3
- package/internals/social.js +8 -6
- package/internals/url.d.ts +69 -1
- package/internals/url.js +120 -2
- package/internals/validation.d.ts +9 -0
- package/internals/validation.js +16 -0
- package/package.json +16 -15
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/blocked.d.ts.map +0 -1
- package/internals/blocked.js.map +0 -1
- package/internals/cheerio.d.ts.map +0 -1
- package/internals/cheerio.js.map +0 -1
- package/internals/chunk.d.ts +0 -2
- package/internals/chunk.d.ts.map +0 -1
- package/internals/chunk.js +0 -40
- package/internals/chunk.js.map +0 -1
- package/internals/debug.d.ts +0 -32
- package/internals/debug.d.ts.map +0 -1
- package/internals/debug.js +0 -41
- package/internals/debug.js.map +0 -1
- package/internals/extract-urls.d.ts.map +0 -1
- package/internals/extract-urls.js.map +0 -1
- package/internals/general.d.ts.map +0 -1
- package/internals/general.js.map +0 -1
- package/internals/iterables.d.ts.map +0 -1
- package/internals/iterables.js.map +0 -1
- package/internals/open_graph_parser.d.ts.map +0 -1
- package/internals/open_graph_parser.js.map +0 -1
- package/internals/robots.d.ts.map +0 -1
- package/internals/robots.js.map +0 -1
- package/internals/sitemap.d.ts.map +0 -1
- package/internals/sitemap.js.map +0 -1
- package/internals/social.d.ts.map +0 -1
- package/internals/social.js.map +0 -1
- package/internals/system-info/cpu-info.d.ts +0 -64
- package/internals/system-info/cpu-info.d.ts.map +0 -1
- package/internals/system-info/cpu-info.js +0 -211
- package/internals/system-info/cpu-info.js.map +0 -1
- package/internals/system-info/memory-info.d.ts +0 -28
- package/internals/system-info/memory-info.d.ts.map +0 -1
- package/internals/system-info/memory-info.js +0 -118
- package/internals/system-info/memory-info.js.map +0 -1
- package/internals/system-info/ps-tree.d.ts +0 -18
- package/internals/system-info/ps-tree.d.ts.map +0 -1
- package/internals/system-info/ps-tree.js +0 -145
- package/internals/system-info/ps-tree.js.map +0 -1
- package/internals/typedefs.d.ts +0 -5
- package/internals/typedefs.d.ts.map +0 -1
- package/internals/typedefs.js +0 -9
- package/internals/typedefs.js.map +0 -1
- package/internals/url.d.ts.map +0 -1
- package/internals/url.js.map +0 -1
package/internals/robots.d.ts
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
|
-
import type { BaseHttpClient } from '@crawlee/
|
|
1
|
+
import type { BaseHttpClient } from '@crawlee/http-client';
|
|
2
|
+
import type { CrawleeLogger, ISession } from '@crawlee/types';
|
|
2
3
|
import { Sitemap } from './sitemap.js';
|
|
4
|
+
import { type EnqueueStrategy } from './url.js';
|
|
5
|
+
export interface RobotsTxtFileSitemapsOptions {
|
|
6
|
+
/**
|
|
7
|
+
* Keep only sitemap URLs matching this strategy relative to the robots.txt host; non-`http(s)` schemes
|
|
8
|
+
* are always dropped. Pass `'all'` to disable host filtering.
|
|
9
|
+
* @default 'same-hostname'
|
|
10
|
+
*/
|
|
11
|
+
enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
|
|
12
|
+
}
|
|
3
13
|
/**
|
|
4
14
|
* Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
|
|
5
15
|
*
|
|
@@ -19,17 +29,24 @@ import { Sitemap } from './sitemap.js';
|
|
|
19
29
|
* ```
|
|
20
30
|
*/
|
|
21
31
|
export declare class RobotsTxtFile {
|
|
22
|
-
private
|
|
23
|
-
private proxyUrl?;
|
|
32
|
+
#private;
|
|
24
33
|
private constructor();
|
|
25
34
|
/**
|
|
26
35
|
* Determine the location of a robots.txt file for a URL and fetch it.
|
|
27
36
|
* @param url the URL to fetch robots.txt for
|
|
28
|
-
* @param [
|
|
37
|
+
* @param [options] additional options
|
|
38
|
+
* @param [options.signal] an AbortSignal to cancel the request
|
|
39
|
+
* @param [options.timeoutMillis] timeout in milliseconds for the request
|
|
40
|
+
* @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
|
|
41
|
+
* @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
|
|
29
42
|
*/
|
|
30
43
|
static find(url: string, options?: {
|
|
44
|
+
signal?: AbortSignal;
|
|
45
|
+
timeoutMillis?: number;
|
|
31
46
|
proxyUrl?: string;
|
|
32
47
|
httpClient?: BaseHttpClient;
|
|
48
|
+
session?: ISession;
|
|
49
|
+
logger?: CrawleeLogger;
|
|
33
50
|
}): Promise<RobotsTxtFile>;
|
|
34
51
|
/**
|
|
35
52
|
* Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
|
|
@@ -38,10 +55,11 @@ export declare class RobotsTxtFile {
|
|
|
38
55
|
* @param [proxyUrl] a proxy to be used for fetching the robots.txt file
|
|
39
56
|
*/
|
|
40
57
|
static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
58
|
+
/**
|
|
59
|
+
* Get crawl delay for a given user agent.
|
|
60
|
+
* @param [userAgent] relevant user agent, default to `*`
|
|
61
|
+
*/
|
|
62
|
+
getCrawlDelay(userAgent?: string): number | undefined;
|
|
45
63
|
/**
|
|
46
64
|
* Check if a URL should be crawled by robots.
|
|
47
65
|
* @param url the URL to check against the rules in robots.txt
|
|
@@ -49,17 +67,19 @@ export declare class RobotsTxtFile {
|
|
|
49
67
|
*/
|
|
50
68
|
isAllowed(url: string, userAgent?: string): boolean;
|
|
51
69
|
/**
|
|
52
|
-
* Get URLs of sitemaps referenced in the robots file.
|
|
70
|
+
* Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
|
|
71
|
+
* the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
|
|
72
|
+
* always dropped.
|
|
53
73
|
*/
|
|
54
|
-
getSitemaps(): string[];
|
|
74
|
+
getSitemaps(options?: RobotsTxtFileSitemapsOptions): string[];
|
|
55
75
|
/**
|
|
56
|
-
* Parse all the sitemaps referenced in the robots file.
|
|
76
|
+
* Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
|
|
77
|
+
* and the sitemap parser.
|
|
57
78
|
*/
|
|
58
|
-
parseSitemaps(): Promise<Sitemap>;
|
|
79
|
+
parseSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<Sitemap>;
|
|
59
80
|
/**
|
|
60
81
|
* Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
|
|
82
|
+
* `options` are forwarded to `parseSitemaps`.
|
|
61
83
|
*/
|
|
62
|
-
parseUrlsFromSitemaps(): Promise<string[]>;
|
|
84
|
+
parseUrlsFromSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<string[]>;
|
|
63
85
|
}
|
|
64
|
-
export { RobotsTxtFile as RobotsFile };
|
|
65
|
-
//# sourceMappingURL=robots.d.ts.map
|
package/internals/robots.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { FetchHttpClient } from '@crawlee/http-client';
|
|
2
2
|
import robotsParser from 'robots-parser';
|
|
3
3
|
import { Sitemap } from './sitemap.js';
|
|
4
|
+
import { filterUrl } from './url.js';
|
|
4
5
|
/**
|
|
5
6
|
* Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
|
|
6
7
|
*
|
|
@@ -20,22 +21,30 @@ import { Sitemap } from './sitemap.js';
|
|
|
20
21
|
* ```
|
|
21
22
|
*/
|
|
22
23
|
export class RobotsTxtFile {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
#url;
|
|
25
|
+
#robots;
|
|
26
|
+
#proxyUrl;
|
|
27
|
+
#logger;
|
|
28
|
+
constructor(url, robots, proxyUrl, logger) {
|
|
29
|
+
this.#url = url;
|
|
30
|
+
this.#robots = robots;
|
|
31
|
+
this.#proxyUrl = proxyUrl;
|
|
32
|
+
this.#logger = logger;
|
|
28
33
|
}
|
|
29
34
|
/**
|
|
30
35
|
* Determine the location of a robots.txt file for a URL and fetch it.
|
|
31
36
|
* @param url the URL to fetch robots.txt for
|
|
32
|
-
* @param [
|
|
37
|
+
* @param [options] additional options
|
|
38
|
+
* @param [options.signal] an AbortSignal to cancel the request
|
|
39
|
+
* @param [options.timeoutMillis] timeout in milliseconds for the request
|
|
40
|
+
* @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
|
|
41
|
+
* @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
|
|
33
42
|
*/
|
|
34
43
|
static async find(url, options) {
|
|
35
44
|
const robotsTxtFileUrl = new URL(url);
|
|
36
45
|
robotsTxtFileUrl.pathname = '/robots.txt';
|
|
37
46
|
robotsTxtFileUrl.search = '';
|
|
38
|
-
return RobotsTxtFile
|
|
47
|
+
return RobotsTxtFile.#load(robotsTxtFileUrl.toString(), options);
|
|
39
48
|
}
|
|
40
49
|
/**
|
|
41
50
|
* Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
|
|
@@ -45,28 +54,43 @@ export class RobotsTxtFile {
|
|
|
45
54
|
*/
|
|
46
55
|
static from(url, content, proxyUrl) {
|
|
47
56
|
// @ts-ignore
|
|
48
|
-
return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
|
|
57
|
+
return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
|
|
49
58
|
}
|
|
50
|
-
static async load(url, options) {
|
|
51
|
-
const { proxyUrl, httpClient = new FetchHttpClient() } = options || {};
|
|
59
|
+
static async #load(url, options) {
|
|
60
|
+
const { proxyUrl, logger, session, httpClient = new FetchHttpClient() } = options || {};
|
|
52
61
|
const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
|
|
62
|
+
session,
|
|
53
63
|
proxyUrl,
|
|
64
|
+
timeoutMillis: options?.timeoutMillis,
|
|
65
|
+
signal: options?.signal,
|
|
54
66
|
});
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
return new RobotsTxtFile({
|
|
67
|
+
// RFC 9309: an unavailable (4xx) robots.txt allows everything, an unreachable (5xx) one disallows everything.
|
|
68
|
+
if (response.status >= 400 && response.status < 600) {
|
|
69
|
+
const allowed = response.status < 500;
|
|
70
|
+
return new RobotsTxtFile(url, {
|
|
60
71
|
isAllowed() {
|
|
61
|
-
return
|
|
72
|
+
return allowed;
|
|
62
73
|
},
|
|
63
74
|
getSitemaps() {
|
|
64
75
|
return [];
|
|
65
76
|
},
|
|
66
|
-
|
|
77
|
+
getCrawlDelay() {
|
|
78
|
+
return undefined;
|
|
79
|
+
},
|
|
80
|
+
}, proxyUrl, logger);
|
|
81
|
+
}
|
|
82
|
+
if (response.status < 200 || response.status >= 300) {
|
|
83
|
+
throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
|
|
67
84
|
}
|
|
68
85
|
// @ts-ignore
|
|
69
|
-
return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl);
|
|
86
|
+
return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Get crawl delay for a given user agent.
|
|
90
|
+
* @param [userAgent] relevant user agent, default to `*`
|
|
91
|
+
*/
|
|
92
|
+
getCrawlDelay(userAgent = '*') {
|
|
93
|
+
return this.#robots.getCrawlDelay(userAgent);
|
|
70
94
|
}
|
|
71
95
|
/**
|
|
72
96
|
* Check if a URL should be crawled by robots.
|
|
@@ -74,27 +98,39 @@ export class RobotsTxtFile {
|
|
|
74
98
|
* @param [userAgent] relevant user agent, default to `*`
|
|
75
99
|
*/
|
|
76
100
|
isAllowed(url, userAgent = '*') {
|
|
77
|
-
return this
|
|
101
|
+
return this.#robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
|
|
78
102
|
}
|
|
79
103
|
/**
|
|
80
|
-
* Get URLs of sitemaps referenced in the robots file.
|
|
104
|
+
* Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
|
|
105
|
+
* the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
|
|
106
|
+
* always dropped.
|
|
81
107
|
*/
|
|
82
|
-
getSitemaps() {
|
|
83
|
-
|
|
108
|
+
getSitemaps(options = {}) {
|
|
109
|
+
const { enqueueStrategy = 'same-hostname' } = options;
|
|
110
|
+
const sitemaps = [];
|
|
111
|
+
for (const sitemapUrl of this.#robots.getSitemaps()) {
|
|
112
|
+
// `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.
|
|
113
|
+
const { allowed, reason } = filterUrl(sitemapUrl, this.#url, enqueueStrategy);
|
|
114
|
+
if (!allowed) {
|
|
115
|
+
this.#logger?.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.#url}: ${reason}.`);
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
sitemaps.push(sitemapUrl);
|
|
119
|
+
}
|
|
120
|
+
return sitemaps;
|
|
84
121
|
}
|
|
85
122
|
/**
|
|
86
|
-
* Parse all the sitemaps referenced in the robots file.
|
|
123
|
+
* Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
|
|
124
|
+
* and the sitemap parser.
|
|
87
125
|
*/
|
|
88
|
-
async parseSitemaps() {
|
|
89
|
-
return Sitemap.load(this.
|
|
126
|
+
async parseSitemaps(options = {}) {
|
|
127
|
+
return Sitemap.load(this.getSitemaps(options), this.#proxyUrl, { ...options, logger: this.#logger });
|
|
90
128
|
}
|
|
91
129
|
/**
|
|
92
130
|
* Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
|
|
131
|
+
* `options` are forwarded to `parseSitemaps`.
|
|
93
132
|
*/
|
|
94
|
-
async parseUrlsFromSitemaps() {
|
|
95
|
-
return (await this.parseSitemaps()).urls;
|
|
133
|
+
async parseUrlsFromSitemaps(options = {}) {
|
|
134
|
+
return (await this.parseSitemaps(options)).urls;
|
|
96
135
|
}
|
|
97
136
|
}
|
|
98
|
-
// to stay backwards compatible
|
|
99
|
-
export { RobotsTxtFile as RobotsFile };
|
|
100
|
-
//# sourceMappingURL=robots.js.map
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { BaseHttpClient } from '@crawlee/http-client';
|
|
2
|
+
import type { Dictionary } from '@crawlee/types';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
/**
|
|
5
|
+
* Accepts any object (including arrays and functions).
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
export declare const anyObject: z.ZodCustom<Dictionary, Dictionary>;
|
|
9
|
+
/**
|
|
10
|
+
* Accepts any array without validating its items (cheap for huge arrays).
|
|
11
|
+
* @internal
|
|
12
|
+
*/
|
|
13
|
+
export declare const anyArray: z.ZodCustom<unknown[], unknown[]>;
|
|
14
|
+
/**
|
|
15
|
+
* Accepts any function.
|
|
16
|
+
* @internal
|
|
17
|
+
*/
|
|
18
|
+
export declare const anyFunction: z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>;
|
|
19
|
+
/**
|
|
20
|
+
* Mirrors `ow.number`: `Infinity` is a valid number, `NaN` is not.
|
|
21
|
+
* @internal
|
|
22
|
+
*/
|
|
23
|
+
export declare const anyNumber: z.ZodCustom<number, number>;
|
|
24
|
+
/**
|
|
25
|
+
* Accepts any object (including functions) that has all the given keys, own or inherited.
|
|
26
|
+
* @internal
|
|
27
|
+
*/
|
|
28
|
+
export declare function objectWithKeys(keys: string[], message?: string): z.ZodType<Dictionary>;
|
|
29
|
+
/**
|
|
30
|
+
* Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
|
|
31
|
+
* @internal
|
|
32
|
+
*/
|
|
33
|
+
export declare const httpClient: z.ZodInstanceOf<BaseHttpClient>;
|
|
34
|
+
/**
|
|
35
|
+
* Accepts any object implementing the CrawleeLogger interface.
|
|
36
|
+
* @internal
|
|
37
|
+
*/
|
|
38
|
+
export declare const logger: z.ZodType<Dictionary, unknown, z.core.$ZodTypeInternals<Dictionary, unknown>>;
|
|
39
|
+
/**
|
|
40
|
+
* Accepts any typed array (`Uint8Array`, `Float64Array`, ...), but not a `DataView`.
|
|
41
|
+
* @internal
|
|
42
|
+
*/
|
|
43
|
+
export declare const typedArray: z.ZodCustom<NodeJS.TypedArray<ArrayBufferLike>, NodeJS.TypedArray<ArrayBufferLike>>;
|
|
44
|
+
/**
|
|
45
|
+
* Accepts any non-null, non-array object.
|
|
46
|
+
* @internal
|
|
47
|
+
*/
|
|
48
|
+
export declare const plainObject: z.ZodCustom<Record<string, unknown>, Record<string, unknown>>;
|
|
49
|
+
/**
|
|
50
|
+
* Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
|
|
51
|
+
* @internal
|
|
52
|
+
*/
|
|
53
|
+
export declare const storageRequest: z.ZodObject<{
|
|
54
|
+
id: z.ZodString;
|
|
55
|
+
url: z.ZodURL;
|
|
56
|
+
uniqueKey: z.ZodString;
|
|
57
|
+
method: z.ZodOptional<z.ZodString>;
|
|
58
|
+
retryCount: z.ZodOptional<z.ZodNumber>;
|
|
59
|
+
handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
|
|
60
|
+
}, z.core.$loose>;
|
|
61
|
+
/**
|
|
62
|
+
* {@link storageRequest} before an id is assigned.
|
|
63
|
+
* @internal
|
|
64
|
+
*/
|
|
65
|
+
export declare const storageRequestWithoutId: z.ZodObject<{
|
|
66
|
+
url: z.ZodURL;
|
|
67
|
+
uniqueKey: z.ZodString;
|
|
68
|
+
method: z.ZodOptional<z.ZodString>;
|
|
69
|
+
retryCount: z.ZodOptional<z.ZodNumber>;
|
|
70
|
+
handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
|
|
71
|
+
}, z.core.$loose>;
|
|
72
|
+
/**
|
|
73
|
+
* `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
|
|
74
|
+
* instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
|
|
75
|
+
* is a human-readable plural (`'numbers'`, `'URL patterns'`), since element types cannot be introspected.
|
|
76
|
+
* @internal
|
|
77
|
+
*/
|
|
78
|
+
export declare function arrayOf<TItem extends z.ZodType>(item: TItem, elements: string): z.ZodArray<TItem>;
|
|
79
|
+
/**
|
|
80
|
+
* Batch of {@link storageRequestWithoutId}.
|
|
81
|
+
* @internal
|
|
82
|
+
*/
|
|
83
|
+
export declare const storageRequestBatch: z.ZodArray<z.ZodObject<{
|
|
84
|
+
url: z.ZodURL;
|
|
85
|
+
uniqueKey: z.ZodString;
|
|
86
|
+
method: z.ZodOptional<z.ZodString>;
|
|
87
|
+
retryCount: z.ZodOptional<z.ZodNumber>;
|
|
88
|
+
handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
|
|
89
|
+
}, z.core.$loose>>;
|
|
90
|
+
/**
|
|
91
|
+
* Options of request queue add/update operations.
|
|
92
|
+
* @internal
|
|
93
|
+
*/
|
|
94
|
+
export declare const requestQueueOperationOptions: z.ZodObject<{
|
|
95
|
+
forefront: z.ZodOptional<z.ZodBoolean>;
|
|
96
|
+
}, z.core.$strip>;
|
|
97
|
+
/**
|
|
98
|
+
* Options of key-value store `listKeys`.
|
|
99
|
+
* @internal
|
|
100
|
+
*/
|
|
101
|
+
export declare const keyValueStoreListKeysOptions: z.ZodObject<{
|
|
102
|
+
prefix: z.ZodOptional<z.ZodString>;
|
|
103
|
+
exclusiveStartKey: z.ZodOptional<z.ZodString>;
|
|
104
|
+
limit: z.ZodOptional<z.ZodNumber>;
|
|
105
|
+
}, z.core.$strip>;
|
|
106
|
+
/**
|
|
107
|
+
* Options of dataset item listing.
|
|
108
|
+
* @internal
|
|
109
|
+
*/
|
|
110
|
+
export declare const datasetListItemsOptions: z.ZodObject<{
|
|
111
|
+
desc: z.ZodOptional<z.ZodBoolean>;
|
|
112
|
+
limit: z.ZodOptional<z.ZodNumber>;
|
|
113
|
+
offset: z.ZodOptional<z.ZodNumber>;
|
|
114
|
+
}, z.core.$strip>;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { BaseHttpClient } from '@crawlee/http-client';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
/**
|
|
4
|
+
* Accepts any object (including arrays and functions).
|
|
5
|
+
* @internal
|
|
6
|
+
*/
|
|
7
|
+
export const anyObject = z.custom((value) => (typeof value === 'object' && value !== null) || typeof value === 'function', { message: 'Invalid input: expected object' });
|
|
8
|
+
/**
|
|
9
|
+
* Accepts any array without validating its items (cheap for huge arrays).
|
|
10
|
+
* @internal
|
|
11
|
+
*/
|
|
12
|
+
export const anyArray = z.custom(Array.isArray, { message: 'Invalid input: expected array' });
|
|
13
|
+
/**
|
|
14
|
+
* Accepts any function.
|
|
15
|
+
* @internal
|
|
16
|
+
*/
|
|
17
|
+
export const anyFunction = z.custom((value) => typeof value === 'function', {
|
|
18
|
+
message: 'Invalid input: expected function',
|
|
19
|
+
});
|
|
20
|
+
/**
|
|
21
|
+
* Mirrors `ow.number`: `Infinity` is a valid number, `NaN` is not.
|
|
22
|
+
* @internal
|
|
23
|
+
*/
|
|
24
|
+
export const anyNumber = z.custom((value) => typeof value === 'number' && !Number.isNaN(value), {
|
|
25
|
+
message: 'Invalid input: expected number',
|
|
26
|
+
});
|
|
27
|
+
/**
|
|
28
|
+
* Accepts any object (including functions) that has all the given keys, own or inherited.
|
|
29
|
+
* @internal
|
|
30
|
+
*/
|
|
31
|
+
export function objectWithKeys(keys, message) {
|
|
32
|
+
return z.custom((value) => ((typeof value === 'object' && value !== null) || typeof value === 'function') &&
|
|
33
|
+
keys.every((key) => key in value), {
|
|
34
|
+
message: message ?? `Invalid input: expected an object with keys ${keys.map((key) => `'${key}'`).join(', ')}`,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
|
|
39
|
+
* @internal
|
|
40
|
+
*/
|
|
41
|
+
export const httpClient = z.instanceof(BaseHttpClient);
|
|
42
|
+
/**
|
|
43
|
+
* Accepts any object implementing the CrawleeLogger interface.
|
|
44
|
+
* @internal
|
|
45
|
+
*/
|
|
46
|
+
export const logger = objectWithKeys(['child', 'info', 'error', 'warning'], "Expected an object implementing the CrawleeLogger interface (missing one of 'child', 'info', 'error', 'warning'), got something else.");
|
|
47
|
+
/**
|
|
48
|
+
* Accepts any typed array (`Uint8Array`, `Float64Array`, ...), but not a `DataView`.
|
|
49
|
+
* @internal
|
|
50
|
+
*/
|
|
51
|
+
export const typedArray = z.custom((value) => ArrayBuffer.isView(value) && !(value instanceof DataView), { message: 'Invalid input: expected a typed array' });
|
|
52
|
+
/**
|
|
53
|
+
* Accepts any non-null, non-array object.
|
|
54
|
+
* @internal
|
|
55
|
+
*/
|
|
56
|
+
export const plainObject = z.custom((value) => typeof value === 'object' && value !== null && !Array.isArray(value), { message: 'Invalid input: expected an object' });
|
|
57
|
+
const storageRequestShape = z.looseObject({
|
|
58
|
+
id: z.string(),
|
|
59
|
+
url: z.url({ protocol: /^https?$/ }),
|
|
60
|
+
uniqueKey: z.string(),
|
|
61
|
+
method: z.string().optional(),
|
|
62
|
+
retryCount: z.number().int().optional(),
|
|
63
|
+
handledAt: z.union([z.string(), z.date()]).optional(),
|
|
64
|
+
});
|
|
65
|
+
/**
|
|
66
|
+
* Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
|
|
67
|
+
* @internal
|
|
68
|
+
*/
|
|
69
|
+
export const storageRequest = z.compile(storageRequestShape);
|
|
70
|
+
/**
|
|
71
|
+
* {@link storageRequest} before an id is assigned.
|
|
72
|
+
* @internal
|
|
73
|
+
*/
|
|
74
|
+
export const storageRequestWithoutId = z.compile(storageRequestShape.omit({ id: true }));
|
|
75
|
+
/**
|
|
76
|
+
* `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
|
|
77
|
+
* instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
|
|
78
|
+
* is a human-readable plural (`'numbers'`, `'URL patterns'`), since element types cannot be introspected.
|
|
79
|
+
* @internal
|
|
80
|
+
*/
|
|
81
|
+
export function arrayOf(item, elements) {
|
|
82
|
+
return z.array(item, {
|
|
83
|
+
error: (issue) => issue.code === 'invalid_type' ? `Invalid input: expected an array of ${elements}` : undefined,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Batch of {@link storageRequestWithoutId}.
|
|
88
|
+
* @internal
|
|
89
|
+
*/
|
|
90
|
+
export const storageRequestBatch = z.compile(arrayOf(storageRequestWithoutId, 'requests'));
|
|
91
|
+
/**
|
|
92
|
+
* Options of request queue add/update operations.
|
|
93
|
+
* @internal
|
|
94
|
+
*/
|
|
95
|
+
export const requestQueueOperationOptions = z.object({
|
|
96
|
+
forefront: z.boolean().optional(),
|
|
97
|
+
});
|
|
98
|
+
/**
|
|
99
|
+
* Options of key-value store `listKeys`.
|
|
100
|
+
* @internal
|
|
101
|
+
*/
|
|
102
|
+
export const keyValueStoreListKeysOptions = z.object({
|
|
103
|
+
prefix: z.string().optional(),
|
|
104
|
+
exclusiveStartKey: z.string().optional(),
|
|
105
|
+
limit: z.number().int().gt(0).optional(),
|
|
106
|
+
});
|
|
107
|
+
/**
|
|
108
|
+
* Options of dataset item listing.
|
|
109
|
+
* @internal
|
|
110
|
+
*/
|
|
111
|
+
export const datasetListItemsOptions = z.object({
|
|
112
|
+
desc: z.boolean().optional(),
|
|
113
|
+
limit: z.number().int().optional(),
|
|
114
|
+
offset: z.number().int().optional(),
|
|
115
|
+
});
|
package/internals/sitemap.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import type { BaseHttpClient } from '@crawlee/
|
|
1
|
+
import type { BaseHttpClient } from '@crawlee/http-client';
|
|
2
|
+
import type { CrawleeLogger } from '@crawlee/types';
|
|
3
|
+
import { type EnqueueStrategy } from './url.js';
|
|
2
4
|
interface SitemapUrlData {
|
|
3
5
|
loc: string;
|
|
4
6
|
lastmod?: Date;
|
|
@@ -47,6 +49,24 @@ export interface ParseSitemapOptions {
|
|
|
47
49
|
* Custom HTTP client to be used for fetching sitemaps.
|
|
48
50
|
*/
|
|
49
51
|
httpClient?: BaseHttpClient;
|
|
52
|
+
/**
|
|
53
|
+
* Optional filter for nested sitemap URLs discovered in sitemap index files.
|
|
54
|
+
* Called with the URL of each child sitemap before it is fetched.
|
|
55
|
+
* Return `true` to include the sitemap, `false` to skip it.
|
|
56
|
+
* If not provided, all nested sitemaps are followed.
|
|
57
|
+
*/
|
|
58
|
+
nestedSitemapFilter?: (sitemapUrl: string) => boolean;
|
|
59
|
+
/**
|
|
60
|
+
* Keep only sitemap-derived URLs (nested `<sitemap>` and `<url>` entries) matching this strategy
|
|
61
|
+
* relative to the parent sitemap URL; non-`http(s)` schemes are always dropped. Skipped for raw string
|
|
62
|
+
* sources (no parent URL). Pass `'all'` to disable host filtering.
|
|
63
|
+
* @default 'same-hostname'
|
|
64
|
+
*/
|
|
65
|
+
enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
|
|
66
|
+
/**
|
|
67
|
+
* Optional logger for reporting warnings during sitemap parsing.
|
|
68
|
+
*/
|
|
69
|
+
logger?: CrawleeLogger;
|
|
50
70
|
}
|
|
51
71
|
export declare function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
|
|
52
72
|
/**
|
|
@@ -62,6 +82,7 @@ export declare function parseSitemap<T extends ParseSitemapOptions>(initialSourc
|
|
|
62
82
|
* ```
|
|
63
83
|
*/
|
|
64
84
|
export declare class Sitemap {
|
|
85
|
+
#private;
|
|
65
86
|
readonly urls: string[];
|
|
66
87
|
constructor(urls: string[]);
|
|
67
88
|
/**
|
|
@@ -83,7 +104,6 @@ export declare class Sitemap {
|
|
|
83
104
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
84
105
|
*/
|
|
85
106
|
static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
|
86
|
-
protected static parse(sources: SitemapSource[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
|
87
107
|
}
|
|
88
108
|
/**
|
|
89
109
|
* Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
|
|
@@ -97,10 +117,31 @@ export declare function discoverValidSitemaps(urls: string[], options?: {
|
|
|
97
117
|
* Proxy URL to be used for network requests.
|
|
98
118
|
*/
|
|
99
119
|
proxyUrl?: string;
|
|
120
|
+
/**
|
|
121
|
+
* Timeout in milliseconds for the entire `discoverValidSitemaps` call.
|
|
122
|
+
* An `AbortController` is created internally and its signal is passed to every HTTP request,
|
|
123
|
+
* so the whole discovery operation is cancelled once the timeout elapses.
|
|
124
|
+
* Defaults to `60_000` ms (60 seconds) to prevent indefinite hangs.
|
|
125
|
+
*/
|
|
126
|
+
timeoutMillis?: number;
|
|
127
|
+
/**
|
|
128
|
+
* An external `AbortSignal` to cancel the entire discovery operation.
|
|
129
|
+
* If both `signal` and `timeout` are provided, the operation is cancelled
|
|
130
|
+
* when either the signal is aborted or the timeout elapses (whichever comes first).
|
|
131
|
+
*/
|
|
132
|
+
signal?: AbortSignal;
|
|
133
|
+
/**
|
|
134
|
+
* Timeout in milliseconds for each individual HTTP request during discovery.
|
|
135
|
+
* Defaults to `20000` ms (20 seconds).
|
|
136
|
+
*/
|
|
137
|
+
requestTimeoutMillis?: number;
|
|
100
138
|
/**
|
|
101
139
|
* HTTP client to be used for network requests.
|
|
102
140
|
*/
|
|
103
141
|
httpClient?: BaseHttpClient;
|
|
142
|
+
/**
|
|
143
|
+
* Optional logger for reporting warnings during sitemap discovery.
|
|
144
|
+
*/
|
|
145
|
+
logger?: CrawleeLogger;
|
|
104
146
|
}): AsyncIterable<string>;
|
|
105
147
|
export {};
|
|
106
|
-
//# sourceMappingURL=sitemap.d.ts.map
|