@crawlee/utils 4.0.0-rc.0 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.d.ts +6 -4
- package/index.js +5 -4
- package/internal.d.ts +5 -2
- package/internal.js +4 -1
- package/internals/blocked.d.ts +7 -0
- package/internals/blocked.js +11 -0
- package/internals/cheerio.d.ts +1 -2
- package/internals/cheerio.js +30 -19
- package/internals/extract-microdata.d.ts +24 -0
- package/internals/extract-microdata.js +122 -0
- package/internals/extract-urls.js +1 -7
- package/internals/http-request.d.ts +5 -0
- package/internals/http-request.js +36 -0
- package/internals/mutation-journal.d.ts +29 -0
- package/internals/mutation-journal.js +173 -0
- package/internals/{open_graph_parser.js → open-graph-parser.js} +1 -1
- package/internals/robots.d.ts +3 -2
- package/internals/robots.js +12 -8
- package/internals/schemas.d.ts +2 -2
- package/internals/schemas.js +8 -7
- package/internals/sitemap.d.ts +1 -1
- package/internals/sitemap.js +90 -65
- package/internals/social.d.ts +3 -1
- package/internals/social.js +30 -7
- package/internals/url.d.ts +2 -4
- package/internals/url.js +4 -6
- package/internals/validation.d.ts +6 -22
- package/internals/validation.js +12 -136
- package/package.json +14 -14
- /package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +0 -0
package/internals/robots.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { BaseHttpClient } from '@crawlee/http-client';
|
|
2
|
-
import type { CrawleeLogger } from '@crawlee/types';
|
|
2
|
+
import type { CrawleeLogger, ISession } from '@crawlee/types';
|
|
3
3
|
import { Sitemap } from './sitemap.js';
|
|
4
4
|
import { type EnqueueStrategy } from './url.js';
|
|
5
5
|
export interface RobotsTxtFileSitemapsOptions {
|
|
@@ -38,12 +38,14 @@ export declare class RobotsTxtFile {
|
|
|
38
38
|
* @param [options.signal] an AbortSignal to cancel the request
|
|
39
39
|
* @param [options.timeoutMillis] timeout in milliseconds for the request
|
|
40
40
|
* @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
|
|
41
|
+
* @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
|
|
41
42
|
*/
|
|
42
43
|
static find(url: string, options?: {
|
|
43
44
|
signal?: AbortSignal;
|
|
44
45
|
timeoutMillis?: number;
|
|
45
46
|
proxyUrl?: string;
|
|
46
47
|
httpClient?: BaseHttpClient;
|
|
48
|
+
session?: ISession;
|
|
47
49
|
logger?: CrawleeLogger;
|
|
48
50
|
}): Promise<RobotsTxtFile>;
|
|
49
51
|
/**
|
|
@@ -53,7 +55,6 @@ export declare class RobotsTxtFile {
|
|
|
53
55
|
* @param [proxyUrl] a proxy to be used for fetching the robots.txt file
|
|
54
56
|
*/
|
|
55
57
|
static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
|
|
56
|
-
private static load;
|
|
57
58
|
/**
|
|
58
59
|
* Get crawl delay for a given user agent.
|
|
59
60
|
* @param [userAgent] relevant user agent, default to `*`
|
package/internals/robots.js
CHANGED
|
@@ -38,12 +38,13 @@ export class RobotsTxtFile {
|
|
|
38
38
|
* @param [options.signal] an AbortSignal to cancel the request
|
|
39
39
|
* @param [options.timeoutMillis] timeout in milliseconds for the request
|
|
40
40
|
* @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
|
|
41
|
+
* @param [options.session] a session whose fingerprint and cookies the request uses, like a page request would
|
|
41
42
|
*/
|
|
42
43
|
static async find(url, options) {
|
|
43
44
|
const robotsTxtFileUrl = new URL(url);
|
|
44
45
|
robotsTxtFileUrl.pathname = '/robots.txt';
|
|
45
46
|
robotsTxtFileUrl.search = '';
|
|
46
|
-
return RobotsTxtFile
|
|
47
|
+
return RobotsTxtFile.#load(robotsTxtFileUrl.toString(), options);
|
|
47
48
|
}
|
|
48
49
|
/**
|
|
49
50
|
* Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
|
|
@@ -55,20 +56,20 @@ export class RobotsTxtFile {
|
|
|
55
56
|
// @ts-ignore
|
|
56
57
|
return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
|
|
57
58
|
}
|
|
58
|
-
static async load(url, options) {
|
|
59
|
-
const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
|
|
59
|
+
static async #load(url, options) {
|
|
60
|
+
const { proxyUrl, logger, session, httpClient = new FetchHttpClient() } = options || {};
|
|
60
61
|
const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
|
|
62
|
+
session,
|
|
61
63
|
proxyUrl,
|
|
62
64
|
timeoutMillis: options?.timeoutMillis,
|
|
63
65
|
signal: options?.signal,
|
|
64
66
|
});
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
if (response.status === 404) {
|
|
67
|
+
// RFC 9309: an unavailable (4xx) robots.txt allows everything, an unreachable (5xx) one disallows everything.
|
|
68
|
+
if (response.status >= 400 && response.status < 600) {
|
|
69
|
+
const allowed = response.status < 500;
|
|
69
70
|
return new RobotsTxtFile(url, {
|
|
70
71
|
isAllowed() {
|
|
71
|
-
return
|
|
72
|
+
return allowed;
|
|
72
73
|
},
|
|
73
74
|
getSitemaps() {
|
|
74
75
|
return [];
|
|
@@ -78,6 +79,9 @@ export class RobotsTxtFile {
|
|
|
78
79
|
},
|
|
79
80
|
}, proxyUrl, logger);
|
|
80
81
|
}
|
|
82
|
+
if (response.status < 200 || response.status >= 300) {
|
|
83
|
+
throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
|
|
84
|
+
}
|
|
81
85
|
// @ts-ignore
|
|
82
86
|
return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
|
|
83
87
|
}
|
package/internals/schemas.d.ts
CHANGED
|
@@ -30,7 +30,7 @@ export declare function objectWithKeys(keys: string[], message?: string): z.ZodT
|
|
|
30
30
|
* Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
|
|
31
31
|
* @internal
|
|
32
32
|
*/
|
|
33
|
-
export declare const httpClient: z.
|
|
33
|
+
export declare const httpClient: z.ZodInstanceOf<BaseHttpClient>;
|
|
34
34
|
/**
|
|
35
35
|
* Accepts any object implementing the CrawleeLogger interface.
|
|
36
36
|
* @internal
|
|
@@ -47,7 +47,7 @@ export declare const typedArray: z.ZodCustom<NodeJS.TypedArray<ArrayBufferLike>,
|
|
|
47
47
|
*/
|
|
48
48
|
export declare const plainObject: z.ZodCustom<Record<string, unknown>, Record<string, unknown>>;
|
|
49
49
|
/**
|
|
50
|
-
* Shape of a request stored in a request queue.
|
|
50
|
+
* Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
|
|
51
51
|
* @internal
|
|
52
52
|
*/
|
|
53
53
|
export declare const storageRequest: z.ZodObject<{
|
package/internals/schemas.js
CHANGED
|
@@ -54,11 +54,7 @@ export const typedArray = z.custom((value) => ArrayBuffer.isView(value) && !(val
|
|
|
54
54
|
* @internal
|
|
55
55
|
*/
|
|
56
56
|
export const plainObject = z.custom((value) => typeof value === 'object' && value !== null && !Array.isArray(value), { message: 'Invalid input: expected an object' });
|
|
57
|
-
|
|
58
|
-
* Shape of a request stored in a request queue.
|
|
59
|
-
* @internal
|
|
60
|
-
*/
|
|
61
|
-
export const storageRequest = z.looseObject({
|
|
57
|
+
const storageRequestShape = z.looseObject({
|
|
62
58
|
id: z.string(),
|
|
63
59
|
url: z.url({ protocol: /^https?$/ }),
|
|
64
60
|
uniqueKey: z.string(),
|
|
@@ -66,11 +62,16 @@ export const storageRequest = z.looseObject({
|
|
|
66
62
|
retryCount: z.number().int().optional(),
|
|
67
63
|
handledAt: z.union([z.string(), z.date()]).optional(),
|
|
68
64
|
});
|
|
65
|
+
/**
|
|
66
|
+
* Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
|
|
67
|
+
* @internal
|
|
68
|
+
*/
|
|
69
|
+
export const storageRequest = z.compile(storageRequestShape);
|
|
69
70
|
/**
|
|
70
71
|
* {@link storageRequest} before an id is assigned.
|
|
71
72
|
* @internal
|
|
72
73
|
*/
|
|
73
|
-
export const storageRequestWithoutId =
|
|
74
|
+
export const storageRequestWithoutId = z.compile(storageRequestShape.omit({ id: true }));
|
|
74
75
|
/**
|
|
75
76
|
* `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
|
|
76
77
|
* instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
|
|
@@ -86,7 +87,7 @@ export function arrayOf(item, elements) {
|
|
|
86
87
|
* Batch of {@link storageRequestWithoutId}.
|
|
87
88
|
* @internal
|
|
88
89
|
*/
|
|
89
|
-
export const storageRequestBatch = arrayOf(storageRequestWithoutId, 'requests');
|
|
90
|
+
export const storageRequestBatch = z.compile(arrayOf(storageRequestWithoutId, 'requests'));
|
|
90
91
|
/**
|
|
91
92
|
* Options of request queue add/update operations.
|
|
92
93
|
* @internal
|
package/internals/sitemap.d.ts
CHANGED
|
@@ -82,6 +82,7 @@ export declare function parseSitemap<T extends ParseSitemapOptions>(initialSourc
|
|
|
82
82
|
* ```
|
|
83
83
|
*/
|
|
84
84
|
export declare class Sitemap {
|
|
85
|
+
#private;
|
|
85
86
|
readonly urls: string[];
|
|
86
87
|
constructor(urls: string[]);
|
|
87
88
|
/**
|
|
@@ -103,7 +104,6 @@ export declare class Sitemap {
|
|
|
103
104
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
104
105
|
*/
|
|
105
106
|
static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
|
106
|
-
private static parse;
|
|
107
107
|
}
|
|
108
108
|
/**
|
|
109
109
|
* Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
|
package/internals/sitemap.js
CHANGED
|
@@ -3,7 +3,7 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
|
|
|
3
3
|
import { StringDecoder } from 'node:string_decoder';
|
|
4
4
|
import { createGunzip } from 'node:zlib';
|
|
5
5
|
import { FetchHttpClient } from '@crawlee/http-client';
|
|
6
|
-
import MIMEType from 'whatwg-mimetype';
|
|
6
|
+
import { MIMEType } from 'whatwg-mimetype';
|
|
7
7
|
import { mergeAsyncIterables } from './iterables.js';
|
|
8
8
|
import { RobotsTxtFile } from './robots.js';
|
|
9
9
|
import { filterUrl } from './url.js';
|
|
@@ -14,33 +14,23 @@ class SitemapTxtParser extends Transform {
|
|
|
14
14
|
super({
|
|
15
15
|
readableObjectMode: true,
|
|
16
16
|
transform: (chunk, _encoding, callback) => {
|
|
17
|
-
this
|
|
17
|
+
this.#processBuffer(this.#decoder.write(chunk), false);
|
|
18
18
|
callback();
|
|
19
19
|
},
|
|
20
20
|
flush: (callback) => {
|
|
21
|
-
this
|
|
21
|
+
this.#processBuffer(this.#decoder.end(), true);
|
|
22
22
|
callback();
|
|
23
23
|
},
|
|
24
24
|
});
|
|
25
25
|
}
|
|
26
|
-
processBuffer(input, finalize) {
|
|
27
|
-
this.#buffer
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
for (const url of parts) {
|
|
35
|
-
this.push({ type: 'url', loc: url });
|
|
36
|
-
}
|
|
37
|
-
this.#buffer = '';
|
|
38
|
-
}
|
|
39
|
-
else if (parts.length > 0) {
|
|
40
|
-
for (const url of parts.slice(0, -1)) {
|
|
41
|
-
this.push({ type: 'url', loc: url });
|
|
42
|
-
}
|
|
43
|
-
this.#buffer = parts.at(-1);
|
|
26
|
+
#processBuffer(input, finalize) {
|
|
27
|
+
const lines = (this.#buffer + input).split('\n');
|
|
28
|
+
// The text after the last line break may be a partial line, so it waits for the next chunk.
|
|
29
|
+
this.#buffer = finalize ? '' : lines.pop();
|
|
30
|
+
for (const line of lines) {
|
|
31
|
+
const url = line.trim();
|
|
32
|
+
if (url.length > 0) {
|
|
33
|
+
this.push({ type: 'url', loc: url });
|
|
44
34
|
}
|
|
45
35
|
}
|
|
46
36
|
}
|
|
@@ -52,7 +42,7 @@ class SitemapXmlParser extends Transform {
|
|
|
52
42
|
#currentTag = undefined;
|
|
53
43
|
#url = {};
|
|
54
44
|
static async create() {
|
|
55
|
-
const { SAXParser } = await import('sax');
|
|
45
|
+
const { SAXParser } = (await import('sax')).default;
|
|
56
46
|
return new SitemapXmlParser(new SAXParser(true));
|
|
57
47
|
}
|
|
58
48
|
constructor(parser) {
|
|
@@ -72,13 +62,13 @@ class SitemapXmlParser extends Transform {
|
|
|
72
62
|
},
|
|
73
63
|
});
|
|
74
64
|
this.#parser = parser;
|
|
75
|
-
this.#parser.onopentag = this
|
|
76
|
-
this.#parser.onclosetag = this
|
|
77
|
-
this.#parser.ontext = this
|
|
78
|
-
this.#parser.oncdata = this
|
|
65
|
+
this.#parser.onopentag = this.#onOpenTag.bind(this);
|
|
66
|
+
this.#parser.onclosetag = this.#onCloseTag.bind(this);
|
|
67
|
+
this.#parser.ontext = this.#onText.bind(this);
|
|
68
|
+
this.#parser.oncdata = this.#onText.bind(this);
|
|
79
69
|
this.#parser.onerror = this.destroy.bind(this);
|
|
80
70
|
}
|
|
81
|
-
onOpenTag(node) {
|
|
71
|
+
#onOpenTag(node) {
|
|
82
72
|
if (this.#rootTagName !== undefined) {
|
|
83
73
|
if (node.name === 'loc' ||
|
|
84
74
|
node.name === 'lastmod' ||
|
|
@@ -94,7 +84,7 @@ class SitemapXmlParser extends Transform {
|
|
|
94
84
|
this.#rootTagName = 'sitemapindex';
|
|
95
85
|
}
|
|
96
86
|
}
|
|
97
|
-
onCloseTag(name) {
|
|
87
|
+
#onCloseTag(name) {
|
|
98
88
|
if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
|
|
99
89
|
this.#currentTag = undefined;
|
|
100
90
|
}
|
|
@@ -105,17 +95,21 @@ class SitemapXmlParser extends Transform {
|
|
|
105
95
|
this.#url = {};
|
|
106
96
|
}
|
|
107
97
|
}
|
|
108
|
-
onText(text) {
|
|
98
|
+
#onText(text) {
|
|
99
|
+
text = text.trim();
|
|
100
|
+
// Whitespace around a CDATA section arrives as separate text events.
|
|
101
|
+
if (text === '') {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
109
104
|
if (this.#currentTag === 'loc') {
|
|
110
105
|
if (this.#rootTagName === 'sitemapindex') {
|
|
111
|
-
this.push({ type: 'sitemapUrl', url: text
|
|
106
|
+
this.push({ type: 'sitemapUrl', url: text });
|
|
112
107
|
}
|
|
113
108
|
if (this.#rootTagName === 'urlset') {
|
|
114
109
|
this.#url ??= {};
|
|
115
|
-
this.#url.loc = text
|
|
110
|
+
this.#url.loc = text;
|
|
116
111
|
}
|
|
117
112
|
}
|
|
118
|
-
text = text.trim();
|
|
119
113
|
if (this.#currentTag === 'lastmod') {
|
|
120
114
|
const lastmod = new Date(text);
|
|
121
115
|
if (!Number.isNaN(lastmod.getTime())) {
|
|
@@ -150,6 +144,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
150
144
|
if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
|
|
151
145
|
return new SitemapTxtParser();
|
|
152
146
|
}
|
|
147
|
+
// A name like `sitemap.gz` says nothing about what was compressed. Sitemaps are XML unless named otherwise.
|
|
148
|
+
if (mimeType?.essence === 'application/gzip') {
|
|
149
|
+
return SitemapXmlParser.create();
|
|
150
|
+
}
|
|
153
151
|
throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
|
|
154
152
|
};
|
|
155
153
|
while (sources.length > 0) {
|
|
@@ -241,44 +239,66 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
241
239
|
// URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
|
|
242
240
|
// the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
|
|
243
241
|
let droppedUrlEntries = 0;
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
242
|
+
const iterator = items[Symbol.asyncIterator]();
|
|
243
|
+
try {
|
|
244
|
+
while (true) {
|
|
245
|
+
let next;
|
|
246
|
+
try {
|
|
247
|
+
next = await iterator.next();
|
|
249
248
|
}
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
|
|
256
|
-
continue;
|
|
249
|
+
catch (e) {
|
|
250
|
+
// Parsing failed: give up on this sitemap, keep the others.
|
|
251
|
+
// Raw sources already logged the error in their pipeline callback.
|
|
252
|
+
if (source.type !== 'raw') {
|
|
253
|
+
logger?.warning(`Malformed sitemap content: ${source.url} (${e})`);
|
|
257
254
|
}
|
|
255
|
+
break;
|
|
258
256
|
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
// Keep only URL entries that match the enqueue strategy relative to the parent (see above).
|
|
266
|
-
if (source.type === 'url') {
|
|
267
|
-
const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
|
|
268
|
-
if (!allowed) {
|
|
269
|
-
droppedUrlEntries++;
|
|
270
|
-
logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
|
|
257
|
+
if (next.done)
|
|
258
|
+
break;
|
|
259
|
+
const item = next.value;
|
|
260
|
+
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
|
|
261
|
+
if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
|
|
262
|
+
logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
|
|
271
263
|
continue;
|
|
272
264
|
}
|
|
265
|
+
// Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
|
|
266
|
+
// parent. Raw string sources have no parent URL, so the check is skipped.
|
|
267
|
+
if (source.type === 'url') {
|
|
268
|
+
const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
|
|
269
|
+
if (!allowed) {
|
|
270
|
+
logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
|
|
275
|
+
if (emitNestedSitemaps) {
|
|
276
|
+
yield { loc: item.url, originSitemapUrl: null };
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
if (item.type === 'url') {
|
|
280
|
+
// Keep only URL entries that match the enqueue strategy relative to the parent (see above).
|
|
281
|
+
if (source.type === 'url') {
|
|
282
|
+
const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
|
|
283
|
+
if (!allowed) {
|
|
284
|
+
droppedUrlEntries++;
|
|
285
|
+
logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
yield {
|
|
290
|
+
...item,
|
|
291
|
+
originSitemapUrl: source.type === 'url'
|
|
292
|
+
? source.url
|
|
293
|
+
: `raw://${createHash('sha256').update(source.content).digest('base64')}`,
|
|
294
|
+
};
|
|
273
295
|
}
|
|
274
|
-
yield {
|
|
275
|
-
...item,
|
|
276
|
-
originSitemapUrl: source.type === 'url'
|
|
277
|
-
? source.url
|
|
278
|
-
: `raw://${createHash('sha256').update(source.content).digest('base64')}`,
|
|
279
|
-
};
|
|
280
296
|
}
|
|
281
297
|
}
|
|
298
|
+
finally {
|
|
299
|
+
// Mirrors `for await` cleanup: destroys the stream when we stop early or the consumer bails out.
|
|
300
|
+
await iterator.return?.();
|
|
301
|
+
}
|
|
282
302
|
if (droppedUrlEntries > 0 && source.type === 'url') {
|
|
283
303
|
logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
|
|
284
304
|
}
|
|
@@ -323,7 +343,7 @@ export class Sitemap {
|
|
|
323
343
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
324
344
|
*/
|
|
325
345
|
static async load(urls, proxyUrl, parseSitemapOptions) {
|
|
326
|
-
return await this
|
|
346
|
+
return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
|
|
327
347
|
}
|
|
328
348
|
/**
|
|
329
349
|
* Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
|
|
@@ -331,9 +351,9 @@ export class Sitemap {
|
|
|
331
351
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
332
352
|
*/
|
|
333
353
|
static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
|
|
334
|
-
return await this
|
|
354
|
+
return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
|
|
335
355
|
}
|
|
336
|
-
static async parse(sources, proxyUrl, parseSitemapOptions) {
|
|
356
|
+
static async #parse(sources, proxyUrl, parseSitemapOptions) {
|
|
337
357
|
const urls = [];
|
|
338
358
|
try {
|
|
339
359
|
for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
|
|
@@ -423,6 +443,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
423
443
|
}
|
|
424
444
|
else {
|
|
425
445
|
const firstUrl = new URL(domainUrls[0]);
|
|
446
|
+
// The guessed candidates are well-known paths on the same origin, so the query string
|
|
447
|
+
// and fragment of the input URL are not part of them - carrying them over probes (and
|
|
448
|
+
// yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
|
|
449
|
+
firstUrl.search = '';
|
|
450
|
+
firstUrl.hash = '';
|
|
426
451
|
const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
|
|
427
452
|
const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
|
|
428
453
|
firstUrl.pathname = pathname;
|
package/internals/social.d.ts
CHANGED
|
@@ -19,6 +19,8 @@ export declare function emailsFromText(text: string): string[];
|
|
|
19
19
|
/**
|
|
20
20
|
* The function extracts email addresses from a list of URLs.
|
|
21
21
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
22
|
+
* A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
|
|
23
|
+
* such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
|
|
22
24
|
* Note that the function preserves the order of emails and keep duplicates.
|
|
23
25
|
* @param urls Array of URLs.
|
|
24
26
|
* @return Array of emails addresses found.
|
|
@@ -225,7 +227,7 @@ export declare const TWITTER_REGEX: RegExp;
|
|
|
225
227
|
* ```
|
|
226
228
|
* import { social } from 'crawlee';
|
|
227
229
|
*
|
|
228
|
-
* const matches = text.match(social.
|
|
230
|
+
* const matches = text.match(social.TWITTER_REGEX_GLOBAL);
|
|
229
231
|
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
230
232
|
* ```
|
|
231
233
|
*/
|
package/internals/social.js
CHANGED
|
@@ -30,6 +30,8 @@ export function emailsFromText(text) {
|
|
|
30
30
|
/**
|
|
31
31
|
* The function extracts email addresses from a list of URLs.
|
|
32
32
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
33
|
+
* A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
|
|
34
|
+
* such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
|
|
33
35
|
* Note that the function preserves the order of emails and keep duplicates.
|
|
34
36
|
* @param urls Array of URLs.
|
|
35
37
|
* @return Array of emails addresses found.
|
|
@@ -44,9 +46,20 @@ export function emailsFromUrls(urls) {
|
|
|
44
46
|
continue;
|
|
45
47
|
if (!EMAIL_URL_PREFIX_REGEX.test(url))
|
|
46
48
|
continue;
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
49
|
+
// Characters like `?` and `,` inside an address must be percent-encoded, so it's safe
|
|
50
|
+
// to split on them before decoding.
|
|
51
|
+
const [addresses] = url.replace(EMAIL_URL_PREFIX_REGEX, '').split('?');
|
|
52
|
+
for (const address of addresses.split(',')) {
|
|
53
|
+
let email = address.trim();
|
|
54
|
+
try {
|
|
55
|
+
email = decodeURIComponent(email).trim();
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// Keep the raw value if it isn't valid percent-encoding
|
|
59
|
+
}
|
|
60
|
+
if (EMAIL_REGEX.test(email))
|
|
61
|
+
emails.push(email);
|
|
62
|
+
}
|
|
50
63
|
}
|
|
51
64
|
return emails;
|
|
52
65
|
}
|
|
@@ -138,8 +151,18 @@ export function phonesFromUrls(urls) {
|
|
|
138
151
|
continue;
|
|
139
152
|
if (!PHONE_URL_PREFIX_REGEX.test(url))
|
|
140
153
|
continue;
|
|
141
|
-
|
|
142
|
-
|
|
154
|
+
// Per RFC 3966, parameters such as `;ext=89` can follow the number, and `-`, `.`, `(` and `)`
|
|
155
|
+
// are only visual separators. They are checked without separators too, so a one-digit country
|
|
156
|
+
// code like in `tel:+1-555-123-4567` is accepted.
|
|
157
|
+
const [number] = url.replace(PHONE_URL_PREFIX_REGEX, '').split(';');
|
|
158
|
+
let phone = number.trim();
|
|
159
|
+
try {
|
|
160
|
+
phone = decodeURIComponent(phone).trim();
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
// Keep the raw value if it isn't valid percent-encoding
|
|
164
|
+
}
|
|
165
|
+
if (PHONE_REGEX.test(phone) || PHONE_REGEX.test(phone.replace(/[-.() ]/g, '')))
|
|
143
166
|
phones.push(phone);
|
|
144
167
|
}
|
|
145
168
|
return phones;
|
|
@@ -326,7 +349,7 @@ export const TWITTER_REGEX = new RegExp(`^${TWITTER_REGEX_STRING}$`, 'i');
|
|
|
326
349
|
* ```
|
|
327
350
|
* import { social } from 'crawlee';
|
|
328
351
|
*
|
|
329
|
-
* const matches = text.match(social.
|
|
352
|
+
* const matches = text.match(social.TWITTER_REGEX_GLOBAL);
|
|
330
353
|
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
331
354
|
* ```
|
|
332
355
|
*/
|
|
@@ -590,7 +613,7 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
|
|
|
590
613
|
* @return An object with the social handles.
|
|
591
614
|
*/
|
|
592
615
|
export async function parseHandlesFromHtml(html, data = null) {
|
|
593
|
-
const cheerio = await import('cheerio');
|
|
616
|
+
const cheerio = await import('cheerio/slim');
|
|
594
617
|
const result = {
|
|
595
618
|
emails: [],
|
|
596
619
|
phones: [],
|
package/internals/url.d.ts
CHANGED
|
@@ -54,10 +54,8 @@ export declare enum EnqueueStrategy {
|
|
|
54
54
|
export declare const UNSUPPORTED_SCHEME_MESSAGE = "unsupported URL scheme (only http and https are allowed)";
|
|
55
55
|
/**
|
|
56
56
|
* Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
|
|
57
|
-
* considered here (use {@link filterUrl} for the combined scheme + strategy check).
|
|
58
|
-
*
|
|
59
|
-
* The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
|
|
60
|
-
* (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
|
|
57
|
+
* considered here (use {@link filterUrl} for the combined scheme + strategy check). This is the single
|
|
58
|
+
* definition of the strategies, shared by `enqueueLinks`, sitemaps and robots.txt.
|
|
61
59
|
*/
|
|
62
60
|
export declare function matchesEnqueueStrategy(strategy: EnqueueStrategy | `${EnqueueStrategy}`, target: URL, origin: URL): boolean;
|
|
63
61
|
/**
|
package/internals/url.js
CHANGED
|
@@ -71,10 +71,8 @@ function normalizeHostname(hostname) {
|
|
|
71
71
|
}
|
|
72
72
|
/**
|
|
73
73
|
* Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
|
|
74
|
-
* considered here (use {@link filterUrl} for the combined scheme + strategy check).
|
|
75
|
-
*
|
|
76
|
-
* The `enqueueLinks` implementation in `@crawlee/core` matches the same strategies via glob patterns
|
|
77
|
-
* (see `packages/core/src/enqueue_links/enqueue_links.ts`) — keep the two in sync when changing either.
|
|
74
|
+
* considered here (use {@link filterUrl} for the combined scheme + strategy check). This is the single
|
|
75
|
+
* definition of the strategies, shared by `enqueueLinks`, sitemaps and robots.txt.
|
|
78
76
|
*/
|
|
79
77
|
export function matchesEnqueueStrategy(strategy, target, origin) {
|
|
80
78
|
switch (strategy) {
|
|
@@ -87,8 +85,8 @@ export function matchesEnqueueStrategy(strategy, target, origin) {
|
|
|
87
85
|
if (originDomain) {
|
|
88
86
|
return originDomain === getDomain(target.hostname, { mixedInputs: false });
|
|
89
87
|
}
|
|
90
|
-
// No registrable domain (e.g. an IP address), fall back to
|
|
91
|
-
return target
|
|
88
|
+
// No registrable domain (e.g. an IP address or `localhost`), fall back to the same hostname.
|
|
89
|
+
return matchesEnqueueStrategy('same-hostname', target, origin);
|
|
92
90
|
}
|
|
93
91
|
case 'same-origin':
|
|
94
92
|
// Compare scheme/host/port directly so a trailing-dot host is normalized.
|
|
@@ -1,25 +1,9 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Dictionary } from '@crawlee/types';
|
|
2
|
+
export { ArgumentValidationError, parseArgument } from '@apify/validations';
|
|
2
3
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* value it received (rather than a raw JSON dump). The structured
|
|
7
|
-
* {@link https://zod.dev | zod} issues are available on `issues`, and the
|
|
8
|
-
* original `ZodError` on `cause`, for programmatic inspection.
|
|
9
|
-
*/
|
|
10
|
-
export declare class ArgumentValidationError extends Error {
|
|
11
|
-
/** Structured issues from the underlying schema check. */
|
|
12
|
-
readonly issues: z.ZodError['issues'];
|
|
13
|
-
/** The raw zod error that triggered this. */
|
|
14
|
-
readonly cause: z.ZodError;
|
|
15
|
-
constructor(error: z.ZodError, value: unknown, label?: string);
|
|
16
|
-
}
|
|
17
|
-
/**
|
|
18
|
-
* Parses `value` with `schema`, returning the typed result (with schema defaults applied).
|
|
19
|
-
* Throws {@link ArgumentValidationError} on failure.
|
|
20
|
-
*
|
|
21
|
-
* The optional `label` names the interface being validated and is appended to every error line
|
|
22
|
-
* (e.g. ``… at `maxRequestRetries` in `BasicCrawlerOptions` ``).
|
|
4
|
+
* Rejects options that exist only to configure the browser pool the crawler would have built for itself.
|
|
5
|
+
* Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
|
|
6
|
+
* into a second, half-working way of configuring the same pool.
|
|
23
7
|
* @internal
|
|
24
8
|
*/
|
|
25
|
-
export declare function
|
|
9
|
+
export declare function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;
|