@crawlee/utils 4.0.0-beta.26 → 4.0.0-beta.260
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +10 -17
- package/index.js +8 -16
- package/internal.d.ts +12 -0
- package/internal.js +11 -0
- package/internals/blocked.d.ts +0 -1
- package/internals/blocked.js +0 -1
- package/internals/cheerio.d.ts +1 -3
- package/internals/cheerio.js +25 -16
- package/internals/extract-microdata.d.ts +24 -0
- package/internals/extract-microdata.js +118 -0
- package/internals/extract-urls.d.ts +1 -2
- package/internals/extract-urls.js +18 -17
- package/internals/general.d.ts +0 -25
- package/internals/general.js +2 -110
- package/internals/http-request.d.ts +5 -0
- package/internals/http-request.js +36 -0
- package/internals/iterables.d.ts +0 -80
- package/internals/iterables.js +0 -126
- package/internals/mutation-journal.d.ts +29 -0
- package/internals/mutation-journal.js +173 -0
- package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
- package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
- package/internals/robots.d.ts +35 -15
- package/internals/robots.js +66 -30
- package/internals/schemas.d.ts +114 -0
- package/internals/schemas.js +115 -0
- package/internals/sitemap.d.ts +44 -3
- package/internals/sitemap.js +173 -97
- package/internals/social.d.ts +4 -3
- package/internals/social.js +36 -11
- package/internals/url.d.ts +67 -1
- package/internals/url.js +118 -2
- package/internals/validation.d.ts +9 -0
- package/internals/validation.js +16 -0
- package/package.json +16 -15
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/blocked.d.ts.map +0 -1
- package/internals/blocked.js.map +0 -1
- package/internals/cheerio.d.ts.map +0 -1
- package/internals/cheerio.js.map +0 -1
- package/internals/chunk.d.ts +0 -2
- package/internals/chunk.d.ts.map +0 -1
- package/internals/chunk.js +0 -40
- package/internals/chunk.js.map +0 -1
- package/internals/debug.d.ts +0 -32
- package/internals/debug.d.ts.map +0 -1
- package/internals/debug.js +0 -41
- package/internals/debug.js.map +0 -1
- package/internals/extract-urls.d.ts.map +0 -1
- package/internals/extract-urls.js.map +0 -1
- package/internals/general.d.ts.map +0 -1
- package/internals/general.js.map +0 -1
- package/internals/iterables.d.ts.map +0 -1
- package/internals/iterables.js.map +0 -1
- package/internals/open_graph_parser.d.ts.map +0 -1
- package/internals/open_graph_parser.js.map +0 -1
- package/internals/robots.d.ts.map +0 -1
- package/internals/robots.js.map +0 -1
- package/internals/sitemap.d.ts.map +0 -1
- package/internals/sitemap.js.map +0 -1
- package/internals/social.d.ts.map +0 -1
- package/internals/social.js.map +0 -1
- package/internals/system-info/cpu-info.d.ts +0 -64
- package/internals/system-info/cpu-info.d.ts.map +0 -1
- package/internals/system-info/cpu-info.js +0 -211
- package/internals/system-info/cpu-info.js.map +0 -1
- package/internals/system-info/memory-info.d.ts +0 -28
- package/internals/system-info/memory-info.d.ts.map +0 -1
- package/internals/system-info/memory-info.js +0 -118
- package/internals/system-info/memory-info.js.map +0 -1
- package/internals/system-info/ps-tree.d.ts +0 -18
- package/internals/system-info/ps-tree.d.ts.map +0 -1
- package/internals/system-info/ps-tree.js +0 -145
- package/internals/system-info/ps-tree.js.map +0 -1
- package/internals/typedefs.d.ts +0 -5
- package/internals/typedefs.d.ts.map +0 -1
- package/internals/typedefs.js +0 -9
- package/internals/typedefs.js.map +0 -1
- package/internals/url.d.ts.map +0 -1
- package/internals/url.js.map +0 -1
package/internals/sitemap.js
CHANGED
|
@@ -3,132 +3,134 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
|
|
|
3
3
|
import { StringDecoder } from 'node:string_decoder';
|
|
4
4
|
import { createGunzip } from 'node:zlib';
|
|
5
5
|
import { FetchHttpClient } from '@crawlee/http-client';
|
|
6
|
-
import {
|
|
7
|
-
import sax from 'sax';
|
|
8
|
-
import MIMEType from 'whatwg-mimetype';
|
|
9
|
-
import log from '@apify/log';
|
|
6
|
+
import { MIMEType } from 'whatwg-mimetype';
|
|
10
7
|
import { mergeAsyncIterables } from './iterables.js';
|
|
11
|
-
import {
|
|
8
|
+
import { RobotsTxtFile } from './robots.js';
|
|
9
|
+
import { filterUrl } from './url.js';
|
|
12
10
|
class SitemapTxtParser extends Transform {
|
|
13
|
-
decoder = new StringDecoder('utf8');
|
|
14
|
-
buffer = '';
|
|
11
|
+
#decoder = new StringDecoder('utf8');
|
|
12
|
+
#buffer = '';
|
|
15
13
|
constructor() {
|
|
16
14
|
super({
|
|
17
15
|
readableObjectMode: true,
|
|
18
16
|
transform: (chunk, _encoding, callback) => {
|
|
19
|
-
this
|
|
17
|
+
this.#processBuffer(this.#decoder.write(chunk), false);
|
|
20
18
|
callback();
|
|
21
19
|
},
|
|
22
20
|
flush: (callback) => {
|
|
23
|
-
this
|
|
21
|
+
this.#processBuffer(this.#decoder.end(), true);
|
|
24
22
|
callback();
|
|
25
23
|
},
|
|
26
24
|
});
|
|
27
25
|
}
|
|
28
|
-
processBuffer(input, finalize) {
|
|
29
|
-
this
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
for (const url of parts) {
|
|
37
|
-
this.push({ type: 'url', loc: url });
|
|
38
|
-
}
|
|
39
|
-
this.buffer = '';
|
|
40
|
-
}
|
|
41
|
-
else if (parts.length > 0) {
|
|
42
|
-
for (const url of parts.slice(0, -1)) {
|
|
43
|
-
this.push({ type: 'url', loc: url });
|
|
44
|
-
}
|
|
45
|
-
this.buffer = parts.at(-1);
|
|
26
|
+
#processBuffer(input, finalize) {
|
|
27
|
+
const lines = (this.#buffer + input).split('\n');
|
|
28
|
+
// The text after the last line break may be a partial line, so it waits for the next chunk.
|
|
29
|
+
this.#buffer = finalize ? '' : lines.pop();
|
|
30
|
+
for (const line of lines) {
|
|
31
|
+
const url = line.trim();
|
|
32
|
+
if (url.length > 0) {
|
|
33
|
+
this.push({ type: 'url', loc: url });
|
|
46
34
|
}
|
|
47
35
|
}
|
|
48
36
|
}
|
|
49
37
|
}
|
|
50
38
|
class SitemapXmlParser extends Transform {
|
|
51
|
-
decoder = new StringDecoder('utf8');
|
|
52
|
-
parser
|
|
53
|
-
rootTagName;
|
|
54
|
-
currentTag = undefined;
|
|
55
|
-
url = {};
|
|
56
|
-
|
|
39
|
+
#decoder = new StringDecoder('utf8');
|
|
40
|
+
#parser;
|
|
41
|
+
#rootTagName;
|
|
42
|
+
#currentTag = undefined;
|
|
43
|
+
#url = {};
|
|
44
|
+
static async create() {
|
|
45
|
+
const { SAXParser } = (await import('sax')).default;
|
|
46
|
+
return new SitemapXmlParser(new SAXParser(true));
|
|
47
|
+
}
|
|
48
|
+
constructor(parser) {
|
|
57
49
|
super({
|
|
58
50
|
readableObjectMode: true,
|
|
59
51
|
transform: (chunk, _encoding, callback) => {
|
|
60
|
-
this
|
|
52
|
+
this.#parser.write(this.#decoder.write(chunk));
|
|
61
53
|
callback();
|
|
62
54
|
},
|
|
63
55
|
flush: (callback) => {
|
|
64
|
-
const rest = this
|
|
56
|
+
const rest = this.#decoder.end();
|
|
65
57
|
if (rest.length > 0) {
|
|
66
|
-
this
|
|
58
|
+
this.#parser.write(rest);
|
|
67
59
|
}
|
|
68
|
-
this
|
|
60
|
+
this.#parser.end();
|
|
69
61
|
callback();
|
|
70
62
|
},
|
|
71
63
|
});
|
|
72
|
-
this
|
|
73
|
-
this
|
|
74
|
-
this
|
|
75
|
-
this
|
|
76
|
-
this
|
|
64
|
+
this.#parser = parser;
|
|
65
|
+
this.#parser.onopentag = this.#onOpenTag.bind(this);
|
|
66
|
+
this.#parser.onclosetag = this.#onCloseTag.bind(this);
|
|
67
|
+
this.#parser.ontext = this.#onText.bind(this);
|
|
68
|
+
this.#parser.oncdata = this.#onText.bind(this);
|
|
69
|
+
this.#parser.onerror = this.destroy.bind(this);
|
|
77
70
|
}
|
|
78
|
-
onOpenTag(node) {
|
|
79
|
-
if (this
|
|
71
|
+
#onOpenTag(node) {
|
|
72
|
+
if (this.#rootTagName !== undefined) {
|
|
80
73
|
if (node.name === 'loc' ||
|
|
81
74
|
node.name === 'lastmod' ||
|
|
82
75
|
node.name === 'priority' ||
|
|
83
76
|
node.name === 'changefreq') {
|
|
84
|
-
this
|
|
77
|
+
this.#currentTag = node.name;
|
|
85
78
|
}
|
|
86
79
|
}
|
|
87
80
|
if (node.name === 'urlset') {
|
|
88
|
-
this
|
|
81
|
+
this.#rootTagName = 'urlset';
|
|
89
82
|
}
|
|
90
83
|
if (node.name === 'sitemapindex') {
|
|
91
|
-
this
|
|
84
|
+
this.#rootTagName = 'sitemapindex';
|
|
92
85
|
}
|
|
93
86
|
}
|
|
94
|
-
onCloseTag(name) {
|
|
87
|
+
#onCloseTag(name) {
|
|
95
88
|
if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
|
|
96
|
-
this
|
|
89
|
+
this.#currentTag = undefined;
|
|
97
90
|
}
|
|
98
|
-
if (name === 'url'
|
|
99
|
-
|
|
100
|
-
|
|
91
|
+
if (name === 'url') {
|
|
92
|
+
if (this.#url.loc !== undefined) {
|
|
93
|
+
this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
|
|
94
|
+
}
|
|
95
|
+
this.#url = {};
|
|
101
96
|
}
|
|
102
97
|
}
|
|
103
|
-
onText(text) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
98
|
+
#onText(text) {
|
|
99
|
+
text = text.trim();
|
|
100
|
+
// Whitespace around a CDATA section arrives as separate text events.
|
|
101
|
+
if (text === '') {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
if (this.#currentTag === 'loc') {
|
|
105
|
+
if (this.#rootTagName === 'sitemapindex') {
|
|
106
|
+
this.push({ type: 'sitemapUrl', url: text });
|
|
107
107
|
}
|
|
108
|
-
if (this
|
|
109
|
-
this
|
|
110
|
-
this
|
|
108
|
+
if (this.#rootTagName === 'urlset') {
|
|
109
|
+
this.#url ??= {};
|
|
110
|
+
this.#url.loc = text;
|
|
111
111
|
}
|
|
112
112
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
113
|
+
if (this.#currentTag === 'lastmod') {
|
|
114
|
+
const lastmod = new Date(text);
|
|
115
|
+
if (!Number.isNaN(lastmod.getTime())) {
|
|
116
|
+
this.#url.lastmod = lastmod;
|
|
117
|
+
}
|
|
116
118
|
}
|
|
117
|
-
if (this
|
|
118
|
-
this
|
|
119
|
+
if (this.#currentTag === 'priority') {
|
|
120
|
+
this.#url.priority = Number(text);
|
|
119
121
|
}
|
|
120
|
-
if (this
|
|
122
|
+
if (this.#currentTag === 'changefreq') {
|
|
121
123
|
if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
|
|
122
|
-
this
|
|
124
|
+
this.#url.changefreq = text;
|
|
123
125
|
}
|
|
124
126
|
}
|
|
125
127
|
}
|
|
126
128
|
}
|
|
127
129
|
export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
128
|
-
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {};
|
|
130
|
+
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', logger, } = options ?? {};
|
|
129
131
|
const sources = [...initialSources];
|
|
130
132
|
const visitedSitemapUrls = new Set();
|
|
131
|
-
const createParser = (contentType = '', url) => {
|
|
133
|
+
const createParser = async (contentType = '', url) => {
|
|
132
134
|
let mimeType;
|
|
133
135
|
try {
|
|
134
136
|
mimeType = new MIMEType(contentType);
|
|
@@ -137,7 +139,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
137
139
|
mimeType = null;
|
|
138
140
|
}
|
|
139
141
|
if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
|
|
140
|
-
return
|
|
142
|
+
return SitemapXmlParser.create();
|
|
141
143
|
}
|
|
142
144
|
if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
|
|
143
145
|
return new SitemapTxtParser();
|
|
@@ -147,12 +149,13 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
147
149
|
while (sources.length > 0) {
|
|
148
150
|
const source = sources.shift();
|
|
149
151
|
if ((source?.depth ?? 0) > maxDepth) {
|
|
150
|
-
log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
|
|
151
152
|
continue;
|
|
152
153
|
}
|
|
153
154
|
let items = null;
|
|
155
|
+
// Parent URL, parsed once and reused as the origin for the strategy checks below.
|
|
156
|
+
let sitemapUrl;
|
|
154
157
|
if (source.type === 'url') {
|
|
155
|
-
|
|
158
|
+
sitemapUrl = new URL(source.url);
|
|
156
159
|
visitedSitemapUrls.add(sitemapUrl.toString());
|
|
157
160
|
let retriesLeft = sitemapRetries + 1;
|
|
158
161
|
while (retriesLeft-- > 0) {
|
|
@@ -162,11 +165,11 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
162
165
|
sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
|
|
163
166
|
method: 'GET',
|
|
164
167
|
headers: {
|
|
165
|
-
accept: '
|
|
168
|
+
accept: '*/*',
|
|
166
169
|
},
|
|
167
170
|
}), {
|
|
168
171
|
proxyUrl,
|
|
169
|
-
timeout,
|
|
172
|
+
timeoutMillis: timeout,
|
|
170
173
|
});
|
|
171
174
|
}
|
|
172
175
|
catch (error) {
|
|
@@ -178,6 +181,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
178
181
|
if (sitemapResponse.body === null) {
|
|
179
182
|
break;
|
|
180
183
|
}
|
|
184
|
+
const { fileTypeStream } = await import('file-type');
|
|
181
185
|
const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
|
|
182
186
|
if (streamWithType.fileType !== undefined) {
|
|
183
187
|
contentType = streamWithType.fileType.mime;
|
|
@@ -191,7 +195,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
191
195
|
sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
|
|
192
196
|
}
|
|
193
197
|
}
|
|
194
|
-
items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
|
|
198
|
+
items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
|
|
195
199
|
if (e !== undefined && e !== null) {
|
|
196
200
|
error = { type: 'parser', error: e };
|
|
197
201
|
}
|
|
@@ -214,28 +218,53 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
214
218
|
}
|
|
215
219
|
}
|
|
216
220
|
catch (e) {
|
|
217
|
-
|
|
221
|
+
logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
|
|
218
222
|
}
|
|
219
223
|
}
|
|
220
224
|
}
|
|
221
225
|
else if (source.type === 'raw') {
|
|
222
|
-
items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
|
|
226
|
+
items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
|
|
223
227
|
if (error !== undefined) {
|
|
224
|
-
|
|
228
|
+
logger?.warning(`Malformed sitemap content: ${error}`);
|
|
225
229
|
}
|
|
226
230
|
});
|
|
227
231
|
}
|
|
228
232
|
if (items === null) {
|
|
229
233
|
continue;
|
|
230
234
|
}
|
|
235
|
+
// URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
|
|
236
|
+
// the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
|
|
237
|
+
let droppedUrlEntries = 0;
|
|
231
238
|
for await (const item of items) {
|
|
232
239
|
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
|
|
240
|
+
if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
|
|
241
|
+
logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
// Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
|
|
245
|
+
// parent. Raw string sources have no parent URL, so the check is skipped.
|
|
246
|
+
if (source.type === 'url') {
|
|
247
|
+
const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
|
|
248
|
+
if (!allowed) {
|
|
249
|
+
logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
|
|
250
|
+
continue;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
233
253
|
sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
|
|
234
254
|
if (emitNestedSitemaps) {
|
|
235
255
|
yield { loc: item.url, originSitemapUrl: null };
|
|
236
256
|
}
|
|
237
257
|
}
|
|
238
258
|
if (item.type === 'url') {
|
|
259
|
+
// Keep only URL entries that match the enqueue strategy relative to the parent (see above).
|
|
260
|
+
if (source.type === 'url') {
|
|
261
|
+
const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
|
|
262
|
+
if (!allowed) {
|
|
263
|
+
droppedUrlEntries++;
|
|
264
|
+
logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
|
|
265
|
+
continue;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
239
268
|
yield {
|
|
240
269
|
...item,
|
|
241
270
|
originSitemapUrl: source.type === 'url'
|
|
@@ -244,6 +273,9 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
244
273
|
};
|
|
245
274
|
}
|
|
246
275
|
}
|
|
276
|
+
if (droppedUrlEntries > 0 && source.type === 'url') {
|
|
277
|
+
logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
|
|
278
|
+
}
|
|
247
279
|
}
|
|
248
280
|
}
|
|
249
281
|
/**
|
|
@@ -285,7 +317,7 @@ export class Sitemap {
|
|
|
285
317
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
286
318
|
*/
|
|
287
319
|
static async load(urls, proxyUrl, parseSitemapOptions) {
|
|
288
|
-
return await this
|
|
320
|
+
return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
|
|
289
321
|
}
|
|
290
322
|
/**
|
|
291
323
|
* Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
|
|
@@ -293,16 +325,17 @@ export class Sitemap {
|
|
|
293
325
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
294
326
|
*/
|
|
295
327
|
static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
|
|
296
|
-
return await this
|
|
328
|
+
return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
|
|
297
329
|
}
|
|
298
|
-
static async parse(sources, proxyUrl, parseSitemapOptions) {
|
|
330
|
+
static async #parse(sources, proxyUrl, parseSitemapOptions) {
|
|
299
331
|
const urls = [];
|
|
300
332
|
try {
|
|
301
333
|
for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
|
|
302
334
|
urls.push(item.loc);
|
|
303
335
|
}
|
|
304
336
|
}
|
|
305
|
-
catch {
|
|
337
|
+
catch (e) {
|
|
338
|
+
parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
|
|
306
339
|
return new Sitemap([]);
|
|
307
340
|
}
|
|
308
341
|
return new Sitemap(urls);
|
|
@@ -316,7 +349,19 @@ export class Sitemap {
|
|
|
316
349
|
* @returns An async iterable with the discovered sitemap URLs.
|
|
317
350
|
*/
|
|
318
351
|
export async function* discoverValidSitemaps(urls, options = {}) {
|
|
319
|
-
const { proxyUrl, httpClient = new FetchHttpClient() } = options;
|
|
352
|
+
const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
|
|
353
|
+
const controller = new AbortController();
|
|
354
|
+
const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
|
|
355
|
+
const onExternalAbort = () => controller.abort();
|
|
356
|
+
if (externalSignal) {
|
|
357
|
+
if (externalSignal.aborted) {
|
|
358
|
+
controller.abort();
|
|
359
|
+
}
|
|
360
|
+
else {
|
|
361
|
+
externalSignal.addEventListener('abort', onExternalAbort, { once: true });
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
const signal = controller.signal;
|
|
320
365
|
const sitemapUrls = new Set();
|
|
321
366
|
const addSitemapUrl = (url) => {
|
|
322
367
|
const sizeBefore = sitemapUrls.size;
|
|
@@ -331,7 +376,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
331
376
|
return false;
|
|
332
377
|
}
|
|
333
378
|
try {
|
|
334
|
-
const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
|
|
379
|
+
const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
|
|
380
|
+
proxyUrl,
|
|
381
|
+
timeoutMillis: requestTimeoutMillis,
|
|
382
|
+
signal,
|
|
383
|
+
});
|
|
335
384
|
return response.status >= 200 && response.status < 400;
|
|
336
385
|
}
|
|
337
386
|
catch {
|
|
@@ -343,17 +392,24 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
343
392
|
return;
|
|
344
393
|
}
|
|
345
394
|
try {
|
|
346
|
-
const robotsFile = await
|
|
347
|
-
|
|
395
|
+
const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
|
|
396
|
+
proxyUrl,
|
|
397
|
+
timeoutMillis: requestTimeoutMillis,
|
|
398
|
+
signal,
|
|
399
|
+
httpClient,
|
|
400
|
+
logger,
|
|
401
|
+
});
|
|
402
|
+
// Surface all referenced sitemaps, including cross-host; scoping happens at load time.
|
|
403
|
+
for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
|
|
348
404
|
if (addSitemapUrl(sitemapUrl)) {
|
|
349
405
|
yield sitemapUrl;
|
|
350
406
|
}
|
|
351
407
|
}
|
|
352
408
|
}
|
|
353
409
|
catch (err) {
|
|
354
|
-
|
|
410
|
+
logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
|
|
355
411
|
}
|
|
356
|
-
const sitemapUrl = domainUrls.find((url) => /sitemap
|
|
412
|
+
const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
|
|
357
413
|
if (sitemapUrl !== undefined) {
|
|
358
414
|
if (addSitemapUrl(sitemapUrl)) {
|
|
359
415
|
yield sitemapUrl;
|
|
@@ -361,14 +417,29 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
361
417
|
}
|
|
362
418
|
else {
|
|
363
419
|
const firstUrl = new URL(domainUrls[0]);
|
|
420
|
+
// The guessed candidates are well-known paths on the same origin, so the query string
|
|
421
|
+
// and fragment of the input URL are not part of them - carrying them over probes (and
|
|
422
|
+
// yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
|
|
423
|
+
firstUrl.search = '';
|
|
424
|
+
firstUrl.hash = '';
|
|
364
425
|
const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
|
|
365
|
-
|
|
426
|
+
const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
|
|
366
427
|
firstUrl.pathname = pathname;
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
428
|
+
return firstUrl.toString();
|
|
429
|
+
});
|
|
430
|
+
const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
|
|
431
|
+
for (const [index, result] of candidateResults.entries()) {
|
|
432
|
+
const candidateSitemapUrl = candidateSitemapUrls[index];
|
|
433
|
+
if (result.status === 'fulfilled') {
|
|
434
|
+
if (result.value && addSitemapUrl(candidateSitemapUrl)) {
|
|
435
|
+
yield candidateSitemapUrl;
|
|
370
436
|
}
|
|
371
437
|
}
|
|
438
|
+
else {
|
|
439
|
+
logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
|
|
440
|
+
error: result.reason,
|
|
441
|
+
});
|
|
442
|
+
}
|
|
372
443
|
}
|
|
373
444
|
}
|
|
374
445
|
};
|
|
@@ -380,12 +451,17 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
380
451
|
}, {});
|
|
381
452
|
const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
|
|
382
453
|
const discoveredUrls = new Set();
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
454
|
+
try {
|
|
455
|
+
for await (const url of mergeAsyncIterables(...iterables)) {
|
|
456
|
+
if (discoveredUrls.has(url)) {
|
|
457
|
+
continue;
|
|
458
|
+
}
|
|
459
|
+
discoveredUrls.add(url);
|
|
460
|
+
yield url;
|
|
386
461
|
}
|
|
387
|
-
|
|
388
|
-
|
|
462
|
+
}
|
|
463
|
+
finally {
|
|
464
|
+
clearTimeout(timeoutHandle);
|
|
465
|
+
externalSignal?.removeEventListener('abort', onExternalAbort);
|
|
389
466
|
}
|
|
390
467
|
}
|
|
391
|
-
//# sourceMappingURL=sitemap.js.map
|
package/internals/social.d.ts
CHANGED
|
@@ -19,6 +19,8 @@ export declare function emailsFromText(text: string): string[];
|
|
|
19
19
|
/**
|
|
20
20
|
* The function extracts email addresses from a list of URLs.
|
|
21
21
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
22
|
+
* A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
|
|
23
|
+
* such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
|
|
22
24
|
* Note that the function preserves the order of emails and keep duplicates.
|
|
23
25
|
* @param urls Array of URLs.
|
|
24
26
|
* @return Array of emails addresses found.
|
|
@@ -225,7 +227,7 @@ export declare const TWITTER_REGEX: RegExp;
|
|
|
225
227
|
* ```
|
|
226
228
|
* import { social } from 'crawlee';
|
|
227
229
|
*
|
|
228
|
-
* const matches = text.match(social.
|
|
230
|
+
* const matches = text.match(social.TWITTER_REGEX_GLOBAL);
|
|
229
231
|
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
230
232
|
* ```
|
|
231
233
|
*/
|
|
@@ -488,5 +490,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
|
|
|
488
490
|
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
489
491
|
* @return An object with the social handles.
|
|
490
492
|
*/
|
|
491
|
-
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles
|
|
492
|
-
//# sourceMappingURL=social.d.ts.map
|
|
493
|
+
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;
|
package/internals/social.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import * as cheerio from 'cheerio';
|
|
2
1
|
import { htmlToText } from './cheerio.js';
|
|
3
2
|
// Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
|
|
4
|
-
|
|
3
|
+
// The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
|
|
4
|
+
// instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
|
|
5
|
+
// dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
|
|
6
|
+
const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
|
|
5
7
|
/**
|
|
6
8
|
* Regular expression to exactly match a single email address.
|
|
7
9
|
* It has the following form: `/^...$/i`.
|
|
@@ -28,6 +30,8 @@ export function emailsFromText(text) {
|
|
|
28
30
|
/**
|
|
29
31
|
* The function extracts email addresses from a list of URLs.
|
|
30
32
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
33
|
+
* A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
|
|
34
|
+
* such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
|
|
31
35
|
* Note that the function preserves the order of emails and keep duplicates.
|
|
32
36
|
* @param urls Array of URLs.
|
|
33
37
|
* @return Array of emails addresses found.
|
|
@@ -42,9 +46,20 @@ export function emailsFromUrls(urls) {
|
|
|
42
46
|
continue;
|
|
43
47
|
if (!EMAIL_URL_PREFIX_REGEX.test(url))
|
|
44
48
|
continue;
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
49
|
+
// Characters like `?` and `,` inside an address must be percent-encoded, so it's safe
|
|
50
|
+
// to split on them before decoding.
|
|
51
|
+
const [addresses] = url.replace(EMAIL_URL_PREFIX_REGEX, '').split('?');
|
|
52
|
+
for (const address of addresses.split(',')) {
|
|
53
|
+
let email = address.trim();
|
|
54
|
+
try {
|
|
55
|
+
email = decodeURIComponent(email).trim();
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// Keep the raw value if it isn't valid percent-encoding
|
|
59
|
+
}
|
|
60
|
+
if (EMAIL_REGEX.test(email))
|
|
61
|
+
emails.push(email);
|
|
62
|
+
}
|
|
48
63
|
}
|
|
49
64
|
return emails;
|
|
50
65
|
}
|
|
@@ -136,8 +151,18 @@ export function phonesFromUrls(urls) {
|
|
|
136
151
|
continue;
|
|
137
152
|
if (!PHONE_URL_PREFIX_REGEX.test(url))
|
|
138
153
|
continue;
|
|
139
|
-
|
|
140
|
-
|
|
154
|
+
// Per RFC 3966, parameters such as `;ext=89` can follow the number, and `-`, `.`, `(` and `)`
|
|
155
|
+
// are only visual separators. They are checked without separators too, so a one-digit country
|
|
156
|
+
// code like in `tel:+1-555-123-4567` is accepted.
|
|
157
|
+
const [number] = url.replace(PHONE_URL_PREFIX_REGEX, '').split(';');
|
|
158
|
+
let phone = number.trim();
|
|
159
|
+
try {
|
|
160
|
+
phone = decodeURIComponent(phone).trim();
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
// Keep the raw value if it isn't valid percent-encoding
|
|
164
|
+
}
|
|
165
|
+
if (PHONE_REGEX.test(phone) || PHONE_REGEX.test(phone.replace(/[-.() ]/g, '')))
|
|
141
166
|
phones.push(phone);
|
|
142
167
|
}
|
|
143
168
|
return phones;
|
|
@@ -324,7 +349,7 @@ export const TWITTER_REGEX = new RegExp(`^${TWITTER_REGEX_STRING}$`, 'i');
|
|
|
324
349
|
* ```
|
|
325
350
|
* import { social } from 'crawlee';
|
|
326
351
|
*
|
|
327
|
-
* const matches = text.match(social.
|
|
352
|
+
* const matches = text.match(social.TWITTER_REGEX_GLOBAL);
|
|
328
353
|
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
329
354
|
* ```
|
|
330
355
|
*/
|
|
@@ -587,7 +612,8 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
|
|
|
587
612
|
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
588
613
|
* @return An object with the social handles.
|
|
589
614
|
*/
|
|
590
|
-
export function parseHandlesFromHtml(html, data = null) {
|
|
615
|
+
export async function parseHandlesFromHtml(html, data = null) {
|
|
616
|
+
const cheerio = await import('cheerio/slim');
|
|
591
617
|
const result = {
|
|
592
618
|
emails: [],
|
|
593
619
|
phones: [],
|
|
@@ -606,7 +632,7 @@ export function parseHandlesFromHtml(html, data = null) {
|
|
|
606
632
|
const $ = cheerio.load(html, { xml: { decodeEntities: true } });
|
|
607
633
|
if (data)
|
|
608
634
|
data.$ = $;
|
|
609
|
-
const text = htmlToText($);
|
|
635
|
+
const text = await htmlToText($);
|
|
610
636
|
if (data)
|
|
611
637
|
data.text = text;
|
|
612
638
|
// NOTE: we need to parse each text separately, orherwise we might concatenate unrelated texts
|
|
@@ -644,4 +670,3 @@ export function parseHandlesFromHtml(html, data = null) {
|
|
|
644
670
|
}
|
|
645
671
|
return result;
|
|
646
672
|
}
|
|
647
|
-
//# sourceMappingURL=social.js.map
|