@crawlee/utils 4.0.0-beta.28 → 4.0.0-beta.281
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +10 -17
- package/index.js +8 -16
- package/internal.d.ts +12 -0
- package/internal.js +11 -0
- package/internals/blocked.d.ts +7 -1
- package/internals/blocked.js +11 -1
- package/internals/cheerio.d.ts +1 -3
- package/internals/cheerio.js +32 -22
- package/internals/extract-microdata.d.ts +24 -0
- package/internals/extract-microdata.js +122 -0
- package/internals/extract-urls.d.ts +1 -2
- package/internals/extract-urls.js +18 -23
- package/internals/general.d.ts +0 -25
- package/internals/general.js +2 -110
- package/internals/http-request.d.ts +5 -0
- package/internals/http-request.js +36 -0
- package/internals/iterables.d.ts +0 -80
- package/internals/iterables.js +0 -126
- package/internals/mutation-journal.d.ts +29 -0
- package/internals/mutation-journal.js +173 -0
- package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
- package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
- package/internals/robots.d.ts +35 -15
- package/internals/robots.js +66 -30
- package/internals/schemas.d.ts +114 -0
- package/internals/schemas.js +115 -0
- package/internals/sitemap.d.ts +44 -3
- package/internals/sitemap.js +214 -110
- package/internals/social.d.ts +4 -3
- package/internals/social.js +36 -11
- package/internals/url.d.ts +67 -1
- package/internals/url.js +118 -2
- package/internals/validation.d.ts +9 -0
- package/internals/validation.js +16 -0
- package/package.json +16 -15
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/blocked.d.ts.map +0 -1
- package/internals/blocked.js.map +0 -1
- package/internals/cheerio.d.ts.map +0 -1
- package/internals/cheerio.js.map +0 -1
- package/internals/chunk.d.ts +0 -2
- package/internals/chunk.d.ts.map +0 -1
- package/internals/chunk.js +0 -40
- package/internals/chunk.js.map +0 -1
- package/internals/debug.d.ts +0 -32
- package/internals/debug.d.ts.map +0 -1
- package/internals/debug.js +0 -41
- package/internals/debug.js.map +0 -1
- package/internals/extract-urls.d.ts.map +0 -1
- package/internals/extract-urls.js.map +0 -1
- package/internals/general.d.ts.map +0 -1
- package/internals/general.js.map +0 -1
- package/internals/iterables.d.ts.map +0 -1
- package/internals/iterables.js.map +0 -1
- package/internals/open_graph_parser.d.ts.map +0 -1
- package/internals/open_graph_parser.js.map +0 -1
- package/internals/robots.d.ts.map +0 -1
- package/internals/robots.js.map +0 -1
- package/internals/sitemap.d.ts.map +0 -1
- package/internals/sitemap.js.map +0 -1
- package/internals/social.d.ts.map +0 -1
- package/internals/social.js.map +0 -1
- package/internals/system-info/cpu-info.d.ts +0 -64
- package/internals/system-info/cpu-info.d.ts.map +0 -1
- package/internals/system-info/cpu-info.js +0 -211
- package/internals/system-info/cpu-info.js.map +0 -1
- package/internals/system-info/memory-info.d.ts +0 -28
- package/internals/system-info/memory-info.d.ts.map +0 -1
- package/internals/system-info/memory-info.js +0 -118
- package/internals/system-info/memory-info.js.map +0 -1
- package/internals/system-info/ps-tree.d.ts +0 -18
- package/internals/system-info/ps-tree.d.ts.map +0 -1
- package/internals/system-info/ps-tree.js +0 -145
- package/internals/system-info/ps-tree.js.map +0 -1
- package/internals/typedefs.d.ts +0 -5
- package/internals/typedefs.d.ts.map +0 -1
- package/internals/typedefs.js +0 -9
- package/internals/typedefs.js.map +0 -1
- package/internals/url.d.ts.map +0 -1
- package/internals/url.js.map +0 -1
package/internals/sitemap.js
CHANGED
|
@@ -3,132 +3,134 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
|
|
|
3
3
|
import { StringDecoder } from 'node:string_decoder';
|
|
4
4
|
import { createGunzip } from 'node:zlib';
|
|
5
5
|
import { FetchHttpClient } from '@crawlee/http-client';
|
|
6
|
-
import {
|
|
7
|
-
import sax from 'sax';
|
|
8
|
-
import MIMEType from 'whatwg-mimetype';
|
|
9
|
-
import log from '@apify/log';
|
|
6
|
+
import { MIMEType } from 'whatwg-mimetype';
|
|
10
7
|
import { mergeAsyncIterables } from './iterables.js';
|
|
11
|
-
import {
|
|
8
|
+
import { RobotsTxtFile } from './robots.js';
|
|
9
|
+
import { filterUrl } from './url.js';
|
|
12
10
|
class SitemapTxtParser extends Transform {
|
|
13
|
-
decoder = new StringDecoder('utf8');
|
|
14
|
-
buffer = '';
|
|
11
|
+
#decoder = new StringDecoder('utf8');
|
|
12
|
+
#buffer = '';
|
|
15
13
|
constructor() {
|
|
16
14
|
super({
|
|
17
15
|
readableObjectMode: true,
|
|
18
16
|
transform: (chunk, _encoding, callback) => {
|
|
19
|
-
this
|
|
17
|
+
this.#processBuffer(this.#decoder.write(chunk), false);
|
|
20
18
|
callback();
|
|
21
19
|
},
|
|
22
20
|
flush: (callback) => {
|
|
23
|
-
this
|
|
21
|
+
this.#processBuffer(this.#decoder.end(), true);
|
|
24
22
|
callback();
|
|
25
23
|
},
|
|
26
24
|
});
|
|
27
25
|
}
|
|
28
|
-
processBuffer(input, finalize) {
|
|
29
|
-
this
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
for (const url of parts) {
|
|
37
|
-
this.push({ type: 'url', loc: url });
|
|
38
|
-
}
|
|
39
|
-
this.buffer = '';
|
|
40
|
-
}
|
|
41
|
-
else if (parts.length > 0) {
|
|
42
|
-
for (const url of parts.slice(0, -1)) {
|
|
43
|
-
this.push({ type: 'url', loc: url });
|
|
44
|
-
}
|
|
45
|
-
this.buffer = parts.at(-1);
|
|
26
|
+
#processBuffer(input, finalize) {
|
|
27
|
+
const lines = (this.#buffer + input).split('\n');
|
|
28
|
+
// The text after the last line break may be a partial line, so it waits for the next chunk.
|
|
29
|
+
this.#buffer = finalize ? '' : lines.pop();
|
|
30
|
+
for (const line of lines) {
|
|
31
|
+
const url = line.trim();
|
|
32
|
+
if (url.length > 0) {
|
|
33
|
+
this.push({ type: 'url', loc: url });
|
|
46
34
|
}
|
|
47
35
|
}
|
|
48
36
|
}
|
|
49
37
|
}
|
|
50
38
|
class SitemapXmlParser extends Transform {
|
|
51
|
-
decoder = new StringDecoder('utf8');
|
|
52
|
-
parser
|
|
53
|
-
rootTagName;
|
|
54
|
-
currentTag = undefined;
|
|
55
|
-
url = {};
|
|
56
|
-
|
|
39
|
+
#decoder = new StringDecoder('utf8');
|
|
40
|
+
#parser;
|
|
41
|
+
#rootTagName;
|
|
42
|
+
#currentTag = undefined;
|
|
43
|
+
#url = {};
|
|
44
|
+
static async create() {
|
|
45
|
+
const { SAXParser } = (await import('sax')).default;
|
|
46
|
+
return new SitemapXmlParser(new SAXParser(true));
|
|
47
|
+
}
|
|
48
|
+
constructor(parser) {
|
|
57
49
|
super({
|
|
58
50
|
readableObjectMode: true,
|
|
59
51
|
transform: (chunk, _encoding, callback) => {
|
|
60
|
-
this
|
|
52
|
+
this.#parser.write(this.#decoder.write(chunk));
|
|
61
53
|
callback();
|
|
62
54
|
},
|
|
63
55
|
flush: (callback) => {
|
|
64
|
-
const rest = this
|
|
56
|
+
const rest = this.#decoder.end();
|
|
65
57
|
if (rest.length > 0) {
|
|
66
|
-
this
|
|
58
|
+
this.#parser.write(rest);
|
|
67
59
|
}
|
|
68
|
-
this
|
|
60
|
+
this.#parser.end();
|
|
69
61
|
callback();
|
|
70
62
|
},
|
|
71
63
|
});
|
|
72
|
-
this
|
|
73
|
-
this
|
|
74
|
-
this
|
|
75
|
-
this
|
|
76
|
-
this
|
|
64
|
+
this.#parser = parser;
|
|
65
|
+
this.#parser.onopentag = this.#onOpenTag.bind(this);
|
|
66
|
+
this.#parser.onclosetag = this.#onCloseTag.bind(this);
|
|
67
|
+
this.#parser.ontext = this.#onText.bind(this);
|
|
68
|
+
this.#parser.oncdata = this.#onText.bind(this);
|
|
69
|
+
this.#parser.onerror = this.destroy.bind(this);
|
|
77
70
|
}
|
|
78
|
-
onOpenTag(node) {
|
|
79
|
-
if (this
|
|
71
|
+
#onOpenTag(node) {
|
|
72
|
+
if (this.#rootTagName !== undefined) {
|
|
80
73
|
if (node.name === 'loc' ||
|
|
81
74
|
node.name === 'lastmod' ||
|
|
82
75
|
node.name === 'priority' ||
|
|
83
76
|
node.name === 'changefreq') {
|
|
84
|
-
this
|
|
77
|
+
this.#currentTag = node.name;
|
|
85
78
|
}
|
|
86
79
|
}
|
|
87
80
|
if (node.name === 'urlset') {
|
|
88
|
-
this
|
|
81
|
+
this.#rootTagName = 'urlset';
|
|
89
82
|
}
|
|
90
83
|
if (node.name === 'sitemapindex') {
|
|
91
|
-
this
|
|
84
|
+
this.#rootTagName = 'sitemapindex';
|
|
92
85
|
}
|
|
93
86
|
}
|
|
94
|
-
onCloseTag(name) {
|
|
87
|
+
#onCloseTag(name) {
|
|
95
88
|
if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
|
|
96
|
-
this
|
|
89
|
+
this.#currentTag = undefined;
|
|
97
90
|
}
|
|
98
|
-
if (name === 'url'
|
|
99
|
-
|
|
100
|
-
|
|
91
|
+
if (name === 'url') {
|
|
92
|
+
if (this.#url.loc !== undefined) {
|
|
93
|
+
this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
|
|
94
|
+
}
|
|
95
|
+
this.#url = {};
|
|
101
96
|
}
|
|
102
97
|
}
|
|
103
|
-
onText(text) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
98
|
+
#onText(text) {
|
|
99
|
+
text = text.trim();
|
|
100
|
+
// Whitespace around a CDATA section arrives as separate text events.
|
|
101
|
+
if (text === '') {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
if (this.#currentTag === 'loc') {
|
|
105
|
+
if (this.#rootTagName === 'sitemapindex') {
|
|
106
|
+
this.push({ type: 'sitemapUrl', url: text });
|
|
107
107
|
}
|
|
108
|
-
if (this
|
|
109
|
-
this
|
|
110
|
-
this
|
|
108
|
+
if (this.#rootTagName === 'urlset') {
|
|
109
|
+
this.#url ??= {};
|
|
110
|
+
this.#url.loc = text;
|
|
111
111
|
}
|
|
112
112
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
113
|
+
if (this.#currentTag === 'lastmod') {
|
|
114
|
+
const lastmod = new Date(text);
|
|
115
|
+
if (!Number.isNaN(lastmod.getTime())) {
|
|
116
|
+
this.#url.lastmod = lastmod;
|
|
117
|
+
}
|
|
116
118
|
}
|
|
117
|
-
if (this
|
|
118
|
-
this
|
|
119
|
+
if (this.#currentTag === 'priority') {
|
|
120
|
+
this.#url.priority = Number(text);
|
|
119
121
|
}
|
|
120
|
-
if (this
|
|
122
|
+
if (this.#currentTag === 'changefreq') {
|
|
121
123
|
if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
|
|
122
|
-
this
|
|
124
|
+
this.#url.changefreq = text;
|
|
123
125
|
}
|
|
124
126
|
}
|
|
125
127
|
}
|
|
126
128
|
}
|
|
127
129
|
export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
128
|
-
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {};
|
|
130
|
+
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', logger, } = options ?? {};
|
|
129
131
|
const sources = [...initialSources];
|
|
130
132
|
const visitedSitemapUrls = new Set();
|
|
131
|
-
const createParser = (contentType = '', url) => {
|
|
133
|
+
const createParser = async (contentType = '', url) => {
|
|
132
134
|
let mimeType;
|
|
133
135
|
try {
|
|
134
136
|
mimeType = new MIMEType(contentType);
|
|
@@ -137,22 +139,27 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
137
139
|
mimeType = null;
|
|
138
140
|
}
|
|
139
141
|
if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
|
|
140
|
-
return
|
|
142
|
+
return SitemapXmlParser.create();
|
|
141
143
|
}
|
|
142
144
|
if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
|
|
143
145
|
return new SitemapTxtParser();
|
|
144
146
|
}
|
|
147
|
+
// A name like `sitemap.gz` says nothing about what was compressed. Sitemaps are XML unless named otherwise.
|
|
148
|
+
if (mimeType?.essence === 'application/gzip') {
|
|
149
|
+
return SitemapXmlParser.create();
|
|
150
|
+
}
|
|
145
151
|
throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
|
|
146
152
|
};
|
|
147
153
|
while (sources.length > 0) {
|
|
148
154
|
const source = sources.shift();
|
|
149
155
|
if ((source?.depth ?? 0) > maxDepth) {
|
|
150
|
-
log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
|
|
151
156
|
continue;
|
|
152
157
|
}
|
|
153
158
|
let items = null;
|
|
159
|
+
// Parent URL, parsed once and reused as the origin for the strategy checks below.
|
|
160
|
+
let sitemapUrl;
|
|
154
161
|
if (source.type === 'url') {
|
|
155
|
-
|
|
162
|
+
sitemapUrl = new URL(source.url);
|
|
156
163
|
visitedSitemapUrls.add(sitemapUrl.toString());
|
|
157
164
|
let retriesLeft = sitemapRetries + 1;
|
|
158
165
|
while (retriesLeft-- > 0) {
|
|
@@ -162,11 +169,11 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
162
169
|
sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
|
|
163
170
|
method: 'GET',
|
|
164
171
|
headers: {
|
|
165
|
-
accept: '
|
|
172
|
+
accept: '*/*',
|
|
166
173
|
},
|
|
167
174
|
}), {
|
|
168
175
|
proxyUrl,
|
|
169
|
-
timeout,
|
|
176
|
+
timeoutMillis: timeout,
|
|
170
177
|
});
|
|
171
178
|
}
|
|
172
179
|
catch (error) {
|
|
@@ -178,6 +185,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
178
185
|
if (sitemapResponse.body === null) {
|
|
179
186
|
break;
|
|
180
187
|
}
|
|
188
|
+
const { fileTypeStream } = await import('file-type');
|
|
181
189
|
const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
|
|
182
190
|
if (streamWithType.fileType !== undefined) {
|
|
183
191
|
contentType = streamWithType.fileType.mime;
|
|
@@ -191,7 +199,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
191
199
|
sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
|
|
192
200
|
}
|
|
193
201
|
}
|
|
194
|
-
items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
|
|
202
|
+
items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
|
|
195
203
|
if (e !== undefined && e !== null) {
|
|
196
204
|
error = { type: 'parser', error: e };
|
|
197
205
|
}
|
|
@@ -214,35 +222,87 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
214
222
|
}
|
|
215
223
|
}
|
|
216
224
|
catch (e) {
|
|
217
|
-
|
|
225
|
+
logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
|
|
218
226
|
}
|
|
219
227
|
}
|
|
220
228
|
}
|
|
221
229
|
else if (source.type === 'raw') {
|
|
222
|
-
items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
|
|
230
|
+
items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
|
|
223
231
|
if (error !== undefined) {
|
|
224
|
-
|
|
232
|
+
logger?.warning(`Malformed sitemap content: ${error}`);
|
|
225
233
|
}
|
|
226
234
|
});
|
|
227
235
|
}
|
|
228
236
|
if (items === null) {
|
|
229
237
|
continue;
|
|
230
238
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
239
|
+
// URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
|
|
240
|
+
// the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
|
|
241
|
+
let droppedUrlEntries = 0;
|
|
242
|
+
// The same for every entry of this source, so compute it once.
|
|
243
|
+
const originSitemapUrl = source.type === 'url'
|
|
244
|
+
? source.url
|
|
245
|
+
: `raw://${createHash('sha256').update(source.content).digest('base64')}`;
|
|
246
|
+
const iterator = items[Symbol.asyncIterator]();
|
|
247
|
+
try {
|
|
248
|
+
while (true) {
|
|
249
|
+
let next;
|
|
250
|
+
try {
|
|
251
|
+
next = await iterator.next();
|
|
252
|
+
}
|
|
253
|
+
catch (e) {
|
|
254
|
+
// Parsing failed: give up on this sitemap, keep the others.
|
|
255
|
+
// Raw sources already logged the error in their pipeline callback.
|
|
256
|
+
if (source.type !== 'raw') {
|
|
257
|
+
logger?.warning(`Malformed sitemap content: ${source.url} (${e})`);
|
|
258
|
+
}
|
|
259
|
+
break;
|
|
260
|
+
}
|
|
261
|
+
if (next.done)
|
|
262
|
+
break;
|
|
263
|
+
const item = next.value;
|
|
264
|
+
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
|
|
265
|
+
if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
|
|
266
|
+
logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
// Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
|
|
270
|
+
// parent. Raw string sources have no parent URL, so the check is skipped.
|
|
271
|
+
if (source.type === 'url') {
|
|
272
|
+
const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
|
|
273
|
+
if (!allowed) {
|
|
274
|
+
logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
|
|
279
|
+
if (emitNestedSitemaps) {
|
|
280
|
+
yield { loc: item.url, originSitemapUrl: null };
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
if (item.type === 'url') {
|
|
284
|
+
// Keep only URL entries that match the enqueue strategy relative to the parent (see above).
|
|
285
|
+
if (source.type === 'url') {
|
|
286
|
+
const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
|
|
287
|
+
if (!allowed) {
|
|
288
|
+
droppedUrlEntries++;
|
|
289
|
+
logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
yield {
|
|
294
|
+
...item,
|
|
295
|
+
originSitemapUrl,
|
|
296
|
+
};
|
|
236
297
|
}
|
|
237
298
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
}
|
|
299
|
+
}
|
|
300
|
+
finally {
|
|
301
|
+
// Mirrors `for await` cleanup: destroys the stream when we stop early or the consumer bails out.
|
|
302
|
+
await iterator.return?.();
|
|
303
|
+
}
|
|
304
|
+
if (droppedUrlEntries > 0 && source.type === 'url') {
|
|
305
|
+
logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
|
|
246
306
|
}
|
|
247
307
|
}
|
|
248
308
|
}
|
|
@@ -285,7 +345,7 @@ export class Sitemap {
|
|
|
285
345
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
286
346
|
*/
|
|
287
347
|
static async load(urls, proxyUrl, parseSitemapOptions) {
|
|
288
|
-
return await this
|
|
348
|
+
return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
|
|
289
349
|
}
|
|
290
350
|
/**
|
|
291
351
|
* Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
|
|
@@ -293,16 +353,17 @@ export class Sitemap {
|
|
|
293
353
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
294
354
|
*/
|
|
295
355
|
static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
|
|
296
|
-
return await this
|
|
356
|
+
return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
|
|
297
357
|
}
|
|
298
|
-
static async parse(sources, proxyUrl, parseSitemapOptions) {
|
|
358
|
+
static async #parse(sources, proxyUrl, parseSitemapOptions) {
|
|
299
359
|
const urls = [];
|
|
300
360
|
try {
|
|
301
361
|
for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
|
|
302
362
|
urls.push(item.loc);
|
|
303
363
|
}
|
|
304
364
|
}
|
|
305
|
-
catch {
|
|
365
|
+
catch (e) {
|
|
366
|
+
parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
|
|
306
367
|
return new Sitemap([]);
|
|
307
368
|
}
|
|
308
369
|
return new Sitemap(urls);
|
|
@@ -316,7 +377,19 @@ export class Sitemap {
|
|
|
316
377
|
* @returns An async iterable with the discovered sitemap URLs.
|
|
317
378
|
*/
|
|
318
379
|
export async function* discoverValidSitemaps(urls, options = {}) {
|
|
319
|
-
const { proxyUrl, httpClient = new FetchHttpClient() } = options;
|
|
380
|
+
const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
|
|
381
|
+
const controller = new AbortController();
|
|
382
|
+
const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
|
|
383
|
+
const onExternalAbort = () => controller.abort();
|
|
384
|
+
if (externalSignal) {
|
|
385
|
+
if (externalSignal.aborted) {
|
|
386
|
+
controller.abort();
|
|
387
|
+
}
|
|
388
|
+
else {
|
|
389
|
+
externalSignal.addEventListener('abort', onExternalAbort, { once: true });
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
const signal = controller.signal;
|
|
320
393
|
const sitemapUrls = new Set();
|
|
321
394
|
const addSitemapUrl = (url) => {
|
|
322
395
|
const sizeBefore = sitemapUrls.size;
|
|
@@ -331,7 +404,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
331
404
|
return false;
|
|
332
405
|
}
|
|
333
406
|
try {
|
|
334
|
-
const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
|
|
407
|
+
const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
|
|
408
|
+
proxyUrl,
|
|
409
|
+
timeoutMillis: requestTimeoutMillis,
|
|
410
|
+
signal,
|
|
411
|
+
});
|
|
335
412
|
return response.status >= 200 && response.status < 400;
|
|
336
413
|
}
|
|
337
414
|
catch {
|
|
@@ -343,17 +420,24 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
343
420
|
return;
|
|
344
421
|
}
|
|
345
422
|
try {
|
|
346
|
-
const robotsFile = await
|
|
347
|
-
|
|
423
|
+
const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
|
|
424
|
+
proxyUrl,
|
|
425
|
+
timeoutMillis: requestTimeoutMillis,
|
|
426
|
+
signal,
|
|
427
|
+
httpClient,
|
|
428
|
+
logger,
|
|
429
|
+
});
|
|
430
|
+
// Surface all referenced sitemaps, including cross-host; scoping happens at load time.
|
|
431
|
+
for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
|
|
348
432
|
if (addSitemapUrl(sitemapUrl)) {
|
|
349
433
|
yield sitemapUrl;
|
|
350
434
|
}
|
|
351
435
|
}
|
|
352
436
|
}
|
|
353
437
|
catch (err) {
|
|
354
|
-
|
|
438
|
+
logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
|
|
355
439
|
}
|
|
356
|
-
const sitemapUrl = domainUrls.find((url) => /sitemap
|
|
440
|
+
const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
|
|
357
441
|
if (sitemapUrl !== undefined) {
|
|
358
442
|
if (addSitemapUrl(sitemapUrl)) {
|
|
359
443
|
yield sitemapUrl;
|
|
@@ -361,14 +445,29 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
361
445
|
}
|
|
362
446
|
else {
|
|
363
447
|
const firstUrl = new URL(domainUrls[0]);
|
|
448
|
+
// The guessed candidates are well-known paths on the same origin, so the query string
|
|
449
|
+
// and fragment of the input URL are not part of them - carrying them over probes (and
|
|
450
|
+
// yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
|
|
451
|
+
firstUrl.search = '';
|
|
452
|
+
firstUrl.hash = '';
|
|
364
453
|
const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
|
|
365
|
-
|
|
454
|
+
const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
|
|
366
455
|
firstUrl.pathname = pathname;
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
456
|
+
return firstUrl.toString();
|
|
457
|
+
});
|
|
458
|
+
const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
|
|
459
|
+
for (const [index, result] of candidateResults.entries()) {
|
|
460
|
+
const candidateSitemapUrl = candidateSitemapUrls[index];
|
|
461
|
+
if (result.status === 'fulfilled') {
|
|
462
|
+
if (result.value && addSitemapUrl(candidateSitemapUrl)) {
|
|
463
|
+
yield candidateSitemapUrl;
|
|
370
464
|
}
|
|
371
465
|
}
|
|
466
|
+
else {
|
|
467
|
+
logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
|
|
468
|
+
error: result.reason,
|
|
469
|
+
});
|
|
470
|
+
}
|
|
372
471
|
}
|
|
373
472
|
}
|
|
374
473
|
};
|
|
@@ -380,12 +479,17 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
380
479
|
}, {});
|
|
381
480
|
const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
|
|
382
481
|
const discoveredUrls = new Set();
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
482
|
+
try {
|
|
483
|
+
for await (const url of mergeAsyncIterables(...iterables)) {
|
|
484
|
+
if (discoveredUrls.has(url)) {
|
|
485
|
+
continue;
|
|
486
|
+
}
|
|
487
|
+
discoveredUrls.add(url);
|
|
488
|
+
yield url;
|
|
386
489
|
}
|
|
387
|
-
|
|
388
|
-
|
|
490
|
+
}
|
|
491
|
+
finally {
|
|
492
|
+
clearTimeout(timeoutHandle);
|
|
493
|
+
externalSignal?.removeEventListener('abort', onExternalAbort);
|
|
389
494
|
}
|
|
390
495
|
}
|
|
391
|
-
//# sourceMappingURL=sitemap.js.map
|
package/internals/social.d.ts
CHANGED
|
@@ -19,6 +19,8 @@ export declare function emailsFromText(text: string): string[];
|
|
|
19
19
|
/**
|
|
20
20
|
* The function extracts email addresses from a list of URLs.
|
|
21
21
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
22
|
+
* A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
|
|
23
|
+
* such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
|
|
22
24
|
* Note that the function preserves the order of emails and keep duplicates.
|
|
23
25
|
* @param urls Array of URLs.
|
|
24
26
|
* @return Array of emails addresses found.
|
|
@@ -225,7 +227,7 @@ export declare const TWITTER_REGEX: RegExp;
|
|
|
225
227
|
* ```
|
|
226
228
|
* import { social } from 'crawlee';
|
|
227
229
|
*
|
|
228
|
-
* const matches = text.match(social.
|
|
230
|
+
* const matches = text.match(social.TWITTER_REGEX_GLOBAL);
|
|
229
231
|
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
230
232
|
* ```
|
|
231
233
|
*/
|
|
@@ -488,5 +490,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
|
|
|
488
490
|
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
489
491
|
* @return An object with the social handles.
|
|
490
492
|
*/
|
|
491
|
-
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles
|
|
492
|
-
//# sourceMappingURL=social.d.ts.map
|
|
493
|
+
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;
|
package/internals/social.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import * as cheerio from 'cheerio';
|
|
2
1
|
import { htmlToText } from './cheerio.js';
|
|
3
2
|
// Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
|
|
4
|
-
|
|
3
|
+
// The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
|
|
4
|
+
// instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
|
|
5
|
+
// dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
|
|
6
|
+
const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
|
|
5
7
|
/**
|
|
6
8
|
* Regular expression to exactly match a single email address.
|
|
7
9
|
* It has the following form: `/^...$/i`.
|
|
@@ -28,6 +30,8 @@ export function emailsFromText(text) {
|
|
|
28
30
|
/**
|
|
29
31
|
* The function extracts email addresses from a list of URLs.
|
|
30
32
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
33
|
+
* A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
|
|
34
|
+
* such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
|
|
31
35
|
* Note that the function preserves the order of emails and keep duplicates.
|
|
32
36
|
* @param urls Array of URLs.
|
|
33
37
|
* @return Array of emails addresses found.
|
|
@@ -42,9 +46,20 @@ export function emailsFromUrls(urls) {
|
|
|
42
46
|
continue;
|
|
43
47
|
if (!EMAIL_URL_PREFIX_REGEX.test(url))
|
|
44
48
|
continue;
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
49
|
+
// Characters like `?` and `,` inside an address must be percent-encoded, so it's safe
|
|
50
|
+
// to split on them before decoding.
|
|
51
|
+
const [addresses] = url.replace(EMAIL_URL_PREFIX_REGEX, '').split('?');
|
|
52
|
+
for (const address of addresses.split(',')) {
|
|
53
|
+
let email = address.trim();
|
|
54
|
+
try {
|
|
55
|
+
email = decodeURIComponent(email).trim();
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// Keep the raw value if it isn't valid percent-encoding
|
|
59
|
+
}
|
|
60
|
+
if (EMAIL_REGEX.test(email))
|
|
61
|
+
emails.push(email);
|
|
62
|
+
}
|
|
48
63
|
}
|
|
49
64
|
return emails;
|
|
50
65
|
}
|
|
@@ -136,8 +151,18 @@ export function phonesFromUrls(urls) {
|
|
|
136
151
|
continue;
|
|
137
152
|
if (!PHONE_URL_PREFIX_REGEX.test(url))
|
|
138
153
|
continue;
|
|
139
|
-
|
|
140
|
-
|
|
154
|
+
// Per RFC 3966, parameters such as `;ext=89` can follow the number, and `-`, `.`, `(` and `)`
|
|
155
|
+
// are only visual separators. They are checked without separators too, so a one-digit country
|
|
156
|
+
// code like in `tel:+1-555-123-4567` is accepted.
|
|
157
|
+
const [number] = url.replace(PHONE_URL_PREFIX_REGEX, '').split(';');
|
|
158
|
+
let phone = number.trim();
|
|
159
|
+
try {
|
|
160
|
+
phone = decodeURIComponent(phone).trim();
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
// Keep the raw value if it isn't valid percent-encoding
|
|
164
|
+
}
|
|
165
|
+
if (PHONE_REGEX.test(phone) || PHONE_REGEX.test(phone.replace(/[-.() ]/g, '')))
|
|
141
166
|
phones.push(phone);
|
|
142
167
|
}
|
|
143
168
|
return phones;
|
|
@@ -324,7 +349,7 @@ export const TWITTER_REGEX = new RegExp(`^${TWITTER_REGEX_STRING}$`, 'i');
|
|
|
324
349
|
* ```
|
|
325
350
|
* import { social } from 'crawlee';
|
|
326
351
|
*
|
|
327
|
-
* const matches = text.match(social.
|
|
352
|
+
* const matches = text.match(social.TWITTER_REGEX_GLOBAL);
|
|
328
353
|
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
329
354
|
* ```
|
|
330
355
|
*/
|
|
@@ -587,7 +612,8 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
|
|
|
587
612
|
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
588
613
|
* @return An object with the social handles.
|
|
589
614
|
*/
|
|
590
|
-
export function parseHandlesFromHtml(html, data = null) {
|
|
615
|
+
export async function parseHandlesFromHtml(html, data = null) {
|
|
616
|
+
const cheerio = await import('cheerio/slim');
|
|
591
617
|
const result = {
|
|
592
618
|
emails: [],
|
|
593
619
|
phones: [],
|
|
@@ -606,7 +632,7 @@ export function parseHandlesFromHtml(html, data = null) {
|
|
|
606
632
|
const $ = cheerio.load(html, { xml: { decodeEntities: true } });
|
|
607
633
|
if (data)
|
|
608
634
|
data.$ = $;
|
|
609
|
-
const text = htmlToText($);
|
|
635
|
+
const text = await htmlToText($);
|
|
610
636
|
if (data)
|
|
611
637
|
data.text = text;
|
|
612
638
|
// NOTE: we need to parse each text separately, orherwise we might concatenate unrelated texts
|
|
@@ -644,4 +670,3 @@ export function parseHandlesFromHtml(html, data = null) {
|
|
|
644
670
|
}
|
|
645
671
|
return result;
|
|
646
672
|
}
|
|
647
|
-
//# sourceMappingURL=social.js.map
|