@crawlee/utils 4.0.0-beta.29 → 4.0.0-beta.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +10 -17
- package/index.js +8 -16
- package/internal.d.ts +12 -0
- package/internal.js +11 -0
- package/internals/blocked.d.ts +7 -1
- package/internals/blocked.js +11 -1
- package/internals/cheerio.d.ts +1 -3
- package/internals/cheerio.js +33 -23
- package/internals/extract-microdata.d.ts +24 -0
- package/internals/extract-microdata.js +136 -0
- package/internals/extract-urls.d.ts +1 -2
- package/internals/extract-urls.js +18 -23
- package/internals/general.d.ts +0 -25
- package/internals/general.js +2 -110
- package/internals/http-request.d.ts +5 -0
- package/internals/http-request.js +36 -0
- package/internals/iterables.d.ts +0 -80
- package/internals/iterables.js +0 -126
- package/internals/mutation-journal.d.ts +29 -0
- package/internals/mutation-journal.js +173 -0
- package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
- package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
- package/internals/robots.d.ts +35 -15
- package/internals/robots.js +66 -30
- package/internals/schemas.d.ts +114 -0
- package/internals/schemas.js +115 -0
- package/internals/sitemap.d.ts +44 -3
- package/internals/sitemap.js +243 -116
- package/internals/social.d.ts +4 -3
- package/internals/social.js +36 -11
- package/internals/url.d.ts +67 -1
- package/internals/url.js +118 -2
- package/internals/validation.d.ts +9 -0
- package/internals/validation.js +16 -0
- package/package.json +16 -15
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/blocked.d.ts.map +0 -1
- package/internals/blocked.js.map +0 -1
- package/internals/cheerio.d.ts.map +0 -1
- package/internals/cheerio.js.map +0 -1
- package/internals/chunk.d.ts +0 -2
- package/internals/chunk.d.ts.map +0 -1
- package/internals/chunk.js +0 -40
- package/internals/chunk.js.map +0 -1
- package/internals/debug.d.ts +0 -32
- package/internals/debug.d.ts.map +0 -1
- package/internals/debug.js +0 -41
- package/internals/debug.js.map +0 -1
- package/internals/extract-urls.d.ts.map +0 -1
- package/internals/extract-urls.js.map +0 -1
- package/internals/general.d.ts.map +0 -1
- package/internals/general.js.map +0 -1
- package/internals/iterables.d.ts.map +0 -1
- package/internals/iterables.js.map +0 -1
- package/internals/open_graph_parser.d.ts.map +0 -1
- package/internals/open_graph_parser.js.map +0 -1
- package/internals/robots.d.ts.map +0 -1
- package/internals/robots.js.map +0 -1
- package/internals/sitemap.d.ts.map +0 -1
- package/internals/sitemap.js.map +0 -1
- package/internals/social.d.ts.map +0 -1
- package/internals/social.js.map +0 -1
- package/internals/system-info/cpu-info.d.ts +0 -64
- package/internals/system-info/cpu-info.d.ts.map +0 -1
- package/internals/system-info/cpu-info.js +0 -211
- package/internals/system-info/cpu-info.js.map +0 -1
- package/internals/system-info/memory-info.d.ts +0 -28
- package/internals/system-info/memory-info.d.ts.map +0 -1
- package/internals/system-info/memory-info.js +0 -118
- package/internals/system-info/memory-info.js.map +0 -1
- package/internals/system-info/ps-tree.d.ts +0 -18
- package/internals/system-info/ps-tree.d.ts.map +0 -1
- package/internals/system-info/ps-tree.js +0 -145
- package/internals/system-info/ps-tree.js.map +0 -1
- package/internals/typedefs.d.ts +0 -5
- package/internals/typedefs.d.ts.map +0 -1
- package/internals/typedefs.js +0 -9
- package/internals/typedefs.js.map +0 -1
- package/internals/url.d.ts.map +0 -1
- package/internals/url.js.map +0 -1
package/internals/sitemap.js
CHANGED
|
@@ -3,132 +3,157 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
|
|
|
3
3
|
import { StringDecoder } from 'node:string_decoder';
|
|
4
4
|
import { createGunzip } from 'node:zlib';
|
|
5
5
|
import { FetchHttpClient } from '@crawlee/http-client';
|
|
6
|
-
import {
|
|
7
|
-
import sax from 'sax';
|
|
8
|
-
import MIMEType from 'whatwg-mimetype';
|
|
9
|
-
import log from '@apify/log';
|
|
6
|
+
import { MIMEType } from 'whatwg-mimetype';
|
|
10
7
|
import { mergeAsyncIterables } from './iterables.js';
|
|
11
|
-
import {
|
|
8
|
+
import { RobotsTxtFile } from './robots.js';
|
|
9
|
+
import { filterUrl } from './url.js';
|
|
12
10
|
class SitemapTxtParser extends Transform {
|
|
13
|
-
decoder = new StringDecoder('utf8');
|
|
14
|
-
buffer = '';
|
|
11
|
+
#decoder = new StringDecoder('utf8');
|
|
12
|
+
#buffer = '';
|
|
15
13
|
constructor() {
|
|
16
14
|
super({
|
|
17
15
|
readableObjectMode: true,
|
|
18
16
|
transform: (chunk, _encoding, callback) => {
|
|
19
|
-
this
|
|
17
|
+
this.#processBuffer(this.#decoder.write(chunk), false);
|
|
20
18
|
callback();
|
|
21
19
|
},
|
|
22
20
|
flush: (callback) => {
|
|
23
|
-
this
|
|
21
|
+
this.#processBuffer(this.#decoder.end(), true);
|
|
24
22
|
callback();
|
|
25
23
|
},
|
|
26
24
|
});
|
|
27
25
|
}
|
|
28
|
-
processBuffer(input, finalize) {
|
|
29
|
-
this
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
for (const url of parts) {
|
|
37
|
-
this.push({ type: 'url', loc: url });
|
|
38
|
-
}
|
|
39
|
-
this.buffer = '';
|
|
40
|
-
}
|
|
41
|
-
else if (parts.length > 0) {
|
|
42
|
-
for (const url of parts.slice(0, -1)) {
|
|
43
|
-
this.push({ type: 'url', loc: url });
|
|
44
|
-
}
|
|
45
|
-
this.buffer = parts.at(-1);
|
|
26
|
+
#processBuffer(input, finalize) {
|
|
27
|
+
const lines = (this.#buffer + input).split('\n');
|
|
28
|
+
// The text after the last line break may be a partial line, so it waits for the next chunk.
|
|
29
|
+
this.#buffer = finalize ? '' : lines.pop();
|
|
30
|
+
for (const line of lines) {
|
|
31
|
+
const url = line.trim();
|
|
32
|
+
if (url.length > 0) {
|
|
33
|
+
this.push({ type: 'url', loc: url });
|
|
46
34
|
}
|
|
47
35
|
}
|
|
48
36
|
}
|
|
49
37
|
}
|
|
38
|
+
/** Namespaces of the sitemap protocol (https://www.sitemaps.org/protocol.html), including Google's legacy ones. */
|
|
39
|
+
const SITEMAP_NAMESPACES = new Set([
|
|
40
|
+
'http://www.sitemaps.org/schemas/sitemap/0.9',
|
|
41
|
+
'https://www.sitemaps.org/schemas/sitemap/0.9',
|
|
42
|
+
'http://www.google.com/schemas/sitemap/0.9',
|
|
43
|
+
'http://www.google.com/schemas/sitemap/0.84',
|
|
44
|
+
]);
|
|
50
45
|
class SitemapXmlParser extends Transform {
|
|
51
|
-
decoder = new StringDecoder('utf8');
|
|
52
|
-
parser
|
|
53
|
-
rootTagName;
|
|
54
|
-
currentTag = undefined;
|
|
55
|
-
url = {};
|
|
56
|
-
|
|
46
|
+
#decoder = new StringDecoder('utf8');
|
|
47
|
+
#parser;
|
|
48
|
+
#rootTagName;
|
|
49
|
+
#currentTag = undefined;
|
|
50
|
+
#url = {};
|
|
51
|
+
/** Prefixes bound to the sitemap namespace, e.g. `ns0` for `<ns0:urlset xmlns:ns0="...">`. */
|
|
52
|
+
#sitemapPrefixes = new Set();
|
|
53
|
+
static async create() {
|
|
54
|
+
const { SAXParser } = (await import('sax')).default;
|
|
55
|
+
return new SitemapXmlParser(new SAXParser(true));
|
|
56
|
+
}
|
|
57
|
+
constructor(parser) {
|
|
57
58
|
super({
|
|
58
59
|
readableObjectMode: true,
|
|
59
60
|
transform: (chunk, _encoding, callback) => {
|
|
60
|
-
this
|
|
61
|
+
this.#parser.write(this.#decoder.write(chunk));
|
|
61
62
|
callback();
|
|
62
63
|
},
|
|
63
64
|
flush: (callback) => {
|
|
64
|
-
const rest = this
|
|
65
|
+
const rest = this.#decoder.end();
|
|
65
66
|
if (rest.length > 0) {
|
|
66
|
-
this
|
|
67
|
+
this.#parser.write(rest);
|
|
67
68
|
}
|
|
68
|
-
this
|
|
69
|
+
this.#parser.end();
|
|
69
70
|
callback();
|
|
70
71
|
},
|
|
71
72
|
});
|
|
72
|
-
this
|
|
73
|
-
this
|
|
74
|
-
this
|
|
75
|
-
this
|
|
76
|
-
this
|
|
73
|
+
this.#parser = parser;
|
|
74
|
+
this.#parser.onopentag = this.#onOpenTag.bind(this);
|
|
75
|
+
this.#parser.onclosetag = this.#onCloseTag.bind(this);
|
|
76
|
+
this.#parser.ontext = this.#onText.bind(this);
|
|
77
|
+
this.#parser.oncdata = this.#onText.bind(this);
|
|
78
|
+
this.#parser.onerror = this.destroy.bind(this);
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Returns the local name of a sitemap protocol element, or `undefined` for an element of another
|
|
82
|
+
* namespace (such as `image:loc`). Unprefixed elements are sitemap elements, as before.
|
|
83
|
+
*/
|
|
84
|
+
#sitemapTagName(qualifiedName) {
|
|
85
|
+
const colon = qualifiedName.indexOf(':');
|
|
86
|
+
if (colon === -1)
|
|
87
|
+
return qualifiedName;
|
|
88
|
+
return this.#sitemapPrefixes.has(qualifiedName.slice(0, colon)) ? qualifiedName.slice(colon + 1) : undefined;
|
|
77
89
|
}
|
|
78
|
-
onOpenTag(node) {
|
|
79
|
-
|
|
80
|
-
if (
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
90
|
+
#onOpenTag(node) {
|
|
91
|
+
for (const [attribute, value] of Object.entries(node.attributes)) {
|
|
92
|
+
if (attribute.startsWith('xmlns:') && SITEMAP_NAMESPACES.has(String(value).trim())) {
|
|
93
|
+
this.#sitemapPrefixes.add(attribute.slice('xmlns:'.length));
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const name = this.#sitemapTagName(node.name);
|
|
97
|
+
if (this.#rootTagName !== undefined) {
|
|
98
|
+
if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
|
|
99
|
+
this.#currentTag = name;
|
|
85
100
|
}
|
|
86
101
|
}
|
|
87
|
-
if (
|
|
88
|
-
this
|
|
102
|
+
if (name === 'urlset') {
|
|
103
|
+
this.#rootTagName = 'urlset';
|
|
89
104
|
}
|
|
90
|
-
if (
|
|
91
|
-
this
|
|
105
|
+
if (name === 'sitemapindex') {
|
|
106
|
+
this.#rootTagName = 'sitemapindex';
|
|
92
107
|
}
|
|
93
108
|
}
|
|
94
|
-
onCloseTag(
|
|
109
|
+
#onCloseTag(qualifiedName) {
|
|
110
|
+
const name = this.#sitemapTagName(qualifiedName);
|
|
95
111
|
if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
|
|
96
|
-
this
|
|
112
|
+
this.#currentTag = undefined;
|
|
97
113
|
}
|
|
98
|
-
if (name === 'url'
|
|
99
|
-
|
|
100
|
-
|
|
114
|
+
if (name === 'url') {
|
|
115
|
+
if (this.#url.loc !== undefined) {
|
|
116
|
+
this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
|
|
117
|
+
}
|
|
118
|
+
this.#url = {};
|
|
101
119
|
}
|
|
102
120
|
}
|
|
103
|
-
onText(text) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
121
|
+
#onText(text) {
|
|
122
|
+
text = text.trim();
|
|
123
|
+
// Whitespace around a CDATA section arrives as separate text events.
|
|
124
|
+
if (text === '') {
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
if (this.#currentTag === 'loc') {
|
|
128
|
+
if (this.#rootTagName === 'sitemapindex') {
|
|
129
|
+
this.push({ type: 'sitemapUrl', url: text });
|
|
107
130
|
}
|
|
108
|
-
if (this
|
|
109
|
-
this
|
|
110
|
-
this
|
|
131
|
+
if (this.#rootTagName === 'urlset') {
|
|
132
|
+
this.#url ??= {};
|
|
133
|
+
this.#url.loc = text;
|
|
111
134
|
}
|
|
112
135
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
136
|
+
if (this.#currentTag === 'lastmod') {
|
|
137
|
+
const lastmod = new Date(text);
|
|
138
|
+
if (!Number.isNaN(lastmod.getTime())) {
|
|
139
|
+
this.#url.lastmod = lastmod;
|
|
140
|
+
}
|
|
116
141
|
}
|
|
117
|
-
if (this
|
|
118
|
-
this
|
|
142
|
+
if (this.#currentTag === 'priority') {
|
|
143
|
+
this.#url.priority = Number(text);
|
|
119
144
|
}
|
|
120
|
-
if (this
|
|
145
|
+
if (this.#currentTag === 'changefreq') {
|
|
121
146
|
if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
|
|
122
|
-
this
|
|
147
|
+
this.#url.changefreq = text;
|
|
123
148
|
}
|
|
124
149
|
}
|
|
125
150
|
}
|
|
126
151
|
}
|
|
127
152
|
export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
128
|
-
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {};
|
|
153
|
+
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', logger, } = options ?? {};
|
|
129
154
|
const sources = [...initialSources];
|
|
130
155
|
const visitedSitemapUrls = new Set();
|
|
131
|
-
const createParser = (contentType = '', url) => {
|
|
156
|
+
const createParser = async (contentType = '', url) => {
|
|
132
157
|
let mimeType;
|
|
133
158
|
try {
|
|
134
159
|
mimeType = new MIMEType(contentType);
|
|
@@ -137,22 +162,27 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
137
162
|
mimeType = null;
|
|
138
163
|
}
|
|
139
164
|
if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
|
|
140
|
-
return
|
|
165
|
+
return SitemapXmlParser.create();
|
|
141
166
|
}
|
|
142
167
|
if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
|
|
143
168
|
return new SitemapTxtParser();
|
|
144
169
|
}
|
|
170
|
+
// A name like `sitemap.gz` says nothing about what was compressed. Sitemaps are XML unless named otherwise.
|
|
171
|
+
if (mimeType?.essence === 'application/gzip') {
|
|
172
|
+
return SitemapXmlParser.create();
|
|
173
|
+
}
|
|
145
174
|
throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
|
|
146
175
|
};
|
|
147
176
|
while (sources.length > 0) {
|
|
148
177
|
const source = sources.shift();
|
|
149
178
|
if ((source?.depth ?? 0) > maxDepth) {
|
|
150
|
-
log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
|
|
151
179
|
continue;
|
|
152
180
|
}
|
|
153
181
|
let items = null;
|
|
182
|
+
// Parent URL, parsed once and reused as the origin for the strategy checks below.
|
|
183
|
+
let sitemapUrl;
|
|
154
184
|
if (source.type === 'url') {
|
|
155
|
-
|
|
185
|
+
sitemapUrl = new URL(source.url);
|
|
156
186
|
visitedSitemapUrls.add(sitemapUrl.toString());
|
|
157
187
|
let retriesLeft = sitemapRetries + 1;
|
|
158
188
|
while (retriesLeft-- > 0) {
|
|
@@ -162,11 +192,11 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
162
192
|
sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
|
|
163
193
|
method: 'GET',
|
|
164
194
|
headers: {
|
|
165
|
-
accept: '
|
|
195
|
+
accept: '*/*',
|
|
166
196
|
},
|
|
167
197
|
}), {
|
|
168
198
|
proxyUrl,
|
|
169
|
-
timeout,
|
|
199
|
+
timeoutMillis: timeout,
|
|
170
200
|
});
|
|
171
201
|
}
|
|
172
202
|
catch (error) {
|
|
@@ -178,6 +208,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
178
208
|
if (sitemapResponse.body === null) {
|
|
179
209
|
break;
|
|
180
210
|
}
|
|
211
|
+
const { fileTypeStream } = await import('file-type');
|
|
181
212
|
const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
|
|
182
213
|
if (streamWithType.fileType !== undefined) {
|
|
183
214
|
contentType = streamWithType.fileType.mime;
|
|
@@ -191,7 +222,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
191
222
|
sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
|
|
192
223
|
}
|
|
193
224
|
}
|
|
194
|
-
items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
|
|
225
|
+
items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
|
|
195
226
|
if (e !== undefined && e !== null) {
|
|
196
227
|
error = { type: 'parser', error: e };
|
|
197
228
|
}
|
|
@@ -214,35 +245,87 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
214
245
|
}
|
|
215
246
|
}
|
|
216
247
|
catch (e) {
|
|
217
|
-
|
|
248
|
+
logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
|
|
218
249
|
}
|
|
219
250
|
}
|
|
220
251
|
}
|
|
221
252
|
else if (source.type === 'raw') {
|
|
222
|
-
items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
|
|
253
|
+
items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
|
|
223
254
|
if (error !== undefined) {
|
|
224
|
-
|
|
255
|
+
logger?.warning(`Malformed sitemap content: ${error}`);
|
|
225
256
|
}
|
|
226
257
|
});
|
|
227
258
|
}
|
|
228
259
|
if (items === null) {
|
|
229
260
|
continue;
|
|
230
261
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
262
|
+
// URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
|
|
263
|
+
// the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
|
|
264
|
+
let droppedUrlEntries = 0;
|
|
265
|
+
// The same for every entry of this source, so compute it once.
|
|
266
|
+
const originSitemapUrl = source.type === 'url'
|
|
267
|
+
? source.url
|
|
268
|
+
: `raw://${createHash('sha256').update(source.content).digest('base64')}`;
|
|
269
|
+
const iterator = items[Symbol.asyncIterator]();
|
|
270
|
+
try {
|
|
271
|
+
while (true) {
|
|
272
|
+
let next;
|
|
273
|
+
try {
|
|
274
|
+
next = await iterator.next();
|
|
275
|
+
}
|
|
276
|
+
catch (e) {
|
|
277
|
+
// Parsing failed: give up on this sitemap, keep the others.
|
|
278
|
+
// Raw sources already logged the error in their pipeline callback.
|
|
279
|
+
if (source.type !== 'raw') {
|
|
280
|
+
logger?.warning(`Malformed sitemap content: ${source.url} (${e})`);
|
|
281
|
+
}
|
|
282
|
+
break;
|
|
283
|
+
}
|
|
284
|
+
if (next.done)
|
|
285
|
+
break;
|
|
286
|
+
const item = next.value;
|
|
287
|
+
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
|
|
288
|
+
if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
|
|
289
|
+
logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
// Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
|
|
293
|
+
// parent. Raw string sources have no parent URL, so the check is skipped.
|
|
294
|
+
if (source.type === 'url') {
|
|
295
|
+
const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
|
|
296
|
+
if (!allowed) {
|
|
297
|
+
logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
|
|
302
|
+
if (emitNestedSitemaps) {
|
|
303
|
+
yield { loc: item.url, originSitemapUrl: null };
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
if (item.type === 'url') {
|
|
307
|
+
// Keep only URL entries that match the enqueue strategy relative to the parent (see above).
|
|
308
|
+
if (source.type === 'url') {
|
|
309
|
+
const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
|
|
310
|
+
if (!allowed) {
|
|
311
|
+
droppedUrlEntries++;
|
|
312
|
+
logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
yield {
|
|
317
|
+
...item,
|
|
318
|
+
originSitemapUrl,
|
|
319
|
+
};
|
|
236
320
|
}
|
|
237
321
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
}
|
|
322
|
+
}
|
|
323
|
+
finally {
|
|
324
|
+
// Mirrors `for await` cleanup: destroys the stream when we stop early or the consumer bails out.
|
|
325
|
+
await iterator.return?.();
|
|
326
|
+
}
|
|
327
|
+
if (droppedUrlEntries > 0 && source.type === 'url') {
|
|
328
|
+
logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
|
|
246
329
|
}
|
|
247
330
|
}
|
|
248
331
|
}
|
|
@@ -285,7 +368,7 @@ export class Sitemap {
|
|
|
285
368
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
286
369
|
*/
|
|
287
370
|
static async load(urls, proxyUrl, parseSitemapOptions) {
|
|
288
|
-
return await this
|
|
371
|
+
return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
|
|
289
372
|
}
|
|
290
373
|
/**
|
|
291
374
|
* Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
|
|
@@ -293,16 +376,17 @@ export class Sitemap {
|
|
|
293
376
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
294
377
|
*/
|
|
295
378
|
static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
|
|
296
|
-
return await this
|
|
379
|
+
return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
|
|
297
380
|
}
|
|
298
|
-
static async parse(sources, proxyUrl, parseSitemapOptions) {
|
|
381
|
+
static async #parse(sources, proxyUrl, parseSitemapOptions) {
|
|
299
382
|
const urls = [];
|
|
300
383
|
try {
|
|
301
384
|
for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
|
|
302
385
|
urls.push(item.loc);
|
|
303
386
|
}
|
|
304
387
|
}
|
|
305
|
-
catch {
|
|
388
|
+
catch (e) {
|
|
389
|
+
parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
|
|
306
390
|
return new Sitemap([]);
|
|
307
391
|
}
|
|
308
392
|
return new Sitemap(urls);
|
|
@@ -316,7 +400,19 @@ export class Sitemap {
|
|
|
316
400
|
* @returns An async iterable with the discovered sitemap URLs.
|
|
317
401
|
*/
|
|
318
402
|
export async function* discoverValidSitemaps(urls, options = {}) {
|
|
319
|
-
const { proxyUrl, httpClient = new FetchHttpClient() } = options;
|
|
403
|
+
const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
|
|
404
|
+
const controller = new AbortController();
|
|
405
|
+
const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
|
|
406
|
+
const onExternalAbort = () => controller.abort();
|
|
407
|
+
if (externalSignal) {
|
|
408
|
+
if (externalSignal.aborted) {
|
|
409
|
+
controller.abort();
|
|
410
|
+
}
|
|
411
|
+
else {
|
|
412
|
+
externalSignal.addEventListener('abort', onExternalAbort, { once: true });
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
const signal = controller.signal;
|
|
320
416
|
const sitemapUrls = new Set();
|
|
321
417
|
const addSitemapUrl = (url) => {
|
|
322
418
|
const sizeBefore = sitemapUrls.size;
|
|
@@ -331,7 +427,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
331
427
|
return false;
|
|
332
428
|
}
|
|
333
429
|
try {
|
|
334
|
-
const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
|
|
430
|
+
const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
|
|
431
|
+
proxyUrl,
|
|
432
|
+
timeoutMillis: requestTimeoutMillis,
|
|
433
|
+
signal,
|
|
434
|
+
});
|
|
335
435
|
return response.status >= 200 && response.status < 400;
|
|
336
436
|
}
|
|
337
437
|
catch {
|
|
@@ -343,17 +443,24 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
343
443
|
return;
|
|
344
444
|
}
|
|
345
445
|
try {
|
|
346
|
-
const robotsFile = await
|
|
347
|
-
|
|
446
|
+
const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
|
|
447
|
+
proxyUrl,
|
|
448
|
+
timeoutMillis: requestTimeoutMillis,
|
|
449
|
+
signal,
|
|
450
|
+
httpClient,
|
|
451
|
+
logger,
|
|
452
|
+
});
|
|
453
|
+
// Surface all referenced sitemaps, including cross-host; scoping happens at load time.
|
|
454
|
+
for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
|
|
348
455
|
if (addSitemapUrl(sitemapUrl)) {
|
|
349
456
|
yield sitemapUrl;
|
|
350
457
|
}
|
|
351
458
|
}
|
|
352
459
|
}
|
|
353
460
|
catch (err) {
|
|
354
|
-
|
|
461
|
+
logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
|
|
355
462
|
}
|
|
356
|
-
const sitemapUrl = domainUrls.find((url) => /sitemap
|
|
463
|
+
const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
|
|
357
464
|
if (sitemapUrl !== undefined) {
|
|
358
465
|
if (addSitemapUrl(sitemapUrl)) {
|
|
359
466
|
yield sitemapUrl;
|
|
@@ -361,14 +468,29 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
361
468
|
}
|
|
362
469
|
else {
|
|
363
470
|
const firstUrl = new URL(domainUrls[0]);
|
|
471
|
+
// The guessed candidates are well-known paths on the same origin, so the query string
|
|
472
|
+
// and fragment of the input URL are not part of them - carrying them over probes (and
|
|
473
|
+
// yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
|
|
474
|
+
firstUrl.search = '';
|
|
475
|
+
firstUrl.hash = '';
|
|
364
476
|
const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
|
|
365
|
-
|
|
477
|
+
const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
|
|
366
478
|
firstUrl.pathname = pathname;
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
479
|
+
return firstUrl.toString();
|
|
480
|
+
});
|
|
481
|
+
const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
|
|
482
|
+
for (const [index, result] of candidateResults.entries()) {
|
|
483
|
+
const candidateSitemapUrl = candidateSitemapUrls[index];
|
|
484
|
+
if (result.status === 'fulfilled') {
|
|
485
|
+
if (result.value && addSitemapUrl(candidateSitemapUrl)) {
|
|
486
|
+
yield candidateSitemapUrl;
|
|
370
487
|
}
|
|
371
488
|
}
|
|
489
|
+
else {
|
|
490
|
+
logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
|
|
491
|
+
error: result.reason,
|
|
492
|
+
});
|
|
493
|
+
}
|
|
372
494
|
}
|
|
373
495
|
}
|
|
374
496
|
};
|
|
@@ -380,12 +502,17 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
380
502
|
}, {});
|
|
381
503
|
const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
|
|
382
504
|
const discoveredUrls = new Set();
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
505
|
+
try {
|
|
506
|
+
for await (const url of mergeAsyncIterables(...iterables)) {
|
|
507
|
+
if (discoveredUrls.has(url)) {
|
|
508
|
+
continue;
|
|
509
|
+
}
|
|
510
|
+
discoveredUrls.add(url);
|
|
511
|
+
yield url;
|
|
386
512
|
}
|
|
387
|
-
|
|
388
|
-
|
|
513
|
+
}
|
|
514
|
+
finally {
|
|
515
|
+
clearTimeout(timeoutHandle);
|
|
516
|
+
externalSignal?.removeEventListener('abort', onExternalAbort);
|
|
389
517
|
}
|
|
390
518
|
}
|
|
391
|
-
//# sourceMappingURL=sitemap.js.map
|
package/internals/social.d.ts
CHANGED
|
@@ -19,6 +19,8 @@ export declare function emailsFromText(text: string): string[];
|
|
|
19
19
|
/**
|
|
20
20
|
* The function extracts email addresses from a list of URLs.
|
|
21
21
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
22
|
+
* A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
|
|
23
|
+
* such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
|
|
22
24
|
* Note that the function preserves the order of emails and keep duplicates.
|
|
23
25
|
* @param urls Array of URLs.
|
|
24
26
|
* @return Array of emails addresses found.
|
|
@@ -225,7 +227,7 @@ export declare const TWITTER_REGEX: RegExp;
|
|
|
225
227
|
* ```
|
|
226
228
|
* import { social } from 'crawlee';
|
|
227
229
|
*
|
|
228
|
-
* const matches = text.match(social.
|
|
230
|
+
* const matches = text.match(social.TWITTER_REGEX_GLOBAL);
|
|
229
231
|
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
230
232
|
* ```
|
|
231
233
|
*/
|
|
@@ -488,5 +490,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
|
|
|
488
490
|
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
489
491
|
* @return An object with the social handles.
|
|
490
492
|
*/
|
|
491
|
-
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles
|
|
492
|
-
//# sourceMappingURL=social.d.ts.map
|
|
493
|
+
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;
|