@crawlee/utils 4.0.0-beta.25 → 4.0.0-beta.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +10 -17
- package/index.js +8 -16
- package/internal.d.ts +11 -0
- package/internal.js +10 -0
- package/internals/blocked.d.ts +0 -1
- package/internals/blocked.js +0 -1
- package/internals/cheerio.d.ts +1 -3
- package/internals/cheerio.js +25 -16
- package/internals/extract-microdata.d.ts +24 -0
- package/internals/extract-microdata.js +118 -0
- package/internals/extract-urls.d.ts +1 -2
- package/internals/extract-urls.js +18 -17
- package/internals/general.d.ts +0 -25
- package/internals/general.js +2 -110
- package/internals/iterables.d.ts +0 -80
- package/internals/iterables.js +0 -126
- package/internals/mutation-journal.d.ts +29 -0
- package/internals/mutation-journal.js +173 -0
- package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
- package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
- package/internals/robots.d.ts +35 -15
- package/internals/robots.js +66 -30
- package/internals/schemas.d.ts +114 -0
- package/internals/schemas.js +115 -0
- package/internals/sitemap.d.ts +44 -3
- package/internals/sitemap.js +171 -85
- package/internals/social.d.ts +2 -3
- package/internals/social.js +8 -6
- package/internals/url.d.ts +69 -1
- package/internals/url.js +120 -2
- package/internals/validation.d.ts +9 -0
- package/internals/validation.js +16 -0
- package/package.json +16 -15
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/blocked.d.ts.map +0 -1
- package/internals/blocked.js.map +0 -1
- package/internals/cheerio.d.ts.map +0 -1
- package/internals/cheerio.js.map +0 -1
- package/internals/chunk.d.ts +0 -2
- package/internals/chunk.d.ts.map +0 -1
- package/internals/chunk.js +0 -40
- package/internals/chunk.js.map +0 -1
- package/internals/debug.d.ts +0 -32
- package/internals/debug.d.ts.map +0 -1
- package/internals/debug.js +0 -41
- package/internals/debug.js.map +0 -1
- package/internals/extract-urls.d.ts.map +0 -1
- package/internals/extract-urls.js.map +0 -1
- package/internals/general.d.ts.map +0 -1
- package/internals/general.js.map +0 -1
- package/internals/iterables.d.ts.map +0 -1
- package/internals/iterables.js.map +0 -1
- package/internals/open_graph_parser.d.ts.map +0 -1
- package/internals/open_graph_parser.js.map +0 -1
- package/internals/robots.d.ts.map +0 -1
- package/internals/robots.js.map +0 -1
- package/internals/sitemap.d.ts.map +0 -1
- package/internals/sitemap.js.map +0 -1
- package/internals/social.d.ts.map +0 -1
- package/internals/social.js.map +0 -1
- package/internals/system-info/cpu-info.d.ts +0 -64
- package/internals/system-info/cpu-info.d.ts.map +0 -1
- package/internals/system-info/cpu-info.js +0 -211
- package/internals/system-info/cpu-info.js.map +0 -1
- package/internals/system-info/memory-info.d.ts +0 -28
- package/internals/system-info/memory-info.d.ts.map +0 -1
- package/internals/system-info/memory-info.js +0 -118
- package/internals/system-info/memory-info.js.map +0 -1
- package/internals/system-info/ps-tree.d.ts +0 -18
- package/internals/system-info/ps-tree.d.ts.map +0 -1
- package/internals/system-info/ps-tree.js +0 -145
- package/internals/system-info/ps-tree.js.map +0 -1
- package/internals/typedefs.d.ts +0 -5
- package/internals/typedefs.d.ts.map +0 -1
- package/internals/typedefs.js +0 -9
- package/internals/typedefs.js.map +0 -1
- package/internals/url.d.ts.map +0 -1
- package/internals/url.js.map +0 -1
package/internals/sitemap.js
CHANGED
|
@@ -3,32 +3,30 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
|
|
|
3
3
|
import { StringDecoder } from 'node:string_decoder';
|
|
4
4
|
import { createGunzip } from 'node:zlib';
|
|
5
5
|
import { FetchHttpClient } from '@crawlee/http-client';
|
|
6
|
-
import {
|
|
7
|
-
import sax from 'sax';
|
|
8
|
-
import MIMEType from 'whatwg-mimetype';
|
|
9
|
-
import log from '@apify/log';
|
|
6
|
+
import { MIMEType } from 'whatwg-mimetype';
|
|
10
7
|
import { mergeAsyncIterables } from './iterables.js';
|
|
11
|
-
import {
|
|
8
|
+
import { RobotsTxtFile } from './robots.js';
|
|
9
|
+
import { filterUrl } from './url.js';
|
|
12
10
|
class SitemapTxtParser extends Transform {
|
|
13
|
-
decoder = new StringDecoder('utf8');
|
|
14
|
-
buffer = '';
|
|
11
|
+
#decoder = new StringDecoder('utf8');
|
|
12
|
+
#buffer = '';
|
|
15
13
|
constructor() {
|
|
16
14
|
super({
|
|
17
15
|
readableObjectMode: true,
|
|
18
16
|
transform: (chunk, _encoding, callback) => {
|
|
19
|
-
this
|
|
17
|
+
this.#processBuffer(this.#decoder.write(chunk), false);
|
|
20
18
|
callback();
|
|
21
19
|
},
|
|
22
20
|
flush: (callback) => {
|
|
23
|
-
this
|
|
21
|
+
this.#processBuffer(this.#decoder.end(), true);
|
|
24
22
|
callback();
|
|
25
23
|
},
|
|
26
24
|
});
|
|
27
25
|
}
|
|
28
|
-
processBuffer(input, finalize) {
|
|
29
|
-
this
|
|
30
|
-
if (finalize || this
|
|
31
|
-
const parts = this
|
|
26
|
+
#processBuffer(input, finalize) {
|
|
27
|
+
this.#buffer += input;
|
|
28
|
+
if (finalize || this.#buffer.includes('\n')) {
|
|
29
|
+
const parts = this.#buffer
|
|
32
30
|
.split('\n')
|
|
33
31
|
.map((part) => part.trim())
|
|
34
32
|
.filter((part) => part.length > 0);
|
|
@@ -36,99 +34,113 @@ class SitemapTxtParser extends Transform {
|
|
|
36
34
|
for (const url of parts) {
|
|
37
35
|
this.push({ type: 'url', loc: url });
|
|
38
36
|
}
|
|
39
|
-
this
|
|
37
|
+
this.#buffer = '';
|
|
40
38
|
}
|
|
41
39
|
else if (parts.length > 0) {
|
|
42
40
|
for (const url of parts.slice(0, -1)) {
|
|
43
41
|
this.push({ type: 'url', loc: url });
|
|
44
42
|
}
|
|
45
|
-
this
|
|
43
|
+
this.#buffer = parts.at(-1);
|
|
46
44
|
}
|
|
47
45
|
}
|
|
48
46
|
}
|
|
49
47
|
}
|
|
50
48
|
class SitemapXmlParser extends Transform {
|
|
51
|
-
decoder = new StringDecoder('utf8');
|
|
52
|
-
parser
|
|
53
|
-
rootTagName;
|
|
54
|
-
currentTag = undefined;
|
|
55
|
-
url = {};
|
|
56
|
-
|
|
49
|
+
#decoder = new StringDecoder('utf8');
|
|
50
|
+
#parser;
|
|
51
|
+
#rootTagName;
|
|
52
|
+
#currentTag = undefined;
|
|
53
|
+
#url = {};
|
|
54
|
+
static async create() {
|
|
55
|
+
const { SAXParser } = (await import('sax')).default;
|
|
56
|
+
return new SitemapXmlParser(new SAXParser(true));
|
|
57
|
+
}
|
|
58
|
+
constructor(parser) {
|
|
57
59
|
super({
|
|
58
60
|
readableObjectMode: true,
|
|
59
61
|
transform: (chunk, _encoding, callback) => {
|
|
60
|
-
this
|
|
62
|
+
this.#parser.write(this.#decoder.write(chunk));
|
|
61
63
|
callback();
|
|
62
64
|
},
|
|
63
65
|
flush: (callback) => {
|
|
64
|
-
const rest = this
|
|
66
|
+
const rest = this.#decoder.end();
|
|
65
67
|
if (rest.length > 0) {
|
|
66
|
-
this
|
|
68
|
+
this.#parser.write(rest);
|
|
67
69
|
}
|
|
68
|
-
this
|
|
70
|
+
this.#parser.end();
|
|
69
71
|
callback();
|
|
70
72
|
},
|
|
71
73
|
});
|
|
72
|
-
this
|
|
73
|
-
this
|
|
74
|
-
this
|
|
75
|
-
this
|
|
76
|
-
this
|
|
74
|
+
this.#parser = parser;
|
|
75
|
+
this.#parser.onopentag = this.#onOpenTag.bind(this);
|
|
76
|
+
this.#parser.onclosetag = this.#onCloseTag.bind(this);
|
|
77
|
+
this.#parser.ontext = this.#onText.bind(this);
|
|
78
|
+
this.#parser.oncdata = this.#onText.bind(this);
|
|
79
|
+
this.#parser.onerror = this.destroy.bind(this);
|
|
77
80
|
}
|
|
78
|
-
onOpenTag(node) {
|
|
79
|
-
if (this
|
|
81
|
+
#onOpenTag(node) {
|
|
82
|
+
if (this.#rootTagName !== undefined) {
|
|
80
83
|
if (node.name === 'loc' ||
|
|
81
84
|
node.name === 'lastmod' ||
|
|
82
85
|
node.name === 'priority' ||
|
|
83
86
|
node.name === 'changefreq') {
|
|
84
|
-
this
|
|
87
|
+
this.#currentTag = node.name;
|
|
85
88
|
}
|
|
86
89
|
}
|
|
87
90
|
if (node.name === 'urlset') {
|
|
88
|
-
this
|
|
91
|
+
this.#rootTagName = 'urlset';
|
|
89
92
|
}
|
|
90
93
|
if (node.name === 'sitemapindex') {
|
|
91
|
-
this
|
|
94
|
+
this.#rootTagName = 'sitemapindex';
|
|
92
95
|
}
|
|
93
96
|
}
|
|
94
|
-
onCloseTag(name) {
|
|
97
|
+
#onCloseTag(name) {
|
|
95
98
|
if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
|
|
96
|
-
this
|
|
99
|
+
this.#currentTag = undefined;
|
|
97
100
|
}
|
|
98
|
-
if (name === 'url'
|
|
99
|
-
|
|
100
|
-
|
|
101
|
+
if (name === 'url') {
|
|
102
|
+
if (this.#url.loc !== undefined) {
|
|
103
|
+
this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
|
|
104
|
+
}
|
|
105
|
+
this.#url = {};
|
|
101
106
|
}
|
|
102
107
|
}
|
|
103
|
-
onText(text) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
108
|
+
#onText(text) {
|
|
109
|
+
text = text.trim();
|
|
110
|
+
// Whitespace around a CDATA section arrives as separate text events.
|
|
111
|
+
if (text === '') {
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
if (this.#currentTag === 'loc') {
|
|
115
|
+
if (this.#rootTagName === 'sitemapindex') {
|
|
116
|
+
this.push({ type: 'sitemapUrl', url: text });
|
|
107
117
|
}
|
|
108
|
-
if (this
|
|
109
|
-
this
|
|
110
|
-
this
|
|
118
|
+
if (this.#rootTagName === 'urlset') {
|
|
119
|
+
this.#url ??= {};
|
|
120
|
+
this.#url.loc = text;
|
|
111
121
|
}
|
|
112
122
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
123
|
+
if (this.#currentTag === 'lastmod') {
|
|
124
|
+
const lastmod = new Date(text);
|
|
125
|
+
if (!Number.isNaN(lastmod.getTime())) {
|
|
126
|
+
this.#url.lastmod = lastmod;
|
|
127
|
+
}
|
|
116
128
|
}
|
|
117
|
-
if (this
|
|
118
|
-
this
|
|
129
|
+
if (this.#currentTag === 'priority') {
|
|
130
|
+
this.#url.priority = Number(text);
|
|
119
131
|
}
|
|
120
|
-
if (this
|
|
132
|
+
if (this.#currentTag === 'changefreq') {
|
|
121
133
|
if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
|
|
122
|
-
this
|
|
134
|
+
this.#url.changefreq = text;
|
|
123
135
|
}
|
|
124
136
|
}
|
|
125
137
|
}
|
|
126
138
|
}
|
|
127
139
|
export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
128
|
-
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {};
|
|
140
|
+
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, enqueueStrategy = 'same-hostname', logger, } = options ?? {};
|
|
129
141
|
const sources = [...initialSources];
|
|
130
142
|
const visitedSitemapUrls = new Set();
|
|
131
|
-
const createParser = (contentType = '', url) => {
|
|
143
|
+
const createParser = async (contentType = '', url) => {
|
|
132
144
|
let mimeType;
|
|
133
145
|
try {
|
|
134
146
|
mimeType = new MIMEType(contentType);
|
|
@@ -137,7 +149,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
137
149
|
mimeType = null;
|
|
138
150
|
}
|
|
139
151
|
if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
|
|
140
|
-
return
|
|
152
|
+
return SitemapXmlParser.create();
|
|
141
153
|
}
|
|
142
154
|
if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
|
|
143
155
|
return new SitemapTxtParser();
|
|
@@ -147,12 +159,13 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
147
159
|
while (sources.length > 0) {
|
|
148
160
|
const source = sources.shift();
|
|
149
161
|
if ((source?.depth ?? 0) > maxDepth) {
|
|
150
|
-
log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
|
|
151
162
|
continue;
|
|
152
163
|
}
|
|
153
164
|
let items = null;
|
|
165
|
+
// Parent URL, parsed once and reused as the origin for the strategy checks below.
|
|
166
|
+
let sitemapUrl;
|
|
154
167
|
if (source.type === 'url') {
|
|
155
|
-
|
|
168
|
+
sitemapUrl = new URL(source.url);
|
|
156
169
|
visitedSitemapUrls.add(sitemapUrl.toString());
|
|
157
170
|
let retriesLeft = sitemapRetries + 1;
|
|
158
171
|
while (retriesLeft-- > 0) {
|
|
@@ -162,11 +175,11 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
162
175
|
sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
|
|
163
176
|
method: 'GET',
|
|
164
177
|
headers: {
|
|
165
|
-
accept: '
|
|
178
|
+
accept: '*/*',
|
|
166
179
|
},
|
|
167
180
|
}), {
|
|
168
181
|
proxyUrl,
|
|
169
|
-
timeout,
|
|
182
|
+
timeoutMillis: timeout,
|
|
170
183
|
});
|
|
171
184
|
}
|
|
172
185
|
catch (error) {
|
|
@@ -178,6 +191,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
178
191
|
if (sitemapResponse.body === null) {
|
|
179
192
|
break;
|
|
180
193
|
}
|
|
194
|
+
const { fileTypeStream } = await import('file-type');
|
|
181
195
|
const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
|
|
182
196
|
if (streamWithType.fileType !== undefined) {
|
|
183
197
|
contentType = streamWithType.fileType.mime;
|
|
@@ -191,7 +205,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
191
205
|
sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
|
|
192
206
|
}
|
|
193
207
|
}
|
|
194
|
-
items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
|
|
208
|
+
items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
|
|
195
209
|
if (e !== undefined && e !== null) {
|
|
196
210
|
error = { type: 'parser', error: e };
|
|
197
211
|
}
|
|
@@ -214,28 +228,53 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
214
228
|
}
|
|
215
229
|
}
|
|
216
230
|
catch (e) {
|
|
217
|
-
|
|
231
|
+
logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
|
|
218
232
|
}
|
|
219
233
|
}
|
|
220
234
|
}
|
|
221
235
|
else if (source.type === 'raw') {
|
|
222
|
-
items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
|
|
236
|
+
items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
|
|
223
237
|
if (error !== undefined) {
|
|
224
|
-
|
|
238
|
+
logger?.warning(`Malformed sitemap content: ${error}`);
|
|
225
239
|
}
|
|
226
240
|
});
|
|
227
241
|
}
|
|
228
242
|
if (items === null) {
|
|
229
243
|
continue;
|
|
230
244
|
}
|
|
245
|
+
// URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
|
|
246
|
+
// the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
|
|
247
|
+
let droppedUrlEntries = 0;
|
|
231
248
|
for await (const item of items) {
|
|
232
249
|
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
|
|
250
|
+
if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
|
|
251
|
+
logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
// Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
|
|
255
|
+
// parent. Raw string sources have no parent URL, so the check is skipped.
|
|
256
|
+
if (source.type === 'url') {
|
|
257
|
+
const { allowed, reason } = filterUrl(item.url, sitemapUrl, enqueueStrategy);
|
|
258
|
+
if (!allowed) {
|
|
259
|
+
logger?.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
233
263
|
sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
|
|
234
264
|
if (emitNestedSitemaps) {
|
|
235
265
|
yield { loc: item.url, originSitemapUrl: null };
|
|
236
266
|
}
|
|
237
267
|
}
|
|
238
268
|
if (item.type === 'url') {
|
|
269
|
+
// Keep only URL entries that match the enqueue strategy relative to the parent (see above).
|
|
270
|
+
if (source.type === 'url') {
|
|
271
|
+
const { allowed, reason } = filterUrl(item.loc, sitemapUrl, enqueueStrategy);
|
|
272
|
+
if (!allowed) {
|
|
273
|
+
droppedUrlEntries++;
|
|
274
|
+
logger?.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
239
278
|
yield {
|
|
240
279
|
...item,
|
|
241
280
|
originSitemapUrl: source.type === 'url'
|
|
@@ -244,6 +283,9 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
244
283
|
};
|
|
245
284
|
}
|
|
246
285
|
}
|
|
286
|
+
if (droppedUrlEntries > 0 && source.type === 'url') {
|
|
287
|
+
logger?.warning(`Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`);
|
|
288
|
+
}
|
|
247
289
|
}
|
|
248
290
|
}
|
|
249
291
|
/**
|
|
@@ -285,7 +327,7 @@ export class Sitemap {
|
|
|
285
327
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
286
328
|
*/
|
|
287
329
|
static async load(urls, proxyUrl, parseSitemapOptions) {
|
|
288
|
-
return await this
|
|
330
|
+
return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
|
|
289
331
|
}
|
|
290
332
|
/**
|
|
291
333
|
* Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
|
|
@@ -293,16 +335,17 @@ export class Sitemap {
|
|
|
293
335
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
294
336
|
*/
|
|
295
337
|
static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
|
|
296
|
-
return await this
|
|
338
|
+
return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
|
|
297
339
|
}
|
|
298
|
-
static async parse(sources, proxyUrl, parseSitemapOptions) {
|
|
340
|
+
static async #parse(sources, proxyUrl, parseSitemapOptions) {
|
|
299
341
|
const urls = [];
|
|
300
342
|
try {
|
|
301
343
|
for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
|
|
302
344
|
urls.push(item.loc);
|
|
303
345
|
}
|
|
304
346
|
}
|
|
305
|
-
catch {
|
|
347
|
+
catch (e) {
|
|
348
|
+
parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
|
|
306
349
|
return new Sitemap([]);
|
|
307
350
|
}
|
|
308
351
|
return new Sitemap(urls);
|
|
@@ -316,7 +359,19 @@ export class Sitemap {
|
|
|
316
359
|
* @returns An async iterable with the discovered sitemap URLs.
|
|
317
360
|
*/
|
|
318
361
|
export async function* discoverValidSitemaps(urls, options = {}) {
|
|
319
|
-
const { proxyUrl, httpClient = new FetchHttpClient() } = options;
|
|
362
|
+
const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
|
|
363
|
+
const controller = new AbortController();
|
|
364
|
+
const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
|
|
365
|
+
const onExternalAbort = () => controller.abort();
|
|
366
|
+
if (externalSignal) {
|
|
367
|
+
if (externalSignal.aborted) {
|
|
368
|
+
controller.abort();
|
|
369
|
+
}
|
|
370
|
+
else {
|
|
371
|
+
externalSignal.addEventListener('abort', onExternalAbort, { once: true });
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
const signal = controller.signal;
|
|
320
375
|
const sitemapUrls = new Set();
|
|
321
376
|
const addSitemapUrl = (url) => {
|
|
322
377
|
const sizeBefore = sitemapUrls.size;
|
|
@@ -331,7 +386,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
331
386
|
return false;
|
|
332
387
|
}
|
|
333
388
|
try {
|
|
334
|
-
const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
|
|
389
|
+
const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
|
|
390
|
+
proxyUrl,
|
|
391
|
+
timeoutMillis: requestTimeoutMillis,
|
|
392
|
+
signal,
|
|
393
|
+
});
|
|
335
394
|
return response.status >= 200 && response.status < 400;
|
|
336
395
|
}
|
|
337
396
|
catch {
|
|
@@ -343,17 +402,24 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
343
402
|
return;
|
|
344
403
|
}
|
|
345
404
|
try {
|
|
346
|
-
const robotsFile = await
|
|
347
|
-
|
|
405
|
+
const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
|
|
406
|
+
proxyUrl,
|
|
407
|
+
timeoutMillis: requestTimeoutMillis,
|
|
408
|
+
signal,
|
|
409
|
+
httpClient,
|
|
410
|
+
logger,
|
|
411
|
+
});
|
|
412
|
+
// Surface all referenced sitemaps, including cross-host; scoping happens at load time.
|
|
413
|
+
for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
|
|
348
414
|
if (addSitemapUrl(sitemapUrl)) {
|
|
349
415
|
yield sitemapUrl;
|
|
350
416
|
}
|
|
351
417
|
}
|
|
352
418
|
}
|
|
353
419
|
catch (err) {
|
|
354
|
-
|
|
420
|
+
logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
|
|
355
421
|
}
|
|
356
|
-
const sitemapUrl = domainUrls.find((url) => /sitemap
|
|
422
|
+
const sitemapUrl = domainUrls.find((url) => /sitemap(?:_index)?\.(?:xml|txt)(?:\.gz)?$/i.test(url));
|
|
357
423
|
if (sitemapUrl !== undefined) {
|
|
358
424
|
if (addSitemapUrl(sitemapUrl)) {
|
|
359
425
|
yield sitemapUrl;
|
|
@@ -361,14 +427,29 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
361
427
|
}
|
|
362
428
|
else {
|
|
363
429
|
const firstUrl = new URL(domainUrls[0]);
|
|
430
|
+
// The guessed candidates are well-known paths on the same origin, so the query string
|
|
431
|
+
// and fragment of the input URL are not part of them - carrying them over probes (and
|
|
432
|
+
// yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
|
|
433
|
+
firstUrl.search = '';
|
|
434
|
+
firstUrl.hash = '';
|
|
364
435
|
const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
|
|
365
|
-
|
|
436
|
+
const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
|
|
366
437
|
firstUrl.pathname = pathname;
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
438
|
+
return firstUrl.toString();
|
|
439
|
+
});
|
|
440
|
+
const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
|
|
441
|
+
for (const [index, result] of candidateResults.entries()) {
|
|
442
|
+
const candidateSitemapUrl = candidateSitemapUrls[index];
|
|
443
|
+
if (result.status === 'fulfilled') {
|
|
444
|
+
if (result.value && addSitemapUrl(candidateSitemapUrl)) {
|
|
445
|
+
yield candidateSitemapUrl;
|
|
370
446
|
}
|
|
371
447
|
}
|
|
448
|
+
else {
|
|
449
|
+
logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
|
|
450
|
+
error: result.reason,
|
|
451
|
+
});
|
|
452
|
+
}
|
|
372
453
|
}
|
|
373
454
|
}
|
|
374
455
|
};
|
|
@@ -380,12 +461,17 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
380
461
|
}, {});
|
|
381
462
|
const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
|
|
382
463
|
const discoveredUrls = new Set();
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
464
|
+
try {
|
|
465
|
+
for await (const url of mergeAsyncIterables(...iterables)) {
|
|
466
|
+
if (discoveredUrls.has(url)) {
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
discoveredUrls.add(url);
|
|
470
|
+
yield url;
|
|
386
471
|
}
|
|
387
|
-
|
|
388
|
-
|
|
472
|
+
}
|
|
473
|
+
finally {
|
|
474
|
+
clearTimeout(timeoutHandle);
|
|
475
|
+
externalSignal?.removeEventListener('abort', onExternalAbort);
|
|
389
476
|
}
|
|
390
477
|
}
|
|
391
|
-
//# sourceMappingURL=sitemap.js.map
|
package/internals/social.d.ts
CHANGED
|
@@ -225,7 +225,7 @@ export declare const TWITTER_REGEX: RegExp;
|
|
|
225
225
|
* ```
|
|
226
226
|
* import { social } from 'crawlee';
|
|
227
227
|
*
|
|
228
|
-
* const matches = text.match(social.
|
|
228
|
+
* const matches = text.match(social.TWITTER_REGEX_GLOBAL);
|
|
229
229
|
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
230
230
|
* ```
|
|
231
231
|
*/
|
|
@@ -488,5 +488,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
|
|
|
488
488
|
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
489
489
|
* @return An object with the social handles.
|
|
490
490
|
*/
|
|
491
|
-
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles
|
|
492
|
-
//# sourceMappingURL=social.d.ts.map
|
|
491
|
+
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;
|
package/internals/social.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import * as cheerio from 'cheerio';
|
|
2
1
|
import { htmlToText } from './cheerio.js';
|
|
3
2
|
// Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
|
|
4
|
-
|
|
3
|
+
// The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
|
|
4
|
+
// instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
|
|
5
|
+
// dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
|
|
6
|
+
const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
|
|
5
7
|
/**
|
|
6
8
|
* Regular expression to exactly match a single email address.
|
|
7
9
|
* It has the following form: `/^...$/i`.
|
|
@@ -324,7 +326,7 @@ export const TWITTER_REGEX = new RegExp(`^${TWITTER_REGEX_STRING}$`, 'i');
|
|
|
324
326
|
* ```
|
|
325
327
|
* import { social } from 'crawlee';
|
|
326
328
|
*
|
|
327
|
-
* const matches = text.match(social.
|
|
329
|
+
* const matches = text.match(social.TWITTER_REGEX_GLOBAL);
|
|
328
330
|
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
329
331
|
* ```
|
|
330
332
|
*/
|
|
@@ -587,7 +589,8 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
|
|
|
587
589
|
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
588
590
|
* @return An object with the social handles.
|
|
589
591
|
*/
|
|
590
|
-
export function parseHandlesFromHtml(html, data = null) {
|
|
592
|
+
export async function parseHandlesFromHtml(html, data = null) {
|
|
593
|
+
const cheerio = await import('cheerio/slim');
|
|
591
594
|
const result = {
|
|
592
595
|
emails: [],
|
|
593
596
|
phones: [],
|
|
@@ -606,7 +609,7 @@ export function parseHandlesFromHtml(html, data = null) {
|
|
|
606
609
|
const $ = cheerio.load(html, { xml: { decodeEntities: true } });
|
|
607
610
|
if (data)
|
|
608
611
|
data.$ = $;
|
|
609
|
-
const text = htmlToText($);
|
|
612
|
+
const text = await htmlToText($);
|
|
610
613
|
if (data)
|
|
611
614
|
data.text = text;
|
|
612
615
|
// NOTE: we need to parse each text separately, orherwise we might concatenate unrelated texts
|
|
@@ -644,4 +647,3 @@ export function parseHandlesFromHtml(html, data = null) {
|
|
|
644
647
|
}
|
|
645
648
|
return result;
|
|
646
649
|
}
|
|
647
|
-
//# sourceMappingURL=social.js.map
|
package/internals/url.d.ts
CHANGED
|
@@ -1,4 +1,73 @@
|
|
|
1
1
|
import type { SearchParams } from '@crawlee/types';
|
|
2
|
+
/**
|
|
3
|
+
* The different enqueueing strategies available.
|
|
4
|
+
*
|
|
5
|
+
* Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
|
|
6
|
+
*
|
|
7
|
+
* ```md
|
|
8
|
+
* Protocol Domain
|
|
9
|
+
* ┌────┐ ┌─────────┐
|
|
10
|
+
* https://example.crawlee.dev/...
|
|
11
|
+
* │ └─────────────────┤
|
|
12
|
+
* │ Hostname │
|
|
13
|
+
* │ │
|
|
14
|
+
* └─────────────────────────┘
|
|
15
|
+
* Origin
|
|
16
|
+
*```
|
|
17
|
+
*
|
|
18
|
+
* - The `Protocol` is usually `http` or `https`
|
|
19
|
+
* - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
|
|
20
|
+
* - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
|
|
21
|
+
* - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
|
|
22
|
+
*/
|
|
23
|
+
export declare enum EnqueueStrategy {
|
|
24
|
+
/**
|
|
25
|
+
* Matches any URLs found
|
|
26
|
+
*/
|
|
27
|
+
All = "all",
|
|
28
|
+
/**
|
|
29
|
+
* Matches any URLs that have the same hostname.
|
|
30
|
+
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
|
|
31
|
+
* `https://example.com/hello` will not be matched.
|
|
32
|
+
*
|
|
33
|
+
* > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
|
|
34
|
+
*/
|
|
35
|
+
SameHostname = "same-hostname",
|
|
36
|
+
/**
|
|
37
|
+
* Matches any URLs that have the same domain as the base URL.
|
|
38
|
+
* For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
|
|
39
|
+
* `https://example.com`.
|
|
40
|
+
*
|
|
41
|
+
* > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
|
|
42
|
+
*/
|
|
43
|
+
SameDomain = "same-domain",
|
|
44
|
+
/**
|
|
45
|
+
* Matches any URLs that have the same hostname and protocol.
|
|
46
|
+
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
|
|
47
|
+
* `http://wow.example.com/hello` will not be matched.
|
|
48
|
+
*
|
|
49
|
+
* > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
|
|
50
|
+
*/
|
|
51
|
+
SameOrigin = "same-origin"
|
|
52
|
+
}
|
|
53
|
+
/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
|
|
54
|
+
export declare const UNSUPPORTED_SCHEME_MESSAGE = "unsupported URL scheme (only http and https are allowed)";
|
|
55
|
+
/**
|
|
56
|
+
* Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
|
|
57
|
+
* considered here (use {@link filterUrl} for the combined scheme + strategy check).
|
|
58
|
+
*
|
|
59
|
+
* The `enqueueLinks` implementation in `@crawlee/basic` matches the same strategies via glob patterns
|
|
60
|
+
* (see `packages/basic-crawler/src/internals/enqueue-links/enqueue-links.ts`) — keep the two in sync when changing either.
|
|
61
|
+
*/
|
|
62
|
+
export declare function matchesEnqueueStrategy(strategy: EnqueueStrategy | `${EnqueueStrategy}`, target: URL, origin: URL): boolean;
|
|
63
|
+
/**
|
|
64
|
+
* Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
|
|
65
|
+
* scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
|
|
66
|
+
*/
|
|
67
|
+
export declare function filterUrl(target: string | URL, origin: string | URL, strategy: EnqueueStrategy | `${EnqueueStrategy}`): {
|
|
68
|
+
allowed: boolean;
|
|
69
|
+
reason?: string;
|
|
70
|
+
};
|
|
2
71
|
/**
|
|
3
72
|
* Appends search (query string) parameters to a URL, replacing the original value (if any).
|
|
4
73
|
*
|
|
@@ -7,4 +76,3 @@ import type { SearchParams } from '@crawlee/types';
|
|
|
7
76
|
* @internal
|
|
8
77
|
*/
|
|
9
78
|
export declare function applySearchParams(url: URL, searchParams: SearchParams | undefined): void;
|
|
10
|
-
//# sourceMappingURL=url.d.ts.map
|