@crawlee/utils 3.10.6-beta.9 → 3.11.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.mjs +1 -0
- package/internals/cheerio.js +2 -3
- package/internals/cheerio.js.map +1 -1
- package/internals/chunk.js +1 -2
- package/internals/chunk.js.map +1 -1
- package/internals/debug.d.ts +0 -2
- package/internals/debug.d.ts.map +1 -1
- package/internals/debug.js +1 -2
- package/internals/debug.js.map +1 -1
- package/internals/extract-urls.d.ts +0 -2
- package/internals/extract-urls.d.ts.map +1 -1
- package/internals/extract-urls.js +3 -4
- package/internals/extract-urls.js.map +1 -1
- package/internals/general.js +6 -6
- package/internals/general.js.map +1 -1
- package/internals/gotScraping.d.ts.map +1 -1
- package/internals/memory-info.js +1 -2
- package/internals/memory-info.js.map +1 -1
- package/internals/open_graph_parser.js +1 -2
- package/internals/open_graph_parser.js.map +1 -1
- package/internals/sitemap.d.ts +27 -27
- package/internals/sitemap.d.ts.map +1 -1
- package/internals/sitemap.js +215 -155
- package/internals/sitemap.js.map +1 -1
- package/internals/social.js +6 -6
- package/internals/social.js.map +1 -1
- package/internals/typedefs.d.ts.map +1 -1
- package/internals/typedefs.js +2 -3
- package/internals/typedefs.js.map +1 -1
- package/package.json +4 -4
- package/tsconfig.build.tsbuildinfo +1 -1
package/internals/sitemap.js
CHANGED
|
@@ -1,104 +1,255 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Sitemap = void 0;
|
|
4
|
+
exports.parseSitemap = parseSitemap;
|
|
4
5
|
const tslib_1 = require("tslib");
|
|
6
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
7
|
const node_stream_1 = require("node:stream");
|
|
6
8
|
const node_string_decoder_1 = require("node:string_decoder");
|
|
7
9
|
const node_zlib_1 = require("node:zlib");
|
|
8
10
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
9
11
|
const sax_1 = tslib_1.__importDefault(require("sax"));
|
|
10
12
|
const whatwg_mimetype_1 = tslib_1.__importDefault(require("whatwg-mimetype"));
|
|
11
|
-
class
|
|
13
|
+
class SitemapTxtParser extends node_stream_1.Transform {
|
|
12
14
|
constructor() {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
super({
|
|
16
|
+
readableObjectMode: true,
|
|
17
|
+
transform: (chunk, _encoding, callback) => {
|
|
18
|
+
this.processBuffer(this.decoder.write(chunk), false);
|
|
19
|
+
callback();
|
|
20
|
+
},
|
|
21
|
+
flush: (callback) => {
|
|
22
|
+
this.processBuffer(this.decoder.end(), true);
|
|
23
|
+
callback();
|
|
24
|
+
},
|
|
18
25
|
});
|
|
19
|
-
Object.defineProperty(this, "
|
|
26
|
+
Object.defineProperty(this, "decoder", {
|
|
20
27
|
enumerable: true,
|
|
21
28
|
configurable: true,
|
|
22
29
|
writable: true,
|
|
23
|
-
value:
|
|
30
|
+
value: new node_string_decoder_1.StringDecoder('utf8')
|
|
24
31
|
});
|
|
25
|
-
Object.defineProperty(this, "
|
|
32
|
+
Object.defineProperty(this, "buffer", {
|
|
26
33
|
enumerable: true,
|
|
27
34
|
configurable: true,
|
|
28
35
|
writable: true,
|
|
29
|
-
value:
|
|
36
|
+
value: ''
|
|
30
37
|
});
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
38
|
+
}
|
|
39
|
+
processBuffer(input, finalize) {
|
|
40
|
+
this.buffer += input;
|
|
41
|
+
if (finalize || this.buffer.includes('\n')) {
|
|
42
|
+
const parts = this.buffer
|
|
43
|
+
.split('\n')
|
|
44
|
+
.map((part) => part.trim())
|
|
45
|
+
.filter((part) => part.length > 0);
|
|
46
|
+
if (finalize) {
|
|
47
|
+
for (const url of parts) {
|
|
48
|
+
this.push({ type: 'url', loc: url });
|
|
49
|
+
}
|
|
50
|
+
this.buffer = '';
|
|
51
|
+
}
|
|
52
|
+
else if (parts.length > 0) {
|
|
53
|
+
for (const url of parts.slice(0, -1)) {
|
|
54
|
+
this.push({ type: 'url', loc: url });
|
|
55
|
+
}
|
|
56
|
+
this.buffer = parts.at(-1);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
class SitemapXmlParser extends node_stream_1.Transform {
|
|
62
|
+
constructor() {
|
|
63
|
+
super({
|
|
64
|
+
readableObjectMode: true,
|
|
65
|
+
transform: (chunk, _encoding, callback) => {
|
|
66
|
+
this.parser.write(this.decoder.write(chunk));
|
|
67
|
+
callback();
|
|
68
|
+
},
|
|
69
|
+
flush: (callback) => {
|
|
70
|
+
const rest = this.decoder.end();
|
|
71
|
+
if (rest.length > 0) {
|
|
72
|
+
this.parser.write(rest);
|
|
73
|
+
}
|
|
74
|
+
this.parser.end();
|
|
75
|
+
callback();
|
|
76
|
+
},
|
|
36
77
|
});
|
|
37
|
-
Object.defineProperty(this, "
|
|
78
|
+
Object.defineProperty(this, "decoder", {
|
|
38
79
|
enumerable: true,
|
|
39
80
|
configurable: true,
|
|
40
81
|
writable: true,
|
|
41
|
-
value:
|
|
82
|
+
value: new node_string_decoder_1.StringDecoder('utf8')
|
|
42
83
|
});
|
|
43
|
-
|
|
44
|
-
resetContext() {
|
|
45
|
-
this.context = undefined;
|
|
46
|
-
this.loc = false;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
class SitemapTxtParser extends node_stream_1.Writable {
|
|
50
|
-
constructor(parsingState, onEnd) {
|
|
51
|
-
super();
|
|
52
|
-
Object.defineProperty(this, "parsingState", {
|
|
84
|
+
Object.defineProperty(this, "parser", {
|
|
53
85
|
enumerable: true,
|
|
54
86
|
configurable: true,
|
|
55
87
|
writable: true,
|
|
56
|
-
value:
|
|
88
|
+
value: new sax_1.default.SAXParser(true)
|
|
57
89
|
});
|
|
58
|
-
Object.defineProperty(this, "
|
|
90
|
+
Object.defineProperty(this, "rootTagName", {
|
|
59
91
|
enumerable: true,
|
|
60
92
|
configurable: true,
|
|
61
93
|
writable: true,
|
|
62
|
-
value:
|
|
94
|
+
value: void 0
|
|
63
95
|
});
|
|
64
|
-
Object.defineProperty(this, "
|
|
96
|
+
Object.defineProperty(this, "currentTag", {
|
|
65
97
|
enumerable: true,
|
|
66
98
|
configurable: true,
|
|
67
99
|
writable: true,
|
|
68
|
-
value:
|
|
100
|
+
value: undefined
|
|
69
101
|
});
|
|
70
|
-
Object.defineProperty(this, "
|
|
102
|
+
Object.defineProperty(this, "url", {
|
|
71
103
|
enumerable: true,
|
|
72
104
|
configurable: true,
|
|
73
105
|
writable: true,
|
|
74
|
-
value:
|
|
106
|
+
value: {}
|
|
75
107
|
});
|
|
108
|
+
this.parser.onopentag = this.onOpenTag.bind(this);
|
|
109
|
+
this.parser.onclosetag = this.onCloseTag.bind(this);
|
|
110
|
+
this.parser.ontext = this.onText.bind(this);
|
|
111
|
+
this.parser.oncdata = this.onText.bind(this);
|
|
112
|
+
this.parser.onerror = this.destroy.bind(this);
|
|
76
113
|
}
|
|
77
|
-
|
|
78
|
-
this.
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
.
|
|
82
|
-
.
|
|
83
|
-
.
|
|
84
|
-
if (finalize) {
|
|
85
|
-
this.parsingState.urls.push(...parts);
|
|
86
|
-
this.buffer = '';
|
|
87
|
-
}
|
|
88
|
-
else if (parts.length > 0) {
|
|
89
|
-
this.parsingState.urls.push(...parts.slice(0, -1));
|
|
90
|
-
this.buffer = parts.at(-1);
|
|
114
|
+
onOpenTag(node) {
|
|
115
|
+
if (this.rootTagName !== undefined) {
|
|
116
|
+
if (node.name === 'loc' ||
|
|
117
|
+
node.name === 'lastmod' ||
|
|
118
|
+
node.name === 'priority' ||
|
|
119
|
+
node.name === 'changefreq') {
|
|
120
|
+
this.currentTag = node.name;
|
|
91
121
|
}
|
|
92
122
|
}
|
|
123
|
+
if (node.name === 'urlset') {
|
|
124
|
+
this.rootTagName = 'urlset';
|
|
125
|
+
}
|
|
126
|
+
if (node.name === 'sitemapindex') {
|
|
127
|
+
this.rootTagName = 'sitemapindex';
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
onCloseTag(name) {
|
|
131
|
+
if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
|
|
132
|
+
this.currentTag = undefined;
|
|
133
|
+
}
|
|
134
|
+
if (name === 'url' && this.url.loc !== undefined) {
|
|
135
|
+
this.push({ type: 'url', ...this.url, loc: this.url.loc });
|
|
136
|
+
this.url = {};
|
|
137
|
+
}
|
|
93
138
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
139
|
+
onText(text) {
|
|
140
|
+
if (this.currentTag === 'loc') {
|
|
141
|
+
if (this.rootTagName === 'sitemapindex') {
|
|
142
|
+
this.push({ type: 'sitemapUrl', url: text });
|
|
143
|
+
}
|
|
144
|
+
if (this.rootTagName === 'urlset') {
|
|
145
|
+
this.url ?? (this.url = {});
|
|
146
|
+
this.url.loc = text;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
text = text.trim();
|
|
150
|
+
if (this.currentTag === 'lastmod') {
|
|
151
|
+
this.url.lastmod = new Date(text);
|
|
152
|
+
}
|
|
153
|
+
if (this.currentTag === 'priority') {
|
|
154
|
+
this.url.priority = Number(text);
|
|
155
|
+
}
|
|
156
|
+
if (this.currentTag === 'changefreq') {
|
|
157
|
+
if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
|
|
158
|
+
this.url.changefreq = text;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
97
161
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
162
|
+
}
|
|
163
|
+
async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
164
|
+
const { gotScraping } = await import('got-scraping');
|
|
165
|
+
const { fileTypeStream } = await import('file-type');
|
|
166
|
+
const sources = [...initialSources];
|
|
167
|
+
const visitedSitemapUrls = new Set();
|
|
168
|
+
const createParser = (contentType = '', url) => {
|
|
169
|
+
let mimeType;
|
|
170
|
+
try {
|
|
171
|
+
mimeType = new whatwg_mimetype_1.default(contentType);
|
|
172
|
+
}
|
|
173
|
+
catch (e) {
|
|
174
|
+
mimeType = null;
|
|
175
|
+
}
|
|
176
|
+
if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
|
|
177
|
+
return new SitemapXmlParser();
|
|
178
|
+
}
|
|
179
|
+
if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
|
|
180
|
+
return new SitemapTxtParser();
|
|
181
|
+
}
|
|
182
|
+
throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
|
|
183
|
+
};
|
|
184
|
+
while (sources.length > 0) {
|
|
185
|
+
const source = sources.shift();
|
|
186
|
+
if ((source?.depth ?? 0) > (options?.maxDepth ?? Infinity)) {
|
|
187
|
+
log_1.default.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${options.maxDepth}.`);
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
let items = null;
|
|
191
|
+
if (source.type === 'url') {
|
|
192
|
+
const sitemapUrl = new URL(source.url);
|
|
193
|
+
visitedSitemapUrls.add(sitemapUrl.toString());
|
|
194
|
+
try {
|
|
195
|
+
const sitemapStream = await new Promise((resolve, reject) => {
|
|
196
|
+
const request = gotScraping.stream({ url: sitemapUrl, proxyUrl, method: 'GET' });
|
|
197
|
+
request.on('response', () => resolve(request));
|
|
198
|
+
request.on('error', reject);
|
|
199
|
+
});
|
|
200
|
+
if (sitemapStream.response.statusCode === 200) {
|
|
201
|
+
let contentType = sitemapStream.response.headers['content-type'];
|
|
202
|
+
const streamWithType = await fileTypeStream(sitemapStream);
|
|
203
|
+
if (streamWithType.fileType !== undefined) {
|
|
204
|
+
contentType = streamWithType.fileType.mime;
|
|
205
|
+
}
|
|
206
|
+
let isGzipped = false;
|
|
207
|
+
if (contentType !== undefined
|
|
208
|
+
? contentType === 'application/gzip'
|
|
209
|
+
: sitemapUrl.pathname.endsWith('.gz')) {
|
|
210
|
+
isGzipped = true;
|
|
211
|
+
if (sitemapUrl.pathname.endsWith('.gz')) {
|
|
212
|
+
sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
items = (0, node_stream_1.pipeline)(streamWithType, isGzipped ? (0, node_zlib_1.createGunzip)() : new node_stream_1.PassThrough(), createParser(contentType, sitemapUrl), (error) => {
|
|
216
|
+
if (error !== undefined) {
|
|
217
|
+
log_1.default.warning(`Malformed sitemap content: ${sitemapUrl}, ${error}`);
|
|
218
|
+
}
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
catch (e) {
|
|
223
|
+
log_1.default.warning(`Malformed sitemap content: ${sitemapUrl}, ${e}`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
else if (source.type === 'raw') {
|
|
227
|
+
items = (0, node_stream_1.pipeline)(node_stream_1.Readable.from([source.content]), createParser('text/xml'), (error) => {
|
|
228
|
+
if (error !== undefined) {
|
|
229
|
+
log_1.default.warning(`Malformed sitemap content: ${error}`);
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
if (items === null) {
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
for await (const item of items) {
|
|
237
|
+
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
|
|
238
|
+
sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
|
|
239
|
+
if (options?.emitNestedSitemaps) {
|
|
240
|
+
// @ts-ignore
|
|
241
|
+
yield { loc: item.url, originSitemapUrl: null };
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
if (item.type === 'url') {
|
|
245
|
+
yield {
|
|
246
|
+
...item,
|
|
247
|
+
originSitemapUrl: source.type === 'url'
|
|
248
|
+
? source.url
|
|
249
|
+
: `raw://${(0, node_crypto_1.createHash)('sha256').update(source.content).digest('base64')}`,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
}
|
|
102
253
|
}
|
|
103
254
|
}
|
|
104
255
|
/**
|
|
@@ -122,42 +273,6 @@ class Sitemap {
|
|
|
122
273
|
value: urls
|
|
123
274
|
});
|
|
124
275
|
}
|
|
125
|
-
static createXmlParser(parsingState, onEnd, onError) {
|
|
126
|
-
const parser = sax_1.default.createStream(true);
|
|
127
|
-
parser.on('opentag', (node) => {
|
|
128
|
-
if (node.name === 'loc' && parsingState.context !== undefined) {
|
|
129
|
-
parsingState.loc = true;
|
|
130
|
-
}
|
|
131
|
-
if (node.name === 'urlset') {
|
|
132
|
-
parsingState.context = 'urlset';
|
|
133
|
-
}
|
|
134
|
-
if (node.name === 'sitemapindex') {
|
|
135
|
-
parsingState.context = 'sitemapindex';
|
|
136
|
-
}
|
|
137
|
-
});
|
|
138
|
-
parser.on('closetag', (name) => {
|
|
139
|
-
if (name === 'loc') {
|
|
140
|
-
parsingState.loc = false;
|
|
141
|
-
}
|
|
142
|
-
});
|
|
143
|
-
const onText = (text) => {
|
|
144
|
-
if (parsingState.loc) {
|
|
145
|
-
if (parsingState.context === 'sitemapindex') {
|
|
146
|
-
if (!parsingState.visitedSitemapUrls.includes(text)) {
|
|
147
|
-
parsingState.sources.push({ type: 'url', url: text });
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
if (parsingState.context === 'urlset') {
|
|
151
|
-
parsingState.urls.push(text);
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
};
|
|
155
|
-
parser.on('text', onText);
|
|
156
|
-
parser.on('cdata', onText);
|
|
157
|
-
parser.on('end', onEnd);
|
|
158
|
-
parser.on('error', onError);
|
|
159
|
-
return parser;
|
|
160
|
-
}
|
|
161
276
|
/**
|
|
162
277
|
* Try to load sitemap from the most common locations - `/sitemap.xml` and `/sitemap.txt`.
|
|
163
278
|
* For loading based on `Sitemap` entries in `robots.txt`, the {@apilink RobotsFile} class should be used.
|
|
@@ -180,9 +295,7 @@ class Sitemap {
|
|
|
180
295
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
181
296
|
*/
|
|
182
297
|
static async load(urls, proxyUrl) {
|
|
183
|
-
|
|
184
|
-
parsingState.sources = (Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url }));
|
|
185
|
-
return await this.parse(parsingState, proxyUrl);
|
|
298
|
+
return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl);
|
|
186
299
|
}
|
|
187
300
|
/**
|
|
188
301
|
* Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
|
|
@@ -190,72 +303,19 @@ class Sitemap {
|
|
|
190
303
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
191
304
|
*/
|
|
192
305
|
static async fromXmlString(content, proxyUrl) {
|
|
193
|
-
|
|
194
|
-
parsingState.sources = [{ type: 'raw', content }];
|
|
195
|
-
return await this.parse(parsingState, proxyUrl);
|
|
196
|
-
}
|
|
197
|
-
static async parse(parsingState, proxyUrl) {
|
|
198
|
-
const { gotScraping } = await import('got-scraping');
|
|
199
|
-
const { fileTypeStream } = await import('file-type');
|
|
200
|
-
while (parsingState.sources.length > 0) {
|
|
201
|
-
const source = parsingState.sources.pop();
|
|
202
|
-
parsingState.resetContext();
|
|
203
|
-
if (source.type === 'url') {
|
|
204
|
-
const sitemapUrl = new URL(source.url);
|
|
205
|
-
parsingState.visitedSitemapUrls.push(sitemapUrl.toString());
|
|
206
|
-
try {
|
|
207
|
-
const sitemapStream = await new Promise((resolve, reject) => {
|
|
208
|
-
const request = gotScraping.stream({ url: sitemapUrl, proxyUrl, method: 'GET' });
|
|
209
|
-
request.on('response', () => resolve(request));
|
|
210
|
-
request.on('error', reject);
|
|
211
|
-
});
|
|
212
|
-
if (sitemapStream.response.statusCode === 200) {
|
|
213
|
-
let contentType = sitemapStream.response.headers['content-type'];
|
|
214
|
-
const streamWithType = await fileTypeStream(sitemapStream);
|
|
215
|
-
if (streamWithType.fileType !== undefined) {
|
|
216
|
-
contentType = streamWithType.fileType.mime;
|
|
217
|
-
}
|
|
218
|
-
await new Promise((resolve, reject) => {
|
|
219
|
-
let stream = streamWithType;
|
|
220
|
-
if (contentType !== undefined
|
|
221
|
-
? contentType === 'application/gzip'
|
|
222
|
-
: sitemapUrl.pathname.endsWith('.gz')) {
|
|
223
|
-
stream = stream.pipe((0, node_zlib_1.createGunzip)()).on('error', reject);
|
|
224
|
-
if (sitemapUrl.pathname.endsWith('.gz')) {
|
|
225
|
-
sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
stream.pipe(this.createParser(resolve, reject, parsingState, contentType, sitemapUrl));
|
|
229
|
-
});
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
catch (e) {
|
|
233
|
-
log_1.default.warning(`Malformed sitemap content: ${sitemapUrl}, ${e}`);
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
if (source.type === 'raw') {
|
|
237
|
-
await new Promise((resolve, reject) => {
|
|
238
|
-
node_stream_1.Readable.from([source.content]).pipe(this.createParser(resolve, reject, parsingState, 'text/xml'));
|
|
239
|
-
});
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
return new Sitemap(parsingState.urls);
|
|
306
|
+
return await this.parse([{ type: 'raw', content }], proxyUrl);
|
|
243
307
|
}
|
|
244
|
-
static
|
|
245
|
-
|
|
308
|
+
static async parse(sources, proxyUrl) {
|
|
309
|
+
const urls = [];
|
|
246
310
|
try {
|
|
247
|
-
|
|
311
|
+
for await (const item of parseSitemap(sources, proxyUrl)) {
|
|
312
|
+
urls.push(item.loc);
|
|
313
|
+
}
|
|
248
314
|
}
|
|
249
315
|
catch (e) {
|
|
250
|
-
|
|
251
|
-
}
|
|
252
|
-
if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
|
|
253
|
-
return Sitemap.createXmlParser(parsingState, () => resolve(undefined), reject);
|
|
254
|
-
}
|
|
255
|
-
if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
|
|
256
|
-
return new SitemapTxtParser(parsingState, () => resolve(undefined));
|
|
316
|
+
return new Sitemap([]);
|
|
257
317
|
}
|
|
258
|
-
|
|
318
|
+
return new Sitemap(urls);
|
|
259
319
|
}
|
|
260
320
|
}
|
|
261
321
|
exports.Sitemap = Sitemap;
|
package/internals/sitemap.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":";;;AAmLA,oCA0HC;;AA7SD,6CAAyC;AAEzC,6CAAyE;AACzE,6DAAoD;AACpD,yCAAyC;AAEzC,6DAA6B;AAC7B,sDAAsB;AACtB,8EAAuC;AAqBvC,MAAM,gBAAiB,SAAQ,uBAAS;IAIpC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBACrD,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAdC;;;;mBAAyB,IAAI,mCAAa,CAAC,MAAM,CAAC;WAAC;QACnD;;;;mBAAiB,EAAE;WAAC;IAc5B,CAAC;IAEO,aAAa,CAAC,KAAa,EAAE,QAAiB;QAClD,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAErB,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM;iBACpB,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEvC,IAAI,QAAQ,EAAE,CAAC;gBACX,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;YAChC,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAED,MAAM,gBAAiB,SAAQ,uBAAS;IAQpC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAClB,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAvBC;;;;mBAAyB,IAAI,mCAAa,CAAC,MAAM,CAAC;WAAC;QACnD;;;;mBAAS,IAAI,aAAG,CAAC,SAAS,CAAC,IAAI,CAAC;WAAC;QAEjC;;;;;WAAwC;QACxC;;;;mBAA6D,SAAS;WAAC;QACvE;;;;mBAA2B,EAAE;WAAC;QAoBlC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAgC;QAC9C,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACjC,IACI,IAAI,CAAC,IAAI,KAAK,KAAK;gBACnB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,IAAI,CAAC,IAAI,KAAK,YAAY,EAC5B,CAAC;gBACC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;YAChC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC,WAAW,GAAG,cAAc,CAAC;QACtC,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACvF,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAwB,CAAC,CAAC;YACjF,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,IAAY;QACvB,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,EAAwB,CAAC,CAAC;YACvE,CAAC;YAED,IAAI,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,KAAR,IAAI,CAAC,GAAG,GAAK,EAAE,EAAC;gBAChB,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC;YACxB,CAAC;QACL,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvF,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAgC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAaM,KAAK,SAAS,CAAC,CAAC,YAAY,CAC/B,cAA+B,EAC/B,QAAiB,EACjB,OAAW;IAEX,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IAErD,MAAM,OAAO,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC;IACpC,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,MAAM,YAAY,GAAG,CAAC,cAAsB,EAAE,EAAE,GAAS,EAAU,EAAE;QACjE,IAAI,QAAyB,CAAC;QAE9B,IAAI,CAAC;YACD,QAAQ,GAAG,IAAI,yBAAQ,CAAC,WAAW,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,IAAI,QAAQ,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,QAAQ,EAAE,OAAO,KAAK,YAAY,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,WAAW,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IACjH,CAAC,CAAC;IAEF,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,EAAG,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;YACzD,aAAG,CAAC,KAAK,CACL,oBAAoB,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,iCAAiC,OAAQ,CAAC,QAAS,GAAG,CACpH,CAAC;YACF,SAAS;QACb,CAAC;QAED,IAAI,KAAK,GAAsC,IAAI,CAAC;QAEpD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE9C,IAAI,CAAC;gBACD,MAAM,aAAa,GAAG,MAAM,IAAI,OAAO,CAAwC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBAC/F,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;oBACjF,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC/C,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAChC,CAAC,CAAC,CAAC;gBAEH,IAAI,aAAa,CAAC,QAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC7C,IAAI,WAAW,GAAG,aAAa,CAAC,QAAS,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;oBAElE,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,CAAC,CAAC;oBAC3D,IAAI,cAAc,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;wBACxC,WAAW,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAC/C,CAAC;oBAED,IAAI,SAAS,GAAG,KAAK,CAAC;oBAEtB,IACI,WAAW,KAAK,SAAS;wBACrB,CAAC,CAAC,WAAW,KAAK,kBAAkB;wBACpC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAC3C,CAAC;wBACC,SAAS,GAAG,IAAI,CAAC;wBAEjB,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;4BACtC,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBAC3F,CAAC;oBACL,CAAC;oBAED,KAAK,GAAG,IAAA,sBAAQ,EACZ,cAAc,EACd,SAAS,CAAC,CAAC,CAAC,IAAA,wBAAY,GAAE,CAAC,CAAC,CAAC,IAAI,yBAAW,EAAE,EAC9C,YAAY,CAAC,WAAW,EAAE,UAAU,CAAC,EACrC,CAAC,KAAK,EAAE,EAAE;wBACN,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;4BACtB,aAAG,CAAC,OAAO,CAAC,8BAA8B,UAAU,KAAK,KAAK,EAAE,CAAC,CAAC;wBACtE,CAAC;oBACL,CAAC,CACJ,CAAC;gBACN,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,aAAG,CAAC,OAAO,CAAC,8BAA8B,UAAU,KAAK,CAAC,EAAE,CAAC,CAAC;YAClE,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC/B,KAAK,GAAG,IAAA,sBAAQ,EAAC,sBAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACtB,aAAG,CAAC,OAAO,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;gBACvD,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7E,IAAI,OAAO,EAAE,kBAAkB,EAAE,CAAC;oBAC9B,aAAa;oBACb,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;gBACpD,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACtB,MAAM;oBACF,GAAG,IAAI;oBACP,gBAAgB,EACZ,MAAM,CAAC,IAAI,KAAK,KAAK;wBACjB,CAAC,CAAC,MAAM,CAAC,GAAG;wBACZ,CAAC,CAAC,SAAS,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;iBACpF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAa,OAAO;IAChB,YAAqB,IAAc;QAAvB;;;;mBAAS,IAAI;WAAU;IAAG,CAAC;IAEvC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,GAAW,EAAE,QAAiB;QACtD,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC;QAEvB,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAuB,EAAE,QAAiB;QACxD,OAAO,MAAM,IAAI,CAAC,KAAK,CACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAC1E,QAAQ,CACX,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,QAAiB;QACzD,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAwB,EAAE,QAAiB;QACpE,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACvD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACJ;AA1DD,0BA0DC"}
|
package/internals/social.js
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DISCORD_REGEX_GLOBAL = exports.DISCORD_REGEX = exports.PINTEREST_REGEX_GLOBAL = exports.PINTEREST_REGEX = exports.TIKTOK_REGEX_GLOBAL = exports.TIKTOK_REGEX = exports.YOUTUBE_REGEX_GLOBAL = exports.YOUTUBE_REGEX = exports.FACEBOOK_REGEX_GLOBAL = exports.FACEBOOK_REGEX = exports.TWITTER_REGEX_GLOBAL = exports.TWITTER_REGEX = exports.INSTAGRAM_REGEX_GLOBAL = exports.INSTAGRAM_REGEX = exports.LINKEDIN_REGEX_GLOBAL = exports.LINKEDIN_REGEX = exports.
|
|
3
|
+
exports.DISCORD_REGEX_GLOBAL = exports.DISCORD_REGEX = exports.PINTEREST_REGEX_GLOBAL = exports.PINTEREST_REGEX = exports.TIKTOK_REGEX_GLOBAL = exports.TIKTOK_REGEX = exports.YOUTUBE_REGEX_GLOBAL = exports.YOUTUBE_REGEX = exports.FACEBOOK_REGEX_GLOBAL = exports.FACEBOOK_REGEX = exports.TWITTER_REGEX_GLOBAL = exports.TWITTER_REGEX = exports.INSTAGRAM_REGEX_GLOBAL = exports.INSTAGRAM_REGEX = exports.LINKEDIN_REGEX_GLOBAL = exports.LINKEDIN_REGEX = exports.EMAIL_REGEX_GLOBAL = exports.EMAIL_REGEX = void 0;
|
|
4
|
+
exports.emailsFromText = emailsFromText;
|
|
5
|
+
exports.emailsFromUrls = emailsFromUrls;
|
|
6
|
+
exports.phonesFromText = phonesFromText;
|
|
7
|
+
exports.phonesFromUrls = phonesFromUrls;
|
|
8
|
+
exports.parseHandlesFromHtml = parseHandlesFromHtml;
|
|
4
9
|
const tslib_1 = require("tslib");
|
|
5
10
|
const cheerio_1 = tslib_1.__importDefault(require("cheerio"));
|
|
6
11
|
const cheerio_2 = require("./cheerio");
|
|
@@ -29,7 +34,6 @@ function emailsFromText(text) {
|
|
|
29
34
|
return [];
|
|
30
35
|
return text.match(exports.EMAIL_REGEX_GLOBAL) || [];
|
|
31
36
|
}
|
|
32
|
-
exports.emailsFromText = emailsFromText;
|
|
33
37
|
/**
|
|
34
38
|
* The function extracts email addresses from a list of URLs.
|
|
35
39
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
@@ -53,7 +57,6 @@ function emailsFromUrls(urls) {
|
|
|
53
57
|
}
|
|
54
58
|
return emails;
|
|
55
59
|
}
|
|
56
|
-
exports.emailsFromUrls = emailsFromUrls;
|
|
57
60
|
// Supports URLs starting with `tel://`, `tel:/` and `tel:`, and similarly `phone`, `telephone` and `callto`
|
|
58
61
|
const PHONE_URL_PREFIX_REGEX = /^(tel|phone|telephone|callto):(\/)?(\/)?/i;
|
|
59
62
|
// It's pretty much impossible (and unmaintainable) to have just one large regular expression for all possible phone numbers.
|
|
@@ -126,7 +129,6 @@ function phonesFromText(text) {
|
|
|
126
129
|
});
|
|
127
130
|
return phones;
|
|
128
131
|
}
|
|
129
|
-
exports.phonesFromText = phonesFromText;
|
|
130
132
|
/**
|
|
131
133
|
* Finds phone number links in an array of URLs and extracts the phone numbers from them.
|
|
132
134
|
* Note that the phone number links look like `tel://123456789`, `tel:/123456789` or `tel:123456789`.
|
|
@@ -149,7 +151,6 @@ function phonesFromUrls(urls) {
|
|
|
149
151
|
}
|
|
150
152
|
return phones;
|
|
151
153
|
}
|
|
152
|
-
exports.phonesFromUrls = phonesFromUrls;
|
|
153
154
|
// NOTEs about the regular expressions
|
|
154
155
|
// - They have just a single matching group for the profile username, all other groups are non-matching
|
|
155
156
|
// - They use a negative lookbehind and lookahead assertions, which are only supported in Node 8+.
|
|
@@ -244,7 +245,6 @@ function parseHandlesFromHtml(html, data = null) {
|
|
|
244
245
|
}
|
|
245
246
|
return result;
|
|
246
247
|
}
|
|
247
|
-
exports.parseHandlesFromHtml = parseHandlesFromHtml;
|
|
248
248
|
/**
|
|
249
249
|
* Regular expression to exactly match a single LinkedIn profile URL.
|
|
250
250
|
* It has the following form: `/^...$/i` and matches URLs such as:
|
package/internals/social.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"social.js","sourceRoot":"","sources":["../../src/internals/social.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"social.js","sourceRoot":"","sources":["../../src/internals/social.ts"],"names":[],"mappings":";;;AA6BA,wCAGC;AAUD,wCAaC;AAoED,wCAiBC;AASD,wCAYC;AAkFD,oDAqDC;;AAxSD,8DAA8B;AAE9B,uCAAuC;AAEvC,6EAA6E;AAC7E,MAAM,kBAAkB,GACpB,ubAAub,CAAC;AAE5b;;;GAGG;AACU,QAAA,WAAW,GAAG,IAAI,MAAM,CAAC,IAAI,kBAAkB,GAAG,EAAE,GAAG,CAAC,CAAC;AAEtE;;;GAGG;AACU,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;AAEvE,MAAM,sBAAsB,GAAG,WAAW,CAAC;AAE3C;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,IAAY;IACvC,IAAK,OAAO,IAAgB,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACrD,OAAO,IAAI,CAAC,KAAK,CAAC,0BAAkB,CAAC,IAAI,EAAE,CAAC;AAChD,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,cAAc,CAAC,IAAc;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACnF,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAEhD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,IAAI,mBAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,4GAA4G;AAC5G,MAAM,sBAAsB,GAAG,2CAA2C,CAAC;AAE3E,6HAA6H;AAC7H,iHAAiH;AACjH,mEAAmE;AACnE,qHAAqH;AACrH,MAAM,oBAAoB,GAAG;IACzB,YAAY;IACZ,aAAa;IAEb,6FAA6F;IAC7F,2EAA2E;IAE3E,mCAAmC;IACnC,uCAAuC;IAEvC,sCAAsC;IACtC,2EAA2E;IAE3E,mBAAmB;IACnB,6CAA6C;IAC7C,eAAe;IACf,kCAAkC;IAClC,UAAU;IACV,uBAAuB;IAEvB,mBAAmB;IACnB,mDAAmD;IACnD,eAAe;IACf,sCAAsC;IACtC,UAAU;IACV,yBAAyB;IAEzB,mBAAmB;IACnB,6CAA6C;IAC7C,eAAe;IACf,kCAAkC;IAClC,WAAW;IACX,uBAAuB;IACvB,gDAAgD;CACnD,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,KAAK,EAAE,CAAC,CAAC;AAEtC,2DAA2D;AAC3D,uGAAuG;AACvG,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAE3B,oEAAoE;AACpE,4EAA4E;AAC5E,MAAM,iBAAiB,GAAG;IACtB,aAAa;IACb,8BAA8B;CACjC,CAAC;AAEF,MAAM,kBAAkB,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACnF,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,KAAK,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAC7E,MAAM,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAE/E;;;;;;;GAOG;AACH,SAAgB,cAAc,CAAC,IAAY;IACvC,IAAK,OAAO,IAAgB,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,MAAM,GAAI,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAc,IAAI,EAAE,CAAC;IAChE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAEzB,wDAAwD;QACxD,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,MAAM,GAAG,gBAAgB;YAAE,OAAO,KAAK,CAAC;QAEnE,gDAAgD;QAChD,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/C,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,IAAc;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAEnF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAEhD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,sCAAsC;AACtC,uGAAuG;AACvG,kGAAkG;AAClG,wFAAwF;AAExF,MAAM,qBAAqB,GACvB,4IAA4I,CAAC;AAEjJ,MAAM,sBAAsB,GACxB,yIAAyI,CAAC;AAE9I,MAAM,sBAAsB,GACxB,sGAAsG,CAAC;AAE3G,MAAM,oBAAoB,GAAG,kEAAkE,sBAAsB,+DAA+D,CAAC;AAErL,MAAM,uBAAuB,GACzB,kWAAkW,CAAC;AAEvW,MAAM,qBAAqB,GAAG,0EAA0E,uBAAuB,qHAAqH,CAAC;AAErP,MAAM,oBAAoB,GACtB,2KAA2K,CAAC;AAEhL,MAAM,mBAAmB,GACrB,2MAA2M,CAAC;AAEhN,MAAM,sBAAsB,GACxB,kNAAkN,CAAC;AAEvN,MAAM,oBAAoB,GACtB,6QAA6Q,CAAC;AAmBlR;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,SAAgB,oBAAoB,CAAC,IAAY,EAAE,OAAuC,IAAI;IAC1F,MAAM,MAAM,GAAkB;QAC1B,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,EAAE;QACV,eAAe,EAAE,EAAE;QACnB,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,EAAE;QACZ,UAAU,EAAE,EAAE;QACd,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,EAAE;QACZ,OAAO,EAAE,EAAE;QACX,UAAU,EAAE,EAAE;QACd,QAAQ,EAAE,EAAE;KACf,CAAC;IAEF,IAAK,OAAO,IAAgB,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAEzD,MAAM,CAAC,GAAG,iBAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,IAAI,IAAI;QAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAErB,MAAM,IAAI,GAAG,IAAA,oBAAU,EAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,IAAI;QAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IAE3B,mCAAmC;IACnC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE;QAC/B,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,CAAC,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAE9C,8EAA8E;IAC9E,2EAA2E;IAC3E,mBAAmB;IACnB,mDAAmD;IACnD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,6BAAqB,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,4BAAoB,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,8BAAsB,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,6BAAqB,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,4BAAoB,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,2BAAmB,CAAC,IAAI,EAAE,CAAC;IACvD,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,8BAAsB,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,4BAAoB,CAAC,IAAI,EAAE,CAAC;IAEzD,+BAA+B;IAC/B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAA4B,EAAE,CAAC;QAC/D,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACU,QAAA,cAAc,GAAG,IAAI,MAAM,CAAC,IAAI,qBAAqB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE5E;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACU,QAAA,qBAAqB,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;AAE7E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACU,QAAA,eAAe,GAAG,IAAI,MAAM,CAAC,IAAI,sBAAsB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACU,QAAA,sBAAsB,GAAG,IAAI,MAAM,CAAC,sBAAsB,EAAE,IAAI,CAAC,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACU,QAAA,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACU,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;AAE3E;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACU,QAAA,cAAc,GAAG,IAAI,MAAM,CAAC,IAAI,qBAAqB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE5E;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACU,QAAA,qBAAqB,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;AAE7E;;;;;;;;;;;;;;;;;;;;;GAqBG;AACU,QAAA,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;GAoBG;AACU,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;AAC3E;;;;;;;;;;;;;;;;;;;;GAoBG;AACU,QAAA,YAAY,GAAG,IAAI,MAAM,CAAC,IAAI,mBAAmB,GAAG,EAAE,GAAG,CAAC,CAAC;AAExE;;;;;;;;;;;;;;;;;;;GAmBG;AACU,QAAA,mBAAmB,GAAG,IAAI,MAAM,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;AAEzE;;;;;;;;;;;;;;;;;;;;;GAqBG;AACU,QAAA,eAAe,GAAG,IAAI,MAAM,CAAC,IAAI,sBAAsB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;;;GAoBG;AACU,QAAA,sBAAsB,GAAG,IAAI,MAAM,CAAC,sBAAsB,EAAE,IAAI,CAAC,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;GAmBG;AACU,QAAA,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;;;;;;;;;;;;;;;GAkBG;AACU,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../../src/internals/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../../src/internals/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACxD;AAED,cAAc;AACd,wBAAgB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,CAAC,EAAE,CACzC"}
|
package/internals/typedefs.js
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.entries = entries;
|
|
4
|
+
exports.keys = keys;
|
|
4
5
|
/** @ignore */
|
|
5
6
|
function entries(obj) {
|
|
6
7
|
return Object.entries(obj);
|
|
7
8
|
}
|
|
8
|
-
exports.entries = entries;
|
|
9
9
|
/** @ignore */
|
|
10
10
|
function keys(obj) {
|
|
11
11
|
return Object.keys(obj);
|
|
12
12
|
}
|
|
13
|
-
exports.keys = keys;
|
|
14
13
|
//# sourceMappingURL=typedefs.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"typedefs.js","sourceRoot":"","sources":["../../src/internals/typedefs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"typedefs.js","sourceRoot":"","sources":["../../src/internals/typedefs.ts"],"names":[],"mappings":";;AACA,0BAEC;AAGD,oBAEC;AARD,cAAc;AACd,SAAgB,OAAO,CAAe,GAAM;IACxC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAA4B,CAAC;AAC1D,CAAC;AAED,cAAc;AACd,SAAgB,IAAI,CAAe,GAAM;IACrC,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAgB,CAAC;AAC3C,CAAC"}
|