@crawlee/utils 3.11.2-beta.9 → 3.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/cheerio.d.ts +1 -1
- package/internals/cheerio.js +3 -3
- package/internals/cheerio.js.map +1 -1
- package/internals/sitemap.d.ts +14 -4
- package/internals/sitemap.d.ts.map +1 -1
- package/internals/sitemap.js +54 -34
- package/internals/sitemap.js.map +1 -1
- package/internals/social.js +4 -4
- package/internals/social.js.map +1 -1
- package/package.json +4 -4
- package/tsconfig.build.tsbuildinfo +1 -1
package/internals/cheerio.d.ts
CHANGED
|
@@ -21,7 +21,7 @@ export type CheerioRoot = ReturnType<typeof load>;
|
|
|
21
21
|
* with the `decodeEntities` option set to `true`. For example:
|
|
22
22
|
*
|
|
23
23
|
* ```javascript
|
|
24
|
-
* import cheerio from 'cheerio';
|
|
24
|
+
* import * as cheerio from 'cheerio';
|
|
25
25
|
* const html = '<html><body>Some text</body></html>';
|
|
26
26
|
* const text = htmlToText(cheerio.load(html, { decodeEntities: true }));
|
|
27
27
|
* ```
|
package/internals/cheerio.js
CHANGED
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.htmlToText = htmlToText;
|
|
4
4
|
exports.extractUrlsFromCheerio = extractUrlsFromCheerio;
|
|
5
5
|
const tslib_1 = require("tslib");
|
|
6
|
-
const
|
|
6
|
+
const cheerio = tslib_1.__importStar(require("cheerio"));
|
|
7
7
|
const extract_urls_1 = require("./extract-urls");
|
|
8
8
|
// NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.
|
|
9
9
|
const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;
|
|
@@ -29,7 +29,7 @@ const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|
|
|
|
29
29
|
* with the `decodeEntities` option set to `true`. For example:
|
|
30
30
|
*
|
|
31
31
|
* ```javascript
|
|
32
|
-
* import cheerio from 'cheerio';
|
|
32
|
+
* import * as cheerio from 'cheerio';
|
|
33
33
|
* const html = '<html><body>Some text</body></html>';
|
|
34
34
|
* const text = htmlToText(cheerio.load(html, { decodeEntities: true }));
|
|
35
35
|
* ```
|
|
@@ -41,7 +41,7 @@ function htmlToText(htmlOrCheerioElement) {
|
|
|
41
41
|
return '';
|
|
42
42
|
const $ = typeof htmlOrCheerioElement === 'function'
|
|
43
43
|
? htmlOrCheerioElement
|
|
44
|
-
:
|
|
44
|
+
: cheerio.load(htmlOrCheerioElement, { decodeEntities: true });
|
|
45
45
|
let text = '';
|
|
46
46
|
const process = (elems) => {
|
|
47
47
|
const len = elems ? elems.length : 0;
|
package/internals/cheerio.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":";;AAyCA,gCA2CC;AAWD,wDAwBC;;AArHD,
|
|
1
|
+
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":";;AAyCA,gCA2CC;AAWD,wDAwBC;;AArHD,yDAAmC;AAEnC,iDAAgD;AAIhD,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAClB,sGAAsG,CAAC;AAE3G;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,SAAgB,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,CAAC,GACH,OAAO,oBAAoB,KAAK,UAAU;QACtC,CAAC,CAAC,oBAAoB;QACtB,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IACvE,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9E,IAAI,IAAI,KAAK,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,qCAAqC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;YACzD,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,sBAAsB,CAAC,CAAa,EAAE,WAAmB,GAAG,EAAE,UAAkB,EAAE;IAC9F,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,IAAI,IAAA,6BAAc,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,uCAAuC;gBAC5D,2DAA2D,CAClE,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,IAAA,6BAAc,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAa,CAAC;AACrC,CAAC"}
|
package/internals/sitemap.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
|
+
import type { Delays } from 'got-scraping';
|
|
1
3
|
interface SitemapUrlData {
|
|
2
4
|
loc: string;
|
|
3
5
|
lastmod?: Date;
|
|
@@ -20,15 +22,23 @@ type SitemapSource = ({
|
|
|
20
22
|
}) & {
|
|
21
23
|
depth?: number;
|
|
22
24
|
};
|
|
23
|
-
interface ParseSitemapOptions {
|
|
25
|
+
export interface ParseSitemapOptions {
|
|
24
26
|
/**
|
|
25
|
-
* If set to `true`, elements referring to other sitemaps will be emitted as special objects with
|
|
27
|
+
* If set to `true`, elements referring to other sitemaps will be emitted as special objects with `originSitemapUrl` set to `null`.
|
|
26
28
|
*/
|
|
27
29
|
emitNestedSitemaps?: true | false;
|
|
28
30
|
/**
|
|
29
31
|
* Maximum depth of nested sitemaps to follow.
|
|
30
32
|
*/
|
|
31
33
|
maxDepth?: number;
|
|
34
|
+
/**
|
|
35
|
+
* Number of retries for fetching sitemaps. The counter resets for each nested sitemap.
|
|
36
|
+
*/
|
|
37
|
+
sitemapRetries?: number;
|
|
38
|
+
/**
|
|
39
|
+
* Network timeouts for sitemap fetching. See [Got documentation](https://github.com/sindresorhus/got/blob/main/documentation/6-timeout.md) for more details.
|
|
40
|
+
*/
|
|
41
|
+
networkTimeouts?: Delays;
|
|
32
42
|
}
|
|
33
43
|
export declare function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
|
|
34
44
|
/**
|
|
@@ -58,14 +68,14 @@ export declare class Sitemap {
|
|
|
58
68
|
* @param urls sitemap URL(s)
|
|
59
69
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
60
70
|
*/
|
|
61
|
-
static load(urls: string | string[], proxyUrl?: string): Promise<Sitemap>;
|
|
71
|
+
static load(urls: string | string[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
|
62
72
|
/**
|
|
63
73
|
* Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
|
|
64
74
|
* @param content XML sitemap content
|
|
65
75
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
66
76
|
*/
|
|
67
77
|
static fromXmlString(content: string, proxyUrl?: string): Promise<Sitemap>;
|
|
68
|
-
protected static parse(sources: SitemapSource[], proxyUrl?: string): Promise<Sitemap>;
|
|
78
|
+
protected static parse(sources: SitemapSource[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
|
69
79
|
}
|
|
70
80
|
export {};
|
|
71
81
|
//# sourceMappingURL=sitemap.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAI3C,UAAU,cAAc;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,IAAI,CAAC;IACf,UAAU,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG;IACtC,gBAAgB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,UAAU,aAAa;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,IAAI,CAAC;CAC1B;AAED,KAAK,aAAa,GAAG,CAAC;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA8I5G,MAAM,WAAW,mBAAmB;IAChC;;OAEG;IACH,kBAAkB,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC;IAClC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,wBAAuB,YAAY,CAAC,CAAC,SAAS,mBAAmB,EAC7D,cAAc,EAAE,aAAa,EAAE,EAC/B,QAAQ,CAAC,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,CAAC,GACZ,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,SAAS,IAAI,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC,CAiJ/F;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,OAAO;IACJ,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE;gBAAd,IAAI,EAAE,MAAM,EAAE;IAEnC;;;;;OAKG;WACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAe7E;;;;OAIG;WACU,IAAI,CACb,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;IAQnB;;;;OAIG;WACU,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;qBAIzD,KAAK,CACxB,OAAO,EAAE,aAAa,EAAE,EACxB,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;CAatB"}
|
package/internals/sitemap.js
CHANGED
|
@@ -163,6 +163,7 @@ class SitemapXmlParser extends node_stream_1.Transform {
|
|
|
163
163
|
async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
164
164
|
const { gotScraping } = await import('got-scraping');
|
|
165
165
|
const { fileTypeStream } = await import('file-type');
|
|
166
|
+
const { emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, networkTimeouts } = options ?? {};
|
|
166
167
|
const sources = [...initialSources];
|
|
167
168
|
const visitedSitemapUrls = new Set();
|
|
168
169
|
const createParser = (contentType = '', url) => {
|
|
@@ -183,44 +184,63 @@ async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
183
184
|
};
|
|
184
185
|
while (sources.length > 0) {
|
|
185
186
|
const source = sources.shift();
|
|
186
|
-
if ((source?.depth ?? 0) >
|
|
187
|
-
log_1.default.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${
|
|
187
|
+
if ((source?.depth ?? 0) > maxDepth) {
|
|
188
|
+
log_1.default.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
|
|
188
189
|
continue;
|
|
189
190
|
}
|
|
190
191
|
let items = null;
|
|
191
192
|
if (source.type === 'url') {
|
|
192
193
|
const sitemapUrl = new URL(source.url);
|
|
193
194
|
visitedSitemapUrls.add(sitemapUrl.toString());
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
195
|
+
let retriesLeft = sitemapRetries + 1;
|
|
196
|
+
while (retriesLeft-- > 0) {
|
|
197
|
+
try {
|
|
198
|
+
const sitemapStream = await new Promise((resolve, reject) => {
|
|
199
|
+
const request = gotScraping.stream({
|
|
200
|
+
url: sitemapUrl,
|
|
201
|
+
proxyUrl,
|
|
202
|
+
method: 'GET',
|
|
203
|
+
timeout: networkTimeouts,
|
|
204
|
+
headers: {
|
|
205
|
+
'accept': 'text/plain, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8',
|
|
206
|
+
},
|
|
207
|
+
});
|
|
208
|
+
request.on('response', () => resolve(request));
|
|
209
|
+
request.on('error', reject);
|
|
210
|
+
});
|
|
211
|
+
let error = null;
|
|
212
|
+
if (sitemapStream.response.statusCode >= 200 && sitemapStream.response.statusCode < 300) {
|
|
213
|
+
let contentType = sitemapStream.response.headers['content-type'];
|
|
214
|
+
const streamWithType = await fileTypeStream(sitemapStream);
|
|
215
|
+
if (streamWithType.fileType !== undefined) {
|
|
216
|
+
contentType = streamWithType.fileType.mime;
|
|
213
217
|
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
+
let isGzipped = false;
|
|
219
|
+
if (contentType !== undefined
|
|
220
|
+
? contentType === 'application/gzip'
|
|
221
|
+
: sitemapUrl.pathname.endsWith('.gz')) {
|
|
222
|
+
isGzipped = true;
|
|
223
|
+
if (sitemapUrl.pathname.endsWith('.gz')) {
|
|
224
|
+
sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
|
|
225
|
+
}
|
|
218
226
|
}
|
|
219
|
-
|
|
227
|
+
items = (0, node_stream_1.pipeline)(streamWithType, isGzipped ? (0, node_zlib_1.createGunzip)() : new node_stream_1.PassThrough(), createParser(contentType, sitemapUrl), (e) => {
|
|
228
|
+
if (e !== undefined) {
|
|
229
|
+
error = e;
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
error = new Error(`Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapStream.response.statusCode}`);
|
|
235
|
+
}
|
|
236
|
+
if (error !== null) {
|
|
237
|
+
throw error;
|
|
238
|
+
}
|
|
239
|
+
break;
|
|
240
|
+
}
|
|
241
|
+
catch (e) {
|
|
242
|
+
log_1.default.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
|
|
220
243
|
}
|
|
221
|
-
}
|
|
222
|
-
catch (e) {
|
|
223
|
-
log_1.default.warning(`Malformed sitemap content: ${sitemapUrl}, ${e}`);
|
|
224
244
|
}
|
|
225
245
|
}
|
|
226
246
|
else if (source.type === 'raw') {
|
|
@@ -236,7 +256,7 @@ async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
236
256
|
for await (const item of items) {
|
|
237
257
|
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
|
|
238
258
|
sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
|
|
239
|
-
if (
|
|
259
|
+
if (emitNestedSitemaps) {
|
|
240
260
|
// @ts-ignore
|
|
241
261
|
yield { loc: item.url, originSitemapUrl: null };
|
|
242
262
|
}
|
|
@@ -294,8 +314,8 @@ class Sitemap {
|
|
|
294
314
|
* @param urls sitemap URL(s)
|
|
295
315
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
296
316
|
*/
|
|
297
|
-
static async load(urls, proxyUrl) {
|
|
298
|
-
return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl);
|
|
317
|
+
static async load(urls, proxyUrl, parseSitemapOptions) {
|
|
318
|
+
return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
|
|
299
319
|
}
|
|
300
320
|
/**
|
|
301
321
|
* Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
|
|
@@ -305,10 +325,10 @@ class Sitemap {
|
|
|
305
325
|
static async fromXmlString(content, proxyUrl) {
|
|
306
326
|
return await this.parse([{ type: 'raw', content }], proxyUrl);
|
|
307
327
|
}
|
|
308
|
-
static async parse(sources, proxyUrl) {
|
|
328
|
+
static async parse(sources, proxyUrl, parseSitemapOptions) {
|
|
309
329
|
const urls = [];
|
|
310
330
|
try {
|
|
311
|
-
for await (const item of parseSitemap(sources, proxyUrl)) {
|
|
331
|
+
for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {
|
|
312
332
|
urls.push(item.loc);
|
|
313
333
|
}
|
|
314
334
|
}
|
package/internals/sitemap.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":";;;AAmLA,oCA0HC;;AA7SD,6CAAyC;AAEzC,6CAAyE;AACzE,6DAAoD;AACpD,yCAAyC;AAEzC,6DAA6B;AAC7B,sDAAsB;AACtB,8EAAuC;AAqBvC,MAAM,gBAAiB,SAAQ,uBAAS;IAIpC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBACrD,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAdC;;;;mBAAyB,IAAI,mCAAa,CAAC,MAAM,CAAC;WAAC;QACnD;;;;mBAAiB,EAAE;WAAC;IAc5B,CAAC;IAEO,aAAa,CAAC,KAAa,EAAE,QAAiB;QAClD,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAErB,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM;iBACpB,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEvC,IAAI,QAAQ,EAAE,CAAC;gBACX,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;YAChC,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAED,MAAM,gBAAiB,SAAQ,uBAAS;IAQpC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAClB,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAvBC;;;;mBAAyB,IAAI,mCAAa,CAAC,MAAM,CAAC;WAAC;QACnD;;;;mBAAS,IAAI,aAAG,CAAC,SAAS,CAAC,IAAI,CAAC;WAAC;QAEjC;;;;;WAAwC;QACxC;;;;mBAA6D,SAAS;WAAC;QACvE;;;;mBAA2B,EAAE;WAAC;QAoBlC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAgC;QAC9C,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACjC,IACI,IAAI,CAAC,IAAI,KAAK,KAAK;gBACnB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,IAAI,CAAC,IAAI,KAAK,YAAY,EAC5B,CAAC;gBACC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;YAChC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC,WAAW,GAAG,cAAc,CAAC;QACtC,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACvF,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAwB,CAAC,CAAC;YACjF,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,IAAY;QACvB,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,EAAwB,CAAC,CAAC;YACvE,CAAC;YAED,IAAI,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,KAAR,IAAI,CAAC,GAAG,GAAK,EAAE,EAAC;gBAChB,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC;YACxB,CAAC;QACL,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvF,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAgC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAaM,KAAK,SAAS,CAAC,CAAC,YAAY,CAC/B,cAA+B,EAC/B,QAAiB,EACjB,OAAW;IAEX,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IAErD,MAAM,OAAO,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC;IACpC,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,MAAM,YAAY,GAAG,CAAC,cAAsB,EAAE,EAAE,GAAS,EAAU,EAAE;QACjE,IAAI,QAAyB,CAAC;QAE9B,IAAI,CAAC;YACD,QAAQ,GAAG,IAAI,yBAAQ,CAAC,WAAW,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,IAAI,QAAQ,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,QAAQ,EAAE,OAAO,KAAK,YAAY,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,WAAW,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IACjH,CAAC,CAAC;IAEF,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,EAAG,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;YACzD,aAAG,CAAC,KAAK,CACL,oBAAoB,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,iCAAiC,OAAQ,CAAC,QAAS,GAAG,CACpH,CAAC;YACF,SAAS;QACb,CAAC;QAED,IAAI,KAAK,GAAsC,IAAI,CAAC;QAEpD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE9C,IAAI,CAAC;gBACD,MAAM,aAAa,GAAG,MAAM,IAAI,OAAO,CAAwC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBAC/F,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;oBACjF,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC/C,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAChC,CAAC,CAAC,CAAC;gBAEH,IAAI,aAAa,CAAC,QAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC7C,IAAI,WAAW,GAAG,aAAa,CAAC,QAAS,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;oBAElE,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,CAAC,CAAC;oBAC3D,IAAI,cAAc,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;wBACxC,WAAW,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAC/C,CAAC;oBAED,IAAI,SAAS,GAAG,KAAK,CAAC;oBAEtB,IACI,WAAW,KAAK,SAAS;wBACrB,CAAC,CAAC,WAAW,KAAK,kBAAkB;wBACpC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAC3C,CAAC;wBACC,SAAS,GAAG,IAAI,CAAC;wBAEjB,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;4BACtC,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBAC3F,CAAC;oBACL,CAAC;oBAED,KAAK,GAAG,IAAA,sBAAQ,EACZ,cAAc,EACd,SAAS,CAAC,CAAC,CAAC,IAAA,wBAAY,GAAE,CAAC,CAAC,CAAC,IAAI,yBAAW,EAAE,EAC9C,YAAY,CAAC,WAAW,EAAE,UAAU,CAAC,EACrC,CAAC,KAAK,EAAE,EAAE;wBACN,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;4BACtB,aAAG,CAAC,OAAO,CAAC,8BAA8B,UAAU,KAAK,KAAK,EAAE,CAAC,CAAC;wBACtE,CAAC;oBACL,CAAC,CACJ,CAAC;gBACN,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,aAAG,CAAC,OAAO,CAAC,8BAA8B,UAAU,KAAK,CAAC,EAAE,CAAC,CAAC;YAClE,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC/B,KAAK,GAAG,IAAA,sBAAQ,EAAC,sBAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACtB,aAAG,CAAC,OAAO,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;gBACvD,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7E,IAAI,OAAO,EAAE,kBAAkB,EAAE,CAAC;oBAC9B,aAAa;oBACb,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;gBACpD,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACtB,MAAM;oBACF,GAAG,IAAI;oBACP,gBAAgB,EACZ,MAAM,CAAC,IAAI,KAAK,KAAK;wBACjB,CAAC,CAAC,MAAM,CAAC,GAAG;wBACZ,CAAC,CAAC,SAAS,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;iBACpF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAa,OAAO;IAChB,YAAqB,IAAc;QAAvB;;;;mBAAS,IAAI;WAAU;IAAG,CAAC;IAEvC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,GAAW,EAAE,QAAiB;QACtD,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC;QAEvB,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAuB,EAAE,QAAiB;QACxD,OAAO,MAAM,IAAI,CAAC,KAAK,CACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAC1E,QAAQ,CACX,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,QAAiB;QACzD,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAwB,EAAE,QAAiB;QACpE,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACvD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACJ;AA1DD,0BA0DC"}
|
|
1
|
+
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":";;;AA6LA,oCAqJC;;AAlVD,6CAAyC;AAEzC,6CAAyE;AACzE,6DAAoD;AACpD,yCAAyC;AAEzC,6DAA6B;AAG7B,sDAAsB;AACtB,8EAAuC;AAqBvC,MAAM,gBAAiB,SAAQ,uBAAS;IAIpC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBACrD,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAdC;;;;mBAAyB,IAAI,mCAAa,CAAC,MAAM,CAAC;WAAC;QACnD;;;;mBAAiB,EAAE;WAAC;IAc5B,CAAC;IAEO,aAAa,CAAC,KAAa,EAAE,QAAiB;QAClD,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAErB,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM;iBACpB,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEvC,IAAI,QAAQ,EAAE,CAAC;gBACX,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;YAChC,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAED,MAAM,gBAAiB,SAAQ,uBAAS;IAQpC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAClB,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAvBC;;;;mBAAyB,IAAI,mCAAa,CAAC,MAAM,CAAC;WAAC;QACnD;;;;mBAAS,IAAI,aAAG,CAAC,SAAS,CAAC,IAAI,CAAC;WAAC;QAEjC;;;;;WAAwC;QACxC;;;;mBAA6D,SAAS;WAAC;QACvE;;;;mBAA2B,EAAE;WAAC;QAoBlC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAgC;QAC9C,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACjC,IACI,IAAI,CAAC,IAAI,KAAK,KAAK;gBACnB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,IAAI,CAAC,IAAI,KAAK,YAAY,EAC5B,CAAC;gBACC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;YAChC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC,WAAW,GAAG,cAAc,CAAC;QACtC,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACvF,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAwB,CAAC,CAAC;YACjF,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,IAAY;QACvB,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,EAAwB,CAAC,CAAC;YACvE,CAAC;YAED,IAAI,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,KAAR,IAAI,CAAC,GAAG,GAAK,EAAE,EAAC;gBAChB,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC;YACxB,CAAC;QACL,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvF,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAgC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAqBM,KAAK,SAAS,CAAC,CAAC,YAAY,CAC/B,cAA+B,EAC/B,QAAiB,EACjB,OAAW;IAEX,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IACrD,MAAM,EAAE,kBAAkB,GAAG,KAAK,EAAE,QAAQ,GAAG,QAAQ,EAAE,cAAc,GAAG,CAAC,EAAE,eAAe,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;IAE/G,MAAM,OAAO,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC;IACpC,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,MAAM,YAAY,GAAG,CAAC,cAAsB,EAAE,EAAE,GAAS,EAAU,EAAE;QACjE,IAAI,QAAyB,CAAC;QAE9B,IAAI,CAAC;YACD,QAAQ,GAAG,IAAI,yBAAQ,CAAC,WAAW,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,IAAI,QAAQ,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,QAAQ,EAAE,OAAO,KAAK,YAAY,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,WAAW,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IACjH,CAAC,CAAC;IAEF,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,EAAG,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,QAAQ,EAAE,CAAC;YAClC,aAAG,CAAC,KAAK,CACL,oBAAoB,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,iCAAiC,QAAQ,GAAG,CAC1G,CAAC;YACF,SAAS;QACb,CAAC;QAED,IAAI,KAAK,GAAsC,IAAI,CAAC;QAEpD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9C,IAAI,WAAW,GAAG,cAAc,GAAG,CAAC,CAAC;YAErC,OAAO,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACD,MAAM,aAAa,GAAG,MAAM,IAAI,OAAO,CACnC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;wBAChB,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC;4BAC/B,GAAG,EAAE,UAAU;4BACf,QAAQ;4BACR,MAAM,EAAE,KAAK;4BACb,OAAO,EAAE,eAAe;4BACxB,OAAO,EAAE;gCACL,QAAQ,EAAE,qEAAqE;6BAClF;yBACJ,CAAC,CAAC;wBACH,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC/C,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;oBAChC,CAAC,CACJ,CAAC;oBAEF,IAAI,KAAK,GAAiB,IAAI,CAAC;oBAE/B,IAAI,aAAa,CAAC,QAAS,CAAC,UAAU,IAAI,GAAG,IAAI,aAAa,CAAC,QAAS,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;wBACxF,IAAI,WAAW,GAAG,aAAa,CAAC,QAAS,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;wBAElE,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,CAAC,CAAC;wBAC3D,IAAI,cAAc,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;4BACxC,WAAW,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC;wBAC/C,CAAC;wBAED,IAAI,SAAS,GAAG,KAAK,CAAC;wBAEtB,IACI,WAAW,KAAK,SAAS;4BACrB,CAAC,CAAC,WAAW,KAAK,kBAAkB;4BACpC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAC3C,CAAC;4BACC,SAAS,GAAG,IAAI,CAAC;4BAEjB,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gCACtC,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;4BAC3F,CAAC;wBACL,CAAC;wBAED,KAAK,GAAG,IAAA,sBAAQ,EACZ,cAAc,EACd,SAAS,CAAC,CAAC,CAAC,IAAA,wBAAY,GAAE,CAAC,CAAC,CAAC,IAAI,yBAAW,EAAE,EAC9C,YAAY,CAAC,WAAW,EAAE,UAAU,CAAC,EACrC,CAAC,CAAC,EAAE,EAAE;4BACF,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;gCAClB,KAAK,GAAG,CAAC,CAAC;4BACd,CAAC;wBACL,CAAC,CACJ,CAAC;oBACN,CAAC;yBAAM,CAAC;wBACJ,KAAK,GAAG,IAAI,KAAK,CACb,4BAA4B,UAAU,kBAAkB,aAAa,CAAC,QAAS,CAAC,UAAU,EAAE,CAC/F,CAAC;oBACN,CAAC;oBAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;wBACjB,MAAM,KAAK,CAAC;oBAChB,CAAC;oBACD,MAAM;gBACV,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,aAAG,CAAC,OAAO,CACP,8BAA8B,UAAU,KAAK,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,GAAG,CAC/G,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC/B,KAAK,GAAG,IAAA,sBAAQ,EAAC,sBAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACtB,aAAG,CAAC,OAAO,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;gBACvD,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7E,IAAI,kBAAkB,EAAE,CAAC;oBACrB,aAAa;oBACb,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;gBACpD,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACtB,MAAM;oBACF,GAAG,IAAI;oBACP,gBAAgB,EACZ,MAAM,CAAC,IAAI,KAAK,KAAK;wBACjB,CAAC,CAAC,MAAM,CAAC,GAAG;wBACZ,CAAC,CAAC,SAAS,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;iBACpF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAa,OAAO;IAChB,YAAqB,IAAc;QAAvB;;;;mBAAS,IAAI;WAAU;IAAG,CAAC;IAEvC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,GAAW,EAAE,QAAiB;QACtD,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC;QAEvB,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,IAAuB,EACvB,QAAiB,EACjB,mBAAyC;QAEzC,OAAO,MAAM,IAAI,CAAC,KAAK,CACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAC1E,QAAQ,EACR,mBAAmB,CACtB,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,QAAiB;QACzD,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,KAAK,CACxB,OAAwB,EACxB,QAAiB,EACjB,mBAAyC;QAEzC,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,mBAAmB,CAAC,EAAE,CAAC;gBAC5E,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACJ;AAnED,0BAmEC"}
|
package/internals/social.js
CHANGED
|
@@ -7,8 +7,8 @@ exports.phonesFromText = phonesFromText;
|
|
|
7
7
|
exports.phonesFromUrls = phonesFromUrls;
|
|
8
8
|
exports.parseHandlesFromHtml = parseHandlesFromHtml;
|
|
9
9
|
const tslib_1 = require("tslib");
|
|
10
|
-
const
|
|
11
|
-
const
|
|
10
|
+
const cheerio = tslib_1.__importStar(require("cheerio"));
|
|
11
|
+
const cheerio_1 = require("./cheerio");
|
|
12
12
|
// Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
|
|
13
13
|
const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
|
|
14
14
|
/**
|
|
@@ -211,10 +211,10 @@ function parseHandlesFromHtml(html, data = null) {
|
|
|
211
211
|
};
|
|
212
212
|
if (typeof html !== 'string')
|
|
213
213
|
return result;
|
|
214
|
-
const $ =
|
|
214
|
+
const $ = cheerio.load(html, { decodeEntities: true });
|
|
215
215
|
if (data)
|
|
216
216
|
data.$ = $;
|
|
217
|
-
const text = (0,
|
|
217
|
+
const text = (0, cheerio_1.htmlToText)($);
|
|
218
218
|
if (data)
|
|
219
219
|
data.text = text;
|
|
220
220
|
// Find all <a> links with href tag
|
package/internals/social.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"social.js","sourceRoot":"","sources":["../../src/internals/social.ts"],"names":[],"mappings":";;;AA6BA,wCAGC;AAUD,wCAaC;AAoED,wCAiBC;AASD,wCAYC;AAkFD,oDAqDC;;AAxSD,
|
|
1
|
+
{"version":3,"file":"social.js","sourceRoot":"","sources":["../../src/internals/social.ts"],"names":[],"mappings":";;;AA6BA,wCAGC;AAUD,wCAaC;AAoED,wCAiBC;AASD,wCAYC;AAkFD,oDAqDC;;AAxSD,yDAAmC;AAEnC,uCAAuC;AAEvC,6EAA6E;AAC7E,MAAM,kBAAkB,GACpB,ubAAub,CAAC;AAE5b;;;GAGG;AACU,QAAA,WAAW,GAAG,IAAI,MAAM,CAAC,IAAI,kBAAkB,GAAG,EAAE,GAAG,CAAC,CAAC;AAEtE;;;GAGG;AACU,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;AAEvE,MAAM,sBAAsB,GAAG,WAAW,CAAC;AAE3C;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,IAAY;IACvC,IAAK,OAAO,IAAgB,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACrD,OAAO,IAAI,CAAC,KAAK,CAAC,0BAAkB,CAAC,IAAI,EAAE,CAAC;AAChD,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,cAAc,CAAC,IAAc;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACnF,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAEhD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,IAAI,mBAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,4GAA4G;AAC5G,MAAM,sBAAsB,GAAG,2CAA2C,CAAC;AAE3E,6HAA6H;AAC7H,iHAAiH;AACjH,mEAAmE;AACnE,qHAAqH;AACrH,MAAM,oBAAoB,GAAG;IACzB,YAAY;IACZ,aAAa;IAEb,6FAA6F;IAC7F,2EAA2E;IAE3E,mCAAmC;IACnC,uCAAuC;IAEvC,sCAAsC;IACtC,2EAA2E;IAE3E,mBAAmB;IACnB,6CAA6C;IAC7C,eAAe;IACf,kCAAkC;IAClC,UAAU;IACV,uBAAuB;IAEvB,mBAAmB;IACnB,mDAAmD;IACnD,eAAe;IACf,sCAAsC;IACtC,UAAU;IACV,yBAAyB;IAEzB,mBAAmB;IACnB,6CAA6C;IAC7C,eAAe;IACf,kCAAkC;IAClC,WAAW;IACX,uBAAuB;IACvB,gDAAgD;CACnD,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,KAAK,EAAE,CAAC,CAAC;AAEtC,2DAA2D;AAC3D,uGAAuG;AACvG,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAE3B,oEAAoE;AACpE,4EAA4E;AAC5E,MAAM,iBAAiB,GAAG;IACtB,aAAa;IACb,8BAA8B;CACjC,CAAC;AAEF,MAAM,kBAAkB,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACnF,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,KAAK,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAC7E,MAAM,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAE/E;;;;;;;GAOG;AACH,SAAgB,cAAc,CAAC,IAAY;IACvC,IAAK,OAAO,IAAgB,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,MAAM,GAAI,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAc,IAAI,EAAE,CAAC;IAChE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAEzB,wDAAwD;QACxD,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,MAAM,GAAG,gBAAgB;YAAE,OAAO,KAAK,CAAC;QAEnE,gDAAgD;QAChD,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/C,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,IAAc;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAEnF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAEhD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,sCAAsC;AACtC,uGAAuG;AACvG,kGAAkG;AAClG,wFAAwF;AAExF,MAAM,qBAAqB,GACvB,4IAA4I,CAAC;AAEjJ,MAAM,sBAAsB,GACxB,yIAAyI,CAAC;AAE9I,MAAM,sBAAsB,GACxB,sGAAsG,CAAC;AAE3G,MAAM,oBAAoB,GAAG,kEAAkE,sBAAsB,+DAA+D,CAAC;AAErL,MAAM,uBAAuB,GACzB,kWAAkW,CAAC;AAEvW,MAAM,qBAAqB,GAAG,0EAA0E,uBAAuB,qHAAqH,CAAC;AAErP,MAAM,oBAAoB,GACtB,2KAA2K,CAAC;AAEhL,MAAM,mBAAmB,GACrB,2MAA2M,CAAC;AAEhN,MAAM,sBAAsB,GACxB,kNAAkN,CAAC;AAEvN,MAAM,oBAAoB,GACtB,6QAA6Q,CAAC;AAmBlR;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,SAAgB,oBAAoB,CAAC,IAAY,EAAE,OAAuC,IAAI;IAC1F,MAAM,MAAM,GAAkB;QAC1B,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,EAAE;QACV,eAAe,EAAE,EAAE;QACnB,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,EAAE;QACZ,UAAU,EAAE,EAAE;QACd,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,EAAE;QACZ,OAAO,EAAE,EAAE;QACX,UAAU,EAAE,EAAE;QACd,QAAQ,EAAE,EAAE;KACf,CAAC;IAEF,IAAK,OAAO,IAAgB,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAEzD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,IAAI,IAAI;QAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAErB,MAAM,IAAI,GAAG,IAAA,oBAAU,EAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,IAAI;QAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IAE3B,mCAAmC;IACnC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE;QAC/B,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,CAAC,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAE9C,8EAA8E;IAC9E,2EAA2E;IAC3E,mBAAmB;IACnB,mDAAmD;IACnD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,6BAAqB,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,4BAAoB,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,8BAAsB,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,6BAAqB,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,4BAAoB,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,2BAAmB,CAAC,IAAI,EAAE,CAAC;IACvD,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,8BAAsB,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,4BAAoB,CAAC,IAAI,EAAE,CAAC;IAEzD,+BAA+B;IAC/B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAA4B,EAAE,CAAC;QAC/D,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACU,QAAA,cAAc,GAAG,IAAI,MAAM,CAAC,IAAI,qBAAqB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE5E;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACU,QAAA,qBAAqB,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;AAE7E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACU,QAAA,eAAe,GAAG,IAAI,MAAM,CAAC,IAAI,sBAAsB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACU,QAAA,sBAAsB,GAAG,IAAI,MAAM,CAAC,sBAAsB,EAAE,IAAI,CAAC,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACU,QAAA,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACU,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;AAE3E;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACU,QAAA,cAAc,GAAG,IAAI,MAAM,CAAC,IAAI,qBAAqB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE5E;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACU,QAAA,qBAAqB,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;AAE7E;;;;;;;;;;;;;;;;;;;;;GAqBG;AACU,QAAA,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;GAoBG;AACU,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;AAC3E;;;;;;;;;;;;;;;;;;;;GAoBG;AACU,QAAA,YAAY,GAAG,IAAI,MAAM,CAAC,IAAI,mBAAmB,GAAG,EAAE,GAAG,CAAC,CAAC;AAExE;;;;;;;;;;;;;;;;;;;GAmBG;AACU,QAAA,mBAAmB,GAAG,IAAI,MAAM,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;AAEzE;;;;;;;;;;;;;;;;;;;;;GAqBG;AACU,QAAA,eAAe,GAAG,IAAI,MAAM,CAAC,IAAI,sBAAsB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;;;GAoBG;AACU,QAAA,sBAAsB,GAAG,IAAI,MAAM,CAAC,sBAAsB,EAAE,IAAI,CAAC,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;GAmBG;AACU,QAAA,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;;;;;;;;;;;;;;;GAkBG;AACU,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "3.11.2
|
|
3
|
+
"version": "3.11.2",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -49,9 +49,9 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/log": "^2.4.0",
|
|
51
51
|
"@apify/ps-tree": "^1.2.0",
|
|
52
|
-
"@crawlee/types": "3.11.2
|
|
52
|
+
"@crawlee/types": "3.11.2",
|
|
53
53
|
"@types/sax": "^1.2.7",
|
|
54
|
-
"cheerio": "
|
|
54
|
+
"cheerio": "1.0.0-rc.12",
|
|
55
55
|
"file-type": "^19.0.0",
|
|
56
56
|
"got-scraping": "^4.0.3",
|
|
57
57
|
"ow": "^0.28.1",
|
|
@@ -70,5 +70,5 @@
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"gitHead": "
|
|
73
|
+
"gitHead": "32d97a64e35dc765292f85d21a710c1c014808a3"
|
|
74
74
|
}
|