@crawlee/utils 3.10.6-beta.24 → 3.10.6-beta.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.mjs +1 -0
- package/internals/sitemap.d.ts +27 -23
- package/internals/sitemap.d.ts.map +1 -1
- package/internals/sitemap.js +215 -155
- package/internals/sitemap.js.map +1 -1
- package/package.json +4 -4
- package/tsconfig.build.tsbuildinfo +1 -1
package/index.mjs
CHANGED
|
@@ -21,6 +21,7 @@ export const htmlToText = mod.htmlToText;
|
|
|
21
21
|
export const isDocker = mod.isDocker;
|
|
22
22
|
export const keys = mod.keys;
|
|
23
23
|
export const parseOpenGraph = mod.parseOpenGraph;
|
|
24
|
+
export const parseSitemap = mod.parseSitemap;
|
|
24
25
|
export const sleep = mod.sleep;
|
|
25
26
|
export const snakeCaseToCamelCase = mod.snakeCaseToCamelCase;
|
|
26
27
|
export const social = mod.social;
|
package/internals/sitemap.d.ts
CHANGED
|
@@ -1,30 +1,36 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
interface SitemapUrlData {
|
|
2
|
+
loc: string;
|
|
3
|
+
lastmod?: Date;
|
|
4
|
+
changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
|
|
5
|
+
priority?: number;
|
|
6
|
+
}
|
|
7
|
+
export type SitemapUrl = SitemapUrlData & {
|
|
8
|
+
originSitemapUrl: string;
|
|
9
|
+
};
|
|
10
|
+
interface NestedSitemap {
|
|
11
|
+
loc: string;
|
|
12
|
+
originSitemapUrl: null;
|
|
13
|
+
}
|
|
14
|
+
type SitemapSource = ({
|
|
4
15
|
type: 'url';
|
|
5
16
|
url: string;
|
|
6
17
|
} | {
|
|
7
18
|
type: 'raw';
|
|
8
19
|
content: string;
|
|
20
|
+
}) & {
|
|
21
|
+
depth?: number;
|
|
9
22
|
};
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
private parsingState;
|
|
20
|
-
private onEnd;
|
|
21
|
-
private decoder;
|
|
22
|
-
private buffer;
|
|
23
|
-
constructor(parsingState: ParsingState, onEnd: () => void);
|
|
24
|
-
private processBuffer;
|
|
25
|
-
_write(chunk: any, _encoding: BufferEncoding, callback: (error?: Error | null | undefined) => void): void;
|
|
26
|
-
_final(callback: (error?: Error | null | undefined) => void): void;
|
|
23
|
+
interface ParseSitemapOptions {
|
|
24
|
+
/**
|
|
25
|
+
* If set to `true`, elements reffering to other sitemaps will be emitted as special objects with a `bouba` property.
|
|
26
|
+
*/
|
|
27
|
+
emitNestedSitemaps?: true | false;
|
|
28
|
+
/**
|
|
29
|
+
* Maximum depth of nested sitemaps to follow.
|
|
30
|
+
*/
|
|
31
|
+
maxDepth?: number;
|
|
27
32
|
}
|
|
33
|
+
export declare function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
|
|
28
34
|
/**
|
|
29
35
|
* Loads one or more sitemaps from given URLs, following references in sitemap index files, and exposes the contained URLs.
|
|
30
36
|
*
|
|
@@ -40,7 +46,6 @@ declare class SitemapTxtParser extends Writable {
|
|
|
40
46
|
export declare class Sitemap {
|
|
41
47
|
readonly urls: string[];
|
|
42
48
|
constructor(urls: string[]);
|
|
43
|
-
protected static createXmlParser(parsingState: ParsingState, onEnd: () => void, onError: (error: Error) => void): SAXStream;
|
|
44
49
|
/**
|
|
45
50
|
* Try to load sitemap from the most common locations - `/sitemap.xml` and `/sitemap.txt`.
|
|
46
51
|
* For loading based on `Sitemap` entries in `robots.txt`, the {@apilink RobotsFile} class should be used.
|
|
@@ -60,8 +65,7 @@ export declare class Sitemap {
|
|
|
60
65
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
61
66
|
*/
|
|
62
67
|
static fromXmlString(content: string, proxyUrl?: string): Promise<Sitemap>;
|
|
63
|
-
protected static parse(
|
|
64
|
-
protected static createParser(resolve: (value: unknown) => void, reject: (value: unknown) => void, parsingState: ParsingState, contentType?: string, url?: URL): SitemapTxtParser | SAXStream;
|
|
68
|
+
protected static parse(sources: SitemapSource[], proxyUrl?: string): Promise<Sitemap>;
|
|
65
69
|
}
|
|
66
70
|
export {};
|
|
67
71
|
//# sourceMappingURL=sitemap.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAUA,UAAU,cAAc;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,IAAI,CAAC;IACf,UAAU,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG;IACtC,gBAAgB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,UAAU,aAAa;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,IAAI,CAAC;CAC1B;AAED,KAAK,aAAa,GAAG,CAAC;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA8I5G,UAAU,mBAAmB;IACzB;;OAEG;IACH,kBAAkB,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC;IAClC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wBAAuB,YAAY,CAAC,CAAC,SAAS,mBAAmB,EAC7D,cAAc,EAAE,aAAa,EAAE,EAC/B,QAAQ,CAAC,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,CAAC,GACZ,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,SAAS,IAAI,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC,CAsH/F;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,OAAO;IACJ,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE;gBAAd,IAAI,EAAE,MAAM,EAAE;IAEnC;;;;;OAKG;WACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAe7E;;;;OAIG;WACU,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO/E;;;;OAIG;WACU,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;qBAIzD,KAAK,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAa9F"}
|
package/internals/sitemap.js
CHANGED
|
@@ -1,104 +1,255 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Sitemap = void 0;
|
|
4
|
+
exports.parseSitemap = parseSitemap;
|
|
4
5
|
const tslib_1 = require("tslib");
|
|
6
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
7
|
const node_stream_1 = require("node:stream");
|
|
6
8
|
const node_string_decoder_1 = require("node:string_decoder");
|
|
7
9
|
const node_zlib_1 = require("node:zlib");
|
|
8
10
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
9
11
|
const sax_1 = tslib_1.__importDefault(require("sax"));
|
|
10
12
|
const whatwg_mimetype_1 = tslib_1.__importDefault(require("whatwg-mimetype"));
|
|
11
|
-
class
|
|
13
|
+
class SitemapTxtParser extends node_stream_1.Transform {
|
|
12
14
|
constructor() {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
super({
|
|
16
|
+
readableObjectMode: true,
|
|
17
|
+
transform: (chunk, _encoding, callback) => {
|
|
18
|
+
this.processBuffer(this.decoder.write(chunk), false);
|
|
19
|
+
callback();
|
|
20
|
+
},
|
|
21
|
+
flush: (callback) => {
|
|
22
|
+
this.processBuffer(this.decoder.end(), true);
|
|
23
|
+
callback();
|
|
24
|
+
},
|
|
18
25
|
});
|
|
19
|
-
Object.defineProperty(this, "
|
|
26
|
+
Object.defineProperty(this, "decoder", {
|
|
20
27
|
enumerable: true,
|
|
21
28
|
configurable: true,
|
|
22
29
|
writable: true,
|
|
23
|
-
value:
|
|
30
|
+
value: new node_string_decoder_1.StringDecoder('utf8')
|
|
24
31
|
});
|
|
25
|
-
Object.defineProperty(this, "
|
|
32
|
+
Object.defineProperty(this, "buffer", {
|
|
26
33
|
enumerable: true,
|
|
27
34
|
configurable: true,
|
|
28
35
|
writable: true,
|
|
29
|
-
value:
|
|
36
|
+
value: ''
|
|
30
37
|
});
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
38
|
+
}
|
|
39
|
+
processBuffer(input, finalize) {
|
|
40
|
+
this.buffer += input;
|
|
41
|
+
if (finalize || this.buffer.includes('\n')) {
|
|
42
|
+
const parts = this.buffer
|
|
43
|
+
.split('\n')
|
|
44
|
+
.map((part) => part.trim())
|
|
45
|
+
.filter((part) => part.length > 0);
|
|
46
|
+
if (finalize) {
|
|
47
|
+
for (const url of parts) {
|
|
48
|
+
this.push({ type: 'url', loc: url });
|
|
49
|
+
}
|
|
50
|
+
this.buffer = '';
|
|
51
|
+
}
|
|
52
|
+
else if (parts.length > 0) {
|
|
53
|
+
for (const url of parts.slice(0, -1)) {
|
|
54
|
+
this.push({ type: 'url', loc: url });
|
|
55
|
+
}
|
|
56
|
+
this.buffer = parts.at(-1);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
class SitemapXmlParser extends node_stream_1.Transform {
|
|
62
|
+
constructor() {
|
|
63
|
+
super({
|
|
64
|
+
readableObjectMode: true,
|
|
65
|
+
transform: (chunk, _encoding, callback) => {
|
|
66
|
+
this.parser.write(this.decoder.write(chunk));
|
|
67
|
+
callback();
|
|
68
|
+
},
|
|
69
|
+
flush: (callback) => {
|
|
70
|
+
const rest = this.decoder.end();
|
|
71
|
+
if (rest.length > 0) {
|
|
72
|
+
this.parser.write(rest);
|
|
73
|
+
}
|
|
74
|
+
this.parser.end();
|
|
75
|
+
callback();
|
|
76
|
+
},
|
|
36
77
|
});
|
|
37
|
-
Object.defineProperty(this, "
|
|
78
|
+
Object.defineProperty(this, "decoder", {
|
|
38
79
|
enumerable: true,
|
|
39
80
|
configurable: true,
|
|
40
81
|
writable: true,
|
|
41
|
-
value:
|
|
82
|
+
value: new node_string_decoder_1.StringDecoder('utf8')
|
|
42
83
|
});
|
|
43
|
-
|
|
44
|
-
resetContext() {
|
|
45
|
-
this.context = undefined;
|
|
46
|
-
this.loc = false;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
class SitemapTxtParser extends node_stream_1.Writable {
|
|
50
|
-
constructor(parsingState, onEnd) {
|
|
51
|
-
super();
|
|
52
|
-
Object.defineProperty(this, "parsingState", {
|
|
84
|
+
Object.defineProperty(this, "parser", {
|
|
53
85
|
enumerable: true,
|
|
54
86
|
configurable: true,
|
|
55
87
|
writable: true,
|
|
56
|
-
value:
|
|
88
|
+
value: new sax_1.default.SAXParser(true)
|
|
57
89
|
});
|
|
58
|
-
Object.defineProperty(this, "
|
|
90
|
+
Object.defineProperty(this, "rootTagName", {
|
|
59
91
|
enumerable: true,
|
|
60
92
|
configurable: true,
|
|
61
93
|
writable: true,
|
|
62
|
-
value:
|
|
94
|
+
value: void 0
|
|
63
95
|
});
|
|
64
|
-
Object.defineProperty(this, "
|
|
96
|
+
Object.defineProperty(this, "currentTag", {
|
|
65
97
|
enumerable: true,
|
|
66
98
|
configurable: true,
|
|
67
99
|
writable: true,
|
|
68
|
-
value:
|
|
100
|
+
value: undefined
|
|
69
101
|
});
|
|
70
|
-
Object.defineProperty(this, "
|
|
102
|
+
Object.defineProperty(this, "url", {
|
|
71
103
|
enumerable: true,
|
|
72
104
|
configurable: true,
|
|
73
105
|
writable: true,
|
|
74
|
-
value:
|
|
106
|
+
value: {}
|
|
75
107
|
});
|
|
108
|
+
this.parser.onopentag = this.onOpenTag.bind(this);
|
|
109
|
+
this.parser.onclosetag = this.onCloseTag.bind(this);
|
|
110
|
+
this.parser.ontext = this.onText.bind(this);
|
|
111
|
+
this.parser.oncdata = this.onText.bind(this);
|
|
112
|
+
this.parser.onerror = this.destroy.bind(this);
|
|
76
113
|
}
|
|
77
|
-
|
|
78
|
-
this.
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
.
|
|
82
|
-
.
|
|
83
|
-
.
|
|
84
|
-
if (finalize) {
|
|
85
|
-
this.parsingState.urls.push(...parts);
|
|
86
|
-
this.buffer = '';
|
|
87
|
-
}
|
|
88
|
-
else if (parts.length > 0) {
|
|
89
|
-
this.parsingState.urls.push(...parts.slice(0, -1));
|
|
90
|
-
this.buffer = parts.at(-1);
|
|
114
|
+
onOpenTag(node) {
|
|
115
|
+
if (this.rootTagName !== undefined) {
|
|
116
|
+
if (node.name === 'loc' ||
|
|
117
|
+
node.name === 'lastmod' ||
|
|
118
|
+
node.name === 'priority' ||
|
|
119
|
+
node.name === 'changefreq') {
|
|
120
|
+
this.currentTag = node.name;
|
|
91
121
|
}
|
|
92
122
|
}
|
|
123
|
+
if (node.name === 'urlset') {
|
|
124
|
+
this.rootTagName = 'urlset';
|
|
125
|
+
}
|
|
126
|
+
if (node.name === 'sitemapindex') {
|
|
127
|
+
this.rootTagName = 'sitemapindex';
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
onCloseTag(name) {
|
|
131
|
+
if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
|
|
132
|
+
this.currentTag = undefined;
|
|
133
|
+
}
|
|
134
|
+
if (name === 'url' && this.url.loc !== undefined) {
|
|
135
|
+
this.push({ type: 'url', ...this.url, loc: this.url.loc });
|
|
136
|
+
this.url = {};
|
|
137
|
+
}
|
|
93
138
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
139
|
+
onText(text) {
|
|
140
|
+
if (this.currentTag === 'loc') {
|
|
141
|
+
if (this.rootTagName === 'sitemapindex') {
|
|
142
|
+
this.push({ type: 'sitemapUrl', url: text });
|
|
143
|
+
}
|
|
144
|
+
if (this.rootTagName === 'urlset') {
|
|
145
|
+
this.url ?? (this.url = {});
|
|
146
|
+
this.url.loc = text;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
text = text.trim();
|
|
150
|
+
if (this.currentTag === 'lastmod') {
|
|
151
|
+
this.url.lastmod = new Date(text);
|
|
152
|
+
}
|
|
153
|
+
if (this.currentTag === 'priority') {
|
|
154
|
+
this.url.priority = Number(text);
|
|
155
|
+
}
|
|
156
|
+
if (this.currentTag === 'changefreq') {
|
|
157
|
+
if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
|
|
158
|
+
this.url.changefreq = text;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
97
161
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
162
|
+
}
|
|
163
|
+
async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
164
|
+
const { gotScraping } = await import('got-scraping');
|
|
165
|
+
const { fileTypeStream } = await import('file-type');
|
|
166
|
+
const sources = [...initialSources];
|
|
167
|
+
const visitedSitemapUrls = new Set();
|
|
168
|
+
const createParser = (contentType = '', url) => {
|
|
169
|
+
let mimeType;
|
|
170
|
+
try {
|
|
171
|
+
mimeType = new whatwg_mimetype_1.default(contentType);
|
|
172
|
+
}
|
|
173
|
+
catch (e) {
|
|
174
|
+
mimeType = null;
|
|
175
|
+
}
|
|
176
|
+
if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
|
|
177
|
+
return new SitemapXmlParser();
|
|
178
|
+
}
|
|
179
|
+
if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
|
|
180
|
+
return new SitemapTxtParser();
|
|
181
|
+
}
|
|
182
|
+
throw new Error(`Unsupported sitemap content type (contentType = ${contentType}, url = ${url?.toString()})`);
|
|
183
|
+
};
|
|
184
|
+
while (sources.length > 0) {
|
|
185
|
+
const source = sources.shift();
|
|
186
|
+
if ((source?.depth ?? 0) > (options?.maxDepth ?? Infinity)) {
|
|
187
|
+
log_1.default.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${options.maxDepth}.`);
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
let items = null;
|
|
191
|
+
if (source.type === 'url') {
|
|
192
|
+
const sitemapUrl = new URL(source.url);
|
|
193
|
+
visitedSitemapUrls.add(sitemapUrl.toString());
|
|
194
|
+
try {
|
|
195
|
+
const sitemapStream = await new Promise((resolve, reject) => {
|
|
196
|
+
const request = gotScraping.stream({ url: sitemapUrl, proxyUrl, method: 'GET' });
|
|
197
|
+
request.on('response', () => resolve(request));
|
|
198
|
+
request.on('error', reject);
|
|
199
|
+
});
|
|
200
|
+
if (sitemapStream.response.statusCode === 200) {
|
|
201
|
+
let contentType = sitemapStream.response.headers['content-type'];
|
|
202
|
+
const streamWithType = await fileTypeStream(sitemapStream);
|
|
203
|
+
if (streamWithType.fileType !== undefined) {
|
|
204
|
+
contentType = streamWithType.fileType.mime;
|
|
205
|
+
}
|
|
206
|
+
let isGzipped = false;
|
|
207
|
+
if (contentType !== undefined
|
|
208
|
+
? contentType === 'application/gzip'
|
|
209
|
+
: sitemapUrl.pathname.endsWith('.gz')) {
|
|
210
|
+
isGzipped = true;
|
|
211
|
+
if (sitemapUrl.pathname.endsWith('.gz')) {
|
|
212
|
+
sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
items = (0, node_stream_1.pipeline)(streamWithType, isGzipped ? (0, node_zlib_1.createGunzip)() : new node_stream_1.PassThrough(), createParser(contentType, sitemapUrl), (error) => {
|
|
216
|
+
if (error !== undefined) {
|
|
217
|
+
log_1.default.warning(`Malformed sitemap content: ${sitemapUrl}, ${error}`);
|
|
218
|
+
}
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
catch (e) {
|
|
223
|
+
log_1.default.warning(`Malformed sitemap content: ${sitemapUrl}, ${e}`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
else if (source.type === 'raw') {
|
|
227
|
+
items = (0, node_stream_1.pipeline)(node_stream_1.Readable.from([source.content]), createParser('text/xml'), (error) => {
|
|
228
|
+
if (error !== undefined) {
|
|
229
|
+
log_1.default.warning(`Malformed sitemap content: ${error}`);
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
if (items === null) {
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
for await (const item of items) {
|
|
237
|
+
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
|
|
238
|
+
sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
|
|
239
|
+
if (options?.emitNestedSitemaps) {
|
|
240
|
+
// @ts-ignore
|
|
241
|
+
yield { loc: item.url, originSitemapUrl: null };
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
if (item.type === 'url') {
|
|
245
|
+
yield {
|
|
246
|
+
...item,
|
|
247
|
+
originSitemapUrl: source.type === 'url'
|
|
248
|
+
? source.url
|
|
249
|
+
: `raw://${(0, node_crypto_1.createHash)('sha256').update(source.content).digest('base64')}`,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
}
|
|
102
253
|
}
|
|
103
254
|
}
|
|
104
255
|
/**
|
|
@@ -122,42 +273,6 @@ class Sitemap {
|
|
|
122
273
|
value: urls
|
|
123
274
|
});
|
|
124
275
|
}
|
|
125
|
-
static createXmlParser(parsingState, onEnd, onError) {
|
|
126
|
-
const parser = sax_1.default.createStream(true);
|
|
127
|
-
parser.on('opentag', (node) => {
|
|
128
|
-
if (node.name === 'loc' && parsingState.context !== undefined) {
|
|
129
|
-
parsingState.loc = true;
|
|
130
|
-
}
|
|
131
|
-
if (node.name === 'urlset') {
|
|
132
|
-
parsingState.context = 'urlset';
|
|
133
|
-
}
|
|
134
|
-
if (node.name === 'sitemapindex') {
|
|
135
|
-
parsingState.context = 'sitemapindex';
|
|
136
|
-
}
|
|
137
|
-
});
|
|
138
|
-
parser.on('closetag', (name) => {
|
|
139
|
-
if (name === 'loc') {
|
|
140
|
-
parsingState.loc = false;
|
|
141
|
-
}
|
|
142
|
-
});
|
|
143
|
-
const onText = (text) => {
|
|
144
|
-
if (parsingState.loc) {
|
|
145
|
-
if (parsingState.context === 'sitemapindex') {
|
|
146
|
-
if (!parsingState.visitedSitemapUrls.includes(text)) {
|
|
147
|
-
parsingState.sources.push({ type: 'url', url: text });
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
if (parsingState.context === 'urlset') {
|
|
151
|
-
parsingState.urls.push(text);
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
};
|
|
155
|
-
parser.on('text', onText);
|
|
156
|
-
parser.on('cdata', onText);
|
|
157
|
-
parser.on('end', onEnd);
|
|
158
|
-
parser.on('error', onError);
|
|
159
|
-
return parser;
|
|
160
|
-
}
|
|
161
276
|
/**
|
|
162
277
|
* Try to load sitemap from the most common locations - `/sitemap.xml` and `/sitemap.txt`.
|
|
163
278
|
* For loading based on `Sitemap` entries in `robots.txt`, the {@apilink RobotsFile} class should be used.
|
|
@@ -180,9 +295,7 @@ class Sitemap {
|
|
|
180
295
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
181
296
|
*/
|
|
182
297
|
static async load(urls, proxyUrl) {
|
|
183
|
-
|
|
184
|
-
parsingState.sources = (Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url }));
|
|
185
|
-
return await this.parse(parsingState, proxyUrl);
|
|
298
|
+
return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl);
|
|
186
299
|
}
|
|
187
300
|
/**
|
|
188
301
|
* Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
|
|
@@ -190,72 +303,19 @@ class Sitemap {
|
|
|
190
303
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
191
304
|
*/
|
|
192
305
|
static async fromXmlString(content, proxyUrl) {
|
|
193
|
-
|
|
194
|
-
parsingState.sources = [{ type: 'raw', content }];
|
|
195
|
-
return await this.parse(parsingState, proxyUrl);
|
|
196
|
-
}
|
|
197
|
-
static async parse(parsingState, proxyUrl) {
|
|
198
|
-
const { gotScraping } = await import('got-scraping');
|
|
199
|
-
const { fileTypeStream } = await import('file-type');
|
|
200
|
-
while (parsingState.sources.length > 0) {
|
|
201
|
-
const source = parsingState.sources.pop();
|
|
202
|
-
parsingState.resetContext();
|
|
203
|
-
if (source.type === 'url') {
|
|
204
|
-
const sitemapUrl = new URL(source.url);
|
|
205
|
-
parsingState.visitedSitemapUrls.push(sitemapUrl.toString());
|
|
206
|
-
try {
|
|
207
|
-
const sitemapStream = await new Promise((resolve, reject) => {
|
|
208
|
-
const request = gotScraping.stream({ url: sitemapUrl, proxyUrl, method: 'GET' });
|
|
209
|
-
request.on('response', () => resolve(request));
|
|
210
|
-
request.on('error', reject);
|
|
211
|
-
});
|
|
212
|
-
if (sitemapStream.response.statusCode === 200) {
|
|
213
|
-
let contentType = sitemapStream.response.headers['content-type'];
|
|
214
|
-
const streamWithType = await fileTypeStream(sitemapStream);
|
|
215
|
-
if (streamWithType.fileType !== undefined) {
|
|
216
|
-
contentType = streamWithType.fileType.mime;
|
|
217
|
-
}
|
|
218
|
-
await new Promise((resolve, reject) => {
|
|
219
|
-
let stream = streamWithType;
|
|
220
|
-
if (contentType !== undefined
|
|
221
|
-
? contentType === 'application/gzip'
|
|
222
|
-
: sitemapUrl.pathname.endsWith('.gz')) {
|
|
223
|
-
stream = stream.pipe((0, node_zlib_1.createGunzip)()).on('error', reject);
|
|
224
|
-
if (sitemapUrl.pathname.endsWith('.gz')) {
|
|
225
|
-
sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
stream.pipe(this.createParser(resolve, reject, parsingState, contentType, sitemapUrl));
|
|
229
|
-
});
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
catch (e) {
|
|
233
|
-
log_1.default.warning(`Malformed sitemap content: ${sitemapUrl}, ${e}`);
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
if (source.type === 'raw') {
|
|
237
|
-
await new Promise((resolve, reject) => {
|
|
238
|
-
node_stream_1.Readable.from([source.content]).pipe(this.createParser(resolve, reject, parsingState, 'text/xml'));
|
|
239
|
-
});
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
return new Sitemap(parsingState.urls);
|
|
306
|
+
return await this.parse([{ type: 'raw', content }], proxyUrl);
|
|
243
307
|
}
|
|
244
|
-
static
|
|
245
|
-
|
|
308
|
+
static async parse(sources, proxyUrl) {
|
|
309
|
+
const urls = [];
|
|
246
310
|
try {
|
|
247
|
-
|
|
311
|
+
for await (const item of parseSitemap(sources, proxyUrl)) {
|
|
312
|
+
urls.push(item.loc);
|
|
313
|
+
}
|
|
248
314
|
}
|
|
249
315
|
catch (e) {
|
|
250
|
-
|
|
251
|
-
}
|
|
252
|
-
if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
|
|
253
|
-
return Sitemap.createXmlParser(parsingState, () => resolve(undefined), reject);
|
|
254
|
-
}
|
|
255
|
-
if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
|
|
256
|
-
return new SitemapTxtParser(parsingState, () => resolve(undefined));
|
|
316
|
+
return new Sitemap([]);
|
|
257
317
|
}
|
|
258
|
-
|
|
318
|
+
return new Sitemap(urls);
|
|
259
319
|
}
|
|
260
320
|
}
|
|
261
321
|
exports.Sitemap = Sitemap;
|
package/internals/sitemap.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":";;;AAmLA,oCA0HC;;AA7SD,6CAAyC;AAEzC,6CAAyE;AACzE,6DAAoD;AACpD,yCAAyC;AAEzC,6DAA6B;AAC7B,sDAAsB;AACtB,8EAAuC;AAqBvC,MAAM,gBAAiB,SAAQ,uBAAS;IAIpC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBACrD,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAdC;;;;mBAAyB,IAAI,mCAAa,CAAC,MAAM,CAAC;WAAC;QACnD;;;;mBAAiB,EAAE;WAAC;IAc5B,CAAC;IAEO,aAAa,CAAC,KAAa,EAAE,QAAiB;QAClD,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAErB,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM;iBACpB,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEvC,IAAI,QAAQ,EAAE,CAAC;gBACX,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;YAChC,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAED,MAAM,gBAAiB,SAAQ,uBAAS;IAQpC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAClB,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAvBC;;;;mBAAyB,IAAI,mCAAa,CAAC,MAAM,CAAC;WAAC;QACnD;;;;mBAAS,IAAI,aAAG,CAAC,SAAS,CAAC,IAAI,CAAC;WAAC;QAEjC;;;;;WAAwC;QACxC;;;;mBAA6D,SAAS;WAAC;QACvE;;;;mBAA2B,EAAE;WAAC;QAoBlC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAgC;QAC9C,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACjC,IACI,IAAI,CAAC,IAAI,KAAK,KAAK;gBACnB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,IAAI,CAAC,IAAI,KAAK,YAAY,EAC5B,CAAC;gBACC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;YAChC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC,WAAW,GAAG,cAAc,CAAC;QACtC,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACvF,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAwB,CAAC,CAAC;YACjF,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,IAAY;QACvB,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,EAAwB,CAAC,CAAC;YACvE,CAAC;YAED,IAAI,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,KAAR,IAAI,CAAC,GAAG,GAAK,EAAE,EAAC;gBAChB,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC;YACxB,CAAC;QACL,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvF,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAgC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAaM,KAAK,SAAS,CAAC,CAAC,YAAY,CAC/B,cAA+B,EAC/B,QAAiB,EACjB,OAAW;IAEX,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IAErD,MAAM,OAAO,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC;IACpC,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,MAAM,YAAY,GAAG,CAAC,cAAsB,EAAE,EAAE,GAAS,EAAU,EAAE;QACjE,IAAI,QAAyB,CAAC;QAE9B,IAAI,CAAC;YACD,QAAQ,GAAG,IAAI,yBAAQ,CAAC,WAAW,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,IAAI,QAAQ,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,QAAQ,EAAE,OAAO,KAAK,YAAY,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,WAAW,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IACjH,CAAC,CAAC;IAEF,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,EAAG,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;YACzD,aAAG,CAAC,KAAK,CACL,oBAAoB,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,iCAAiC,OAAQ,CAAC,QAAS,GAAG,CACpH,CAAC;YACF,SAAS;QACb,CAAC;QAED,IAAI,KAAK,GAAsC,IAAI,CAAC;QAEpD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE9C,IAAI,CAAC;gBACD,MAAM,aAAa,GAAG,MAAM,IAAI,OAAO,CAAwC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBAC/F,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;oBACjF,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC/C,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAChC,CAAC,CAAC,CAAC;gBAEH,IAAI,aAAa,CAAC,QAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC7C,IAAI,WAAW,GAAG,aAAa,CAAC,QAAS,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;oBAElE,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,CAAC,CAAC;oBAC3D,IAAI,cAAc,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;wBACxC,WAAW,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAC/C,CAAC;oBAED,IAAI,SAAS,GAAG,KAAK,CAAC;oBAEtB,IACI,WAAW,KAAK,SAAS;wBACrB,CAAC,CAAC,WAAW,KAAK,kBAAkB;wBACpC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAC3C,CAAC;wBACC,SAAS,GAAG,IAAI,CAAC;wBAEjB,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;4BACtC,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBAC3F,CAAC;oBACL,CAAC;oBAED,KAAK,GAAG,IAAA,sBAAQ,EACZ,cAAc,EACd,SAAS,CAAC,CAAC,CAAC,IAAA,wBAAY,GAAE,CAAC,CAAC,CAAC,IAAI,yBAAW,EAAE,EAC9C,YAAY,CAAC,WAAW,EAAE,UAAU,CAAC,EACrC,CAAC,KAAK,EAAE,EAAE;wBACN,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;4BACtB,aAAG,CAAC,OAAO,CAAC,8BAA8B,UAAU,KAAK,KAAK,EAAE,CAAC,CAAC;wBACtE,CAAC;oBACL,CAAC,CACJ,CAAC;gBACN,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,aAAG,CAAC,OAAO,CAAC,8BAA8B,UAAU,KAAK,CAAC,EAAE,CAAC,CAAC;YAClE,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC/B,KAAK,GAAG,IAAA,sBAAQ,EAAC,sBAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACtB,aAAG,CAAC,OAAO,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;gBACvD,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7E,IAAI,OAAO,EAAE,kBAAkB,EAAE,CAAC;oBAC9B,aAAa;oBACb,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;gBACpD,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACtB,MAAM;oBACF,GAAG,IAAI;oBACP,gBAAgB,EACZ,MAAM,CAAC,IAAI,KAAK,KAAK;wBACjB,CAAC,CAAC,MAAM,CAAC,GAAG;wBACZ,CAAC,CAAC,SAAS,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;iBACpF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAa,OAAO;IAChB,YAAqB,IAAc;QAAvB;;;;mBAAS,IAAI;WAAU;IAAG,CAAC;IAEvC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,GAAW,EAAE,QAAiB;QACtD,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC;QAEvB,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAuB,EAAE,QAAiB;QACxD,OAAO,MAAM,IAAI,CAAC,KAAK,CACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAC1E,QAAQ,CACX,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,OAAe,EAAE,QAAiB;QACzD,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAwB,EAAE,QAAiB;QACpE,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACvD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACJ;AA1DD,0BA0DC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "3.10.6-beta.
|
|
3
|
+
"version": "3.10.6-beta.25",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -49,14 +49,14 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/log": "^2.4.0",
|
|
51
51
|
"@apify/ps-tree": "^1.2.0",
|
|
52
|
-
"@crawlee/types": "3.10.6-beta.
|
|
52
|
+
"@crawlee/types": "3.10.6-beta.25",
|
|
53
53
|
"@types/sax": "^1.2.7",
|
|
54
54
|
"cheerio": "^1.0.0-rc.12",
|
|
55
55
|
"file-type": "^19.0.0",
|
|
56
56
|
"got-scraping": "^4.0.3",
|
|
57
57
|
"ow": "^0.28.1",
|
|
58
58
|
"robots-parser": "^3.0.1",
|
|
59
|
-
"sax": "^1.
|
|
59
|
+
"sax": "^1.4.1",
|
|
60
60
|
"tslib": "^2.4.0",
|
|
61
61
|
"whatwg-mimetype": "^4.0.0"
|
|
62
62
|
},
|
|
@@ -70,5 +70,5 @@
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"gitHead": "
|
|
73
|
+
"gitHead": "e8a5d51f0fe0d8f151c570c3ac07d1df68b837bd"
|
|
74
74
|
}
|