@crawlee/utils 4.0.0-beta.21 → 4.0.0-beta.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/extract-urls.js +2 -2
- package/internals/extract-urls.js.map +1 -1
- package/internals/iterables.d.ts +19 -0
- package/internals/iterables.d.ts.map +1 -1
- package/internals/iterables.js +55 -0
- package/internals/iterables.js.map +1 -1
- package/internals/robots.js +2 -2
- package/internals/robots.js.map +1 -1
- package/internals/sitemap.d.ts +17 -0
- package/internals/sitemap.d.ts.map +1 -1
- package/internals/sitemap.js +85 -3
- package/internals/sitemap.js.map +1 -1
- package/package.json +4 -4
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { FetchHttpClient } from '@crawlee/http-client';
|
|
2
2
|
import ow from 'ow';
|
|
3
3
|
import { URL_NO_COMMAS_REGEX } from './general.js';
|
|
4
4
|
/**
|
|
@@ -13,7 +13,7 @@ export async function downloadListOfUrls(options) {
|
|
|
13
13
|
proxyUrl: ow.optional.string,
|
|
14
14
|
httpClient: ow.optional.object,
|
|
15
15
|
}));
|
|
16
|
-
const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl, httpClient = new
|
|
16
|
+
const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl, httpClient = new FetchHttpClient(), } = options;
|
|
17
17
|
// Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.
|
|
18
18
|
const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/);
|
|
19
19
|
let fixedUrl = url;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AA8BnD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACvE,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KACjC,CAAC,CACL,CAAC;IACF,MAAM,EACF,GAAG,EACH,QAAQ,GAAG,MAAM,EACjB,SAAS,GAAG,mBAAmB,EAC/B,QAAQ,EACR,UAAU,GAAG,IAAI,eAAe,EAAE,GACrC,GAAG,OAAO,CAAC;IAEZ,0GAA0G;IAC1G,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACzF,IAAI,QAAQ,GAAG,GAAG,CAAC;IAEnB,IAAI,KAAK,EAAE,CAAC;QACR,QAAQ,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;QACpF,QAAQ;KACX,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAE9F,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC;AAeD;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAA2B;IACnD,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CACL,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,OAAe;IACxD,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC"}
|
package/internals/iterables.d.ts
CHANGED
|
@@ -105,4 +105,23 @@ export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
|
|
|
105
105
|
* ```
|
|
106
106
|
*/
|
|
107
107
|
export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;
|
|
108
|
+
/**
|
|
109
|
+
* Merges multiple async iterables into a single async iterable, yielding values concurrently.
|
|
110
|
+
*
|
|
111
|
+
* **Example usage:**
|
|
112
|
+
* ```ts
|
|
113
|
+
* const asyncIterable1 = async function* () {
|
|
114
|
+
* yield 1; yield 3; yield 5;
|
|
115
|
+
* };
|
|
116
|
+
*
|
|
117
|
+
* const asyncIterable2 = async function* () {
|
|
118
|
+
* yield 2; yield 4; yield 6;
|
|
119
|
+
* };
|
|
120
|
+
*
|
|
121
|
+
* for await (const value of mergeAsyncIterables(asyncIterable1(), asyncIterable2())) {
|
|
122
|
+
* console.log(value);
|
|
123
|
+
* }
|
|
124
|
+
* ```
|
|
125
|
+
*/
|
|
126
|
+
export declare function mergeAsyncIterables<T>(...iterables: AsyncIterable<T>[]): AsyncIterable<T>;
|
|
108
127
|
//# sourceMappingURL=iterables.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"iterables.d.ts","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,CAUlE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC,CAM5E;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAuB,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAErG;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAuB,oBAAoB,CAAC,CAAC,EACzC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACxC,SAAS,EAAE,MAAM,GAClB,aAAa,CAAC,CAAC,EAAE,CAAC,CAmBpB;AAED;;;;GAIG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;IAChF;;;;;OAKG;IACH,IAAI,IAAI,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC;IAC9D,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,CAAC;CACtD;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,qBAAqB,CAAC,CAAC,CAAC,CA+D3G"}
|
|
1
|
+
{"version":3,"file":"iterables.d.ts","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,CAUlE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC,CAM5E;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAuB,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAErG;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAuB,oBAAoB,CAAC,CAAC,EACzC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACxC,SAAS,EAAE,MAAM,GAClB,aAAa,CAAC,CAAC,EAAE,CAAC,CAmBpB;AAED;;;;GAIG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;IAChF;;;;;OAKG;IACH,IAAI,IAAI,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC;IAC9D,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,CAAC;CACtD;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,qBAAqB,CAAC,CAAC,CAAC,CA+D3G;AAGD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAuB,mBAAmB,CAAC,CAAC,EAAE,GAAG,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,CAgChG"}
|
package/internals/iterables.js
CHANGED
|
@@ -164,4 +164,59 @@ export function peekableAsyncIterable(iterable) {
|
|
|
164
164
|
},
|
|
165
165
|
};
|
|
166
166
|
}
|
|
167
|
+
// Source - https://stackoverflow.com/a/71288323
|
|
168
|
+
/**
|
|
169
|
+
* Merges multiple async iterables into a single async iterable, yielding values concurrently.
|
|
170
|
+
*
|
|
171
|
+
* **Example usage:**
|
|
172
|
+
* ```ts
|
|
173
|
+
* const asyncIterable1 = async function* () {
|
|
174
|
+
* yield 1; yield 3; yield 5;
|
|
175
|
+
* };
|
|
176
|
+
*
|
|
177
|
+
* const asyncIterable2 = async function* () {
|
|
178
|
+
* yield 2; yield 4; yield 6;
|
|
179
|
+
* };
|
|
180
|
+
*
|
|
181
|
+
* for await (const value of mergeAsyncIterables(asyncIterable1(), asyncIterable2())) {
|
|
182
|
+
* console.log(value);
|
|
183
|
+
* }
|
|
184
|
+
* ```
|
|
185
|
+
*/
|
|
186
|
+
export async function* mergeAsyncIterables(...iterables) {
|
|
187
|
+
const asyncIterators = iterables.map((iterable) => iterable[Symbol.asyncIterator]());
|
|
188
|
+
const results = [];
|
|
189
|
+
let count = asyncIterators.length;
|
|
190
|
+
const never = new Promise(() => { });
|
|
191
|
+
async function getNext(asyncIterator, index) {
|
|
192
|
+
const result = await asyncIterator.next();
|
|
193
|
+
return {
|
|
194
|
+
index,
|
|
195
|
+
result,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
const nextPromises = asyncIterators.map(getNext);
|
|
199
|
+
try {
|
|
200
|
+
while (count) {
|
|
201
|
+
const { index, result } = await Promise.race(nextPromises);
|
|
202
|
+
if (result.done) {
|
|
203
|
+
nextPromises[index] = never;
|
|
204
|
+
results[index] = result.value;
|
|
205
|
+
count--;
|
|
206
|
+
}
|
|
207
|
+
else {
|
|
208
|
+
nextPromises[index] = getNext(asyncIterators[index], index);
|
|
209
|
+
yield result.value;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
finally {
|
|
214
|
+
for (const [index, iterator] of asyncIterators.entries()) {
|
|
215
|
+
// no await here - see https://github.com/tc39/proposal-async-iteration/issues/126
|
|
216
|
+
if (nextPromises[index] !== never && iterator.return != null)
|
|
217
|
+
void iterator.return();
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return results;
|
|
221
|
+
}
|
|
167
222
|
//# sourceMappingURL=iterables.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"iterables.js","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,UAAU,CAAI,KAAc;IACxC,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC;AAChE,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,eAAe,CAAI,KAAc;IAC7C,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC;AACrE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAI,QAAwC;IAC/E,KAAK,CAAC,CAAC,QAAQ,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACvC,QAAwC,EACxC,SAAiB;IAEjB,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,KAAK,GAAQ,EAAE,CAAC;IAEpB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,KAAK,CAAC;YACZ,KAAK,GAAG,EAAE,CAAC;QACf,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACf,MAAM,KAAK,CAAC;IAChB,CAAC;AACL,CAAC;AAyBD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,qBAAqB,CAAI,QAAwC;IAC7E,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IACpE,IAAI,WAAoD,CAAC;IACzD,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,MAAM,gBAAgB,GAA6B;QAC/C,KAAK,CAAC,IAAI;YACN,0DAA0D;YAC1D,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,MAAM,MAAM,GAAG,WAAW,CAAC;gBAC3B,WAAW,GAAG,SAAS,CAAC;gBAExB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBACd,WAAW,GAAG,IAAI,CAAC;oBACnB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;gBAC5C,CAAC;gBAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAChD,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YAC5C,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAErC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,KAAK,CAAC,IAAI;YACN,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;YAC5D,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrC,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAElE,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;gBACnB,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC;QACxB,CAAC;QAED,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;KACJ,CAAC;IAEF,OAAO;QACH,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,gBAAgB,CAAC;QAC5B,CAAC;KACJ,CAAC;AACN,CAAC"}
|
|
1
|
+
{"version":3,"file":"iterables.js","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,UAAU,CAAI,KAAc;IACxC,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC;AAChE,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,eAAe,CAAI,KAAc;IAC7C,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC;AACrE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAI,QAAwC;IAC/E,KAAK,CAAC,CAAC,QAAQ,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACvC,QAAwC,EACxC,SAAiB;IAEjB,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,KAAK,GAAQ,EAAE,CAAC;IAEpB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,KAAK,CAAC;YACZ,KAAK,GAAG,EAAE,CAAC;QACf,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACf,MAAM,KAAK,CAAC;IAChB,CAAC;AACL,CAAC;AAyBD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,qBAAqB,CAAI,QAAwC;IAC7E,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IACpE,IAAI,WAAoD,CAAC;IACzD,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,MAAM,gBAAgB,GAA6B;QAC/C,KAAK,CAAC,IAAI;YACN,0DAA0D;YAC1D,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,MAAM,MAAM,GAAG,WAAW,CAAC;gBAC3B,WAAW,GAAG,SAAS,CAAC;gBAExB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBACd,WAAW,GAAG,IAAI,CAAC;oBACnB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;gBAC5C,CAAC;gBAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAChD,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YAC5C,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAErC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,KAAK,CAAC,IAAI;YACN,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;YAC5D,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrC,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAElE,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;gBACnB,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC;QACxB,CAAC;QAED,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;KACJ,CAAC;IAEF,OAAO;QACH,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,gBAAgB,CAAC;QAC5B,CAAC;KACJ,CAAC;AACN,CAAC;AAED,gDAAgD;AAChD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAI,GAAG,SAA6B;IAC1E,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACrF,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,IAAI,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC;IAClC,MAAM,KAAK,GAAmB,IAAI,OAAO,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACpD,KAAK,UAAU,OAAO,CAAC,aAA+B,EAAE,KAAa;QACjE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO;YACH,KAAK;YACL,MAAM;SACT,CAAC;IACN,CAAC;IACD,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACjD,IAAI,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3D,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAC5B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC9B,KAAK,EAAE,CAAC;YACZ,CAAC;iBAAM,CAAC;gBACJ,YAAY,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC5D,MAAM,MAAM,CAAC,KAAK,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC;YAAS,CAAC;QACP,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,kFAAkF;YAClF,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI;gBAAE,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QACzF,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC"}
|
package/internals/robots.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { FetchHttpClient } from '@crawlee/http-client';
|
|
2
2
|
import robotsParser from 'robots-parser';
|
|
3
3
|
import { Sitemap } from './sitemap.js';
|
|
4
4
|
/**
|
|
@@ -48,7 +48,7 @@ export class RobotsTxtFile {
|
|
|
48
48
|
return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
|
|
49
49
|
}
|
|
50
50
|
static async load(url, options) {
|
|
51
|
-
const { proxyUrl, httpClient = new
|
|
51
|
+
const { proxyUrl, httpClient = new FetchHttpClient() } = options || {};
|
|
52
52
|
const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
|
|
53
53
|
proxyUrl,
|
|
54
54
|
});
|
package/internals/robots.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAGvD,OAAO,YAAY,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,aAAa;IAEV;IACA;IAFZ,YACY,MAAgD,EAChD,QAAiB;QADjB,WAAM,GAAN,MAAM,CAA0C;QAChD,aAAQ,GAAR,QAAQ,CAAS;IAC1B,CAAC;IAEJ;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,GAAW,EACX,OAA4D;QAE5D,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,gBAAgB,CAAC,QAAQ,GAAG,aAAa,CAAC;QAC1C,gBAAgB,CAAC,MAAM,GAAG,EAAE,CAAC;QAE7B,OAAO,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAW,EAAE,OAAe,EAAE,QAAiB;QACvD,aAAa;QACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,IAAI,CACvB,GAAW,EACX,OAA4D;QAE5D,MAAM,EAAE,QAAQ,EAAE,UAAU,GAAG,IAAI,eAAe,EAAE,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAEvE,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;YAC/E,QAAQ;SACX,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,kCAAkC,GAAG,UAAU,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC1B,OAAO,IAAI,aAAa,CACpB;gBACI,SAAS;oBACL,OAAO,IAAI,CAAC;gBAChB,CAAC;gBACD,WAAW;oBACP,OAAO,EAAE,CAAC;gBACd,CAAC;aACJ,EACD,QAAQ,CACX,CAAC;QACN,CAAC;QAED,aAAa;QACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC5F,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC,+FAA+F;IACzJ,CAAC;IAED;;OAEG;IACH,WAAW;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACvB,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;CACJ;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}
|
package/internals/sitemap.d.ts
CHANGED
|
@@ -85,5 +85,22 @@ export declare class Sitemap {
|
|
|
85
85
|
static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
|
86
86
|
protected static parse(sources: SitemapSource[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
|
87
87
|
}
|
|
88
|
+
/**
|
|
89
|
+
* Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
|
|
90
|
+
* the default `sitemap.xml` & `sitemap.txt` files and the URLs themselves.
|
|
91
|
+
* @param `urls` The list of URLs to discover sitemaps for.
|
|
92
|
+
* @param `options` Options for sitemap discovery
|
|
93
|
+
* @returns An async iterable with the discovered sitemap URLs.
|
|
94
|
+
*/
|
|
95
|
+
export declare function discoverValidSitemaps(urls: string[], options?: {
|
|
96
|
+
/**
|
|
97
|
+
* Proxy URL to be used for network requests.
|
|
98
|
+
*/
|
|
99
|
+
proxyUrl?: string;
|
|
100
|
+
/**
|
|
101
|
+
* HTTP client to be used for network requests.
|
|
102
|
+
*/
|
|
103
|
+
httpClient?: BaseHttpClient;
|
|
104
|
+
}): AsyncIterable<string>;
|
|
88
105
|
export {};
|
|
89
106
|
//# sourceMappingURL=sitemap.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAUrD,UAAU,cAAc;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,IAAI,CAAC;IACf,UAAU,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG;IACtC,gBAAgB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,UAAU,aAAa;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,IAAI,CAAC;CAC1B;AAED,KAAK,aAAa,GAAG,CAAC;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA8I5G,MAAM,WAAW,mBAAmB;IAChC;;OAEG;IACH,kBAAkB,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC;IAClC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B;AAED,wBAAuB,YAAY,CAAC,CAAC,SAAS,mBAAmB,EAC7D,cAAc,EAAE,aAAa,EAAE,EAC/B,QAAQ,CAAC,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,CAAC,GACZ,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,SAAS,IAAI,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC,CAkK/F;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,OAAO;IACJ,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE;gBAAd,IAAI,EAAE,MAAM,EAAE;IAEnC;;;;;OAKG;WACU,cAAc,CACvB,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;IAenB;;;;OAIG;WACU,IAAI,CACb,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;IAQnB;;;;OAIG;WACU,aAAa,CACtB,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;qBAII,KAAK,CACxB,OAAO,EAAE,aAAa,EAAE,EACxB,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;CAatB;AAED;;;;;;GAMG;AACH,wBAAuB,qBAAqB,CACxC,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE;IACL;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC1B,GACP,aAAa,CAAC,MAAM,CAAC,CAwFvB"}
|
package/internals/sitemap.js
CHANGED
|
@@ -2,11 +2,13 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
|
|
3
3
|
import { StringDecoder } from 'node:string_decoder';
|
|
4
4
|
import { createGunzip } from 'node:zlib';
|
|
5
|
-
import {
|
|
5
|
+
import { FetchHttpClient } from '@crawlee/http-client';
|
|
6
6
|
import { fileTypeStream } from 'file-type';
|
|
7
7
|
import sax from 'sax';
|
|
8
8
|
import MIMEType from 'whatwg-mimetype';
|
|
9
9
|
import log from '@apify/log';
|
|
10
|
+
import { mergeAsyncIterables } from './iterables.js';
|
|
11
|
+
import { RobotsFile } from './robots.js';
|
|
10
12
|
class SitemapTxtParser extends Transform {
|
|
11
13
|
decoder = new StringDecoder('utf8');
|
|
12
14
|
buffer = '';
|
|
@@ -123,7 +125,7 @@ class SitemapXmlParser extends Transform {
|
|
|
123
125
|
}
|
|
124
126
|
}
|
|
125
127
|
export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
126
|
-
const { httpClient = new
|
|
128
|
+
const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, } = options ?? {};
|
|
127
129
|
const sources = [...initialSources];
|
|
128
130
|
const visitedSitemapUrls = new Set();
|
|
129
131
|
const createParser = (contentType = '', url) => {
|
|
@@ -157,7 +159,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
157
159
|
try {
|
|
158
160
|
let sitemapResponse;
|
|
159
161
|
try {
|
|
160
|
-
sitemapResponse = await httpClient.
|
|
162
|
+
sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
|
|
161
163
|
method: 'GET',
|
|
162
164
|
headers: {
|
|
163
165
|
accept: 'text/plain, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8',
|
|
@@ -306,4 +308,84 @@ export class Sitemap {
|
|
|
306
308
|
return new Sitemap(urls);
|
|
307
309
|
}
|
|
308
310
|
}
|
|
311
|
+
/**
|
|
312
|
+
* Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
|
|
313
|
+
* the default `sitemap.xml` & `sitemap.txt` files and the URLs themselves.
|
|
314
|
+
* @param `urls` The list of URLs to discover sitemaps for.
|
|
315
|
+
* @param `options` Options for sitemap discovery
|
|
316
|
+
* @returns An async iterable with the discovered sitemap URLs.
|
|
317
|
+
*/
|
|
318
|
+
export async function* discoverValidSitemaps(urls, options = {}) {
|
|
319
|
+
const { proxyUrl, httpClient = new FetchHttpClient() } = options;
|
|
320
|
+
const sitemapUrls = new Set();
|
|
321
|
+
const addSitemapUrl = (url) => {
|
|
322
|
+
const sizeBefore = sitemapUrls.size;
|
|
323
|
+
sitemapUrls.add(url);
|
|
324
|
+
if (sitemapUrls.size > sizeBefore) {
|
|
325
|
+
return url;
|
|
326
|
+
}
|
|
327
|
+
return undefined;
|
|
328
|
+
};
|
|
329
|
+
const urlExists = async (url) => {
|
|
330
|
+
if (!httpClient) {
|
|
331
|
+
return false;
|
|
332
|
+
}
|
|
333
|
+
try {
|
|
334
|
+
const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), { proxyUrl });
|
|
335
|
+
return response.status >= 200 && response.status < 400;
|
|
336
|
+
}
|
|
337
|
+
catch {
|
|
338
|
+
return false;
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
const discoverSitemapsForDomainUrls = async function* (hostname, domainUrls) {
|
|
342
|
+
if (!hostname) {
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
try {
|
|
346
|
+
const robotsFile = await RobotsFile.find(domainUrls[0], { proxyUrl, httpClient });
|
|
347
|
+
for (const sitemapUrl of robotsFile.getSitemaps()) {
|
|
348
|
+
if (addSitemapUrl(sitemapUrl)) {
|
|
349
|
+
yield sitemapUrl;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
catch (err) {
|
|
354
|
+
log.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
|
|
355
|
+
}
|
|
356
|
+
const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
|
|
357
|
+
if (sitemapUrl !== undefined) {
|
|
358
|
+
if (addSitemapUrl(sitemapUrl)) {
|
|
359
|
+
yield sitemapUrl;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
else {
|
|
363
|
+
const firstUrl = new URL(domainUrls[0]);
|
|
364
|
+
const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
|
|
365
|
+
for (const pathname of possibleSitemapPathnames) {
|
|
366
|
+
firstUrl.pathname = pathname;
|
|
367
|
+
if (await urlExists(firstUrl.toString())) {
|
|
368
|
+
if (addSitemapUrl(firstUrl.toString())) {
|
|
369
|
+
yield firstUrl.toString();
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
const groupedUrls = urls.reduce((acc, url) => {
|
|
376
|
+
const hostname = new URL(url)?.hostname ?? '';
|
|
377
|
+
acc[hostname] ??= [];
|
|
378
|
+
acc[hostname].push(url);
|
|
379
|
+
return acc;
|
|
380
|
+
}, {});
|
|
381
|
+
const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
|
|
382
|
+
const discoveredUrls = new Set();
|
|
383
|
+
for await (const url of mergeAsyncIterables(...iterables)) {
|
|
384
|
+
if (discoveredUrls.has(url)) {
|
|
385
|
+
continue;
|
|
386
|
+
}
|
|
387
|
+
discoveredUrls.add(url);
|
|
388
|
+
yield url;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
309
391
|
//# sourceMappingURL=sitemap.js.map
|
package/internals/sitemap.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,QAAQ,MAAM,iBAAiB,CAAC;AAEvC,OAAO,GAAG,MAAM,YAAY,CAAC;AAqB7B,MAAM,gBAAiB,SAAQ,SAAS;IAC5B,OAAO,GAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,GAAG,EAAE,CAAC;IAEpB;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBACrD,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,aAAa,CAAC,KAAa,EAAE,QAAiB;QAClD,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAErB,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM;iBACpB,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEvC,IAAI,QAAQ,EAAE,CAAC;gBACX,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;YAChC,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAED,MAAM,gBAAiB,SAAQ,SAAS;IAC5B,OAAO,GAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAEjC,WAAW,CAA6B;IACxC,UAAU,GAAmD,SAAS,CAAC;IACvE,GAAG,GAAwB,EAAE,CAAC;IAEtC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAClB,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAgC;QAC9C,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACjC,IACI,IAAI,CAAC,IAAI,KAAK,KAAK;gBACnB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,IAAI,CAAC,IAAI,KAAK,YAAY,EAC5B,CAAC;gBACC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;YAChC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC,WAAW,GAAG,cAAc,CAAC;QACtC,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACvF,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAwB,CAAC,CAAC;YACjF,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,IAAY;QACvB,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,EAAwB,CAAC,CAAC;YAC9E,CAAC;YAED,IAAI,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC/B,CAAC;QACL,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvF,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAgC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AA8BD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,YAAY,CAC/B,cAA+B,EAC/B,QAAiB,EACjB,OAAW;IAEX,MAAM,EACF,UAAU,GAAG,IAAI,eAAe,EAAE,EAClC,kBAAkB,GAAG,KAAK,EAC1B,QAAQ,GAAG,QAAQ,EACnB,cAAc,GAAG,CAAC,EAClB,aAAa,EAAE,OAAO,GAAG,KAAK,EAC9B,mBAAmB,GAAG,IAAI,GAC7B,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,MAAM,OAAO,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC;IACpC,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,MAAM,YAAY,GAAG,CAAC,WAAW,GAAG,EAAE,EAAE,GAAS,EAAU,EAAE;QACzD,IAAI,QAAyB,CAAC;QAE9B,IAAI,CAAC;YACD,QAAQ,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACL,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,IAAI,QAAQ,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,QAAQ,EAAE,OAAO,KAAK,YAAY,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,WAAW,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IACjH,CAAC,CAAC;IAEF,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,EAAG,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,QAAQ,EAAE,CAAC;YAClC,GAAG,CAAC,KAAK,CACL,oBAAoB,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,iCAAiC,QAAQ,GAAG,CAC1G,CAAC;YACF,SAAS;QACb,CAAC;QAED,IAAI,KAAK,GAAsC,IAAI,CAAC;QAEpD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9C,IAAI,WAAW,GAAG,cAAc,GAAG,CAAC,CAAC;YAErC,OAAO,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACD,IAAI,eAAgC,CAAC;oBAErC,IAAI,CAAC;wBACD,eAAe,GAAG,MAAM,UAAU,CAAC,MAAM,CACrC,IAAI,OAAO,CAAC,UAAU,EAAE;4BACpB,MAAM,EAAE,KAAK;4BACb,OAAO,EAAE;gCACL,MAAM,EAAE,qEAAqE;6BAChF;yBACJ,CAAC,EACF;4BACI,QAAQ;4BACR,OAAO;yBACV,CACJ,CAAC;oBACN,CAAC;oBAAC,OAAO,KAAU,EAAE,CAAC;wBAClB,eAAe,GAAG,IAAI,CAAC;oBAC3B,CAAC;oBAED,IAAI,KAAK,GAAsD,IAAI,CAAC;oBAEpE,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;wBACnF,IAAI,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;wBAE9D,IAAI,eAAe,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;4BAChC,MAAM;wBACV,CAAC;wBACD,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,IAAW,CAAC,CAAC,CAAC;wBAC3F,IAAI,cAAc,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;4BACxC,WAAW,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC;wBAC/C,CAAC;wBAED,IAAI,SAAS,GAAG,KAAK,CAAC;wBAEtB,IACI,WAAW,KAAK,SAAS;4BACrB,CAAC,CAAC,WAAW,KAAK,kBAAkB;4BACpC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAC3C,CAAC;4BACC,SAAS,GAAG,IAAI,CAAC;4BAEjB,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gCACtC,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;4BAC3F,CAAC;wBACL,CAAC;wBAED,KAAK,GAAG,QAAQ,CACZ,cAAc,EACd,SAAS,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,EAC9C,YAAY,CAAC,WAAW,IAAI,SAAS,EAAE,UAAU,CAAC,EAClD,CAAC,CAAC,EAAE,EAAE;4BACF,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gCAChC,KAAK,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;4BACzC,CAAC;wBACL,CAAC,CACJ,CAAC;oBACN,CAAC;yBAAM,CAAC;wBACJ,KAAK,GAAG;4BACJ,IAAI,EAAE,OAAO;4BACb,KAAK,EAAE,IAAI,KAAK,CACZ,4BAA4B,UAAU,kBAAkB,eAAe,EAAE,MAAM,EAAE,CACpF;yBACJ,CAAC;oBACN,CAAC;oBAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;wBACjB,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,mBAAmB,CAAC;wBACzE,IAAI,CAAC,iBAAiB,EAAE,CAAC;4BACrB,MAAM,KAAK,CAAC,KAAK,CAAC;wBACtB,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACJ,MAAM;oBACV,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,GAAG,CAAC,OAAO,CACP,8BAA8B,UAAU,KAAK,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,GAAG,CAC/G,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC/B,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACtB,GAAG,CAAC,OAAO,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;gBACvD,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7E,IAAI,kBAAkB,EAAE,CAAC;oBACrB,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAS,CAAC;gBAC3D,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACtB,MAAM;oBACF,GAAG,IAAI;oBACP,gBAAgB,EACZ,MAAM,CAAC,IAAI,KAAK,KAAK;wBACjB,CAAC,CAAC,MAAM,CAAC,GAAG;wBACZ,CAAC,CAAC,SAAS,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;iBACpF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,OAAO;IACK;IAArB,YAAqB,IAAc;QAAd,SAAI,GAAJ,IAAI,CAAU;IAAG,CAAC;IAEvC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CACvB,GAAW,EACX,QAAiB,EACjB,mBAAyC;QAEzC,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC;QAEvB,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,mBAAmB,EAAE,CAAC,CAAC;IACvG,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,IAAuB,EACvB,QAAiB,EACjB,mBAAyC;QAEzC,OAAO,MAAM,IAAI,CAAC,KAAK,CACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAC1E,QAAQ,EACR,mBAAmB,CACtB,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CACtB,OAAe,EACf,QAAiB,EACjB,mBAAyC;QAEzC,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,mBAAmB,CAAC,CAAC;IACvF,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,KAAK,CACxB,OAAwB,EACxB,QAAiB,EACjB,mBAAyC;QAEzC,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,mBAAmB,CAAC,EAAE,CAAC;gBAC5E,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,QAAQ,MAAM,iBAAiB,CAAC;AAEvC,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAqBzC,MAAM,gBAAiB,SAAQ,SAAS;IAC5B,OAAO,GAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,GAAG,EAAE,CAAC;IAEpB;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBACrD,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,aAAa,CAAC,KAAa,EAAE,QAAiB;QAClD,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAErB,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM;iBACpB,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEvC,IAAI,QAAQ,EAAE,CAAC;gBACX,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACtB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACnC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAwB,CAAC,CAAC;gBAC/D,CAAC;gBAED,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;YAChC,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAED,MAAM,gBAAiB,SAAQ,SAAS;IAC5B,OAAO,GAAkB,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAEjC,WAAW,CAA6B;IACxC,UAAU,GAAmD,SAAS,CAAC;IACvE,GAAG,GAAwB,EAAE,CAAC;IAEtC;QACI,KAAK,CAAC;YACF,kBAAkB,EAAE,IAAI;YACxB,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7C,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAChB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;gBAClB,QAAQ,EAAE,CAAC;YACf,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7C,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAgC;QAC9C,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACjC,IACI,IAAI,CAAC,IAAI,KAAK,KAAK;gBACnB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,IAAI,CAAC,IAAI,KAAK,YAAY,EAC5B,CAAC;gBACC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;YAChC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC,WAAW,GAAG,cAAc,CAAC;QACtC,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,IAAY;QAC3B,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YACvF,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAwB,CAAC,CAAC;YACjF,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,MAAM,CAAC,IAAY;QACvB,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,EAAwB,CAAC,CAAC;YAC9E,CAAC;YAED,IAAI,IAAI,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC/B,CAAC;QACL,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvF,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAgC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AA8BD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,YAAY,CAC/B,cAA+B,EAC/B,QAAiB,EACjB,OAAW;IAEX,MAAM,EACF,UAAU,GAAG,IAAI,eAAe,EAAE,EAClC,kBAAkB,GAAG,KAAK,EAC1B,QAAQ,GAAG,QAAQ,EACnB,cAAc,GAAG,CAAC,EAClB,aAAa,EAAE,OAAO,GAAG,KAAK,EAC9B,mBAAmB,GAAG,IAAI,GAC7B,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,MAAM,OAAO,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC;IACpC,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,MAAM,YAAY,GAAG,CAAC,WAAW,GAAG,EAAE,EAAE,GAAS,EAAU,EAAE;QACzD,IAAI,QAAyB,CAAC;QAE9B,IAAI,CAAC;YACD,QAAQ,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACL,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,IAAI,QAAQ,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,QAAQ,EAAE,OAAO,KAAK,YAAY,IAAI,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,WAAW,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IACjH,CAAC,CAAC;IAEF,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,EAAG,CAAC;QAEhC,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,QAAQ,EAAE,CAAC;YAClC,GAAG,CAAC,KAAK,CACL,oBAAoB,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,iCAAiC,QAAQ,GAAG,CAC1G,CAAC;YACF,SAAS;QACb,CAAC;QAED,IAAI,KAAK,GAAsC,IAAI,CAAC;QAEpD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9C,IAAI,WAAW,GAAG,cAAc,GAAG,CAAC,CAAC;YAErC,OAAO,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACD,IAAI,eAAgC,CAAC;oBAErC,IAAI,CAAC;wBACD,eAAe,GAAG,MAAM,UAAU,CAAC,WAAW,CAC1C,IAAI,OAAO,CAAC,UAAU,EAAE;4BACpB,MAAM,EAAE,KAAK;4BACb,OAAO,EAAE;gCACL,MAAM,EAAE,qEAAqE;6BAChF;yBACJ,CAAC,EACF;4BACI,QAAQ;4BACR,OAAO;yBACV,CACJ,CAAC;oBACN,CAAC;oBAAC,OAAO,KAAU,EAAE,CAAC;wBAClB,eAAe,GAAG,IAAI,CAAC;oBAC3B,CAAC;oBAED,IAAI,KAAK,GAAsD,IAAI,CAAC;oBAEpE,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,IAAI,GAAG,IAAI,eAAe,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;wBACnF,IAAI,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;wBAE9D,IAAI,eAAe,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;4BAChC,MAAM;wBACV,CAAC;wBACD,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,IAAW,CAAC,CAAC,CAAC;wBAC3F,IAAI,cAAc,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;4BACxC,WAAW,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC;wBAC/C,CAAC;wBAED,IAAI,SAAS,GAAG,KAAK,CAAC;wBAEtB,IACI,WAAW,KAAK,SAAS;4BACrB,CAAC,CAAC,WAAW,KAAK,kBAAkB;4BACpC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAC3C,CAAC;4BACC,SAAS,GAAG,IAAI,CAAC;4BAEjB,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gCACtC,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;4BAC3F,CAAC;wBACL,CAAC;wBAED,KAAK,GAAG,QAAQ,CACZ,cAAc,EACd,SAAS,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,EAC9C,YAAY,CAAC,WAAW,IAAI,SAAS,EAAE,UAAU,CAAC,EAClD,CAAC,CAAC,EAAE,EAAE;4BACF,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gCAChC,KAAK,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;4BACzC,CAAC;wBACL,CAAC,CACJ,CAAC;oBACN,CAAC;yBAAM,CAAC;wBACJ,KAAK,GAAG;4BACJ,IAAI,EAAE,OAAO;4BACb,KAAK,EAAE,IAAI,KAAK,CACZ,4BAA4B,UAAU,kBAAkB,eAAe,EAAE,MAAM,EAAE,CACpF;yBACJ,CAAC;oBACN,CAAC;oBAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;wBACjB,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,mBAAmB,CAAC;wBACzE,IAAI,CAAC,iBAAiB,EAAE,CAAC;4BACrB,MAAM,KAAK,CAAC,KAAK,CAAC;wBACtB,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACJ,MAAM;oBACV,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,GAAG,CAAC,OAAO,CACP,8BAA8B,UAAU,KAAK,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,GAAG,CAC/G,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC/B,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACtB,GAAG,CAAC,OAAO,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;gBACvD,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7E,IAAI,kBAAkB,EAAE,CAAC;oBACrB,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAS,CAAC;gBAC3D,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBACtB,MAAM;oBACF,GAAG,IAAI;oBACP,gBAAgB,EACZ,MAAM,CAAC,IAAI,KAAK,KAAK;wBACjB,CAAC,CAAC,MAAM,CAAC,GAAG;wBACZ,CAAC,CAAC,SAAS,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;iBACpF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,OAAO;IACK;IAArB,YAAqB,IAAc;QAAd,SAAI,GAAJ,IAAI,CAAU;IAAG,CAAC;IAEvC;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CACvB,GAAW,EACX,QAAiB,EACjB,mBAAyC;QAEzC,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC;QAEvB,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,mBAAmB,EAAE,CAAC,CAAC;IACvG,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,IAAuB,EACvB,QAAiB,EACjB,mBAAyC;QAEzC,OAAO,MAAM,IAAI,CAAC,KAAK,CACnB,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAC1E,QAAQ,EACR,mBAAmB,CACtB,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,aAAa,CACtB,OAAe,EACf,QAAiB,EACjB,mBAAyC;QAEzC,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,mBAAmB,CAAC,CAAC;IACvF,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,KAAK,CACxB,OAAwB,EACxB,QAAiB,EACjB,mBAAyC;QAEzC,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,mBAAmB,CAAC,EAAE,CAAC;gBAC5E,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACJ;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,qBAAqB,CACxC,IAAc,EACd,UASI,EAAE;IAEN,MAAM,EAAE,QAAQ,EAAE,UAAU,GAAG,IAAI,eAAe,EAAE,EAAE,GAAG,OAAO,CAAC;IACjE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC,MAAM,aAAa,GAAG,CAAC,GAAW,EAAsB,EAAE;QACtD,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC;QAEpC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAErB,IAAI,WAAW,CAAC,IAAI,GAAG,UAAU,EAAE,CAAC;YAChC,OAAO,GAAG,CAAC;QACf,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,KAAK,EAAE,GAAW,EAAoB,EAAE;QACtD,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;YAClG,OAAO,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,6BAA6B,GAAG,KAAK,SAAS,CAAC,EAAE,QAAgB,EAAE,UAAoB;QACzF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC;YAElF,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC;gBAChD,IAAI,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC5B,MAAM,UAAU,CAAC;gBACrB,CAAC;YACL,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,OAAO,CAAC,uCAAuC,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iCAAiC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzF,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC5B,MAAM,UAAU,CAAC;YACrB,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YACxC,MAAM,wBAAwB,GAAG,CAAC,cAAc,EAAE,cAAc,EAAE,oBAAoB,CAAC,CAAC;YACxF,KAAK,MAAM,QAAQ,IAAI,wBAAwB,EAAE,CAAC;gBAC9C,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;gBAC7B,IAAI,MAAM,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;oBACvC,IAAI,aAAa,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;wBACrC,MAAM,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBAC9B,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAC3B,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,IAAI,EAAE,CAAC;QAC9C,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrB,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,GAAG,CAAC;IACf,CAAC,EACD,EAA8B,CACjC,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAE,CACzE,6BAA6B,CAAC,QAAQ,EAAE,UAAU,CAAC,CACtD,CAAC;IAEF,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAEzC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,mBAAmB,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC;QACxD,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,SAAS;QACb,CAAC;QACD,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,GAAG,CAAC;IACd,CAAC;AACL,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.23",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -43,8 +43,8 @@
|
|
|
43
43
|
"dependencies": {
|
|
44
44
|
"@apify/log": "^2.5.18",
|
|
45
45
|
"@apify/ps-tree": "^1.2.0",
|
|
46
|
-
"@crawlee/
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.23",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.23",
|
|
48
48
|
"@types/sax": "^1.2.7",
|
|
49
49
|
"cheerio": "^1.0.0",
|
|
50
50
|
"domhandler": "^5.0.3",
|
|
@@ -62,5 +62,5 @@
|
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
64
|
},
|
|
65
|
-
"gitHead": "
|
|
65
|
+
"gitHead": "929fd9374b74ee5e9720b3864b39326f014cd45b"
|
|
66
66
|
}
|