@crawlee/utils 3.10.6-beta.9 → 3.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.mjs +1 -0
- package/internals/cheerio.js +2 -3
- package/internals/cheerio.js.map +1 -1
- package/internals/chunk.js +1 -2
- package/internals/chunk.js.map +1 -1
- package/internals/debug.d.ts +0 -2
- package/internals/debug.d.ts.map +1 -1
- package/internals/debug.js +1 -2
- package/internals/debug.js.map +1 -1
- package/internals/extract-urls.d.ts +0 -2
- package/internals/extract-urls.d.ts.map +1 -1
- package/internals/extract-urls.js +3 -4
- package/internals/extract-urls.js.map +1 -1
- package/internals/general.js +6 -6
- package/internals/general.js.map +1 -1
- package/internals/gotScraping.d.ts.map +1 -1
- package/internals/memory-info.js +1 -2
- package/internals/memory-info.js.map +1 -1
- package/internals/open_graph_parser.js +1 -2
- package/internals/open_graph_parser.js.map +1 -1
- package/internals/sitemap.d.ts +27 -27
- package/internals/sitemap.d.ts.map +1 -1
- package/internals/sitemap.js +215 -155
- package/internals/sitemap.js.map +1 -1
- package/internals/social.js +6 -6
- package/internals/social.js.map +1 -1
- package/internals/typedefs.d.ts.map +1 -1
- package/internals/typedefs.js +2 -3
- package/internals/typedefs.js.map +1 -1
- package/package.json +4 -4
- package/tsconfig.build.tsbuildinfo +1 -1
package/index.mjs
CHANGED
|
@@ -21,6 +21,7 @@ export const htmlToText = mod.htmlToText;
|
|
|
21
21
|
export const isDocker = mod.isDocker;
|
|
22
22
|
export const keys = mod.keys;
|
|
23
23
|
export const parseOpenGraph = mod.parseOpenGraph;
|
|
24
|
+
export const parseSitemap = mod.parseSitemap;
|
|
24
25
|
export const sleep = mod.sleep;
|
|
25
26
|
export const snakeCaseToCamelCase = mod.snakeCaseToCamelCase;
|
|
26
27
|
export const social = mod.social;
|
package/internals/cheerio.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.htmlToText = htmlToText;
|
|
4
|
+
exports.extractUrlsFromCheerio = extractUrlsFromCheerio;
|
|
4
5
|
const tslib_1 = require("tslib");
|
|
5
6
|
const cheerio_1 = tslib_1.__importDefault(require("cheerio"));
|
|
6
7
|
const extract_urls_1 = require("./extract-urls");
|
|
@@ -84,7 +85,6 @@ function htmlToText(htmlOrCheerioElement) {
|
|
|
84
85
|
process($body.length > 0 ? $body : $.root());
|
|
85
86
|
return text.trim();
|
|
86
87
|
}
|
|
87
|
-
exports.htmlToText = htmlToText;
|
|
88
88
|
/**
|
|
89
89
|
* Extracts URLs from a given Cheerio object.
|
|
90
90
|
*
|
|
@@ -115,5 +115,4 @@ function extractUrlsFromCheerio($, selector = 'a', baseUrl = '') {
|
|
|
115
115
|
})
|
|
116
116
|
.filter(Boolean);
|
|
117
117
|
}
|
|
118
|
-
exports.extractUrlsFromCheerio = extractUrlsFromCheerio;
|
|
119
118
|
//# sourceMappingURL=cheerio.js.map
|
package/internals/cheerio.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":";;AAyCA,gCA2CC;AAWD,wDAwBC;;AArHD,8DAA8B;AAE9B,iDAAgD;AAIhD,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAClB,sGAAsG,CAAC;AAE3G;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,SAAgB,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,CAAC,GACH,OAAO,oBAAoB,KAAK,UAAU;QACtC,CAAC,CAAC,oBAAoB;QACtB,CAAC,CAAC,iBAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IACvE,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9E,IAAI,IAAI,KAAK,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,qCAAqC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;YACzD,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,sBAAsB,CAAC,CAAa,EAAE,WAAmB,GAAG,EAAE,UAAkB,EAAE;IAC9F,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,IAAI,IAAA,6BAAc,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,uCAAuC;gBAC5D,2DAA2D,CAClE,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,IAAA,6BAAc,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAa,CAAC;AACrC,CAAC"}
|
package/internals/chunk.js
CHANGED
|
@@ -26,7 +26,7 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
|
26
26
|
OTHER DEALINGS IN THE SOFTWARE.
|
|
27
27
|
*/
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.chunk =
|
|
29
|
+
exports.chunk = chunk;
|
|
30
30
|
function chunk(array, chunkSize) {
|
|
31
31
|
if (!Array.isArray(array))
|
|
32
32
|
throw new TypeError('entries must be an array.');
|
|
@@ -40,5 +40,4 @@ function chunk(array, chunkSize) {
|
|
|
40
40
|
chunks.push(clone.splice(0, chunkSize));
|
|
41
41
|
return chunks;
|
|
42
42
|
}
|
|
43
|
-
exports.chunk = chunk;
|
|
44
43
|
//# sourceMappingURL=chunk.js.map
|
package/internals/chunk.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;EAyBE
|
|
1
|
+
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;EAyBE;;AAEF,sBAQC;AARD,SAAgB,KAAK,CAAI,KAAmB,EAAE,SAAiB;IAC3D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,2BAA2B,CAAC,CAAC;IAC5E,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,+BAA+B,CAAC,CAAC;IACvF,IAAI,SAAS,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAQ,KAAK,CAAC,KAAK,EAAE,CAAC;IACjC,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,OAAO,KAAK,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7D,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
package/internals/debug.d.ts
CHANGED
package/internals/debug.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"debug.d.ts","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"debug.d.ts","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAGrE,UAAU,mBAAmB;IACzB,MAAM,IAAI,MAAM,CAAC;CACpB;AAED,UAAU,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACtD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAClC,OAAO,EAAE,OAAO,EAChB,QAAQ,GAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAM,EAC7D,gBAAgB,GAAE,UAAe,GAClC,UAAU,CAmBZ"}
|
package/internals/debug.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.createRequestDebugInfo =
|
|
3
|
+
exports.createRequestDebugInfo = createRequestDebugInfo;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
6
|
/**
|
|
@@ -30,5 +30,4 @@ function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
|
|
|
30
30
|
...additionalFields,
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
|
-
exports.createRequestDebugInfo = createRequestDebugInfo;
|
|
34
33
|
//# sourceMappingURL=debug.js.map
|
package/internals/debug.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"debug.js","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"debug.js","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":";;AAiCA,wDAuBC;;AArDD,oDAAoB;AAqBpB;;;;;;;;GAQG;AACH,SAAgB,sBAAsB,CAClC,OAAgB,EAChB,WAA2D,EAAE,EAC7D,mBAA+B,EAAE;IAEjC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACvB,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACxB,IAAA,YAAE,EAAC,gBAAgB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO;QACH,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,sFAAsF;QACtF,UAAU,EACN,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,YAAY,QAAQ;YACvD,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE;YACnB,CAAC,CAAE,QAA4B,CAAC,UAAU;QAClD,GAAG,gBAAgB;KACtB,CAAC;AACN,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAuB9F;AAED,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,EAAE,CAiBjE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMhF"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.downloadListOfUrls = downloadListOfUrls;
|
|
4
|
+
exports.extractUrls = extractUrls;
|
|
5
|
+
exports.tryAbsoluteURL = tryAbsoluteURL;
|
|
4
6
|
const tslib_1 = require("tslib");
|
|
5
7
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
8
|
const general_1 = require("./general");
|
|
@@ -26,7 +28,6 @@ async function downloadListOfUrls(options) {
|
|
|
26
28
|
const { body: string } = await (0, gotScraping_1.gotScraping)({ url: fixedUrl, encoding, proxyUrl });
|
|
27
29
|
return extractUrls({ string, urlRegExp });
|
|
28
30
|
}
|
|
29
|
-
exports.downloadListOfUrls = downloadListOfUrls;
|
|
30
31
|
/**
|
|
31
32
|
* Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
|
|
32
33
|
*/
|
|
@@ -43,7 +44,6 @@ function extractUrls(options) {
|
|
|
43
44
|
}
|
|
44
45
|
return result;
|
|
45
46
|
}
|
|
46
|
-
exports.extractUrls = extractUrls;
|
|
47
47
|
/**
|
|
48
48
|
* Helper function used to validate URLs used when extracting URLs from a page
|
|
49
49
|
*/
|
|
@@ -55,5 +55,4 @@ function tryAbsoluteURL(href, baseUrl) {
|
|
|
55
55
|
return undefined;
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
|
-
exports.tryAbsoluteURL = tryAbsoluteURL;
|
|
59
58
|
//# sourceMappingURL=extract-urls.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":";;AAgCA,gDAuBC;AAkBD,kCAiBC;AAKD,wCAMC;;AArGD,oDAAoB;AAEpB,uCAAgD;AAChD,+CAA4C;AAyB5C;;;GAGG;AACI,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACvE,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B,CAAC,CACL,CAAC;IACF,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,EAAE,SAAS,GAAG,6BAAmB,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAEtF,0GAA0G;IAC1G,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACzF,IAAI,QAAQ,GAAG,GAAG,CAAC;IAEnB,IAAI,KAAK,EAAE,CAAC;QACR,QAAQ,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjD,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,yBAAW,EAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IAElF,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC;AAeD;;GAEG;AACH,SAAgB,WAAW,CAAC,OAA2B;IACnD,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,MAAM,EAAE,YAAE,CAAC,MAAM;QACjB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CACL,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,6BAAmB,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,IAAY,EAAE,OAAe;IACxD,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC"}
|
package/internals/general.js
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.URL_WITH_COMMAS_REGEX = exports.URL_NO_COMMAS_REGEX = void 0;
|
|
4
|
+
exports.isDocker = isDocker;
|
|
5
|
+
exports.weightedAvg = weightedAvg;
|
|
6
|
+
exports.sleep = sleep;
|
|
7
|
+
exports.snakeCaseToCamelCase = snakeCaseToCamelCase;
|
|
8
|
+
exports.expandShadowRoots = expandShadowRoots;
|
|
4
9
|
const tslib_1 = require("tslib");
|
|
5
10
|
const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
|
|
6
11
|
const promises_2 = require("node:timers/promises");
|
|
@@ -36,7 +41,6 @@ async function isDocker(forceReset) {
|
|
|
36
41
|
isDockerPromiseCache = createIsDockerPromise();
|
|
37
42
|
return isDockerPromiseCache;
|
|
38
43
|
}
|
|
39
|
-
exports.isDocker = isDocker;
|
|
40
44
|
/**
|
|
41
45
|
* Computes a weighted average of an array of numbers, complemented by an array of weights.
|
|
42
46
|
* @ignore
|
|
@@ -51,7 +55,6 @@ function weightedAvg(arrValues, arrWeights) {
|
|
|
51
55
|
.reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
|
|
52
56
|
return result[0] / result[1];
|
|
53
57
|
}
|
|
54
|
-
exports.weightedAvg = weightedAvg;
|
|
55
58
|
/**
|
|
56
59
|
* Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
|
|
57
60
|
* in your code, e.g. to prevent overloading of target website or to avoid bot detection.
|
|
@@ -71,7 +74,6 @@ exports.weightedAvg = weightedAvg;
|
|
|
71
74
|
async function sleep(millis) {
|
|
72
75
|
return (0, promises_2.setTimeout)(millis);
|
|
73
76
|
}
|
|
74
|
-
exports.sleep = sleep;
|
|
75
77
|
/**
|
|
76
78
|
* Converts SNAKE_CASE to camelCase.
|
|
77
79
|
* @ignore
|
|
@@ -85,7 +87,6 @@ function snakeCaseToCamelCase(snakeCaseStr) {
|
|
|
85
87
|
})
|
|
86
88
|
.join('');
|
|
87
89
|
}
|
|
88
|
-
exports.snakeCaseToCamelCase = snakeCaseToCamelCase;
|
|
89
90
|
/**
|
|
90
91
|
* Traverses DOM and expands shadow-root elements (created by custom components).
|
|
91
92
|
* @ignore
|
|
@@ -111,5 +112,4 @@ function expandShadowRoots(document) {
|
|
|
111
112
|
replaceShadowDomsWithHtml(document.body);
|
|
112
113
|
return document.documentElement.outerHTML;
|
|
113
114
|
}
|
|
114
|
-
exports.expandShadowRoots = expandShadowRoots;
|
|
115
115
|
//# sourceMappingURL=general.js.map
|
package/internals/general.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":";;;AAsCA,4BAKC;AAMD,kCAWC;AAkBD,sBAEC;AAMD,oDAQC;AAMD,8CAyBC;;AA7HD,wEAAkC;AAClC,mDAAkD;AAElD;;;GAGG;AACU,QAAA,mBAAmB,GAC5B,2IAA2I,CAAC;AAEhJ;;;GAGG;AACU,QAAA,qBAAqB,GAC9B,4IAA4I,CAAC;AAEjJ,IAAI,oBAAkD,CAAC;AAEvD,KAAK,UAAU,qBAAqB;IAChC,MAAM,QAAQ,GAAG,kBAAE;SACd,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,QAAQ,GAAG,kBAAE;SACd,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEnE,OAAO,OAAO,IAAI,OAAO,CAAC;AAC9B,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,QAAQ,CAAC,UAAoB;IAC/C,wDAAwD;IACxD,IAAI,CAAC,oBAAoB,IAAI,UAAU;QAAE,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;IAExF,OAAO,oBAAoB,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,SAAgB,WAAW,CAAC,SAAmB,EAAE,UAAoB;IACjE,MAAM,MAAM,GAAG,SAAS;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACd,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1D,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACI,KAAK,UAAU,KAAK,CAAC,MAAe;IACvC,OAAO,IAAA,qBAAU,EAAC,MAAM,CAAC,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,YAAoB;IACrD,OAAO,YAAY;SACd,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACjB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,SAAgB,iBAAiB,CAAC,QAAkB;IAChD,oCAAoC;IACpC,SAAS,gBAAgB,CAAC,UAAe;QACrC,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACrC,UAAU,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC;QACrD,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,oDAAoD;IACpD,SAAS,yBAAyB,CAAC,WAAgB;QAC/C,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;gBAChB,yBAAyB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;gBACzC,EAAE,CAAC,SAAS,IAAI,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YAC1D,CAAC;QACL,CAAC;IACL,CAAC;IAED,yBAAyB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEzC,OAAO,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;AAC9C,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gotScraping.d.ts","sourceRoot":"","sources":["../../src/internals/gotScraping.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAGhD,QAAA,IAAI,WAAW,
|
|
1
|
+
{"version":3,"file":"gotScraping.d.ts","sourceRoot":"","sources":["../../src/internals/gotScraping.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAGhD,QAAA,IAAI,WAAW,EAIT,WAAW,CAAC;AAElB,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
package/internals/memory-info.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getMemoryInfo =
|
|
3
|
+
exports.getMemoryInfo = getMemoryInfo;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const node_child_process_1 = require("node:child_process");
|
|
6
6
|
const promises_1 = require("node:fs/promises");
|
|
@@ -132,5 +132,4 @@ async function getMemoryInfo() {
|
|
|
132
132
|
childProcessesBytes,
|
|
133
133
|
};
|
|
134
134
|
}
|
|
135
|
-
exports.getMemoryInfo = getMemoryInfo;
|
|
136
135
|
//# sourceMappingURL=memory-info.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-info.js","sourceRoot":"","sources":["../../src/internals/memory-info.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"memory-info.js","sourceRoot":"","sources":["../../src/internals/memory-info.ts"],"names":[],"mappings":";;AAoDA,sCA+GC;;AAnKD,2DAA8C;AAC9C,+CAAoD;AACpD,qCAA4C;AAC5C,kEAA6B;AAE7B,6DAA6B;AAC7B,6DAA6D;AAC7D,qEAAoC;AAGpC,uCAAqC;AAErC,MAAM,iBAAiB,GAAG;IACtB,KAAK,EAAE;QACH,EAAE,EAAE,6CAA6C;QACjD,EAAE,EAAE,2BAA2B;KAClC;IACD,IAAI,EAAE;QACF,EAAE,EAAE,6CAA6C;QACjD,EAAE,EAAE,+BAA+B;KACtC;CACJ,CAAC;AAsBF;;;;;;;;GAQG;AACI,KAAK,UAAU,aAAa;IAC/B,MAAM,cAAc,GAAG,mBAAI,CAAC,SAAS,CAAC,iBAAM,CAAC,CAAC;IAE9C,2DAA2D;IAC3D,oCAAoC;IACpC,MAAM,mBAAmB,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;IAE1G,MAAM,WAAW,GAAG,CAAC,mBAAmB,IAAI,CAAC,MAAM,IAAA,kBAAQ,GAAE,CAAC,CAAC;IAE/D,IAAI,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAC1B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAE5B,IAAI,mBAAmB,EAAE,CAAC;QACtB,oBAAoB;QACpB,gBAAgB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC;QAE7C,8CAA8C;QAC9C,MAAM,OAAO,GAAG,IAAA,6BAAQ,EAAC,mBAAmB,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACnE,uFAAuF;QACvF,qGAAqG;QACrG,mBAAmB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,GAAG,gBAAgB,CAAC;IAChE,CAAC;SAAM,CAAC;QACJ,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE1D,SAAS,CAAC,OAAO,CAAC,CAAC,GAAuB,EAAE,EAAE;YAC1C,0EAA0E;YAC1E,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,IAAI,GAAG,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;gBACrD,OAAO;YACX,CAAC;YACD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACpC,yCAAyC;YACzC,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC/B,gBAAgB,GAAG,KAAK,CAAC;gBACzB,OAAO;YACX,CAAC;YACD,mBAAmB,IAAI,KAAK,CAAC;QACjC,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,SAAiB,CAAC;IACtB,IAAI,SAAiB,CAAC;IAEtB,IAAI,mBAAmB,EAAE,CAAC;QACtB,sCAAsC;QACtC,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,+BAAgC,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC;QAClF,SAAS,GAAG,gBAAgB,GAAG,mBAAmB,CAAC;QACnD,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QAEnC,aAAG,CAAC,KAAK,CAAC,kBAAkB,UAAU,SAAS,SAAS,aAAa,CAAC,CAAC;IAC3E,CAAC;SAAM,IAAI,WAAW,EAAE,CAAC;QACrB,oEAAoE;QAEpE,yCAAyC;QACzC,IAAI,cAAc,GAAyC,IAAI,CAAC;QAChE,IAAI,CAAC;YACD,uEAAuE;YACvE,MAAM,IAAA,iBAAM,EAAC,wBAAwB,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACL,cAAc,GAAG,IAAI,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC;YACD,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBAClD,IAAA,mBAAQ,EAAC,iBAAiB,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;gBACzD,IAAA,mBAAQ,EAAC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;aAC3D,CAAC,CAAC;YAEH,gHAAgH;YAChH,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAC5D,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE1D,4DAA4D;YAC5D,oIAAoI;YACpI,IAAI,aAAa,KAAK,KAAK,EAAE,CAAC;gBAC1B,UAAU,GAAG,IAAA,kBAAQ,GAAE,CAAC;gBACxB,yFAAyF;gBACzF,8CAA8C;YAClD,CAAC;iBAAM,CAAC;gBACJ,UAAU,GAAG,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;gBACzC,MAAM,gCAAgC,GAAG,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC;gBAC9E,IAAI,gCAAgC;oBAAE,UAAU,GAAG,IAAA,kBAAQ,GAAE,CAAC;YAClE,CAAC;YACD,SAAS,GAAG,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACvC,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QACvC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,0CAA0C;YAC1C,aAAG,CAAC,UAAU,CACV,+EAA+E;gBAC3E,mGAAmG;gBACnG,UAAW,GAAa,CAAC,OAAO,EAAE,CACzC,CAAC;YACF,UAAU,GAAG,IAAA,kBAAQ,GAAE,CAAC;YACxB,SAAS,GAAG,IAAA,iBAAO,GAAE,CAAC;YACtB,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QACvC,CAAC;IACL,CAAC;SAAM,CAAC;QACJ,UAAU,GAAG,IAAA,kBAAQ,GAAE,CAAC;QACxB,SAAS,GAAG,IAAA,iBAAO,GAAE,CAAC;QACtB,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;IACvC,CAAC;IAED,OAAO;QACH,UAAU;QACV,SAAS;QACT,SAAS;QACT,gBAAgB;QAChB,mBAAmB;KACtB,CAAC;AACN,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.parseOpenGraph =
|
|
3
|
+
exports.parseOpenGraph = parseOpenGraph;
|
|
4
4
|
const cheerio_1 = require("cheerio");
|
|
5
5
|
/**
|
|
6
6
|
* To be used with the spread operator. Ensures that the item is defined, and is not empty.
|
|
@@ -378,5 +378,4 @@ function parseOpenGraph(item, additionalProperties) {
|
|
|
378
378
|
};
|
|
379
379
|
}, {});
|
|
380
380
|
}
|
|
381
|
-
exports.parseOpenGraph = parseOpenGraph;
|
|
382
381
|
//# sourceMappingURL=open_graph_parser.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"open_graph_parser.js","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"open_graph_parser.js","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":";;AA+YA,wCAYC;AAzZD,qCAA+B;AAU/B;;;;;;GAMG;AACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE,CAC9C,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAE/E,MAAM,qBAAqB,GAAwB;IAC/C;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,MAAM;QAClB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,eAAe;QACrB,UAAU,EAAE,YAAY;QACxB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,WAAW;QACjB,UAAU,EAAE,QAAQ;QACpB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD,kDAAkD;IAClD,iHAAiH;IACjH,mDAAmD;IACnD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,mBAAmB;wBACzB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,YAAY;gBAClB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,iBAAiB;wBACvB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,sBAAsB;gBAC5B,UAAU,EAAE,eAAe;gBAC3B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,cAAc;gBAC1B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,uBAAuB;gBAC7B,UAAU,EAAE,gBAAgB;gBAC5B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,UAAU;gBAChB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,kBAAkB;gBACxB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;CACJ,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,kBAAkB,IAAI,IAAI,CAAC;AAE3E,MAAM,sBAAsB,GAAG,CAAC,QAA2B,EAAE,CAAa,EAAuC,EAAE;IAC/G,6GAA6G;IAC7G,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnG,4FAA4F;IAC5F,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAE,MAAM,CAAC,CAAC,CAAY,CAAC,CAAC,CAAE,MAAmB,CAAC;IAElF,sEAAsE;IACtE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IAE9C,wGAAwG;IACxG,OAAO;QACH,qGAAqG;QACrG,+EAA+E;QAC/E,wGAAwG;QACxG,uDAAuD;QACvD,GAAG,cAAc,CAAC,GAAG,QAAQ,CAAC,UAAU,OAAO,EAAE,OAAO,CAAC;QACzD,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CACvB,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YACV,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/C,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,GAAG,CAAC;YAErC,OAAO;gBACH,GAAG,GAAG;gBACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;aACtE,CAAC;QACN,CAAC,EACD,EAAqC,CACxC;KACJ,CAAC;AACN,CAAC,CAAC;AAYF,SAAgB,cAAc,CAAC,IAAyB,EAAE,oBAA0C;IAChG,MAAM,CAAC,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAI,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvD,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,EAAE,GAAG,qBAAqB,CAAC,CAAC,MAAM,CACrE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QACV,OAAO;YACH,GAAG,GAAG;YACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;SACtE,CAAC;IACN,CAAC,EACD,EAAiC,CACpC,CAAC;AACN,CAAC"}
|
package/internals/sitemap.d.ts
CHANGED
|
@@ -1,34 +1,36 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
type
|
|
1
|
+
interface SitemapUrlData {
|
|
2
|
+
loc: string;
|
|
3
|
+
lastmod?: Date;
|
|
4
|
+
changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
|
|
5
|
+
priority?: number;
|
|
6
|
+
}
|
|
7
|
+
export type SitemapUrl = SitemapUrlData & {
|
|
8
|
+
originSitemapUrl: string;
|
|
9
|
+
};
|
|
10
|
+
interface NestedSitemap {
|
|
11
|
+
loc: string;
|
|
12
|
+
originSitemapUrl: null;
|
|
13
|
+
}
|
|
14
|
+
type SitemapSource = ({
|
|
8
15
|
type: 'url';
|
|
9
16
|
url: string;
|
|
10
17
|
} | {
|
|
11
18
|
type: 'raw';
|
|
12
19
|
content: string;
|
|
20
|
+
}) & {
|
|
21
|
+
depth?: number;
|
|
13
22
|
};
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
private parsingState;
|
|
24
|
-
private onEnd;
|
|
25
|
-
private decoder;
|
|
26
|
-
private buffer;
|
|
27
|
-
constructor(parsingState: ParsingState, onEnd: () => void);
|
|
28
|
-
private processBuffer;
|
|
29
|
-
_write(chunk: any, _encoding: BufferEncoding, callback: (error?: Error | null | undefined) => void): void;
|
|
30
|
-
_final(callback: (error?: Error | null | undefined) => void): void;
|
|
23
|
+
interface ParseSitemapOptions {
|
|
24
|
+
/**
|
|
25
|
+
* If set to `true`, elements reffering to other sitemaps will be emitted as special objects with a `bouba` property.
|
|
26
|
+
*/
|
|
27
|
+
emitNestedSitemaps?: true | false;
|
|
28
|
+
/**
|
|
29
|
+
* Maximum depth of nested sitemaps to follow.
|
|
30
|
+
*/
|
|
31
|
+
maxDepth?: number;
|
|
31
32
|
}
|
|
33
|
+
export declare function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
|
|
32
34
|
/**
|
|
33
35
|
* Loads one or more sitemaps from given URLs, following references in sitemap index files, and exposes the contained URLs.
|
|
34
36
|
*
|
|
@@ -44,7 +46,6 @@ declare class SitemapTxtParser extends Writable {
|
|
|
44
46
|
export declare class Sitemap {
|
|
45
47
|
readonly urls: string[];
|
|
46
48
|
constructor(urls: string[]);
|
|
47
|
-
protected static createXmlParser(parsingState: ParsingState, onEnd: () => void, onError: (error: Error) => void): SAXStream;
|
|
48
49
|
/**
|
|
49
50
|
* Try to load sitemap from the most common locations - `/sitemap.xml` and `/sitemap.txt`.
|
|
50
51
|
* For loading based on `Sitemap` entries in `robots.txt`, the {@apilink RobotsFile} class should be used.
|
|
@@ -64,8 +65,7 @@ export declare class Sitemap {
|
|
|
64
65
|
* @param proxyUrl URL of a proxy to be used for fetching sitemap contents
|
|
65
66
|
*/
|
|
66
67
|
static fromXmlString(content: string, proxyUrl?: string): Promise<Sitemap>;
|
|
67
|
-
protected static parse(
|
|
68
|
-
protected static createParser(resolve: (value: unknown) => void, reject: (value: unknown) => void, parsingState: ParsingState, contentType?: string, url?: URL): SitemapTxtParser | SAXStream;
|
|
68
|
+
protected static parse(sources: SitemapSource[], proxyUrl?: string): Promise<Sitemap>;
|
|
69
69
|
}
|
|
70
70
|
export {};
|
|
71
71
|
//# sourceMappingURL=sitemap.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAUA,UAAU,cAAc;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,IAAI,CAAC;IACf,UAAU,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG;IACtC,gBAAgB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,UAAU,aAAa;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,IAAI,CAAC;CAC1B;AAED,KAAK,aAAa,GAAG,CAAC;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA8I5G,UAAU,mBAAmB;IACzB;;OAEG;IACH,kBAAkB,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC;IAClC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wBAAuB,YAAY,CAAC,CAAC,SAAS,mBAAmB,EAC7D,cAAc,EAAE,aAAa,EAAE,EAC/B,QAAQ,CAAC,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,CAAC,GACZ,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,SAAS,IAAI,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC,CAsH/F;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,OAAO;IACJ,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE;gBAAd,IAAI,EAAE,MAAM,EAAE;IAEnC;;;;;OAKG;WACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAe7E;;;;OAIG;WACU,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO/E;;;;OAIG;WACU,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;qBAIzD,KAAK,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAa9F"}
|