@crawlee/utils 3.10.6-beta.11 → 3.10.6-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/cheerio.js +2 -3
- package/internals/cheerio.js.map +1 -1
- package/internals/chunk.js +1 -2
- package/internals/chunk.js.map +1 -1
- package/internals/debug.d.ts +0 -2
- package/internals/debug.d.ts.map +1 -1
- package/internals/debug.js +1 -2
- package/internals/debug.js.map +1 -1
- package/internals/extract-urls.d.ts +0 -2
- package/internals/extract-urls.d.ts.map +1 -1
- package/internals/extract-urls.js +3 -4
- package/internals/extract-urls.js.map +1 -1
- package/internals/general.js +6 -6
- package/internals/general.js.map +1 -1
- package/internals/gotScraping.d.ts.map +1 -1
- package/internals/memory-info.js +1 -2
- package/internals/memory-info.js.map +1 -1
- package/internals/open_graph_parser.js +1 -2
- package/internals/open_graph_parser.js.map +1 -1
- package/internals/sitemap.d.ts +0 -4
- package/internals/sitemap.d.ts.map +1 -1
- package/internals/social.js +6 -6
- package/internals/social.js.map +1 -1
- package/internals/typedefs.d.ts.map +1 -1
- package/internals/typedefs.js +2 -3
- package/internals/typedefs.js.map +1 -1
- package/package.json +3 -3
- package/tsconfig.build.tsbuildinfo +1 -1
package/internals/cheerio.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.htmlToText = htmlToText;
|
|
4
|
+
exports.extractUrlsFromCheerio = extractUrlsFromCheerio;
|
|
4
5
|
const tslib_1 = require("tslib");
|
|
5
6
|
const cheerio_1 = tslib_1.__importDefault(require("cheerio"));
|
|
6
7
|
const extract_urls_1 = require("./extract-urls");
|
|
@@ -84,7 +85,6 @@ function htmlToText(htmlOrCheerioElement) {
|
|
|
84
85
|
process($body.length > 0 ? $body : $.root());
|
|
85
86
|
return text.trim();
|
|
86
87
|
}
|
|
87
|
-
exports.htmlToText = htmlToText;
|
|
88
88
|
/**
|
|
89
89
|
* Extracts URLs from a given Cheerio object.
|
|
90
90
|
*
|
|
@@ -115,5 +115,4 @@ function extractUrlsFromCheerio($, selector = 'a', baseUrl = '') {
|
|
|
115
115
|
})
|
|
116
116
|
.filter(Boolean);
|
|
117
117
|
}
|
|
118
|
-
exports.extractUrlsFromCheerio = extractUrlsFromCheerio;
|
|
119
118
|
//# sourceMappingURL=cheerio.js.map
|
package/internals/cheerio.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":";;AAyCA,gCA2CC;AAWD,wDAwBC;;AArHD,8DAA8B;AAE9B,iDAAgD;AAIhD,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAClB,sGAAsG,CAAC;AAE3G;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,SAAgB,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,CAAC,GACH,OAAO,oBAAoB,KAAK,UAAU;QACtC,CAAC,CAAC,oBAAoB;QACtB,CAAC,CAAC,iBAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IACvE,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9E,IAAI,IAAI,KAAK,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,qCAAqC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;YACzD,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,sBAAsB,CAAC,CAAa,EAAE,WAAmB,GAAG,EAAE,UAAkB,EAAE;IAC9F,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,IAAI,IAAA,6BAAc,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,uCAAuC;gBAC5D,2DAA2D,CAClE,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,IAAA,6BAAc,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAa,CAAC;AACrC,CAAC"}
|
package/internals/chunk.js
CHANGED
|
@@ -26,7 +26,7 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
|
26
26
|
OTHER DEALINGS IN THE SOFTWARE.
|
|
27
27
|
*/
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.chunk =
|
|
29
|
+
exports.chunk = chunk;
|
|
30
30
|
function chunk(array, chunkSize) {
|
|
31
31
|
if (!Array.isArray(array))
|
|
32
32
|
throw new TypeError('entries must be an array.');
|
|
@@ -40,5 +40,4 @@ function chunk(array, chunkSize) {
|
|
|
40
40
|
chunks.push(clone.splice(0, chunkSize));
|
|
41
41
|
return chunks;
|
|
42
42
|
}
|
|
43
|
-
exports.chunk = chunk;
|
|
44
43
|
//# sourceMappingURL=chunk.js.map
|
package/internals/chunk.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;EAyBE
|
|
1
|
+
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;EAyBE;;AAEF,sBAQC;AARD,SAAgB,KAAK,CAAI,KAAmB,EAAE,SAAiB;IAC3D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,2BAA2B,CAAC,CAAC;IAC5E,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,+BAA+B,CAAC,CAAC;IACvF,IAAI,SAAS,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAQ,KAAK,CAAC,KAAK,EAAE,CAAC;IACjC,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,OAAO,KAAK,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7D,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
package/internals/debug.d.ts
CHANGED
package/internals/debug.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"debug.d.ts","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"debug.d.ts","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAGrE,UAAU,mBAAmB;IACzB,MAAM,IAAI,MAAM,CAAC;CACpB;AAED,UAAU,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACtD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAClC,OAAO,EAAE,OAAO,EAChB,QAAQ,GAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAM,EAC7D,gBAAgB,GAAE,UAAe,GAClC,UAAU,CAmBZ"}
|
package/internals/debug.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.createRequestDebugInfo =
|
|
3
|
+
exports.createRequestDebugInfo = createRequestDebugInfo;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
6
|
/**
|
|
@@ -30,5 +30,4 @@ function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
|
|
|
30
30
|
...additionalFields,
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
|
-
exports.createRequestDebugInfo = createRequestDebugInfo;
|
|
34
33
|
//# sourceMappingURL=debug.js.map
|
package/internals/debug.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"debug.js","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"debug.js","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":";;AAiCA,wDAuBC;;AArDD,oDAAoB;AAqBpB;;;;;;;;GAQG;AACH,SAAgB,sBAAsB,CAClC,OAAgB,EAChB,WAA2D,EAAE,EAC7D,mBAA+B,EAAE;IAEjC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACvB,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACxB,IAAA,YAAE,EAAC,gBAAgB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO;QACH,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,sFAAsF;QACtF,UAAU,EACN,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,YAAY,QAAQ;YACvD,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE;YACnB,CAAC,CAAE,QAA4B,CAAC,UAAU;QAClD,GAAG,gBAAgB;KACtB,CAAC;AACN,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAuB9F;AAED,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,EAAE,CAiBjE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMhF"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.downloadListOfUrls = downloadListOfUrls;
|
|
4
|
+
exports.extractUrls = extractUrls;
|
|
5
|
+
exports.tryAbsoluteURL = tryAbsoluteURL;
|
|
4
6
|
const tslib_1 = require("tslib");
|
|
5
7
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
8
|
const general_1 = require("./general");
|
|
@@ -26,7 +28,6 @@ async function downloadListOfUrls(options) {
|
|
|
26
28
|
const { body: string } = await (0, gotScraping_1.gotScraping)({ url: fixedUrl, encoding, proxyUrl });
|
|
27
29
|
return extractUrls({ string, urlRegExp });
|
|
28
30
|
}
|
|
29
|
-
exports.downloadListOfUrls = downloadListOfUrls;
|
|
30
31
|
/**
|
|
31
32
|
* Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
|
|
32
33
|
*/
|
|
@@ -43,7 +44,6 @@ function extractUrls(options) {
|
|
|
43
44
|
}
|
|
44
45
|
return result;
|
|
45
46
|
}
|
|
46
|
-
exports.extractUrls = extractUrls;
|
|
47
47
|
/**
|
|
48
48
|
* Helper function used to validate URLs used when extracting URLs from a page
|
|
49
49
|
*/
|
|
@@ -55,5 +55,4 @@ function tryAbsoluteURL(href, baseUrl) {
|
|
|
55
55
|
return undefined;
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
|
-
exports.tryAbsoluteURL = tryAbsoluteURL;
|
|
59
58
|
//# sourceMappingURL=extract-urls.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":";;AAgCA,gDAuBC;AAkBD,kCAiBC;AAKD,wCAMC;;AArGD,oDAAoB;AAEpB,uCAAgD;AAChD,+CAA4C;AAyB5C;;;GAGG;AACI,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACvE,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B,CAAC,CACL,CAAC;IACF,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,EAAE,SAAS,GAAG,6BAAmB,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAEtF,0GAA0G;IAC1G,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACzF,IAAI,QAAQ,GAAG,GAAG,CAAC;IAEnB,IAAI,KAAK,EAAE,CAAC;QACR,QAAQ,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjD,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,yBAAW,EAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IAElF,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC;AAeD;;GAEG;AACH,SAAgB,WAAW,CAAC,OAA2B;IACnD,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,MAAM,EAAE,YAAE,CAAC,MAAM;QACjB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CACL,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,6BAAmB,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,IAAY,EAAE,OAAe;IACxD,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC"}
|
package/internals/general.js
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.URL_WITH_COMMAS_REGEX = exports.URL_NO_COMMAS_REGEX = void 0;
|
|
4
|
+
exports.isDocker = isDocker;
|
|
5
|
+
exports.weightedAvg = weightedAvg;
|
|
6
|
+
exports.sleep = sleep;
|
|
7
|
+
exports.snakeCaseToCamelCase = snakeCaseToCamelCase;
|
|
8
|
+
exports.expandShadowRoots = expandShadowRoots;
|
|
4
9
|
const tslib_1 = require("tslib");
|
|
5
10
|
const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
|
|
6
11
|
const promises_2 = require("node:timers/promises");
|
|
@@ -36,7 +41,6 @@ async function isDocker(forceReset) {
|
|
|
36
41
|
isDockerPromiseCache = createIsDockerPromise();
|
|
37
42
|
return isDockerPromiseCache;
|
|
38
43
|
}
|
|
39
|
-
exports.isDocker = isDocker;
|
|
40
44
|
/**
|
|
41
45
|
* Computes a weighted average of an array of numbers, complemented by an array of weights.
|
|
42
46
|
* @ignore
|
|
@@ -51,7 +55,6 @@ function weightedAvg(arrValues, arrWeights) {
|
|
|
51
55
|
.reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
|
|
52
56
|
return result[0] / result[1];
|
|
53
57
|
}
|
|
54
|
-
exports.weightedAvg = weightedAvg;
|
|
55
58
|
/**
|
|
56
59
|
* Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
|
|
57
60
|
* in your code, e.g. to prevent overloading of target website or to avoid bot detection.
|
|
@@ -71,7 +74,6 @@ exports.weightedAvg = weightedAvg;
|
|
|
71
74
|
async function sleep(millis) {
|
|
72
75
|
return (0, promises_2.setTimeout)(millis);
|
|
73
76
|
}
|
|
74
|
-
exports.sleep = sleep;
|
|
75
77
|
/**
|
|
76
78
|
* Converts SNAKE_CASE to camelCase.
|
|
77
79
|
* @ignore
|
|
@@ -85,7 +87,6 @@ function snakeCaseToCamelCase(snakeCaseStr) {
|
|
|
85
87
|
})
|
|
86
88
|
.join('');
|
|
87
89
|
}
|
|
88
|
-
exports.snakeCaseToCamelCase = snakeCaseToCamelCase;
|
|
89
90
|
/**
|
|
90
91
|
* Traverses DOM and expands shadow-root elements (created by custom components).
|
|
91
92
|
* @ignore
|
|
@@ -111,5 +112,4 @@ function expandShadowRoots(document) {
|
|
|
111
112
|
replaceShadowDomsWithHtml(document.body);
|
|
112
113
|
return document.documentElement.outerHTML;
|
|
113
114
|
}
|
|
114
|
-
exports.expandShadowRoots = expandShadowRoots;
|
|
115
115
|
//# sourceMappingURL=general.js.map
|
package/internals/general.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":";;;AAsCA,4BAKC;AAMD,kCAWC;AAkBD,sBAEC;AAMD,oDAQC;AAMD,8CAyBC;;AA7HD,wEAAkC;AAClC,mDAAkD;AAElD;;;GAGG;AACU,QAAA,mBAAmB,GAC5B,2IAA2I,CAAC;AAEhJ;;;GAGG;AACU,QAAA,qBAAqB,GAC9B,4IAA4I,CAAC;AAEjJ,IAAI,oBAAkD,CAAC;AAEvD,KAAK,UAAU,qBAAqB;IAChC,MAAM,QAAQ,GAAG,kBAAE;SACd,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,QAAQ,GAAG,kBAAE;SACd,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEnE,OAAO,OAAO,IAAI,OAAO,CAAC;AAC9B,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,QAAQ,CAAC,UAAoB;IAC/C,wDAAwD;IACxD,IAAI,CAAC,oBAAoB,IAAI,UAAU;QAAE,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;IAExF,OAAO,oBAAoB,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,SAAgB,WAAW,CAAC,SAAmB,EAAE,UAAoB;IACjE,MAAM,MAAM,GAAG,SAAS;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACd,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1D,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACI,KAAK,UAAU,KAAK,CAAC,MAAe;IACvC,OAAO,IAAA,qBAAU,EAAC,MAAM,CAAC,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,YAAoB;IACrD,OAAO,YAAY;SACd,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACjB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,SAAgB,iBAAiB,CAAC,QAAkB;IAChD,oCAAoC;IACpC,SAAS,gBAAgB,CAAC,UAAe;QACrC,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACrC,UAAU,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC;QACrD,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,oDAAoD;IACpD,SAAS,yBAAyB,CAAC,WAAgB;QAC/C,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;gBAChB,yBAAyB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;gBACzC,EAAE,CAAC,SAAS,IAAI,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YAC1D,CAAC;QACL,CAAC;IACL,CAAC;IAED,yBAAyB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEzC,OAAO,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;AAC9C,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gotScraping.d.ts","sourceRoot":"","sources":["../../src/internals/gotScraping.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAGhD,QAAA,IAAI,WAAW,
|
|
1
|
+
{"version":3,"file":"gotScraping.d.ts","sourceRoot":"","sources":["../../src/internals/gotScraping.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAGhD,QAAA,IAAI,WAAW,EAIT,WAAW,CAAC;AAElB,OAAO,EAAE,WAAW,EAAE,CAAC"}
|
package/internals/memory-info.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getMemoryInfo =
|
|
3
|
+
exports.getMemoryInfo = getMemoryInfo;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const node_child_process_1 = require("node:child_process");
|
|
6
6
|
const promises_1 = require("node:fs/promises");
|
|
@@ -132,5 +132,4 @@ async function getMemoryInfo() {
|
|
|
132
132
|
childProcessesBytes,
|
|
133
133
|
};
|
|
134
134
|
}
|
|
135
|
-
exports.getMemoryInfo = getMemoryInfo;
|
|
136
135
|
//# sourceMappingURL=memory-info.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-info.js","sourceRoot":"","sources":["../../src/internals/memory-info.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"memory-info.js","sourceRoot":"","sources":["../../src/internals/memory-info.ts"],"names":[],"mappings":";;AAoDA,sCA+GC;;AAnKD,2DAA8C;AAC9C,+CAAoD;AACpD,qCAA4C;AAC5C,kEAA6B;AAE7B,6DAA6B;AAC7B,6DAA6D;AAC7D,qEAAoC;AAGpC,uCAAqC;AAErC,MAAM,iBAAiB,GAAG;IACtB,KAAK,EAAE;QACH,EAAE,EAAE,6CAA6C;QACjD,EAAE,EAAE,2BAA2B;KAClC;IACD,IAAI,EAAE;QACF,EAAE,EAAE,6CAA6C;QACjD,EAAE,EAAE,+BAA+B;KACtC;CACJ,CAAC;AAsBF;;;;;;;;GAQG;AACI,KAAK,UAAU,aAAa;IAC/B,MAAM,cAAc,GAAG,mBAAI,CAAC,SAAS,CAAC,iBAAM,CAAC,CAAC;IAE9C,2DAA2D;IAC3D,oCAAoC;IACpC,MAAM,mBAAmB,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;IAE1G,MAAM,WAAW,GAAG,CAAC,mBAAmB,IAAI,CAAC,MAAM,IAAA,kBAAQ,GAAE,CAAC,CAAC;IAE/D,IAAI,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAC1B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAE5B,IAAI,mBAAmB,EAAE,CAAC;QACtB,oBAAoB;QACpB,gBAAgB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC;QAE7C,8CAA8C;QAC9C,MAAM,OAAO,GAAG,IAAA,6BAAQ,EAAC,mBAAmB,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACnE,uFAAuF;QACvF,qGAAqG;QACrG,mBAAmB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,GAAG,gBAAgB,CAAC;IAChE,CAAC;SAAM,CAAC;QACJ,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE1D,SAAS,CAAC,OAAO,CAAC,CAAC,GAAuB,EAAE,EAAE;YAC1C,0EAA0E;YAC1E,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,IAAI,GAAG,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;gBACrD,OAAO;YACX,CAAC;YACD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACpC,yCAAyC;YACzC,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC/B,gBAAgB,GAAG,KAAK,CAAC;gBACzB,OAAO;YACX,CAAC;YACD,mBAAmB,IAAI,KAAK,CAAC;QACjC,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,SAAiB,CAAC;IACtB,IAAI,SAAiB,CAAC;IAEtB,IAAI,mBAAmB,EAAE,CAAC;QACtB,sCAAsC;QACtC,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,+BAAgC,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC;QAClF,SAAS,GAAG,gBAAgB,GAAG,mBAAmB,CAAC;QACnD,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QAEnC,aAAG,CAAC,KAAK,CAAC,kBAAkB,UAAU,SAAS,SAAS,aAAa,CAAC,CAAC;IAC3E,CAAC;SAAM,IAAI,WAAW,EAAE,CAAC;QACrB,oEAAoE;QAEpE,yCAAyC;QACzC,IAAI,cAAc,GAAyC,IAAI,CAAC;QAChE,IAAI,CAAC;YACD,uEAAuE;YACvE,MAAM,IAAA,iBAAM,EAAC,wBAAwB,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACL,cAAc,GAAG,IAAI,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC;YACD,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBAClD,IAAA,mBAAQ,EAAC,iBAAiB,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;gBACzD,IAAA,mBAAQ,EAAC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;aAC3D,CAAC,CAAC;YAEH,gHAAgH;YAChH,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAC5D,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE1D,4DAA4D;YAC5D,oIAAoI;YACpI,IAAI,aAAa,KAAK,KAAK,EAAE,CAAC;gBAC1B,UAAU,GAAG,IAAA,kBAAQ,GAAE,CAAC;gBACxB,yFAAyF;gBACzF,8CAA8C;YAClD,CAAC;iBAAM,CAAC;gBACJ,UAAU,GAAG,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;gBACzC,MAAM,gCAAgC,GAAG,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC;gBAC9E,IAAI,gCAAgC;oBAAE,UAAU,GAAG,IAAA,kBAAQ,GAAE,CAAC;YAClE,CAAC;YACD,SAAS,GAAG,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACvC,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QACvC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,0CAA0C;YAC1C,aAAG,CAAC,UAAU,CACV,+EAA+E;gBAC3E,mGAAmG;gBACnG,UAAW,GAAa,CAAC,OAAO,EAAE,CACzC,CAAC;YACF,UAAU,GAAG,IAAA,kBAAQ,GAAE,CAAC;YACxB,SAAS,GAAG,IAAA,iBAAO,GAAE,CAAC;YACtB,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QACvC,CAAC;IACL,CAAC;SAAM,CAAC;QACJ,UAAU,GAAG,IAAA,kBAAQ,GAAE,CAAC;QACxB,SAAS,GAAG,IAAA,iBAAO,GAAE,CAAC;QACtB,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;IACvC,CAAC;IAED,OAAO;QACH,UAAU;QACV,SAAS;QACT,SAAS;QACT,gBAAgB;QAChB,mBAAmB;KACtB,CAAC;AACN,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.parseOpenGraph =
|
|
3
|
+
exports.parseOpenGraph = parseOpenGraph;
|
|
4
4
|
const cheerio_1 = require("cheerio");
|
|
5
5
|
/**
|
|
6
6
|
* To be used with the spread operator. Ensures that the item is defined, and is not empty.
|
|
@@ -378,5 +378,4 @@ function parseOpenGraph(item, additionalProperties) {
|
|
|
378
378
|
};
|
|
379
379
|
}, {});
|
|
380
380
|
}
|
|
381
|
-
exports.parseOpenGraph = parseOpenGraph;
|
|
382
381
|
//# sourceMappingURL=open_graph_parser.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"open_graph_parser.js","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"open_graph_parser.js","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":";;AA+YA,wCAYC;AAzZD,qCAA+B;AAU/B;;;;;;GAMG;AACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE,CAC9C,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAE/E,MAAM,qBAAqB,GAAwB;IAC/C;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,MAAM;QAClB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,eAAe;QACrB,UAAU,EAAE,YAAY;QACxB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,WAAW;QACjB,UAAU,EAAE,QAAQ;QACpB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD,kDAAkD;IAClD,iHAAiH;IACjH,mDAAmD;IACnD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,mBAAmB;wBACzB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,YAAY;gBAClB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,iBAAiB;wBACvB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,sBAAsB;gBAC5B,UAAU,EAAE,eAAe;gBAC3B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,cAAc;gBAC1B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,uBAAuB;gBAC7B,UAAU,EAAE,gBAAgB;gBAC5B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,UAAU;gBAChB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,kBAAkB;gBACxB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;CACJ,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,kBAAkB,IAAI,IAAI,CAAC;AAE3E,MAAM,sBAAsB,GAAG,CAAC,QAA2B,EAAE,CAAa,EAAuC,EAAE;IAC/G,6GAA6G;IAC7G,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnG,4FAA4F;IAC5F,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAE,MAAM,CAAC,CAAC,CAAY,CAAC,CAAC,CAAE,MAAmB,CAAC;IAElF,sEAAsE;IACtE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IAE9C,wGAAwG;IACxG,OAAO;QACH,qGAAqG;QACrG,+EAA+E;QAC/E,wGAAwG;QACxG,uDAAuD;QACvD,GAAG,cAAc,CAAC,GAAG,QAAQ,CAAC,UAAU,OAAO,EAAE,OAAO,CAAC;QACzD,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CACvB,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YACV,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/C,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,GAAG,CAAC;YAErC,OAAO;gBACH,GAAG,GAAG;gBACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;aACtE,CAAC;QACN,CAAC,EACD,EAAqC,CACxC;KACJ,CAAC;AACN,CAAC,CAAC;AAYF,SAAgB,cAAc,CAAC,IAAyB,EAAE,oBAA0C;IAChG,MAAM,CAAC,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAI,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvD,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,EAAE,GAAG,qBAAqB,CAAC,CAAC,MAAM,CACrE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QACV,OAAO;YACH,GAAG,GAAG;YACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;SACtE,CAAC;IACN,CAAC,EACD,EAAiC,CACpC,CAAC;AACN,CAAC"}
|
package/internals/sitemap.d.ts
CHANGED
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
|
-
/// <reference types="node" />
|
|
3
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
-
/// <reference types="node" />
|
|
5
1
|
import { Writable } from 'node:stream';
|
|
6
2
|
import type { SAXStream } from 'sax';
|
|
7
3
|
type SitemapSource = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,QAAQ,EAAE,MAAM,aAAa,CAAC;AAKjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AAIrC,KAAK,aAAa,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAErF,cAAM,YAAY;IACd,OAAO,EAAE,aAAa,EAAE,CAAM;IAC9B,IAAI,EAAE,MAAM,EAAE,CAAM;IACpB,kBAAkB,EAAE,MAAM,EAAE,CAAM;IAClC,OAAO,CAAC,EAAE,cAAc,GAAG,QAAQ,CAAC;IACpC,GAAG,UAAS;IAEZ,YAAY;CAIf;AAED,cAAM,gBAAiB,SAAQ,QAAQ;IAK/B,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,KAAK;IALjB,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,MAAM,CAAc;gBAGhB,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,IAAI;IAK7B,OAAO,CAAC,aAAa;IAmBZ,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,GAAG,SAAS,KAAK,IAAI,GAAG,IAAI;IAKzG,MAAM,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,GAAG,SAAS,KAAK,IAAI,GAAG,IAAI;CAK9E;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,OAAO;IACJ,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE;gBAAd,IAAI,EAAE,MAAM,EAAE;IAEnC,SAAS,CAAC,MAAM,CAAC,eAAe,CAC5B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,IAAI,EACjB,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAChC,SAAS;IA0CZ;;;;;OAKG;WACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAe7E;;;;OAIG;WACU,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO/E;;;;OAIG;WACU,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;qBAOzD,KAAK,CAAC,YAAY,EAAE,YAAY,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAiE7F,SAAS,CAAC,MAAM,CAAC,YAAY,CACzB,OAAO,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,EACjC,MAAM,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,EAChC,YAAY,EAAE,YAAY,EAC1B,WAAW,GAAE,MAAW,EACxB,GAAG,CAAC,EAAE,GAAG;CAoBhB"}
|
package/internals/social.js
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DISCORD_REGEX_GLOBAL = exports.DISCORD_REGEX = exports.PINTEREST_REGEX_GLOBAL = exports.PINTEREST_REGEX = exports.TIKTOK_REGEX_GLOBAL = exports.TIKTOK_REGEX = exports.YOUTUBE_REGEX_GLOBAL = exports.YOUTUBE_REGEX = exports.FACEBOOK_REGEX_GLOBAL = exports.FACEBOOK_REGEX = exports.TWITTER_REGEX_GLOBAL = exports.TWITTER_REGEX = exports.INSTAGRAM_REGEX_GLOBAL = exports.INSTAGRAM_REGEX = exports.LINKEDIN_REGEX_GLOBAL = exports.LINKEDIN_REGEX = exports.
|
|
3
|
+
exports.DISCORD_REGEX_GLOBAL = exports.DISCORD_REGEX = exports.PINTEREST_REGEX_GLOBAL = exports.PINTEREST_REGEX = exports.TIKTOK_REGEX_GLOBAL = exports.TIKTOK_REGEX = exports.YOUTUBE_REGEX_GLOBAL = exports.YOUTUBE_REGEX = exports.FACEBOOK_REGEX_GLOBAL = exports.FACEBOOK_REGEX = exports.TWITTER_REGEX_GLOBAL = exports.TWITTER_REGEX = exports.INSTAGRAM_REGEX_GLOBAL = exports.INSTAGRAM_REGEX = exports.LINKEDIN_REGEX_GLOBAL = exports.LINKEDIN_REGEX = exports.EMAIL_REGEX_GLOBAL = exports.EMAIL_REGEX = void 0;
|
|
4
|
+
exports.emailsFromText = emailsFromText;
|
|
5
|
+
exports.emailsFromUrls = emailsFromUrls;
|
|
6
|
+
exports.phonesFromText = phonesFromText;
|
|
7
|
+
exports.phonesFromUrls = phonesFromUrls;
|
|
8
|
+
exports.parseHandlesFromHtml = parseHandlesFromHtml;
|
|
4
9
|
const tslib_1 = require("tslib");
|
|
5
10
|
const cheerio_1 = tslib_1.__importDefault(require("cheerio"));
|
|
6
11
|
const cheerio_2 = require("./cheerio");
|
|
@@ -29,7 +34,6 @@ function emailsFromText(text) {
|
|
|
29
34
|
return [];
|
|
30
35
|
return text.match(exports.EMAIL_REGEX_GLOBAL) || [];
|
|
31
36
|
}
|
|
32
|
-
exports.emailsFromText = emailsFromText;
|
|
33
37
|
/**
|
|
34
38
|
* The function extracts email addresses from a list of URLs.
|
|
35
39
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
@@ -53,7 +57,6 @@ function emailsFromUrls(urls) {
|
|
|
53
57
|
}
|
|
54
58
|
return emails;
|
|
55
59
|
}
|
|
56
|
-
exports.emailsFromUrls = emailsFromUrls;
|
|
57
60
|
// Supports URLs starting with `tel://`, `tel:/` and `tel:`, and similarly `phone`, `telephone` and `callto`
|
|
58
61
|
const PHONE_URL_PREFIX_REGEX = /^(tel|phone|telephone|callto):(\/)?(\/)?/i;
|
|
59
62
|
// It's pretty much impossible (and unmaintainable) to have just one large regular expression for all possible phone numbers.
|
|
@@ -126,7 +129,6 @@ function phonesFromText(text) {
|
|
|
126
129
|
});
|
|
127
130
|
return phones;
|
|
128
131
|
}
|
|
129
|
-
exports.phonesFromText = phonesFromText;
|
|
130
132
|
/**
|
|
131
133
|
* Finds phone number links in an array of URLs and extracts the phone numbers from them.
|
|
132
134
|
* Note that the phone number links look like `tel://123456789`, `tel:/123456789` or `tel:123456789`.
|
|
@@ -149,7 +151,6 @@ function phonesFromUrls(urls) {
|
|
|
149
151
|
}
|
|
150
152
|
return phones;
|
|
151
153
|
}
|
|
152
|
-
exports.phonesFromUrls = phonesFromUrls;
|
|
153
154
|
// NOTEs about the regular expressions
|
|
154
155
|
// - They have just a single matching group for the profile username, all other groups are non-matching
|
|
155
156
|
// - They use a negative lookbehind and lookahead assertions, which are only supported in Node 8+.
|
|
@@ -244,7 +245,6 @@ function parseHandlesFromHtml(html, data = null) {
|
|
|
244
245
|
}
|
|
245
246
|
return result;
|
|
246
247
|
}
|
|
247
|
-
exports.parseHandlesFromHtml = parseHandlesFromHtml;
|
|
248
248
|
/**
|
|
249
249
|
* Regular expression to exactly match a single LinkedIn profile URL.
|
|
250
250
|
* It has the following form: `/^...$/i` and matches URLs such as:
|
package/internals/social.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"social.js","sourceRoot":"","sources":["../../src/internals/social.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"social.js","sourceRoot":"","sources":["../../src/internals/social.ts"],"names":[],"mappings":";;;AA6BA,wCAGC;AAUD,wCAaC;AAoED,wCAiBC;AASD,wCAYC;AAkFD,oDAqDC;;AAxSD,8DAA8B;AAE9B,uCAAuC;AAEvC,6EAA6E;AAC7E,MAAM,kBAAkB,GACpB,ubAAub,CAAC;AAE5b;;;GAGG;AACU,QAAA,WAAW,GAAG,IAAI,MAAM,CAAC,IAAI,kBAAkB,GAAG,EAAE,GAAG,CAAC,CAAC;AAEtE;;;GAGG;AACU,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;AAEvE,MAAM,sBAAsB,GAAG,WAAW,CAAC;AAE3C;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,IAAY;IACvC,IAAK,OAAO,IAAgB,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACrD,OAAO,IAAI,CAAC,KAAK,CAAC,0BAAkB,CAAC,IAAI,EAAE,CAAC;AAChD,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,cAAc,CAAC,IAAc;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACnF,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAEhD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,IAAI,mBAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,4GAA4G;AAC5G,MAAM,sBAAsB,GAAG,2CAA2C,CAAC;AAE3E,6HAA6H;AAC7H,iHAAiH;AACjH,mEAAmE;AACnE,qHAAqH;AACrH,MAAM,oBAAoB,GAAG;IACzB,YAAY;IACZ,aAAa;IAEb,6FAA6F;IAC7F,2EAA2E;IAE3E,mCAAmC;IACnC,uCAAuC;IAEvC,sCAAsC;IACtC,2EAA2E;IAE3E,mBAAmB;IACnB,6CAA6C;IAC7C,eAAe;IACf,kCAAkC;IAClC,UAAU;IACV,uBAAuB;IAEvB,mBAAmB;IACnB,mDAAmD;IACnD,eAAe;IACf,sCAAsC;IACtC,UAAU;IACV,yBAAyB;IAEzB,mBAAmB;IACnB,6CAA6C;IAC7C,eAAe;IACf,kCAAkC;IAClC,WAAW;IACX,uBAAuB;IACvB,gDAAgD;CACnD,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,KAAK,EAAE,CAAC,CAAC;AAEtC,2DAA2D;AAC3D,uGAAuG;AACvG,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAE3B,oEAAoE;AACpE,4EAA4E;AAC5E,MAAM,iBAAiB,GAAG;IACtB,aAAa;IACb,8BAA8B;CACjC,CAAC;AAEF,MAAM,kBAAkB,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACnF,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,KAAK,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAC7E,MAAM,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAE/E;;;;;;;GAOG;AACH,SAAgB,cAAc,CAAC,IAAY;IACvC,IAAK,OAAO,IAAgB,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,MAAM,GAAI,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAc,IAAI,EAAE,CAAC;IAChE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7B,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAEzB,wDAAwD;QACxD,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAE,CAAC,MAAM,GAAG,gBAAgB;YAAE,OAAO,KAAK,CAAC;QAEnE,gDAAgD;QAChD,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/C,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,IAAc;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAEnF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QAEhD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7D,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,sCAAsC;AACtC,uGAAuG;AACvG,kGAAkG;AAClG,wFAAwF;AAExF,MAAM,qBAAqB,GACvB,4IAA4I,CAAC;AAEjJ,MAAM,sBAAsB,GACxB,yIAAyI,CAAC;AAE9I,MAAM,sBAAsB,GACxB,sGAAsG,CAAC;AAE3G,MAAM,oBAAoB,GAAG,kEAAkE,sBAAsB,+DAA+D,CAAC;AAErL,MAAM,uBAAuB,GACzB,kWAAkW,CAAC;AAEvW,MAAM,qBAAqB,GAAG,0EAA0E,uBAAuB,qHAAqH,CAAC;AAErP,MAAM,oBAAoB,GACtB,2KAA2K,CAAC;AAEhL,MAAM,mBAAmB,GACrB,2MAA2M,CAAC;AAEhN,MAAM,sBAAsB,GACxB,kNAAkN,CAAC;AAEvN,MAAM,oBAAoB,GACtB,6QAA6Q,CAAC;AAmBlR;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,SAAgB,oBAAoB,CAAC,IAAY,EAAE,OAAuC,IAAI;IAC1F,MAAM,MAAM,GAAkB;QAC1B,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,EAAE;QACV,eAAe,EAAE,EAAE;QACnB,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,EAAE;QACZ,UAAU,EAAE,EAAE;QACd,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,EAAE;QACZ,OAAO,EAAE,EAAE;QACX,UAAU,EAAE,EAAE;QACd,QAAQ,EAAE,EAAE;KACf,CAAC;IAEF,IAAK,OAAO,IAAgB,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAEzD,MAAM,CAAC,GAAG,iBAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,IAAI,IAAI;QAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAErB,MAAM,IAAI,GAAG,IAAA,oBAAU,EAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,IAAI;QAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IAE3B,mCAAmC;IACnC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE;QAC/B,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,CAAC,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAE9C,8EAA8E;IAC9E,2EAA2E;IAC3E,mBAAmB;IACnB,mDAAmD;IACnD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,6BAAqB,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,4BAAoB,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,8BAAsB,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,6BAAqB,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,4BAAoB,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,2BAAmB,CAAC,IAAI,EAAE,CAAC;IACvD,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,8BAAsB,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,4BAAoB,CAAC,IAAI,EAAE,CAAC;IAEzD,+BAA+B;IAC/B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAA4B,EAAE,CAAC;QAC/D,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACU,QAAA,cAAc,GAAG,IAAI,MAAM,CAAC,IAAI,qBAAqB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE5E;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACU,QAAA,qBAAqB,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;AAE7E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACU,QAAA,eAAe,GAAG,IAAI,MAAM,CAAC,IAAI,sBAAsB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACU,QAAA,sBAAsB,GAAG,IAAI,MAAM,CAAC,sBAAsB,EAAE,IAAI,CAAC,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACU,QAAA,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACU,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;AAE3E;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACU,QAAA,cAAc,GAAG,IAAI,MAAM,CAAC,IAAI,qBAAqB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE5E;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACU,QAAA,qBAAqB,GAAG,IAAI,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;AAE7E;;;;;;;;;;;;;;;;;;;;;GAqBG;AACU,QAAA,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;GAoBG;AACU,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;AAC3E;;;;;;;;;;;;;;;;;;;;GAoBG;AACU,QAAA,YAAY,GAAG,IAAI,MAAM,CAAC,IAAI,mBAAmB,GAAG,EAAE,GAAG,CAAC,CAAC;AAExE;;;;;;;;;;;;;;;;;;;GAmBG;AACU,QAAA,mBAAmB,GAAG,IAAI,MAAM,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;AAEzE;;;;;;;;;;;;;;;;;;;;;GAqBG;AACU,QAAA,eAAe,GAAG,IAAI,MAAM,CAAC,IAAI,sBAAsB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;;;GAoBG;AACU,QAAA,sBAAsB,GAAG,IAAI,MAAM,CAAC,sBAAsB,EAAE,IAAI,CAAC,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;GAmBG;AACU,QAAA,aAAa,GAAG,IAAI,MAAM,CAAC,IAAI,oBAAoB,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;;;;;;;;;;;;;;;GAkBG;AACU,QAAA,oBAAoB,GAAG,IAAI,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../../src/internals/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"typedefs.d.ts","sourceRoot":"","sources":["../../src/internals/typedefs.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,wBAAgB,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACxD;AAED,cAAc;AACd,wBAAgB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,GACV,CAAC,MAAM,CAAC,CAAC,EAAE,CACzC"}
|
package/internals/typedefs.js
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.entries = entries;
|
|
4
|
+
exports.keys = keys;
|
|
4
5
|
/** @ignore */
|
|
5
6
|
function entries(obj) {
|
|
6
7
|
return Object.entries(obj);
|
|
7
8
|
}
|
|
8
|
-
exports.entries = entries;
|
|
9
9
|
/** @ignore */
|
|
10
10
|
function keys(obj) {
|
|
11
11
|
return Object.keys(obj);
|
|
12
12
|
}
|
|
13
|
-
exports.keys = keys;
|
|
14
13
|
//# sourceMappingURL=typedefs.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"typedefs.js","sourceRoot":"","sources":["../../src/internals/typedefs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"typedefs.js","sourceRoot":"","sources":["../../src/internals/typedefs.ts"],"names":[],"mappings":";;AACA,0BAEC;AAGD,oBAEC;AARD,cAAc;AACd,SAAgB,OAAO,CAAe,GAAM;IACxC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAA4B,CAAC;AAC1D,CAAC;AAED,cAAc;AACd,SAAgB,IAAI,CAAe,GAAM;IACrC,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAgB,CAAC;AAC3C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "3.10.6-beta.
|
|
3
|
+
"version": "3.10.6-beta.13",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/log": "^2.4.0",
|
|
51
51
|
"@apify/ps-tree": "^1.2.0",
|
|
52
|
-
"@crawlee/types": "3.10.6-beta.
|
|
52
|
+
"@crawlee/types": "3.10.6-beta.13",
|
|
53
53
|
"@types/sax": "^1.2.7",
|
|
54
54
|
"cheerio": "^1.0.0-rc.12",
|
|
55
55
|
"file-type": "^19.0.0",
|
|
@@ -70,5 +70,5 @@
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"gitHead": "
|
|
73
|
+
"gitHead": "15fa44cc09462c67b892d854cb9a705ca4332f60"
|
|
74
74
|
}
|