@crawlee/utils 3.9.3-beta.5 → 3.9.3-beta.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +0 -1
- package/index.d.ts.map +1 -1
- package/index.js +0 -1
- package/index.js.map +1 -1
- package/index.mjs +0 -1
- package/internals/robots.js +1 -1
- package/internals/robots.js.map +1 -1
- package/internals/sitemap.d.ts.map +1 -1
- package/internals/sitemap.js +10 -2
- package/internals/sitemap.js.map +1 -1
- package/package.json +8 -4
- package/tsconfig.build.tsbuildinfo +1 -1
- package/internals/error_tracker.d.ts +0 -44
- package/internals/error_tracker.d.ts.map +0 -1
- package/internals/error_tracker.js +0 -285
- package/internals/error_tracker.js.map +0 -1
package/index.d.ts
CHANGED
|
@@ -7,7 +7,6 @@ export * from './internals/memory-info';
|
|
|
7
7
|
export * from './internals/debug';
|
|
8
8
|
export * as social from './internals/social';
|
|
9
9
|
export * from './internals/typedefs';
|
|
10
|
-
export * from './internals/error_tracker';
|
|
11
10
|
export * from './internals/open_graph_parser';
|
|
12
11
|
export * from './internals/gotScraping';
|
|
13
12
|
export * from './internals/robots';
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,oBAAoB,CAAC;AAC7C,cAAc,sBAAsB,CAAC;AACrC,cAAc
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,oBAAoB,CAAC;AAC7C,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AAEpC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}
|
package/index.js
CHANGED
|
@@ -11,7 +11,6 @@ tslib_1.__exportStar(require("./internals/memory-info"), exports);
|
|
|
11
11
|
tslib_1.__exportStar(require("./internals/debug"), exports);
|
|
12
12
|
exports.social = tslib_1.__importStar(require("./internals/social"));
|
|
13
13
|
tslib_1.__exportStar(require("./internals/typedefs"), exports);
|
|
14
|
-
tslib_1.__exportStar(require("./internals/error_tracker"), exports);
|
|
15
14
|
tslib_1.__exportStar(require("./internals/open_graph_parser"), exports);
|
|
16
15
|
tslib_1.__exportStar(require("./internals/gotScraping"), exports);
|
|
17
16
|
tslib_1.__exportStar(require("./internals/robots"), exports);
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,8DAAoC;AACpC,8DAAoC;AACpC,4DAAkC;AAClC,mEAAyC;AACzC,8DAAoC;AACpC,kEAAwC;AACxC,4DAAkC;AAClC,qEAA6C;AAC7C,+DAAqC;AACrC,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,8DAAoC;AACpC,8DAAoC;AACpC,4DAAkC;AAClC,mEAAyC;AACzC,8DAAoC;AACpC,kEAAwC;AACxC,4DAAkC;AAClC,qEAA6C;AAC7C,+DAAqC;AACrC,wEAA8C;AAC9C,kEAAwC;AACxC,6DAAmC;AACnC,8DAAoC"}
|
package/index.mjs
CHANGED
|
@@ -2,7 +2,6 @@ import mod from "./index.js";
|
|
|
2
2
|
|
|
3
3
|
export default mod;
|
|
4
4
|
export const CLOUDFLARE_RETRY_CSS_SELECTORS = mod.CLOUDFLARE_RETRY_CSS_SELECTORS;
|
|
5
|
-
export const ErrorTracker = mod.ErrorTracker;
|
|
6
5
|
export const RETRY_CSS_SELECTORS = mod.RETRY_CSS_SELECTORS;
|
|
7
6
|
export const ROTATE_PROXY_ERRORS = mod.ROTATE_PROXY_ERRORS;
|
|
8
7
|
export const RobotsFile = mod.RobotsFile;
|
package/internals/robots.js
CHANGED
|
@@ -85,7 +85,7 @@ class RobotsFile {
|
|
|
85
85
|
* @param [userAgent] relevant user agent, default to `*`
|
|
86
86
|
*/
|
|
87
87
|
isAllowed(url, userAgent = '*') {
|
|
88
|
-
return this.robots.isAllowed(url, userAgent) ??
|
|
88
|
+
return this.robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
|
|
89
89
|
}
|
|
90
90
|
/**
|
|
91
91
|
* Get URLs of sitemaps referenced in the robots file.
|
package/internals/robots.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":";;;;AAGA,0EAAyC;AAEzC,+CAA4C;AAC5C,uCAAoC;AAEpC,IAAI,SAAgC,CAAC;AAErC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAa,UAAU;IACnB,YACY,MAAgD,EAChD,QAAiB;QADzB;;;;mBAAQ,MAAM;WAA0C;QACxD;;;;mBAAQ,QAAQ;WAAS;IAC1B,CAAC;IAEJ;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,QAAiB;QAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,aAAa,CAAC,QAAQ,GAAG,aAAa,CAAC;QACvC,aAAa,CAAC,MAAM,GAAG,EAAE,CAAC;QAE1B,OAAO,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAW,EAAE,OAAe,EAAE,QAAiB;QACvD,OAAO,IAAI,UAAU,CAAC,IAAA,uBAAY,EAAC,GAAG,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,QAAiB;QACtD,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,SAAS,GAAG,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,CAAC;QAED,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,IAAA,yBAAW,EAAC;gBAC/B,GAAG;gBACH,QAAQ;gBACR,MAAM,EAAE,KAAK;gBACb,YAAY,EAAE,MAAM;aACvB,CAAC,CAAC;YAEH,OAAO,IAAI,UAAU,CAAC,IAAA,uBAAY,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;QACjF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC1D,OAAO,IAAI,UAAU,CAAC,EAAE,SAAS,KAAK,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YACpG,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,
|
|
1
|
+
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":";;;;AAGA,0EAAyC;AAEzC,+CAA4C;AAC5C,uCAAoC;AAEpC,IAAI,SAAgC,CAAC;AAErC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAa,UAAU;IACnB,YACY,MAAgD,EAChD,QAAiB;QADzB;;;;mBAAQ,MAAM;WAA0C;QACxD;;;;mBAAQ,QAAQ;WAAS;IAC1B,CAAC;IAEJ;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,QAAiB;QAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,aAAa,CAAC,QAAQ,GAAG,aAAa,CAAC;QACvC,aAAa,CAAC,MAAM,GAAG,EAAE,CAAC;QAE1B,OAAO,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAW,EAAE,OAAe,EAAE,QAAiB;QACvD,OAAO,IAAI,UAAU,CAAC,IAAA,uBAAY,EAAC,GAAG,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;IAChE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,QAAiB;QACtD,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,SAAS,GAAG,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,CAAC;QAED,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,IAAA,yBAAW,EAAC;gBAC/B,GAAG;gBACH,QAAQ;gBACR,MAAM,EAAE,KAAK;gBACb,YAAY,EAAE,MAAM;aACvB,CAAC,CAAC;YAEH,OAAO,IAAI,UAAU,CAAC,IAAA,uBAAY,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;QACjF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC1D,OAAO,IAAI,UAAU,CAAC,EAAE,SAAS,KAAK,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YACpG,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC,+FAA+F;IACzJ,CAAC;IAED;;OAEG;IACH,WAAW;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,OAAO,iBAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACvB,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;CACJ;AAhFD,gCAgFC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AAIrC,cAAM,YAAY;IACd,WAAW,EAAE,MAAM,EAAE,CAAM;IAC3B,IAAI,EAAE,MAAM,EAAE,CAAM;IACpB,kBAAkB,EAAE,MAAM,EAAE,CAAM;IAClC,OAAO,CAAC,EAAE,cAAc,GAAG,QAAQ,CAAC;IACpC,GAAG,UAAS;IAEZ,YAAY;CAIf;AAyCD;;;;;;;;;;;GAWG;AACH,qBAAa,OAAO;IACJ,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE;gBAAd,IAAI,EAAE,MAAM,EAAE;IAEnC,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,IAAI,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,SAAS;IA0C3H;;;;;OAKG;WACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAe7E;;;;OAIG;WACU,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAyDlF"}
|
package/internals/sitemap.js
CHANGED
|
@@ -7,6 +7,7 @@ const node_string_decoder_1 = require("node:string_decoder");
|
|
|
7
7
|
const node_zlib_1 = require("node:zlib");
|
|
8
8
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
9
9
|
const sax_1 = tslib_1.__importDefault(require("sax"));
|
|
10
|
+
const whatwg_mimetype_1 = tslib_1.__importDefault(require("whatwg-mimetype"));
|
|
10
11
|
class ParsingState {
|
|
11
12
|
constructor() {
|
|
12
13
|
Object.defineProperty(this, "sitemapUrls", {
|
|
@@ -201,10 +202,17 @@ class Sitemap {
|
|
|
201
202
|
}
|
|
202
203
|
const parser = (() => {
|
|
203
204
|
const contentType = sitemapStream.response.headers['content-type'];
|
|
204
|
-
|
|
205
|
+
let mimeType;
|
|
206
|
+
try {
|
|
207
|
+
mimeType = new whatwg_mimetype_1.default(contentType ?? '');
|
|
208
|
+
}
|
|
209
|
+
catch (e) {
|
|
210
|
+
mimeType = null;
|
|
211
|
+
}
|
|
212
|
+
if (mimeType?.isXML() || sitemapUrl.pathname.endsWith('.xml')) {
|
|
205
213
|
return Sitemap.createXmlParser(parsingState, () => resolve(undefined), reject);
|
|
206
214
|
}
|
|
207
|
-
if (
|
|
215
|
+
if (mimeType?.essence === 'text/plain' || sitemapUrl.pathname.endsWith('.txt')) {
|
|
208
216
|
return new SitemapTxtParser(parsingState, () => resolve(undefined));
|
|
209
217
|
}
|
|
210
218
|
throw new Error('Unsupported sitemap content type');
|
package/internals/sitemap.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":";;;;AACA,6CAAuC;AACvC,6DAAoD;AACpD,yCAAyC;AAEzC,6DAA6B;AAE7B,sDAAsB;
|
|
1
|
+
{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":";;;;AACA,6CAAuC;AACvC,6DAAoD;AACpD,yCAAyC;AAEzC,6DAA6B;AAE7B,sDAAsB;AACtB,8EAAuC;AAEvC,MAAM,YAAY;IAAlB;QACI;;;;mBAAwB,EAAE;WAAC;QAC3B;;;;mBAAiB,EAAE;WAAC;QACpB;;;;mBAA+B,EAAE;WAAC;QAClC;;;;;WAAoC;QACpC;;;;mBAAM,KAAK;WAAC;IAMhB,CAAC;IAJG,YAAY;QACR,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QACzB,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC;IACrB,CAAC;CACJ;AAED,MAAM,gBAAiB,SAAQ,sBAAQ;IAInC,YAAoB,YAA0B,EAAU,KAAiB;QACrE,KAAK,EAAE,CAAC;QADA;;;;mBAAQ,YAAY;WAAc;QAAE;;;;mBAAQ,KAAK;WAAY;QAHjE;;;;mBAAyB,IAAI,mCAAa,CAAC,MAAM,CAAC;WAAC;QACnD;;;;mBAAiB,EAAE;WAAC;IAI5B,CAAC;IAEO,aAAa,CAAC,KAAa,EAAE,QAAiB;QAClD,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAErB,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM;iBACpB,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEvC,IAAI,QAAQ,EAAE,CAAC;gBACX,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACtC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnD,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;YAChC,CAAC;QACL,CAAC;IACL,CAAC;IAEQ,MAAM,CAAC,KAAU,EAAE,SAAyB,EAAE,QAAoD;QACvG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;QACrD,QAAQ,EAAE,CAAC;IACf,CAAC;IAEQ,MAAM,CAAC,QAAoD;QAChE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;QAC7C,QAAQ,EAAE,CAAC;QACX,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;CACJ;AAED;;;;;;;;;;;GAWG;AACH,MAAa,OAAO;IAChB,YAAqB,IAAc;QAAvB;;;;mBAAS,IAAI;WAAU;IAAG,CAAC;IAE7B,MAAM,CAAC,eAAe,CAAC,YAA0B,EAAE,KAAiB,EAAE,OAA+B;QAC3G,MAAM,MAAM,GAAG,aAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAEtC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE;YAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,YAAY,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC5D,YAAY,CAAC,GAAG,GAAG,IAAI,CAAC;YAC5B,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACzB,YAAY,CAAC,OAAO,GAAG,QAAQ,CAAC;YACpC,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBAC/B,YAAY,CAAC,OAAO,GAAG,cAAc,CAAC;YAC1C,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;YAC3B,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACjB,YAAY,CAAC,GAAG,GAAG,KAAK,CAAC;YAC7B,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE;YAC5B,IAAI,YAAY,CAAC,GAAG,EAAE,CAAC;gBACnB,IAAI,YAAY,CAAC,OAAO,KAAK,cAAc,EAAE,CAAC;oBAC1C,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBAClD,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACxC,CAAC;gBACL,CAAC;gBACD,IAAI,YAAY,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACpC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjC,CAAC;YACL,CAAC;QACL,CAAC,CAAC;QACF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAE3B,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACxB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE5B,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,GAAW,EAAE,QAAiB;QACtD,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAC;QAEvB,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,UAAU,CAAC,QAAQ,GAAG,cAAc,CAAC;QACrC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAuB,EAAE,QAAiB;QACxD,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QAErD,MAAM,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;QACxC,YAAY,CAAC,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAE/D,OAAO,YAAY,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,WAAW,CAAC,GAAG,EAAG,CAAC,CAAC;YAC5D,YAAY,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC5D,YAAY,CAAC,YAAY,EAAE,CAAC;YAE5B,IAAI,CAAC;gBACD,MAAM,aAAa,GAAG,MAAM,IAAI,OAAO,CAAwC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBAC/F,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;oBACjF,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC/C,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAChC,CAAC,CAAC,CAAC;gBAEH,IAAI,aAAa,CAAC,QAAS,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC7C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;wBAClC,IAAI,MAAM,GAAW,aAAa,CAAC;wBACnC,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;4BACtC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAA,wBAAY,GAAE,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;4BACzD,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBAC3F,CAAC;wBAED,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;4BACjB,MAAM,WAAW,GAAG,aAAa,CAAC,QAAS,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;4BACpE,IAAI,QAAyB,CAAC;4BAE9B,IAAI,CAAC;gCACD,QAAQ,GAAG,IAAI,yBAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;4BAC/C,CAAC;4BAAC,OAAO,CAAC,EAAE,CAAC;gCACT,QAAQ,GAAG,IAAI,CAAC;4BACpB,CAAC;4BAED,IAAI,QAAQ,EAAE,KAAK,EAAE,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gCAC5D,OAAO,OAAO,CAAC,eAAe,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;4BACnF,CAAC;4BAED,IAAI,QAAQ,EAAE,OAAO,KAAK,YAAY,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gCAC7E,OAAO,IAAI,gBAAgB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;4BACxE,CAAC;4BAED,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;wBACxD,CAAC,CAAC,EAAE,CAAC;wBAEL,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACxB,CAAC,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,aAAG,CAAC,OAAO,CAAC,8BAA8B,UAAU,EAAE,CAAC,CAAC;YAC5D,CAAC;QACL,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;CACJ;AAhID,0BAgIC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "3.9.3-beta.
|
|
3
|
+
"version": "3.9.3-beta.50",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -49,14 +49,18 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/log": "^2.4.0",
|
|
51
51
|
"@apify/ps-tree": "^1.2.0",
|
|
52
|
-
"@crawlee/types": "3.9.3-beta.
|
|
52
|
+
"@crawlee/types": "3.9.3-beta.50",
|
|
53
53
|
"@types/sax": "^1.2.7",
|
|
54
54
|
"cheerio": "^1.0.0-rc.12",
|
|
55
55
|
"got-scraping": "^4.0.3",
|
|
56
56
|
"ow": "^0.28.1",
|
|
57
57
|
"robots-parser": "^3.0.1",
|
|
58
58
|
"sax": "^1.3.0",
|
|
59
|
-
"tslib": "^2.4.0"
|
|
59
|
+
"tslib": "^2.4.0",
|
|
60
|
+
"whatwg-mimetype": "^4.0.0"
|
|
61
|
+
},
|
|
62
|
+
"devDependencies": {
|
|
63
|
+
"@types/whatwg-mimetype": "^3.0.2"
|
|
60
64
|
},
|
|
61
65
|
"lerna": {
|
|
62
66
|
"command": {
|
|
@@ -65,5 +69,5 @@
|
|
|
65
69
|
}
|
|
66
70
|
}
|
|
67
71
|
},
|
|
68
|
-
"gitHead": "
|
|
72
|
+
"gitHead": "b4d36bdb7cb3de644b5c286ff634b110cf9ab580"
|
|
69
73
|
}
|