@crawlee/core 3.0.0-beta.9 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -133
- package/autoscaling/autoscaled_pool.d.ts +5 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +16 -16
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +6 -6
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +12 -8
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +1 -1
- package/autoscaling/system_status.d.ts.map +1 -1
- package/configuration.d.ts +77 -62
- package/configuration.d.ts.map +1 -1
- package/configuration.js +85 -87
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +7 -12
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_utils.d.ts +1 -1
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +1 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +25 -23
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +23 -23
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +11 -6
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +7 -2
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +16 -0
- package/errors.d.ts.map +1 -0
- package/errors.js +22 -0
- package/errors.js.map +1 -0
- package/events/event_manager.d.ts +12 -8
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +17 -6
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +3 -5
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +3 -5
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +4 -0
- package/index.d.ts.map +1 -1
- package/index.js +5 -0
- package/index.js.map +1 -1
- package/index.mjs +6 -0
- package/package.json +22 -27
- package/proxy_configuration.d.ts +7 -14
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +6 -15
- package/proxy_configuration.js.map +1 -1
- package/request.d.ts +11 -6
- package/request.d.ts.map +1 -1
- package/request.js +5 -1
- package/request.js.map +1 -1
- package/router.d.ts +93 -0
- package/router.d.ts.map +1 -0
- package/router.js +137 -0
- package/router.js.map +1 -0
- package/serialization.d.ts.map +1 -1
- package/serialization.js +1 -2
- package/serialization.js.map +1 -1
- package/session_pool/session.d.ts +10 -15
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +18 -18
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +3 -2
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +4 -5
- package/session_pool/session_pool.js.map +1 -1
- package/session_pool/session_utils.d.ts +6 -5
- package/session_pool/session_utils.d.ts.map +1 -1
- package/session_pool/session_utils.js +3 -0
- package/session_pool/session_utils.js.map +1 -1
- package/storages/dataset.d.ts +37 -15
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +44 -12
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +1 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +1 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +121 -23
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +178 -24
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +32 -35
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +32 -24
- package/storages/request_list.js.map +1 -1
- package/storages/request_queue.d.ts +12 -8
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +18 -10
- package/storages/request_queue.js.map +1 -1
- package/storages/storage_manager.d.ts +9 -5
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +28 -18
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts +16 -0
- package/storages/utils.d.ts.map +1 -0
- package/storages/utils.js +26 -0
- package/storages/utils.js.map +1 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/typedefs.d.ts +0 -2
- package/typedefs.d.ts.map +1 -1
- package/typedefs.js.map +1 -1
- package/validators.d.ts +5 -5
- package/validators.d.ts.map +1 -1
- package/validators.js.map +1 -1
|
@@ -2,14 +2,12 @@ import { EventManager } from './event_manager';
|
|
|
2
2
|
export declare class LocalEventManager extends EventManager {
|
|
3
3
|
private previousTicks;
|
|
4
4
|
/**
|
|
5
|
-
* Initializes
|
|
6
|
-
* This is
|
|
5
|
+
* Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
|
|
6
|
+
* This is automatically called at the beginning of `crawler.run()`.
|
|
7
7
|
*/
|
|
8
8
|
init(): Promise<void>;
|
|
9
9
|
/**
|
|
10
|
-
*
|
|
11
|
-
* of Apify package such as `persistState`.
|
|
12
|
-
* This is automatically called at the end of `Actor.main()`.
|
|
10
|
+
* @inheritDoc
|
|
13
11
|
*/
|
|
14
12
|
close(): Promise<void>;
|
|
15
13
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;AAE1D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C;;;OAGG;IACY,IAAI;IAYnB
|
|
1
|
+
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;AAE1D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C;;;OAGG;IACY,IAAI;IAYnB;;OAEG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;YACW,gBAAgB;IAQ9B,OAAO,CAAC,aAAa;YAaP,gBAAgB;IAc9B;;OAEG;YACW,cAAc;CAG/B"}
|
|
@@ -18,8 +18,8 @@ class LocalEventManager extends event_manager_1.EventManager {
|
|
|
18
18
|
});
|
|
19
19
|
}
|
|
20
20
|
/**
|
|
21
|
-
* Initializes
|
|
22
|
-
* This is
|
|
21
|
+
* Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
|
|
22
|
+
* This is automatically called at the beginning of `crawler.run()`.
|
|
23
23
|
*/
|
|
24
24
|
async init() {
|
|
25
25
|
if (this.initialized) {
|
|
@@ -31,9 +31,7 @@ class LocalEventManager extends event_manager_1.EventManager {
|
|
|
31
31
|
this.intervals.systemInfo = (0, utilities_1.betterSetInterval)(this.emitSystemInfoEvent.bind(this), systemInfoIntervalMillis);
|
|
32
32
|
}
|
|
33
33
|
/**
|
|
34
|
-
*
|
|
35
|
-
* of Apify package such as `persistState`.
|
|
36
|
-
* This is automatically called at the end of `Actor.main()`.
|
|
34
|
+
* @inheritDoc
|
|
37
35
|
*/
|
|
38
36
|
async close() {
|
|
39
37
|
if (!this.initialized) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,8DAAyB;AACzB,6DAA6B;AAC7B,gDAA0E;AAC1E,0CAA+C;AAE/C,mDAA0D;AAE1D,MAAa,iBAAkB,SAAQ,4BAAY;IAAnD;;QACI;;;;mBAAwB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;WAAC;
|
|
1
|
+
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,8DAAyB;AACzB,6DAA6B;AAC7B,gDAA0E;AAC1E,0CAA+C;AAE/C,mDAA0D;AAE1D,MAAa,iBAAkB,SAAQ,4BAAY;IAAnD;;QACI;;;;mBAAwB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;WAAC;IAgGlD,CAAC;IA9FG;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAEO,kBAAkB;QACtB,MAAM,IAAI,GAAG,iBAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;YAC9B,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;SACrB,CAAC;IACpB,CAAC;IAEO,aAAa,CAAC,OAAoC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC;QAC7D,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC;QAChE,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEzC,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI;YACA,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5C,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC;YAE1D,OAAO;gBACH,eAAe,EAAE,gBAAgB,GAAG,mBAAmB;aAC1D,CAAC;SACL;QAAC,OAAO,GAAG,EAAE;YACV,aAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;SACb;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,OAAO,IAAA,qBAAa,GAAE,CAAC;IAC3B,CAAC;CACJ;AAjGD,8CAiGC"}
|
package/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export * from './errors';
|
|
1
2
|
export * from './autoscaling';
|
|
2
3
|
export * from './configuration';
|
|
3
4
|
export * from './constants';
|
|
@@ -7,8 +8,11 @@ export * from './events';
|
|
|
7
8
|
export * from './log';
|
|
8
9
|
export * from './proxy_configuration';
|
|
9
10
|
export * from './request';
|
|
11
|
+
export * from './router';
|
|
10
12
|
export * from './serialization';
|
|
11
13
|
export * from './session_pool';
|
|
12
14
|
export * from './storages';
|
|
13
15
|
export * from './validators';
|
|
16
|
+
export { PseudoUrl } from '@apify/pseudo_url';
|
|
17
|
+
export { Dictionary, Awaitable, Constructor, StorageClient } from '@crawlee/types';
|
|
14
18
|
//# sourceMappingURL=index.d.ts.map
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}
|
package/index.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PseudoUrl = void 0;
|
|
3
4
|
const tslib_1 = require("tslib");
|
|
5
|
+
tslib_1.__exportStar(require("./errors"), exports);
|
|
4
6
|
tslib_1.__exportStar(require("./autoscaling"), exports);
|
|
5
7
|
tslib_1.__exportStar(require("./configuration"), exports);
|
|
6
8
|
tslib_1.__exportStar(require("./constants"), exports);
|
|
@@ -10,8 +12,11 @@ tslib_1.__exportStar(require("./events"), exports);
|
|
|
10
12
|
tslib_1.__exportStar(require("./log"), exports);
|
|
11
13
|
tslib_1.__exportStar(require("./proxy_configuration"), exports);
|
|
12
14
|
tslib_1.__exportStar(require("./request"), exports);
|
|
15
|
+
tslib_1.__exportStar(require("./router"), exports);
|
|
13
16
|
tslib_1.__exportStar(require("./serialization"), exports);
|
|
14
17
|
tslib_1.__exportStar(require("./session_pool"), exports);
|
|
15
18
|
tslib_1.__exportStar(require("./storages"), exports);
|
|
16
19
|
tslib_1.__exportStar(require("./validators"), exports);
|
|
20
|
+
var pseudo_url_1 = require("@apify/pseudo_url");
|
|
21
|
+
Object.defineProperty(exports, "PseudoUrl", { enumerable: true, get: function () { return pseudo_url_1.PseudoUrl; } });
|
|
17
22
|
//# sourceMappingURL=index.js.map
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,mDAAyB;AACzB,wDAA8B;AAC9B,0DAAgC;AAChC,sDAA4B;AAC5B,qDAA2B;AAC3B,0DAAgC;AAChC,mDAAyB;AACzB,gDAAsB;AACtB,gEAAsC;AACtC,oDAA0B;AAC1B,mDAAyB;AACzB,0DAAgC;AAChC,yDAA+B;AAC/B,qDAA2B;AAC3B,uDAA6B;AAC7B,gDAA8C;AAArC,uGAAA,SAAS,OAAA"}
|
package/index.mjs
CHANGED
|
@@ -6,6 +6,7 @@ export const AutoscaledPool = mod.AutoscaledPool;
|
|
|
6
6
|
export const Configuration = mod.Configuration;
|
|
7
7
|
export const CookieParseError = mod.CookieParseError;
|
|
8
8
|
export const CrawlerExtension = mod.CrawlerExtension;
|
|
9
|
+
export const CriticalError = mod.CriticalError;
|
|
9
10
|
export const DATASET_ITERATORS_DEFAULT_LIMIT = mod.DATASET_ITERATORS_DEFAULT_LIMIT;
|
|
10
11
|
export const Dataset = mod.Dataset;
|
|
11
12
|
export const EVENT_SESSION_RETIRED = mod.EVENT_SESSION_RETIRED;
|
|
@@ -20,13 +21,17 @@ export const Logger = mod.Logger;
|
|
|
20
21
|
export const LoggerJson = mod.LoggerJson;
|
|
21
22
|
export const LoggerText = mod.LoggerText;
|
|
22
23
|
export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
|
|
24
|
+
export const MissingRouteError = mod.MissingRouteError;
|
|
25
|
+
export const NonRetryableError = mod.NonRetryableError;
|
|
23
26
|
export const ProxyConfiguration = mod.ProxyConfiguration;
|
|
27
|
+
export const PseudoUrl = mod.PseudoUrl;
|
|
24
28
|
export const QUERY_HEAD_BUFFER = mod.QUERY_HEAD_BUFFER;
|
|
25
29
|
export const QUERY_HEAD_MIN_LENGTH = mod.QUERY_HEAD_MIN_LENGTH;
|
|
26
30
|
export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
|
|
27
31
|
export const Request = mod.Request;
|
|
28
32
|
export const RequestList = mod.RequestList;
|
|
29
33
|
export const RequestQueue = mod.RequestQueue;
|
|
34
|
+
export const Router = mod.Router;
|
|
30
35
|
export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
|
|
31
36
|
export const STATUS_CODES_BLOCKED = mod.STATUS_CODES_BLOCKED;
|
|
32
37
|
export const STORAGE_CONSISTENCY_DELAY_MILLIS = mod.STORAGE_CONSISTENCY_DELAY_MILLIS;
|
|
@@ -53,6 +58,7 @@ export const handleRequestTimeout = mod.handleRequestTimeout;
|
|
|
53
58
|
export const log = mod.log;
|
|
54
59
|
export const maybeStringify = mod.maybeStringify;
|
|
55
60
|
export const mergeCookies = mod.mergeCookies;
|
|
61
|
+
export const purgeDefaultStorages = mod.purgeDefaultStorages;
|
|
56
62
|
export const resolveBaseUrl = mod.resolveBaseUrl;
|
|
57
63
|
export const serializeArray = mod.serializeArray;
|
|
58
64
|
export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
|
package/package.json
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.0
|
|
3
|
+
"version": "3.0.0",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
7
7
|
},
|
|
8
|
-
"
|
|
8
|
+
"main": "./index.js",
|
|
9
|
+
"module": "./index.mjs",
|
|
10
|
+
"types": "./index.d.ts",
|
|
9
11
|
"exports": {
|
|
10
12
|
".": {
|
|
11
13
|
"import": "./index.mjs",
|
|
12
|
-
"require": "./index.js"
|
|
14
|
+
"require": "./index.js",
|
|
15
|
+
"types": "./index.d.ts"
|
|
13
16
|
}
|
|
14
17
|
},
|
|
15
18
|
"keywords": [
|
|
@@ -33,12 +36,12 @@
|
|
|
33
36
|
"license": "Apache-2.0",
|
|
34
37
|
"repository": {
|
|
35
38
|
"type": "git",
|
|
36
|
-
"url": "git+https://github.com/apify/
|
|
39
|
+
"url": "git+https://github.com/apify/crawlee"
|
|
37
40
|
},
|
|
38
41
|
"bugs": {
|
|
39
|
-
"url": "https://github.com/apify/
|
|
42
|
+
"url": "https://github.com/apify/crawlee/issues"
|
|
40
43
|
},
|
|
41
|
-
"homepage": "https://
|
|
44
|
+
"homepage": "https://crawlee.dev",
|
|
42
45
|
"scripts": {
|
|
43
46
|
"build": "npm run clean && npm run compile && npm run copy",
|
|
44
47
|
"clean": "rimraf ./dist",
|
|
@@ -49,32 +52,24 @@
|
|
|
49
52
|
"access": "public"
|
|
50
53
|
},
|
|
51
54
|
"dependencies": {
|
|
52
|
-
"@apify/consts": "^
|
|
53
|
-
"@apify/datastructures": "^
|
|
54
|
-
"@apify/log": "^
|
|
55
|
-
"@apify/pseudo_url": "^
|
|
56
|
-
"@apify/timeout": "^0.
|
|
57
|
-
"@apify/utilities": "^
|
|
58
|
-
"@crawlee/browser-pool": "^3.0.0
|
|
59
|
-
"@crawlee/memory-storage": "^3.0.0
|
|
60
|
-
"@crawlee/types": "^3.0.0
|
|
61
|
-
"@crawlee/utils": "^3.0.0
|
|
55
|
+
"@apify/consts": "^2.0.0",
|
|
56
|
+
"@apify/datastructures": "^2.0.0",
|
|
57
|
+
"@apify/log": "^2.0.0",
|
|
58
|
+
"@apify/pseudo_url": "^2.0.0",
|
|
59
|
+
"@apify/timeout": "^0.3.0",
|
|
60
|
+
"@apify/utilities": "^2.0.0",
|
|
61
|
+
"@crawlee/browser-pool": "^3.0.0",
|
|
62
|
+
"@crawlee/memory-storage": "^3.0.0",
|
|
63
|
+
"@crawlee/types": "^3.0.0",
|
|
64
|
+
"@crawlee/utils": "^3.0.0",
|
|
62
65
|
"@types/tough-cookie": "^4.0.2",
|
|
63
|
-
"
|
|
66
|
+
"@vladfrangu/async_event_emitter": "^2.0.0",
|
|
67
|
+
"fs-extra": "^10.1.0",
|
|
64
68
|
"minimatch": "^5.1.0",
|
|
65
69
|
"ow": "^0.28.1",
|
|
66
70
|
"stream-chain": "^2.2.5",
|
|
67
71
|
"stream-json": "^1.7.4",
|
|
68
72
|
"tldts": "^5.7.80",
|
|
69
|
-
"tough-cookie": "^4.0.0"
|
|
70
|
-
"ws": "^8.7.0"
|
|
71
|
-
},
|
|
72
|
-
"peerDependencies": {
|
|
73
|
-
"puppeteer": ">= 9.x <= 13.x"
|
|
74
|
-
},
|
|
75
|
-
"peerDependenciesMeta": {
|
|
76
|
-
"puppeteer": {
|
|
77
|
-
"optional": true
|
|
78
|
-
}
|
|
73
|
+
"tough-cookie": "^4.0.0"
|
|
79
74
|
}
|
|
80
75
|
}
|
package/proxy_configuration.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { Configuration } from './configuration';
|
|
2
1
|
export interface ProxyConfigurationFunction {
|
|
3
2
|
(sessionId: string | number): string | Promise<string>;
|
|
4
3
|
}
|
|
@@ -24,20 +23,18 @@ export interface ProxyConfigurationOptions {
|
|
|
24
23
|
* **Example usage:**
|
|
25
24
|
*
|
|
26
25
|
* ```javascript
|
|
27
|
-
* // TODO fix examples and jsdoc
|
|
28
26
|
* const proxyConfiguration = new ProxyConfiguration({
|
|
29
|
-
*
|
|
30
|
-
* countryCode: 'US',
|
|
27
|
+
* proxyUrls: ['...', '...'] // List of Proxy URLs to rotate
|
|
31
28
|
* });
|
|
32
29
|
*
|
|
33
30
|
* // Getting proxyInfo object by calling class method directly
|
|
34
|
-
* const proxyInfo = proxyConfiguration.newProxyInfo();
|
|
31
|
+
* const proxyInfo = await proxyConfiguration.newProxyInfo();
|
|
35
32
|
*
|
|
36
33
|
* // In crawler
|
|
37
34
|
* const crawler = new CheerioCrawler({
|
|
38
35
|
* // ...
|
|
39
36
|
* proxyConfiguration,
|
|
40
|
-
*
|
|
37
|
+
* requestHandler({ proxyInfo }) {
|
|
41
38
|
* // Getting used proxy URL
|
|
42
39
|
* const proxyUrl = proxyInfo.url;
|
|
43
40
|
*
|
|
@@ -94,7 +91,7 @@ export interface ProxyInfo {
|
|
|
94
91
|
* const crawler = new CheerioCrawler({
|
|
95
92
|
* // ...
|
|
96
93
|
* proxyConfiguration,
|
|
97
|
-
*
|
|
94
|
+
* requestHandler({ proxyInfo }) {
|
|
98
95
|
* const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
|
|
99
96
|
* }
|
|
100
97
|
* })
|
|
@@ -103,7 +100,6 @@ export interface ProxyInfo {
|
|
|
103
100
|
* @category Scaling
|
|
104
101
|
*/
|
|
105
102
|
export declare class ProxyConfiguration {
|
|
106
|
-
readonly config: Configuration;
|
|
107
103
|
isManInTheMiddle: boolean;
|
|
108
104
|
protected nextCustomUrlIndex: number;
|
|
109
105
|
protected proxyUrls?: string[];
|
|
@@ -123,21 +119,18 @@ export declare class ProxyConfiguration {
|
|
|
123
119
|
* const crawler = new CheerioCrawler({
|
|
124
120
|
* // ...
|
|
125
121
|
* proxyConfiguration,
|
|
126
|
-
*
|
|
122
|
+
* requestHandler({ proxyInfo }) {
|
|
127
123
|
* const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
|
|
128
124
|
* }
|
|
129
125
|
* })
|
|
130
126
|
*
|
|
131
127
|
* ```
|
|
132
|
-
* @param options
|
|
133
|
-
* @param config
|
|
134
|
-
* @param validateRequired internal flag to disable validation that requires one of `proxyUrls` or `newUrlFunction`, defaults to true
|
|
135
128
|
*/
|
|
136
|
-
constructor(options?: ProxyConfigurationOptions
|
|
129
|
+
constructor(options?: ProxyConfigurationOptions);
|
|
137
130
|
/**
|
|
138
131
|
* This function creates a new {@link ProxyInfo} info object.
|
|
139
132
|
* It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
|
|
140
|
-
* the currently used proxy via the
|
|
133
|
+
* the currently used proxy via the requestHandler parameter `proxyInfo`.
|
|
141
134
|
* Use it if you want to work with a rich representation of a proxy URL.
|
|
142
135
|
* If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
|
|
143
136
|
* @param [sessionId]
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,0BAA0B;IACvC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,yBAAyB;IACtC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;;;OAIG;IACH,cAAc,CAAC,EAAE,0BAA0B,CAAC;CAC/C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,WAAW,SAAS;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,kBAAkB;IAC3B,gBAAgB,UAAS;IACzB,SAAS,CAAC,kBAAkB,SAAK;IACjC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,aAAa,sBAA6B;IACpD,SAAS,CAAC,cAAc,CAAC,EAAE,0BAA0B,CAAC;IACtD,SAAS,CAAC,GAAG,2BAA+C;IAE5D;;;;;;;;;;;;;;;;;;;OAmBG;gBACS,OAAO,GAAE,yBAA8B;IAgBnD;;;;;;;;;;;;;;;;OAgBG;IACG,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAgBnE;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAU1D;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM;IAiBtD;;OAEG;cACa,mBAAmB,CAAC,SAAS,CAAC,EAAE,MAAM;IAYtD,SAAS,CAAC,2BAA2B,CAAC,GAAG,EAAE,KAAK,GAAI,KAAK;IAIzD,SAAS,CAAC,gCAAgC,IAAK,KAAK;IAIpD,SAAS,CAAC,uBAAuB,IAAK,KAAK;CAG9C"}
|
package/proxy_configuration.js
CHANGED
|
@@ -4,7 +4,6 @@ exports.ProxyConfiguration = void 0;
|
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
6
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
7
|
-
const configuration_1 = require("./configuration");
|
|
8
7
|
/**
|
|
9
8
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
10
9
|
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
@@ -25,7 +24,7 @@ const configuration_1 = require("./configuration");
|
|
|
25
24
|
* const crawler = new CheerioCrawler({
|
|
26
25
|
* // ...
|
|
27
26
|
* proxyConfiguration,
|
|
28
|
-
*
|
|
27
|
+
* requestHandler({ proxyInfo }) {
|
|
29
28
|
* const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
|
|
30
29
|
* }
|
|
31
30
|
* })
|
|
@@ -47,23 +46,14 @@ class ProxyConfiguration {
|
|
|
47
46
|
* const crawler = new CheerioCrawler({
|
|
48
47
|
* // ...
|
|
49
48
|
* proxyConfiguration,
|
|
50
|
-
*
|
|
49
|
+
* requestHandler({ proxyInfo }) {
|
|
51
50
|
* const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
|
|
52
51
|
* }
|
|
53
52
|
* })
|
|
54
53
|
*
|
|
55
54
|
* ```
|
|
56
|
-
* @param options
|
|
57
|
-
* @param config
|
|
58
|
-
* @param validateRequired internal flag to disable validation that requires one of `proxyUrls` or `newUrlFunction`, defaults to true
|
|
59
55
|
*/
|
|
60
|
-
constructor(options = {}
|
|
61
|
-
Object.defineProperty(this, "config", {
|
|
62
|
-
enumerable: true,
|
|
63
|
-
configurable: true,
|
|
64
|
-
writable: true,
|
|
65
|
-
value: config
|
|
66
|
-
});
|
|
56
|
+
constructor(options = {}) {
|
|
67
57
|
Object.defineProperty(this, "isManInTheMiddle", {
|
|
68
58
|
enumerable: true,
|
|
69
59
|
configurable: true,
|
|
@@ -100,7 +90,8 @@ class ProxyConfiguration {
|
|
|
100
90
|
writable: true,
|
|
101
91
|
value: log_1.default.child({ prefix: 'ProxyConfiguration' })
|
|
102
92
|
});
|
|
103
|
-
|
|
93
|
+
const { validateRequired, ...rest } = options;
|
|
94
|
+
(0, ow_1.default)(rest, ow_1.default.object.exactShape({
|
|
104
95
|
proxyUrls: ow_1.default.optional.array.nonEmpty.ofType(ow_1.default.string.url),
|
|
105
96
|
newUrlFunction: ow_1.default.optional.function,
|
|
106
97
|
}));
|
|
@@ -115,7 +106,7 @@ class ProxyConfiguration {
|
|
|
115
106
|
/**
|
|
116
107
|
* This function creates a new {@link ProxyInfo} info object.
|
|
117
108
|
* It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
|
|
118
|
-
* the currently used proxy via the
|
|
109
|
+
* the currently used proxy via the requestHandler parameter `proxyInfo`.
|
|
119
110
|
* Use it if you want to work with a rich representation of a proxy URL.
|
|
120
111
|
* If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
|
|
121
112
|
* @param [sessionId]
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,6DAA6B;
|
|
1
|
+
{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,6DAA6B;AAqF7B;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,kBAAkB;IAQ3B;;;;;;;;;;;;;;;;;;;OAmBG;IACH,YAAY,UAAqC,EAAE;QA3BnD;;;;mBAAmB,KAAK;WAAC;QACzB;;;;mBAA+B,CAAC;WAAC;QACjC;;;;;WAA+B;QAC/B;;;;mBAA0B,IAAI,GAAG,EAAkB;WAAC;QACpD;;;;;WAAsD;QACtD;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;WAAC;QAuBxD,MAAM,EAAE,gBAAgB,EAAE,GAAG,IAAI,EAAE,GAAG,OAAqB,CAAC;QAC5D,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC1B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3D,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAE9C,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACzE,IAAI,CAAC,SAAS,IAAI,CAAC,cAAc,IAAI,gBAAgB;YAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAEtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,YAAY,CAAC,SAA2B;QAC1C,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEzC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO;YACH,SAAS;YACT,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,IAAI,EAAE,IAAK;SACd,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,MAAM,CAAC,SAA2B;QACpC,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAE9D,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAE,CAAC;SAC/C;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,gBAAgB,CAAC,SAAkB;QACzC,IAAI,cAAsB,CAAC;QAE3B,IAAI,CAAC,SAAS,EAAE;YACZ,OAAO,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;SAC9E;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACnC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;SACvD;aAAM;YACH,cAAc,GAAG,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;YACrF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;SACrD;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,SAAkB;QAClD,IAAI,QAAgB,CAAC;QAErB,IAAI;YACA,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,SAAU,CAAC,CAAC;YAClD,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;YAChD,OAAO,QAAQ,CAAC;SACnB;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,2BAA2B,CAAC,GAAY,CAAC,CAAC;SAClD;IACL,CAAC;IAES,2BAA2B,CAAC,GAAU;QAC5C,MAAM,IAAI,KAAK,CAAC,mEAAmE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACtG,CAAC;IAES,gCAAgC;QACtC,MAAM,IAAI,KAAK,CAAC,6GAA6G,CAAC,CAAC;IACnI,CAAC;IAES,uBAAuB;QAC7B,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IACpG,CAAC;CACJ;AAlJD,gDAkJC"}
|
package/request.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import type { Dictionary } from '@crawlee/types';
|
|
4
|
+
import type { BinaryLike } from 'node:crypto';
|
|
5
|
+
import type { AllowedHttpMethods } from './typedefs';
|
|
5
6
|
/**
|
|
6
7
|
* Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
|
|
7
8
|
* The `Request` object also stores information about errors that occurred during processing of the request.
|
|
@@ -30,7 +31,7 @@ import { AllowedHttpMethods } from './typedefs';
|
|
|
30
31
|
* ```
|
|
31
32
|
* @category Sources
|
|
32
33
|
*/
|
|
33
|
-
export declare class Request {
|
|
34
|
+
export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
34
35
|
/** Request ID */
|
|
35
36
|
id?: string;
|
|
36
37
|
/** URL of the web page to crawl. */
|
|
@@ -64,7 +65,7 @@ export declare class Request {
|
|
|
64
65
|
/** Private store for the custom user data assigned to the request. */
|
|
65
66
|
private _userData;
|
|
66
67
|
/** Custom user data assigned to the request. */
|
|
67
|
-
userData:
|
|
68
|
+
userData: UserData;
|
|
68
69
|
/**
|
|
69
70
|
* ISO datetime string that indicates the time when the request has been processed.
|
|
70
71
|
* Is `null` if the request has not been crawled yet.
|
|
@@ -101,7 +102,7 @@ export declare class Request {
|
|
|
101
102
|
/**
|
|
102
103
|
* Specifies required and optional fields for constructing a {@link Request}.
|
|
103
104
|
*/
|
|
104
|
-
export interface RequestOptions {
|
|
105
|
+
export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
105
106
|
/** URL of the web page to crawl. It must be a non-empty string. */
|
|
106
107
|
url: string;
|
|
107
108
|
/**
|
|
@@ -140,7 +141,11 @@ export interface RequestOptions {
|
|
|
140
141
|
* Custom user data assigned to the request. Use this to save any request related data to the
|
|
141
142
|
* request's scope, keeping them accessible on retries, failures etc.
|
|
142
143
|
*/
|
|
143
|
-
userData?:
|
|
144
|
+
userData?: UserData;
|
|
145
|
+
/**
|
|
146
|
+
* Shortcut for setting `userData: { label: '...' }`.
|
|
147
|
+
*/
|
|
148
|
+
label?: string;
|
|
144
149
|
/**
|
|
145
150
|
* If `false` then the hash part of a URL is removed when computing the `uniqueKey` property.
|
|
146
151
|
* For example, this causes the `http://www.example.com#foo` and `http://www.example.com#bar` URLs
|
package/request.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAM9C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAwBrD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc;IA0GnC,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAG5B;IAED,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAKhC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,SAAS,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,EAAE,uBAAuB;IAgBpH,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAQtD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IAEpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CAEtB;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC"}
|
package/request.js
CHANGED
|
@@ -21,6 +21,7 @@ const requestOptionalPredicates = {
|
|
|
21
21
|
errorMessages: ow_1.default.optional.array.ofType(ow_1.default.string),
|
|
22
22
|
headers: ow_1.default.optional.object,
|
|
23
23
|
userData: ow_1.default.optional.object,
|
|
24
|
+
label: ow_1.default.optional.string,
|
|
24
25
|
handledAt: ow_1.default.optional.any(ow_1.default.string.date, ow_1.default.date),
|
|
25
26
|
keepUrlFragment: ow_1.default.optional.boolean,
|
|
26
27
|
useExtendedUniqueKey: ow_1.default.optional.boolean,
|
|
@@ -182,7 +183,7 @@ class Request {
|
|
|
182
183
|
throw new ow_1.ArgumentError(msg, this.constructor);
|
|
183
184
|
}
|
|
184
185
|
});
|
|
185
|
-
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, errorMessages = [], headers = {}, userData = {}, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, } = options;
|
|
186
|
+
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, } = options;
|
|
186
187
|
let { method = 'GET', } = options;
|
|
187
188
|
method = method.toUpperCase();
|
|
188
189
|
if (method === 'GET' && payload)
|
|
@@ -198,6 +199,9 @@ class Request {
|
|
|
198
199
|
this.errorMessages = [...errorMessages];
|
|
199
200
|
this.headers = { ...headers };
|
|
200
201
|
this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
|
|
202
|
+
if (label) {
|
|
203
|
+
userData.label = label;
|
|
204
|
+
}
|
|
201
205
|
Object.defineProperties(this, {
|
|
202
206
|
_userData: {
|
|
203
207
|
value: { __crawlee: {}, ...userData },
|
package/request.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;;;
|
|
1
|
+
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;;;AACA,gDAAgD;AAEhD,sEAAiC;AAEjC,iDAAuC;AACvC,kEAA6B;AAC7B,+BAA0C;AAE1C,yCAAkC;AAElC,4DAA4D;AAC5D,MAAM,GAAG,GAAG,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAEpD,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IACzB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,CAAC,IAAI,EAAE,YAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACzC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,OAAO;IAqDhB;;OAEG;IACH,YAAY,OAAuB;QAvDnC,iBAAiB;QACjB;;;;;WAAY;QAEZ,oCAAoC;QACpC;;;;;WAAY;QAEZ;;;;;;;WAOG;QACH;;;;;WAAmB;QAEnB;;;WAGG;QACH;;;;;WAAkB;QAElB,yCAAyC;QACzC;;;;;WAA2B;QAE3B,oDAAoD;QACpD;;;;;WAAiB;QAEjB,8FAA8F;QAC9F;;;;;WAAiB;QAEjB,2FAA2F;QAC3F;;;;;WAAmB;QAEnB,0DAA0D;QAC1D;;;;;WAAwB;QAExB,wEAAwE;QACxE;;;;;WAAiC;QAEjC,sEAAsE;QACtE;;;;mBAAyC,EAAE;WAAC;QAE5C,gDAAgD;QAChD;;;;mBAAqB,EAAc;WAAC;QAEpC;;;WAGG;QACH;;;;;WAAmB;QAMf,IAAA,YAAE,EAAC,OAAO,EAAE,gBAAgB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACzC,IAAA,YAAE,EAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE;gBACX,IAAA,YAAE,EAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;gBAChE,mDAAmD;aACtD;iBAAM,IAAI,IAAI,KAAK,KAAK,EAAE;gBACvB,MAAM,GAAG,GAAG,6BAA6B,IAAI,sBAAsB,KAAK,iCAAiC,CAAC;gBAC1G,MAAM,IAAI,kBAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;aAClD;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,KAAK,EACL,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,cAAc,GACjB,GAAG,OAA4H,CAAC;QAEjI,IAAI,EACA,MAAM,GAAG,KAAK,GACjB,GAAG,OAAO,CAAC;QAEZ,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACtH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,SAAoB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;QAEvG,IAAI,KAAK,EAAE;YACP,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;SAC1B;QAED,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC1B,SAAS,EAAE;gBACP,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,QAAQ,EAAE;gBACrC,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACjB;YACD,QAAQ,EAAE;gBACN,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;gBACzB,GAAG,EAAE,CAAC,KAA0B,EAAE,EAAE;oBAChC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE;wBAC3B,SAAS,EAAE;4BACP,gDAAgD;4BAChD,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;4BAC/B,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;wBACD,MAAM,EAAE;4BACJ,gDAAgD;4BAChD,KAAK,EAAE,GAAG,EAAE;gCACR,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;oCAClD,OAAO,CAAC;wCACJ,GAAG,IAAI,CAAC,SAAS;wCACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;qCACtC,CAAC,CAAC;iCACN;gCAED,OAAO,IAAI,CAAC,SAAS,CAAC;4BAC1B,CAAC;4BACD,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;qBACJ,CAAC,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,UAAU,EAAE,IAAI;aACnB;SACJ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,cAAc,IAAI,IAAI;YAAE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACrE,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc;QACd,gDAAgD;QAChD,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,IAAI,KAAK,CAAC;IAC5D,CAAC;IAED,IAAI,cAAc,CAAC,KAAc;QAC7B,gDAAgD;QAChD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS;YAAG,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QAClG,gDAAgD;;YAC3C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;IACxD,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAyB;QAC9B,IAAI,CAAC,QAAuB,CAAC,KAAK,GAAG,KAAK,CAAC;IAChD,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,IAAI,CAAC,cAAc,EAAE;gBACjB,OAAO,GAAG,MAAM,CAAC;aACpB;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE;gBACxC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,8BAA8B;oBAC9B,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC;aAC9B;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE;gBACvD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;aAC5D;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE;gBACpE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;aACnD;iBAAM;gBACH,IAAI;oBACA,OAAO,GAAG,mBAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;iBAC1C;gBAAC,OAAO,GAAG,EAAE;oBACV,OAAO,GAAG,yDAAyD,CAAC;iBACvE;aACJ;SACJ;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE;YAC7B,OAAO,GAAG,WAAW,CAAC;SACzB;aAAM;YACH,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;SACnD;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAES,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAA2B;QAChH,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAA,wBAAY,EAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE;YACvB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE;gBACvC,8EAA8E;gBAC9E,GAAG,CAAC,UAAU,CAAC,uBAAuB,gBAAgB,2BAA2B;sBAC3E,kFAAkF;sBAClF,iHAAiH,CAAC,CAAC;aAC5H;YACD,OAAO,aAAa,CAAC;SACxB;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;IAES,YAAY,CAAC,OAAmB;QACtC,OAAO,qBAAM;aACR,UAAU,CAAC,QAAQ,CAAC;aACpB,MAAM,CAAC,OAAO,CAAC;aACf,MAAM,CAAC,QAAQ,CAAC;aAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;aACrB,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,CAAC;CACJ;AA9PD,0BA8PC"}
|
package/router.d.ts
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { CrawlingContext } from './crawlers/crawler_commons';
|
|
2
|
+
import type { Awaitable } from './typedefs';
|
|
3
|
+
export interface RouterHandler<Context extends CrawlingContext = CrawlingContext> extends Router<Context> {
|
|
4
|
+
(ctx: Context): Awaitable<void>;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.
|
|
8
|
+
*
|
|
9
|
+
* ```ts
|
|
10
|
+
* import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
|
|
11
|
+
*
|
|
12
|
+
* const router = Router.create<CheerioCrawlingContext>();
|
|
13
|
+
*
|
|
14
|
+
* // we can also use factory methods for specific crawling contexts, the above equals to:
|
|
15
|
+
* // import { createCheerioRouter } from 'crawlee';
|
|
16
|
+
* // const router = createCheerioRouter();
|
|
17
|
+
*
|
|
18
|
+
* router.addHandler('label-a', async (ctx) => {
|
|
19
|
+
* ctx.log.info('...');
|
|
20
|
+
* });
|
|
21
|
+
* router.addDefaultHandler(async (ctx) => {
|
|
22
|
+
* ctx.log.info('...');
|
|
23
|
+
* });
|
|
24
|
+
*
|
|
25
|
+
* const crawler = new CheerioCrawler({
|
|
26
|
+
* requestHandler: router,
|
|
27
|
+
* });
|
|
28
|
+
* await crawler.run();
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* Alternatively we can use the default router instance from crawler object:
|
|
32
|
+
*
|
|
33
|
+
* ```ts
|
|
34
|
+
* import { CheerioCrawler } from 'crawlee';
|
|
35
|
+
*
|
|
36
|
+
* const crawler = new CheerioCrawler();
|
|
37
|
+
*
|
|
38
|
+
* crawler.router.addHandler('label-a', async (ctx) => {
|
|
39
|
+
* ctx.log.info('...');
|
|
40
|
+
* });
|
|
41
|
+
* crawler.router.addDefaultHandler(async (ctx) => {
|
|
42
|
+
* ctx.log.info('...');
|
|
43
|
+
* });
|
|
44
|
+
*
|
|
45
|
+
* await crawler.run();
|
|
46
|
+
* ```
|
|
47
|
+
*/
|
|
48
|
+
export declare class Router<Context extends CrawlingContext> {
|
|
49
|
+
private readonly routes;
|
|
50
|
+
/**
|
|
51
|
+
* use Router.create() instead!
|
|
52
|
+
* @ignore
|
|
53
|
+
*/
|
|
54
|
+
protected constructor();
|
|
55
|
+
/**
|
|
56
|
+
* Registers new route handler for given label.
|
|
57
|
+
*/
|
|
58
|
+
addHandler(label: string | symbol, handler: (ctx: Context) => Awaitable<void>): void;
|
|
59
|
+
/**
|
|
60
|
+
* Registers default route handler.
|
|
61
|
+
*/
|
|
62
|
+
addDefaultHandler(handler: (ctx: Context) => Awaitable<void>): void;
|
|
63
|
+
/**
|
|
64
|
+
* Returns route handler for given label. If no label is provided, the default request handler will be returned.
|
|
65
|
+
*/
|
|
66
|
+
getHandler(label?: string | symbol): (ctx: Context) => Awaitable<void>;
|
|
67
|
+
/**
|
|
68
|
+
* Throws when the label already exists in our registry.
|
|
69
|
+
*/
|
|
70
|
+
private validate;
|
|
71
|
+
/**
|
|
72
|
+
* Creates new router instance. This instance can then serve as a `requestHandler` of your crawler.
|
|
73
|
+
*
|
|
74
|
+
* ```ts
|
|
75
|
+
* import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
|
|
76
|
+
*
|
|
77
|
+
* const router = Router.create<CheerioCrawlingContext>();
|
|
78
|
+
* router.addHandler('label-a', async (ctx) => {
|
|
79
|
+
* ctx.log.info('...');
|
|
80
|
+
* });
|
|
81
|
+
* router.addDefaultHandler(async (ctx) => {
|
|
82
|
+
* ctx.log.info('...');
|
|
83
|
+
* });
|
|
84
|
+
*
|
|
85
|
+
* const crawler = new CheerioCrawler({
|
|
86
|
+
* requestHandler: router,
|
|
87
|
+
* });
|
|
88
|
+
* await crawler.run();
|
|
89
|
+
* ```
|
|
90
|
+
*/
|
|
91
|
+
static create<Context extends CrawlingContext = CrawlingContext>(): RouterHandler<Context>;
|
|
92
|
+
}
|
|
93
|
+
//# sourceMappingURL=router.d.ts.map
|
package/router.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAK5C,MAAM,WAAW,aAAa,CAAC,OAAO,SAAS,eAAe,GAAG,eAAe,CAAE,SAAQ,MAAM,CAAC,OAAO,CAAC;IACrG,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACnC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,MAAM,CAAC,OAAO,SAAS,eAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsE;IAE7F;;;OAGG;IACH,SAAS;IAET;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAK7E;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAK5D;;OAEG;IACH,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAgBtE;;OAEG;IACH,OAAO,CAAC,QAAQ;IAShB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM,CAAC,OAAO,SAAS,eAAe,GAAG,eAAe,KAAK,aAAa,CAAC,OAAO,CAAC;CAkB7F"}
|