@crawlee/core 3.0.0-alpha.4 → 3.0.0-alpha.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +10 -0
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +50 -1
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +1 -1
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +5 -5
- package/configuration.d.ts.map +1 -1
- package/configuration.js +12 -16
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +2 -0
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_utils.d.ts +0 -5
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -12
- package/crawlers/crawler_utils.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +14 -1
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +31 -2
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +1 -1
- package/enqueue_links/shared.d.ts.map +1 -1
- package/events/event_manager.d.ts +9 -3
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +5 -4
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +0 -1
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +0 -6
- package/events/local_event_manager.js.map +1 -1
- package/index.mjs +1 -4
- package/package.json +5 -14
- package/proxy_configuration.d.ts +8 -8
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +6 -6
- package/proxy_configuration.js.map +1 -1
- package/request.d.ts +17 -2
- package/request.d.ts.map +1 -1
- package/request.js +71 -3
- package/request.js.map +1 -1
- package/storages/dataset.d.ts +1 -42
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +1 -49
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +0 -1
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +0 -2
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +1 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +3 -2
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +8 -0
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +11 -2
- package/storages/request_list.js.map +1 -1
- package/storages/request_queue.d.ts +5 -2
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +40 -18
- package/storages/request_queue.js.map +1 -1
- package/storages/storage_manager.d.ts +4 -4
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/cache_container.d.ts +0 -18
- package/cache_container.d.ts.map +0 -1
- package/cache_container.js +0 -42
- package/cache_container.js.map +0 -1
- package/errors.d.ts +0 -8
- package/errors.d.ts.map +0 -1
- package/errors.js +0 -12
- package/errors.js.map +0 -1
- package/storages/storage.d.ts +0 -260
- package/storages/storage.d.ts.map +0 -1
- package/storages/storage.js +0 -3
- package/storages/storage.js.map +0 -1
|
@@ -55,6 +55,6 @@ export interface RequestTransform {
|
|
|
55
55
|
* @param original Request options to be modified.
|
|
56
56
|
* @returns The modified request options to enqueue.
|
|
57
57
|
*/
|
|
58
|
-
(original: RequestOptions): RequestOptions;
|
|
58
|
+
(original: RequestOptions): RequestOptions | false | undefined | null;
|
|
59
59
|
}
|
|
60
60
|
//# sourceMappingURL=shared.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAYrD,oBAAY,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAExE,oBAAY,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAErH,oBAAY,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,oBAAY,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAEhH,oBAAY,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,oBAAY,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAEpH,oBAAY,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAAE,OAAO,EAAE,YAAY,GAAG,UAAU,GAAG,IAAI,CAMvI;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,YAAY,EAAE,CAiBjG;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,CAiB9E;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBxF;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAAE,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CAyB7H;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,GAAG,cAAc,EAAE,CAcpG;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;OAGG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAYrD,oBAAY,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAExE,oBAAY,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAErH,oBAAY,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,oBAAY,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAEhH,oBAAY,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,oBAAY,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAEpH,oBAAY,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAAE,OAAO,EAAE,YAAY,GAAG,UAAU,GAAG,IAAI,CAMvI;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,YAAY,EAAE,CAiBjG;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,CAiB9E;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBxF;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAAE,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CAyB7H;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,GAAG,cAAc,EAAE,CAcpG;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;OAGG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC;CACzE"}
|
|
@@ -6,14 +6,19 @@ export declare const enum EventType {
|
|
|
6
6
|
PERSIST_STATE = "persistState",
|
|
7
7
|
SYSTEM_INFO = "systemInfo",
|
|
8
8
|
MIGRATING = "migrating",
|
|
9
|
-
ABORTING = "aborting"
|
|
9
|
+
ABORTING = "aborting",
|
|
10
|
+
EXIT = "exit"
|
|
11
|
+
}
|
|
12
|
+
export declare type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit';
|
|
13
|
+
interface Intervals {
|
|
14
|
+
persistState?: BetterIntervalID;
|
|
15
|
+
systemInfo?: BetterIntervalID;
|
|
10
16
|
}
|
|
11
|
-
export declare type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting';
|
|
12
17
|
export declare abstract class EventManager {
|
|
13
18
|
readonly config: Configuration;
|
|
14
19
|
protected events: EventEmitter;
|
|
15
20
|
protected initialized: boolean;
|
|
16
|
-
protected
|
|
21
|
+
protected intervals: Intervals;
|
|
17
22
|
protected log: import("@apify/log").Log;
|
|
18
23
|
constructor(config?: Configuration);
|
|
19
24
|
/**
|
|
@@ -38,4 +43,5 @@ export declare abstract class EventManager {
|
|
|
38
43
|
*/
|
|
39
44
|
listeners(event: EventTypeName): (() => Promise<unknown>)[];
|
|
40
45
|
}
|
|
46
|
+
export {};
|
|
41
47
|
//# sourceMappingURL=event_manager.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAuB,gBAAgB,EAAqB,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;
|
|
1
|
+
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAuB,gBAAgB,EAAqB,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,oBAAY,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,eAAsB;IACtC,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;gBAE3B,MAAM,gBAAkC;IAE7D;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IASX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;CAG9D"}
|
package/events/event_manager.js
CHANGED
|
@@ -12,6 +12,7 @@ var EventType;
|
|
|
12
12
|
EventType["SYSTEM_INFO"] = "systemInfo";
|
|
13
13
|
EventType["MIGRATING"] = "migrating";
|
|
14
14
|
EventType["ABORTING"] = "aborting";
|
|
15
|
+
EventType["EXIT"] = "exit";
|
|
15
16
|
})(EventType = exports.EventType || (exports.EventType = {}));
|
|
16
17
|
class EventManager {
|
|
17
18
|
constructor(config = configuration_1.Configuration.getGlobalConfig()) {
|
|
@@ -33,11 +34,11 @@ class EventManager {
|
|
|
33
34
|
writable: true,
|
|
34
35
|
value: false
|
|
35
36
|
});
|
|
36
|
-
Object.defineProperty(this, "
|
|
37
|
+
Object.defineProperty(this, "intervals", {
|
|
37
38
|
enumerable: true,
|
|
38
39
|
configurable: true,
|
|
39
40
|
writable: true,
|
|
40
|
-
value:
|
|
41
|
+
value: {}
|
|
41
42
|
});
|
|
42
43
|
Object.defineProperty(this, "log", {
|
|
43
44
|
enumerable: true,
|
|
@@ -55,7 +56,7 @@ class EventManager {
|
|
|
55
56
|
return;
|
|
56
57
|
}
|
|
57
58
|
const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
|
|
58
|
-
this.
|
|
59
|
+
this.intervals.persistState = (0, utilities_1.betterSetInterval)((intervalCallback) => {
|
|
59
60
|
this.emit("persistState" /* PERSIST_STATE */, { isMigrating: false });
|
|
60
61
|
intervalCallback();
|
|
61
62
|
}, persistStateIntervalMillis);
|
|
@@ -69,7 +70,7 @@ class EventManager {
|
|
|
69
70
|
if (!this.initialized) {
|
|
70
71
|
return;
|
|
71
72
|
}
|
|
72
|
-
(0, utilities_1.betterClearInterval)(this.
|
|
73
|
+
(0, utilities_1.betterClearInterval)(this.intervals.persistState);
|
|
73
74
|
this.initialized = false;
|
|
74
75
|
}
|
|
75
76
|
on(event, listener) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,mCAAsC;AACtC,gDAA4F;AAC5F,oDAAiD;AAEjD,IAAkB,
|
|
1
|
+
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,mCAAsC;AACtC,gDAA4F;AAC5F,oDAAiD;AAEjD,IAAkB,SAMjB;AAND,WAAkB,SAAS;IACvB,2CAA8B,CAAA;IAC9B,uCAA0B,CAAA;IAC1B,oCAAuB,CAAA;IACvB,kCAAqB,CAAA;IACrB,0BAAa,CAAA;AACjB,CAAC,EANiB,SAAS,GAAT,iBAAS,KAAT,iBAAS,QAM1B;AASD,MAAsB,YAAY;IAM9B,YAAqB,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QALrB;;;;mBAAmB,IAAI,qBAAY,EAAE;WAAC;QACtC;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;mBAAiC,EAAE;WAAC;QACpC;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;WAAC;IAEgB,CAAC;IAEjE;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,IAAI,CAAC,SAAS,CAAC,YAAY,GAAG,IAAA,6BAAiB,EAAC,CAAC,gBAA+B,EAAE,EAAE;YAChF,IAAI,CAAC,IAAI,qCAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,gBAAgB,EAAE,CAAC;QACvB,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC7B,CAAC;IAED,EAAE,CAAC,KAAoB,EAAE,QAAiC;QACtD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CAAC,KAAoB,EAAE,QAAkC;QACxD,IAAI,QAAQ,EAAE;YACV,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;SAC/C;aAAM;YACH,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;SACzC;IACL,CAAC;IAED,IAAI,CAAC,KAAoB,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAoB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAoB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAA+B,CAAC;IACtE,CAAC;CACJ;AAnED,oCAmEC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;AAE1D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C;;;OAGG;IACY,IAAI;IAYnB;;;;OAIG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;YACW,gBAAgB;IAQ9B,OAAO,CAAC,aAAa;YAaP,gBAAgB;IAc9B;;OAEG;YACW,cAAc;CAG/B"}
|
|
@@ -11,12 +11,6 @@ const event_manager_1 = require("./event_manager");
|
|
|
11
11
|
class LocalEventManager extends event_manager_1.EventManager {
|
|
12
12
|
constructor() {
|
|
13
13
|
super(...arguments);
|
|
14
|
-
Object.defineProperty(this, "intervals", {
|
|
15
|
-
enumerable: true,
|
|
16
|
-
configurable: true,
|
|
17
|
-
writable: true,
|
|
18
|
-
value: {}
|
|
19
|
-
});
|
|
20
14
|
Object.defineProperty(this, "previousTicks", {
|
|
21
15
|
enumerable: true,
|
|
22
16
|
configurable: true,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,gDAAgD;AAChD,oDAAoB;AACpB,6DAA6B;AAC7B,
|
|
1
|
+
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,gDAAgD;AAChD,oDAAoB;AACpB,6DAA6B;AAC7B,gDAA0E;AAC1E,0CAA+C;AAE/C,mDAA0D;AAE1D,MAAa,iBAAkB,SAAQ,4BAAY;IAAnD;;QACI;;;;mBAAwB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;WAAC;IAkGlD,CAAC;IAhGG;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;;;OAIG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,iCAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAEO,kBAAkB;QACtB,MAAM,IAAI,GAAG,YAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;YAC9B,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;SACrB,CAAC;IACpB,CAAC;IAEO,aAAa,CAAC,OAAoC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC;QAC7D,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC;QAChE,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEzC,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI;YACA,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5C,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC;YAE1D,OAAO;gBACH,eAAe,EAAE,gBAAgB,GAAG,mBAAmB;aAC1D,CAAC;SACL;QAAC,OAAO,GAAG,EAAE;YACV,aAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;SACb;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,OAAO,IAAA,qBAAa,GAAE,CAAC;IAC3B,CAAC;CACJ;AAnGD,8CAmGC"}
|
package/index.mjs
CHANGED
|
@@ -53,11 +53,8 @@ export const handleRequestTimeout = mod.handleRequestTimeout;
|
|
|
53
53
|
export const log = mod.log;
|
|
54
54
|
export const maybeStringify = mod.maybeStringify;
|
|
55
55
|
export const mergeCookies = mod.mergeCookies;
|
|
56
|
-
export const
|
|
57
|
-
export const pushData = mod.pushData;
|
|
56
|
+
export const resolveBaseUrl = mod.resolveBaseUrl;
|
|
58
57
|
export const serializeArray = mod.serializeArray;
|
|
59
|
-
export const storage = mod.storage;
|
|
60
|
-
export const throwOnBlockedRequest = mod.throwOnBlockedRequest;
|
|
61
58
|
export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
|
|
62
59
|
export const validateGlobPattern = mod.validateGlobPattern;
|
|
63
60
|
export const validators = mod.validators;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.0-alpha.
|
|
3
|
+
"version": "3.0.0-alpha.40",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -51,26 +51,17 @@
|
|
|
51
51
|
"dependencies": {
|
|
52
52
|
"@apify/consts": "^1.4.2",
|
|
53
53
|
"@apify/datastructures": "^1.0.1",
|
|
54
|
-
"@apify/log": "^1.
|
|
55
|
-
"@apify/ps-tree": "^1.1.4",
|
|
54
|
+
"@apify/log": "^1.2.0",
|
|
56
55
|
"@apify/pseudo_url": "^1.1.0",
|
|
57
56
|
"@apify/timeout": "^0.2.1",
|
|
58
57
|
"@apify/utilities": "^1.2.5",
|
|
59
|
-
"@
|
|
60
|
-
"@crawlee/
|
|
61
|
-
"@crawlee/storage": "^3.0.0-alpha.
|
|
62
|
-
"@crawlee/utils": "^3.0.0-alpha.4",
|
|
63
|
-
"@types/mime-types": "^2.1.1",
|
|
64
|
-
"@types/stream-json": "^1.7.1",
|
|
65
|
-
"content-type": "^1.0.4",
|
|
58
|
+
"@crawlee/browser-pool": "^3.0.0-alpha.40",
|
|
59
|
+
"@crawlee/utils": "^3.0.0-alpha.40",
|
|
60
|
+
"@crawlee/memory-storage": "^3.0.0-alpha.40",
|
|
66
61
|
"got-scraping": "^3.2.7",
|
|
67
|
-
"htmlparser2": "^7.0.0",
|
|
68
62
|
"iconv-lite": "^0.6.3",
|
|
69
|
-
"jquery": "^3.6.0",
|
|
70
|
-
"mime-types": "^2.1.34",
|
|
71
63
|
"minimatch": "^5.0.1",
|
|
72
64
|
"ow": "^0.28.0",
|
|
73
|
-
"proxy-chain": "^2.0.1-beta.0",
|
|
74
65
|
"stream-json": "^1.7.3",
|
|
75
66
|
"tldts": "^5.7.69",
|
|
76
67
|
"tough-cookie": "^4.0.0",
|
package/proxy_configuration.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Configuration } from './configuration';
|
|
2
2
|
export interface ProxyConfigurationFunction {
|
|
3
|
-
(sessionId: string | number): string
|
|
3
|
+
(sessionId: string | number): string | Promise<string>;
|
|
4
4
|
}
|
|
5
5
|
export interface ProxyConfigurationOptions {
|
|
6
6
|
/**
|
|
@@ -11,7 +11,7 @@ export interface ProxyConfigurationOptions {
|
|
|
11
11
|
proxyUrls?: string[];
|
|
12
12
|
/**
|
|
13
13
|
* Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter
|
|
14
|
-
* and should always return stringified proxy URL.
|
|
14
|
+
* and should always return stringified proxy URL. Can be asynchronous.
|
|
15
15
|
* This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
|
|
16
16
|
*/
|
|
17
17
|
newUrlFunction?: ProxyConfigurationFunction;
|
|
@@ -151,7 +151,7 @@ export declare class ProxyConfiguration {
|
|
|
151
151
|
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
|
|
152
152
|
* @return Represents information about used proxy and its configuration.
|
|
153
153
|
*/
|
|
154
|
-
newProxyInfo(sessionId?: string | number): ProxyInfo
|
|
154
|
+
newProxyInfo(sessionId?: string | number): Promise<ProxyInfo>;
|
|
155
155
|
/**
|
|
156
156
|
* Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
|
|
157
157
|
* @param [sessionId]
|
|
@@ -165,7 +165,7 @@ export declare class ProxyConfiguration {
|
|
|
165
165
|
* @return A string with a proxy URL, including authentication credentials and port number.
|
|
166
166
|
* For example, `http://bob:password123@proxy.example.com:8000`
|
|
167
167
|
*/
|
|
168
|
-
newUrl(sessionId?: string | number): string
|
|
168
|
+
newUrl(sessionId?: string | number): Promise<string>;
|
|
169
169
|
/**
|
|
170
170
|
* Handles custom url rotation with session
|
|
171
171
|
*/
|
|
@@ -173,9 +173,9 @@ export declare class ProxyConfiguration {
|
|
|
173
173
|
/**
|
|
174
174
|
* Calls the custom newUrlFunction and checks format of its return value
|
|
175
175
|
*/
|
|
176
|
-
protected _callNewUrlFunction(sessionId?: string): string
|
|
177
|
-
protected _throwNewUrlFunctionInvalid(err: Error):
|
|
178
|
-
protected _throwCannotCombineCustomMethods():
|
|
179
|
-
protected _throwNoOptionsProvided():
|
|
176
|
+
protected _callNewUrlFunction(sessionId?: string): Promise<string>;
|
|
177
|
+
protected _throwNewUrlFunctionInvalid(err: Error): never;
|
|
178
|
+
protected _throwCannotCombineCustomMethods(): never;
|
|
179
|
+
protected _throwNoOptionsProvided(): never;
|
|
180
180
|
}
|
|
181
181
|
//# sourceMappingURL=proxy_configuration.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,0BAA0B;IACvC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,0BAA0B;IACvC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,yBAAyB;IACtC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;;;OAIG;IACH,cAAc,CAAC,EAAE,0BAA0B,CAAC;CAC/C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,WAAW,SAAS;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,kBAAkB;IA+B0B,QAAQ,CAAC,MAAM;IA9BpE,gBAAgB,UAAS;IACzB,SAAS,CAAC,kBAAkB,SAAK;IACjC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,aAAa,sBAA6B;IACpD,SAAS,CAAC,cAAc,CAAC,EAAE,0BAA0B,CAAC;IACtD,SAAS,CAAC,GAAG,2BAA+C;IAE5D;;;;;;;;;;;;;;;;;;;;;;OAsBG;gBACS,OAAO,GAAE,yBAA8B,EAAW,MAAM,gBAAkC,EAAE,gBAAgB,UAAO;IAe/H;;;;;;;;;;;;;;;;OAgBG;IACG,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAgBnE;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAU1D;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM;IAiBtD;;OAEG;cACa,mBAAmB,CAAC,SAAS,CAAC,EAAE,MAAM;IAYtD,SAAS,CAAC,2BAA2B,CAAC,GAAG,EAAE,KAAK,GAAI,KAAK;IAIzD,SAAS,CAAC,gCAAgC,IAAK,KAAK;IAIpD,SAAS,CAAC,uBAAuB,IAAK,KAAK;CAG9C"}
|
package/proxy_configuration.js
CHANGED
|
@@ -129,10 +129,10 @@ class ProxyConfiguration {
|
|
|
129
129
|
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
|
|
130
130
|
* @return Represents information about used proxy and its configuration.
|
|
131
131
|
*/
|
|
132
|
-
newProxyInfo(sessionId) {
|
|
132
|
+
async newProxyInfo(sessionId) {
|
|
133
133
|
if (typeof sessionId === 'number')
|
|
134
134
|
sessionId = `${sessionId}`;
|
|
135
|
-
const url = this.newUrl(sessionId);
|
|
135
|
+
const url = await this.newUrl(sessionId);
|
|
136
136
|
const { username, password, port, hostname } = new URL(url);
|
|
137
137
|
return {
|
|
138
138
|
sessionId,
|
|
@@ -156,7 +156,7 @@ class ProxyConfiguration {
|
|
|
156
156
|
* @return A string with a proxy URL, including authentication credentials and port number.
|
|
157
157
|
* For example, `http://bob:password123@proxy.example.com:8000`
|
|
158
158
|
*/
|
|
159
|
-
newUrl(sessionId) {
|
|
159
|
+
async newUrl(sessionId) {
|
|
160
160
|
if (typeof sessionId === 'number')
|
|
161
161
|
sessionId = `${sessionId}`;
|
|
162
162
|
if (this.newUrlFunction) {
|
|
@@ -184,16 +184,16 @@ class ProxyConfiguration {
|
|
|
184
184
|
/**
|
|
185
185
|
* Calls the custom newUrlFunction and checks format of its return value
|
|
186
186
|
*/
|
|
187
|
-
_callNewUrlFunction(sessionId) {
|
|
187
|
+
async _callNewUrlFunction(sessionId) {
|
|
188
188
|
let proxyUrl;
|
|
189
189
|
try {
|
|
190
|
-
proxyUrl = this.newUrlFunction(sessionId);
|
|
190
|
+
proxyUrl = await this.newUrlFunction(sessionId);
|
|
191
191
|
new URL(proxyUrl); // eslint-disable-line no-new
|
|
192
|
+
return proxyUrl;
|
|
192
193
|
}
|
|
193
194
|
catch (err) {
|
|
194
195
|
this._throwNewUrlFunctionInvalid(err);
|
|
195
196
|
}
|
|
196
|
-
return proxyUrl;
|
|
197
197
|
}
|
|
198
198
|
_throwNewUrlFunctionInvalid(err) {
|
|
199
199
|
throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,6DAA6B;AAC7B,mDAAgD;AAsFhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,kBAAkB;IAQ3B;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,YAAY,UAAqC,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE,EAAE,gBAAgB,GAAG,IAAI;;;;;mBAAjE;;QA9B9D;;;;mBAAmB,KAAK;WAAC;QACzB;;;;mBAA+B,CAAC;WAAC;QACjC;;;;;WAA+B;QAC/B;;;;mBAA0B,IAAI,GAAG,EAAkB;WAAC;QACpD;;;;;WAAsD;QACtD;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;WAAC;QA0BxD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3D,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAE9C,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACzE,IAAI,CAAC,SAAS,IAAI,CAAC,cAAc,IAAI,gBAAgB;YAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAEtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,YAAY,CAAC,SAA2B;
|
|
1
|
+
{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,6DAA6B;AAC7B,mDAAgD;AAsFhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,kBAAkB;IAQ3B;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,YAAY,UAAqC,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE,EAAE,gBAAgB,GAAG,IAAI;;;;;mBAAjE;;QA9B9D;;;;mBAAmB,KAAK;WAAC;QACzB;;;;mBAA+B,CAAC;WAAC;QACjC;;;;;WAA+B;QAC/B;;;;mBAA0B,IAAI,GAAG,EAAkB;WAAC;QACpD;;;;;WAAsD;QACtD;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;WAAC;QA0BxD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3D,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAE9C,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACzE,IAAI,CAAC,SAAS,IAAI,CAAC,cAAc,IAAI,gBAAgB;YAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAEtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,YAAY,CAAC,SAA2B;QAC1C,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEzC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO;YACH,SAAS;YACT,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,IAAI,EAAE,IAAK;SACd,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,MAAM,CAAC,SAA2B;QACpC,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAE9D,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAE,CAAC;SAC/C;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,gBAAgB,CAAC,SAAkB;QACzC,IAAI,cAAsB,CAAC;QAE3B,IAAI,CAAC,SAAS,EAAE;YACZ,OAAO,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;SAC9E;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACnC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;SACvD;aAAM;YACH,cAAc,GAAG,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;YACrF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;SACrD;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,SAAkB;QAClD,IAAI,QAAgB,CAAC;QAErB,IAAI;YACA,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,SAAU,CAAC,CAAC;YAClD,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;YAChD,OAAO,QAAQ,CAAC;SACnB;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,2BAA2B,CAAC,GAAY,CAAC,CAAC;SAClD;IACL,CAAC;IAES,2BAA2B,CAAC,GAAU;QAC5C,MAAM,IAAI,KAAK,CAAC,mEAAmE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACtG,CAAC;IAES,gCAAgC;QACtC,MAAM,IAAI,KAAK,CAAC,6GAA6G,CAAC,CAAC;IACnI,CAAC;IAES,uBAAuB;QAC7B,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IACpG,CAAC;CACJ;AApJD,gDAoJC"}
|
package/request.d.ts
CHANGED
|
@@ -60,8 +60,10 @@ export declare class Request {
|
|
|
60
60
|
errorMessages: string[];
|
|
61
61
|
/** Object with HTTP headers. Key is header name, value is the value. */
|
|
62
62
|
headers?: Record<string, string>;
|
|
63
|
+
/** Private store for the custom user data assigned to the request. */
|
|
64
|
+
private _userData;
|
|
63
65
|
/** Custom user data assigned to the request. */
|
|
64
|
-
userData
|
|
66
|
+
userData: Record<string, any>;
|
|
65
67
|
/**
|
|
66
68
|
* ISO datetime string that indicates the time when the request has been processed.
|
|
67
69
|
* Is `null` if the request has not been crawled yet.
|
|
@@ -71,6 +73,13 @@ export declare class Request {
|
|
|
71
73
|
* `Request` parameters including the URL, HTTP method and headers, and others.
|
|
72
74
|
*/
|
|
73
75
|
constructor(options: RequestOptions);
|
|
76
|
+
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
77
|
+
get skipNavigation(): boolean;
|
|
78
|
+
set skipNavigation(value: boolean);
|
|
79
|
+
/** shortcut for getting `request.userData.label` */
|
|
80
|
+
get label(): string | undefined;
|
|
81
|
+
/** shortcut for setting `request.userData.label` */
|
|
82
|
+
set label(value: string | undefined);
|
|
74
83
|
/**
|
|
75
84
|
* Stores information about an error that occurred during processing of this request.
|
|
76
85
|
*
|
|
@@ -130,7 +139,7 @@ export interface RequestOptions {
|
|
|
130
139
|
* Custom user data assigned to the request. Use this to save any request related data to the
|
|
131
140
|
* request's scope, keeping them accessible on retries, failures etc.
|
|
132
141
|
*/
|
|
133
|
-
userData?: Record<string,
|
|
142
|
+
userData?: Record<string, any>;
|
|
134
143
|
/**
|
|
135
144
|
* If `false` then the hash part of a URL is removed when computing the `uniqueKey` property.
|
|
136
145
|
* For example, this causes the `http://www.example.com#foo` and `http://www.example.com#bar` URLs
|
|
@@ -151,6 +160,12 @@ export interface RequestOptions {
|
|
|
151
160
|
* @default false
|
|
152
161
|
*/
|
|
153
162
|
noRetry?: boolean;
|
|
163
|
+
/**
|
|
164
|
+
* If set to `true` then the crawler processing this request evaluates
|
|
165
|
+
* the `handleRequestFunction` immediately without prior browser navigation.
|
|
166
|
+
* @default false
|
|
167
|
+
*/
|
|
168
|
+
skipNavigation?: boolean;
|
|
154
169
|
/** @internal */
|
|
155
170
|
id?: string;
|
|
156
171
|
/** @internal */
|
package/request.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";AACA,OAAe,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAI5C,OAAO,EAAE,kBAAkB,EAAQ,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";AACA,OAAe,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAI5C,OAAO,EAAE,kBAAkB,EAAQ,MAAM,YAAY,CAAC;AAsBtD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,OAAO;IAChB,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAM;IAEnC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAqGnC,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAG5B;IAED,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAKhC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,SAAS,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,EAAE,uBAAuB;IAgBpH,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAQtD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAE3B,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE/B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CAEtB;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC"}
|
package/request.js
CHANGED
|
@@ -24,6 +24,7 @@ const requestOptionalPredicates = {
|
|
|
24
24
|
handledAt: ow_1.default.optional.any(ow_1.default.string.date, ow_1.default.date),
|
|
25
25
|
keepUrlFragment: ow_1.default.optional.boolean,
|
|
26
26
|
useExtendedUniqueKey: ow_1.default.optional.boolean,
|
|
27
|
+
skipNavigation: ow_1.default.optional.boolean,
|
|
27
28
|
};
|
|
28
29
|
/**
|
|
29
30
|
* Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
|
|
@@ -138,12 +139,19 @@ class Request {
|
|
|
138
139
|
writable: true,
|
|
139
140
|
value: void 0
|
|
140
141
|
});
|
|
142
|
+
/** Private store for the custom user data assigned to the request. */
|
|
143
|
+
Object.defineProperty(this, "_userData", {
|
|
144
|
+
enumerable: true,
|
|
145
|
+
configurable: true,
|
|
146
|
+
writable: true,
|
|
147
|
+
value: {}
|
|
148
|
+
});
|
|
141
149
|
/** Custom user data assigned to the request. */
|
|
142
150
|
Object.defineProperty(this, "userData", {
|
|
143
151
|
enumerable: true,
|
|
144
152
|
configurable: true,
|
|
145
153
|
writable: true,
|
|
146
|
-
value:
|
|
154
|
+
value: {}
|
|
147
155
|
});
|
|
148
156
|
/**
|
|
149
157
|
* ISO datetime string that indicates the time when the request has been processed.
|
|
@@ -174,7 +182,7 @@ class Request {
|
|
|
174
182
|
throw new ow_1.ArgumentError(msg, this.constructor);
|
|
175
183
|
}
|
|
176
184
|
});
|
|
177
|
-
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, errorMessages = [], headers = {}, userData = {}, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, } = options;
|
|
185
|
+
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, errorMessages = [], headers = {}, userData = {}, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, } = options;
|
|
178
186
|
let { method = 'GET', } = options;
|
|
179
187
|
method = method.toUpperCase();
|
|
180
188
|
if (method === 'GET' && payload)
|
|
@@ -189,8 +197,68 @@ class Request {
|
|
|
189
197
|
this.retryCount = retryCount;
|
|
190
198
|
this.errorMessages = [...errorMessages];
|
|
191
199
|
this.headers = { ...headers };
|
|
192
|
-
this.userData = { ...userData };
|
|
193
200
|
this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
|
|
201
|
+
Object.defineProperties(this, {
|
|
202
|
+
_userData: {
|
|
203
|
+
value: { __crawlee: {}, ...userData },
|
|
204
|
+
enumerable: false,
|
|
205
|
+
writable: true,
|
|
206
|
+
},
|
|
207
|
+
userData: {
|
|
208
|
+
get: () => this._userData,
|
|
209
|
+
set: (value) => {
|
|
210
|
+
Object.defineProperties(value, {
|
|
211
|
+
__crawlee: {
|
|
212
|
+
// eslint-disable-next-line no-underscore-dangle
|
|
213
|
+
value: this._userData.__crawlee,
|
|
214
|
+
enumerable: false,
|
|
215
|
+
writable: true,
|
|
216
|
+
},
|
|
217
|
+
toJSON: {
|
|
218
|
+
// eslint-disable-next-line no-underscore-dangle
|
|
219
|
+
value: () => {
|
|
220
|
+
if (Object.keys(this._userData.__crawlee).length > 0) {
|
|
221
|
+
return ({
|
|
222
|
+
...this._userData,
|
|
223
|
+
__crawlee: this._userData.__crawlee,
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
return this._userData;
|
|
227
|
+
},
|
|
228
|
+
enumerable: false,
|
|
229
|
+
writable: true,
|
|
230
|
+
},
|
|
231
|
+
});
|
|
232
|
+
this._userData = value;
|
|
233
|
+
},
|
|
234
|
+
enumerable: true,
|
|
235
|
+
},
|
|
236
|
+
});
|
|
237
|
+
// reassign userData to ensure internal `__crawlee` object is non-enumerable
|
|
238
|
+
this.userData = userData;
|
|
239
|
+
if (skipNavigation != null)
|
|
240
|
+
this.skipNavigation = skipNavigation;
|
|
241
|
+
}
|
|
242
|
+
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
243
|
+
get skipNavigation() {
|
|
244
|
+
// eslint-disable-next-line no-underscore-dangle
|
|
245
|
+
return this.userData.__crawlee?.skipNavigation ?? false;
|
|
246
|
+
}
|
|
247
|
+
set skipNavigation(value) {
|
|
248
|
+
// eslint-disable-next-line no-underscore-dangle
|
|
249
|
+
if (!this.userData.__crawlee)
|
|
250
|
+
this.userData.__crawlee = { skipNavigation: value };
|
|
251
|
+
// eslint-disable-next-line no-underscore-dangle
|
|
252
|
+
else
|
|
253
|
+
this.userData.__crawlee.skipNavigation = value;
|
|
254
|
+
}
|
|
255
|
+
/** shortcut for getting `request.userData.label` */
|
|
256
|
+
get label() {
|
|
257
|
+
return this.userData.label;
|
|
258
|
+
}
|
|
259
|
+
/** shortcut for setting `request.userData.label` */
|
|
260
|
+
set label(value) {
|
|
261
|
+
this.userData.label = value;
|
|
194
262
|
}
|
|
195
263
|
/**
|
|
196
264
|
* Stores information about an error that occurred during processing of this request.
|
package/request.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;;;AAAA,gDAAgD;AAChD,4DAA4C;AAC5C,iDAAsD;AACtD,wDAAwB;AACxB,+BAA0C;AAC1C,yCAAsD;AAEtD,4DAA4D;AAC5D,MAAM,GAAG,GAAG,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAEpD,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,CAAC,IAAI,EAAE,YAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;
|
|
1
|
+
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;;;AAAA,gDAAgD;AAChD,4DAA4C;AAC5C,iDAAsD;AACtD,wDAAwB;AACxB,+BAA0C;AAC1C,yCAAsD;AAEtD,4DAA4D;AAC5D,MAAM,GAAG,GAAG,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAEpD,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,CAAC,IAAI,EAAE,YAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACzC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;CACtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,OAAO;IAqDhB;;OAEG;IACH,YAAY,OAAuB;QAvDnC,iBAAiB;QACjB;;;;;WAAY;QAEZ,oCAAoC;QACpC;;;;;WAAY;QAEZ;;;;;;;WAOG;QACH;;;;;WAAmB;QAEnB;;;WAGG;QACH;;;;;WAAkB;QAElB,yCAAyC;QACzC;;;;;WAA2B;QAE3B,oDAAoD;QACpD;;;;;WAAiB;QAEjB,8FAA8F;QAC9F;;;;;WAAiB;QAEjB,2FAA2F;QAC3F;;;;;WAAmB;QAEnB,0DAA0D;QAC1D;;;;;WAAwB;QAExB,wEAAwE;QACxE;;;;;WAAiC;QAEjC,sEAAsE;QACtE;;;;mBAAyC,EAAE;WAAC;QAE5C,gDAAgD;QAChD;;;;mBAAgC,EAAE;WAAC;QAEnC;;;WAGG;QACH;;;;;WAAmB;QAMf,IAAA,YAAE,EAAC,OAAO,EAAE,gBAAgB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACzC,IAAA,YAAE,EAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE;gBACX,IAAA,YAAE,EAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;gBAChE,mDAAmD;aACtD;iBAAM,IAAI,IAAI,KAAK,KAAK,EAAE;gBACvB,MAAM,GAAG,GAAG,6BAA6B,IAAI,sBAAsB,KAAK,iCAAiC,CAAC;gBAC1G,MAAM,IAAI,kBAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;aAClD;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,cAAc,GACjB,GAAG,OAA4H,CAAC;QAEjI,IAAI,EACA,MAAM,GAAG,KAAK,GACjB,GAAG,OAAO,CAAC;QAEZ,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACtH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,SAAoB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;QAEvG,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC1B,SAAS,EAAE;gBACP,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,QAAQ,EAAE;gBACrC,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACjB;YACD,QAAQ,EAAE;gBACN,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;gBACzB,GAAG,EAAE,CAAC,KAA0B,EAAE,EAAE;oBAChC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE;wBAC3B,SAAS,EAAE;4BACP,gDAAgD;4BAChD,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;4BAC/B,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;wBACD,MAAM,EAAE;4BACJ,gDAAgD;4BAChD,KAAK,EAAE,GAAG,EAAE;gCACR,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;oCAClD,OAAO,CAAC;wCACJ,GAAG,IAAI,CAAC,SAAS;wCACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;qCACtC,CAAC,CAAC;iCACN;gCAED,OAAO,IAAI,CAAC,SAAS,CAAC;4BAC1B,CAAC;4BACD,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;qBACJ,CAAC,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,UAAU,EAAE,IAAI;aACnB;SACJ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,cAAc,IAAI,IAAI;YAAE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACrE,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc;QACd,gDAAgD;QAChD,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,IAAI,KAAK,CAAC;IAC5D,CAAC;IAED,IAAI,cAAc,CAAC,KAAc;QAC7B,gDAAgD;QAChD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS;YAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QAClF,gDAAgD;;YAC3C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;IACxD,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAyB;QAC/B,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,IAAI,CAAC,cAAc,EAAE;gBACjB,OAAO,GAAG,MAAM,CAAC;aACpB;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE;gBACxC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,8BAA8B;oBAC9B,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC;aAC9B;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE;gBACvD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;aAC5D;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE;gBACpE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;aACnD;iBAAM;gBACH,IAAI;oBACA,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;iBAC1C;gBAAC,OAAO,GAAG,EAAE;oBACV,OAAO,GAAG,yDAAyD,CAAC;iBACvE;aACJ;SACJ;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE;YAC7B,OAAO,GAAG,WAAW,CAAC;SACzB;aAAM;YACH,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;SACnD;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAES,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAA2B;QAChH,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAA,wBAAY,EAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE;YACvB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE;gBACvC,8EAA8E;gBAC9E,GAAG,CAAC,UAAU,CAAC,uBAAuB,gBAAgB,2BAA2B;sBAC3E,kFAAkF;sBAClF,iHAAiH,CAAC,CAAC;aAC5H;YACD,OAAO,aAAa,CAAC;SACxB;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;IAES,YAAY,CAAC,OAAmB;QACtC,OAAO,gBAAM;aACR,UAAU,CAAC,QAAQ,CAAC;aACpB,MAAM,CAAC,OAAO,CAAC;aACf,MAAM,CAAC,QAAQ,CAAC;aAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;aACrB,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,CAAC;CACJ;AAzPD,0BAyPC"}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import type { DatasetClient, DatasetInfo, PaginatedList, StorageClient } from '@crawlee/types';
|
|
1
2
|
import { Awaitable, Dictionary } from '../typedefs';
|
|
2
3
|
import { StorageManagerOptions } from './storage_manager';
|
|
3
|
-
import { DatasetClient, DatasetInfo, PaginatedList, StorageClient } from './storage';
|
|
4
4
|
/** @internal */
|
|
5
5
|
export declare const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
6
6
|
/**
|
|
@@ -263,47 +263,6 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
263
263
|
*/
|
|
264
264
|
static open<Data extends Dictionary = Dictionary>(datasetIdOrName?: string | null, options?: StorageManagerOptions): Promise<Dataset<Data>>;
|
|
265
265
|
}
|
|
266
|
-
/**
|
|
267
|
-
* Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
|
|
268
|
-
*
|
|
269
|
-
* Datasets are used to store structured data where each object stored has the same attributes,
|
|
270
|
-
* such as online store products or real estate offers.
|
|
271
|
-
* The actual data is stored either on the local filesystem or in the cloud.
|
|
272
|
-
*
|
|
273
|
-
* For more details and code examples, see the {@link Dataset} class.
|
|
274
|
-
*
|
|
275
|
-
* @param [datasetIdOrName]
|
|
276
|
-
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
277
|
-
* the function returns the default dataset associated with the actor run.
|
|
278
|
-
* @param [options] Storage manager options.
|
|
279
|
-
* @deprecated use `Dataset.open()` instead
|
|
280
|
-
*/
|
|
281
|
-
export declare function openDataset<Data extends Dictionary = Dictionary>(datasetIdOrName?: string | null, options?: StorageManagerOptions): Promise<Dataset<Data>>;
|
|
282
|
-
/**
|
|
283
|
-
* Stores an object or an array of objects to the default {@link Dataset} of the current actor run.
|
|
284
|
-
*
|
|
285
|
-
* This is just a convenient shortcut for {@link Dataset.pushData}.
|
|
286
|
-
* For example, calling the following code:
|
|
287
|
-
* ```javascript
|
|
288
|
-
* await Actor.pushData({ myValue: 123 });
|
|
289
|
-
* ```
|
|
290
|
-
*
|
|
291
|
-
* is equivalent to:
|
|
292
|
-
* ```javascript
|
|
293
|
-
* const dataset = await Dataset.open();
|
|
294
|
-
* await dataset.pushData({ myValue: 123 });
|
|
295
|
-
* ```
|
|
296
|
-
*
|
|
297
|
-
* For more information, see {@link Dataset.open} and {@link Dataset.pushData}
|
|
298
|
-
*
|
|
299
|
-
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
300
|
-
* otherwise the actor process might finish before the data are stored!
|
|
301
|
-
*
|
|
302
|
-
* @param {object} item Object or array of objects containing data to be stored in the default dataset.
|
|
303
|
-
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
|
|
304
|
-
* @param [options] Storage manager options.
|
|
305
|
-
*/
|
|
306
|
-
export declare function pushData(item: Dictionary | Dictionary[], options?: StorageManagerOptions): Promise<void>;
|
|
307
266
|
/**
|
|
308
267
|
* User-function used in the `Dataset.forEach()` API.
|
|
309
268
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/F,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAkB,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1E,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC/H,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IACrD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,GAAG,+BAAoC;IAEvC;;OAEG;gBACS,OAAO,EAAE,cAAc;IAMnC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBlD;;;;;;OAMG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAY7E;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAIjD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAWlG;;;;;;;;;;;;;;OAcG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,CAAC;IAoB7G;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;CASxJ;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IAEjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAEhD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAElC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAE7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IAEnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAEtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAGD,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,IAAI,EAAE,OAAO,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;CACjB"}
|