@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +4 -11
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +0 -7
- package/configuration.d.ts.map +1 -1
- package/configuration.js +0 -4
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +1 -1
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +4 -6
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +61 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +99 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +12 -17
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +0 -8
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +11 -2
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +6 -3
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +22 -15
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +33 -14
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +8 -2
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +40 -18
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +12 -0
- package/errors.d.ts.map +1 -1
- package/errors.js +20 -0
- package/errors.js.map +1 -1
- package/events/local_event_manager.d.ts +0 -2
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +3 -32
- package/events/local_event_manager.js.map +1 -1
- package/http_clients/base-http-client.d.ts +6 -128
- package/http_clients/base-http-client.d.ts.map +1 -1
- package/http_clients/base-http-client.js +20 -9
- package/http_clients/base-http-client.js.map +1 -1
- package/http_clients/got-scraping-http-client.d.ts +10 -3
- package/http_clients/got-scraping-http-client.d.ts.map +1 -1
- package/http_clients/got-scraping-http-client.js +86 -39
- package/http_clients/got-scraping-http-client.js.map +1 -1
- package/index.d.ts +1 -0
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/log.d.ts +4 -2
- package/log.d.ts.map +1 -1
- package/log.js.map +1 -1
- package/package.json +6 -6
- package/proxy_configuration.d.ts +16 -93
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +15 -51
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +123 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +140 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +31 -5
- package/request.d.ts.map +1 -1
- package/request.js +35 -12
- package/request.js.map +1 -1
- package/session_pool/session.d.ts +7 -20
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +9 -2
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +10 -3
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +13 -1
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +7 -0
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +10 -1
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +2 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +2 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +13 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +11 -6
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +8 -7
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +8 -2
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +70 -5
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +81 -45
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue_v2.d.ts +2 -2
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +7 -2
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +3 -2
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts +5 -1
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +1 -1
- package/storages/storage_manager.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { Configuration } from '@crawlee/core';
|
|
2
|
+
import type { Log } from '@apify/log';
|
|
3
|
+
export interface RecoverableStatePersistenceOptions {
|
|
4
|
+
/**
|
|
5
|
+
* The key under which the state is stored in the KeyValueStore
|
|
6
|
+
*/
|
|
7
|
+
persistStateKey: string;
|
|
8
|
+
/**
|
|
9
|
+
* Flag to enable or disable state persistence
|
|
10
|
+
*/
|
|
11
|
+
persistenceEnabled?: boolean;
|
|
12
|
+
/**
|
|
13
|
+
* The name of the KeyValueStore to use for persistence.
|
|
14
|
+
* If neither a name nor an id are supplied, the default store will be used.
|
|
15
|
+
*/
|
|
16
|
+
persistStateKvsName?: string;
|
|
17
|
+
/**
|
|
18
|
+
* The identifier of the KeyValueStore to use for persistence.
|
|
19
|
+
* If neither a name nor an id are supplied, the default store will be used.
|
|
20
|
+
*/
|
|
21
|
+
persistStateKvsId?: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Options for configuring the RecoverableState
|
|
25
|
+
*/
|
|
26
|
+
export interface RecoverableStateOptions<TStateModel = Record<string, unknown>> extends RecoverableStatePersistenceOptions {
|
|
27
|
+
/**
|
|
28
|
+
* The default state used if no persisted state is found.
|
|
29
|
+
* A deep copy is made each time the state is used.
|
|
30
|
+
*/
|
|
31
|
+
defaultState: TStateModel;
|
|
32
|
+
/**
|
|
33
|
+
* A logger instance for logging operations related to state persistence
|
|
34
|
+
*/
|
|
35
|
+
logger?: Log;
|
|
36
|
+
/**
|
|
37
|
+
* Configuration instance to use
|
|
38
|
+
*/
|
|
39
|
+
config?: Configuration;
|
|
40
|
+
/**
|
|
41
|
+
* Optional function to transform the state to a JSON string before persistence.
|
|
42
|
+
* If not provided, JSON.stringify will be used.
|
|
43
|
+
*/
|
|
44
|
+
serialize?: (state: TStateModel) => string;
|
|
45
|
+
/**
|
|
46
|
+
* Optional function to transform a JSON-serialized object back to the state model.
|
|
47
|
+
* If not provided, JSON.parse is used.
|
|
48
|
+
* It is advisable to perform validation in this function and to throw an exception if it fails.
|
|
49
|
+
*/
|
|
50
|
+
deserialize?: (serializedState: string) => TStateModel;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* A class for managing persistent recoverable state using a plain JavaScript object.
|
|
54
|
+
*
|
|
55
|
+
* This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
|
|
56
|
+
* across migrations or restarts. It manages the loading, saving, and resetting of state data,
|
|
57
|
+
* with optional persistence capabilities.
|
|
58
|
+
*
|
|
59
|
+
* The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
|
|
60
|
+
* The class automatically hooks into the event system to persist state when needed.
|
|
61
|
+
*/
|
|
62
|
+
export declare class RecoverableState<TStateModel = Record<string, unknown>> {
|
|
63
|
+
private readonly defaultState;
|
|
64
|
+
private state;
|
|
65
|
+
private readonly persistenceEnabled;
|
|
66
|
+
private readonly persistStateKey;
|
|
67
|
+
private readonly persistStateKvsName?;
|
|
68
|
+
private readonly persistStateKvsId?;
|
|
69
|
+
private keyValueStore;
|
|
70
|
+
private readonly log;
|
|
71
|
+
private readonly config;
|
|
72
|
+
private readonly serialize;
|
|
73
|
+
private readonly deserialize;
|
|
74
|
+
/**
|
|
75
|
+
* Initialize a new recoverable state object.
|
|
76
|
+
*
|
|
77
|
+
* @param options Configuration options for the recoverable state
|
|
78
|
+
*/
|
|
79
|
+
constructor(options: RecoverableStateOptions<TStateModel>);
|
|
80
|
+
/**
|
|
81
|
+
* Initialize the recoverable state.
|
|
82
|
+
*
|
|
83
|
+
* This method must be called before using the recoverable state. It loads the saved state
|
|
84
|
+
* if persistence is enabled and registers the object to listen for PERSIST_STATE events.
|
|
85
|
+
*
|
|
86
|
+
* @returns The loaded state object
|
|
87
|
+
*/
|
|
88
|
+
initialize(): Promise<TStateModel>;
|
|
89
|
+
/**
|
|
90
|
+
* Clean up resources used by the recoverable state.
|
|
91
|
+
*
|
|
92
|
+
* If persistence is enabled, this method deregisters the object from PERSIST_STATE events
|
|
93
|
+
* and persists the current state one last time.
|
|
94
|
+
*/
|
|
95
|
+
teardown(): Promise<void>;
|
|
96
|
+
/**
|
|
97
|
+
* Get the current state.
|
|
98
|
+
*/
|
|
99
|
+
get currentValue(): TStateModel;
|
|
100
|
+
/**
|
|
101
|
+
* Reset the state to the default values and clear any persisted state.
|
|
102
|
+
*
|
|
103
|
+
* Resets the current state to the default state and, if persistence is enabled,
|
|
104
|
+
* clears the persisted state from the KeyValueStore.
|
|
105
|
+
*/
|
|
106
|
+
reset(): Promise<void>;
|
|
107
|
+
/**
|
|
108
|
+
* Persist the current state to the KeyValueStore.
|
|
109
|
+
*
|
|
110
|
+
* This method is typically called in response to a PERSIST_STATE event, but can also be called
|
|
111
|
+
* directly when needed.
|
|
112
|
+
*
|
|
113
|
+
* @param eventData Optional data associated with a PERSIST_STATE event
|
|
114
|
+
*/
|
|
115
|
+
persistState(eventData?: {
|
|
116
|
+
isMigrating: boolean;
|
|
117
|
+
}): Promise<void>;
|
|
118
|
+
/**
|
|
119
|
+
* Load the saved state from the KeyValueStore
|
|
120
|
+
*/
|
|
121
|
+
private loadSavedState;
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=recoverable_state.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recoverable_state.d.ts","sourceRoot":"","sources":["../src/recoverable_state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAA4B,MAAM,eAAe,CAAC;AAExE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,MAAM,WAAW,kCAAkC;IAC/C;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC1E,SAAQ,kCAAkC;IAC1C;;;OAGG;IACH,YAAY,EAAE,WAAW,CAAC;IAE1B;;OAEG;IACH,MAAM,CAAC,EAAE,GAAG,CAAC;IAEb;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;;OAGG;IACH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,MAAM,CAAC;IAE3C;;;;OAIG;IACH,WAAW,CAAC,EAAE,CAAC,eAAe,EAAE,MAAM,KAAK,WAAW,CAAC;CAC1D;AAED;;;;;;;;;GASG;AACH,qBAAa,gBAAgB,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAC/D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAc;IAC3C,OAAO,CAAC,KAAK,CAA4B;IACzC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAC7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAS;IAC5C,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiC;IAC3D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2C;IAEvE;;;;OAIG;gBACS,OAAO,EAAE,uBAAuB,CAAC,WAAW,CAAC;IAczD;;;;;;;OAOG;IACG,UAAU,IAAI,OAAO,CAAC,WAAW,CAAC;IAuBxC;;;;;OAKG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B;;OAEG;IACH,IAAI,YAAY,IAAI,WAAW,CAM9B;IAED;;;;;OAKG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAY5B;;;;;;;OAOG;IACG,YAAY,CAAC,SAAS,CAAC,EAAE;QAAE,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAcvE;;OAEG;YACW,cAAc;CAY/B"}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { Configuration, KeyValueStore } from '@crawlee/core';
|
|
2
|
+
import log from '@apify/log';
|
|
3
|
+
/**
|
|
4
|
+
* A class for managing persistent recoverable state using a plain JavaScript object.
|
|
5
|
+
*
|
|
6
|
+
* This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
|
|
7
|
+
* across migrations or restarts. It manages the loading, saving, and resetting of state data,
|
|
8
|
+
* with optional persistence capabilities.
|
|
9
|
+
*
|
|
10
|
+
* The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
|
|
11
|
+
* The class automatically hooks into the event system to persist state when needed.
|
|
12
|
+
*/
|
|
13
|
+
export class RecoverableState {
|
|
14
|
+
defaultState;
|
|
15
|
+
state = null;
|
|
16
|
+
persistenceEnabled;
|
|
17
|
+
persistStateKey;
|
|
18
|
+
persistStateKvsName;
|
|
19
|
+
persistStateKvsId;
|
|
20
|
+
keyValueStore = null;
|
|
21
|
+
log;
|
|
22
|
+
config;
|
|
23
|
+
serialize;
|
|
24
|
+
deserialize;
|
|
25
|
+
/**
|
|
26
|
+
* Initialize a new recoverable state object.
|
|
27
|
+
*
|
|
28
|
+
* @param options Configuration options for the recoverable state
|
|
29
|
+
*/
|
|
30
|
+
constructor(options) {
|
|
31
|
+
this.defaultState = options.defaultState;
|
|
32
|
+
this.persistStateKey = options.persistStateKey;
|
|
33
|
+
this.persistenceEnabled = options.persistenceEnabled ?? false;
|
|
34
|
+
this.persistStateKvsName = options.persistStateKvsName;
|
|
35
|
+
this.persistStateKvsId = options.persistStateKvsId;
|
|
36
|
+
this.log = options.logger ?? log.child({ prefix: 'RecoverableState' });
|
|
37
|
+
this.config = options.config ?? Configuration.getGlobalConfig();
|
|
38
|
+
this.serialize = options.serialize ?? JSON.stringify;
|
|
39
|
+
this.deserialize = options.deserialize ?? JSON.parse;
|
|
40
|
+
this.persistState = this.persistState.bind(this);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Initialize the recoverable state.
|
|
44
|
+
*
|
|
45
|
+
* This method must be called before using the recoverable state. It loads the saved state
|
|
46
|
+
* if persistence is enabled and registers the object to listen for PERSIST_STATE events.
|
|
47
|
+
*
|
|
48
|
+
* @returns The loaded state object
|
|
49
|
+
*/
|
|
50
|
+
async initialize() {
|
|
51
|
+
if (this.state !== null && this.state !== undefined) {
|
|
52
|
+
return this.currentValue;
|
|
53
|
+
}
|
|
54
|
+
if (!this.persistenceEnabled) {
|
|
55
|
+
this.state = this.deserialize(this.serialize(this.defaultState));
|
|
56
|
+
return this.currentValue;
|
|
57
|
+
}
|
|
58
|
+
this.keyValueStore = await KeyValueStore.open(this.persistStateKvsName ?? this.persistStateKvsId, {
|
|
59
|
+
config: this.config,
|
|
60
|
+
});
|
|
61
|
+
await this.loadSavedState();
|
|
62
|
+
// Register for persist state events
|
|
63
|
+
const eventManager = this.config.getEventManager();
|
|
64
|
+
eventManager.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
|
|
65
|
+
return this.currentValue;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Clean up resources used by the recoverable state.
|
|
69
|
+
*
|
|
70
|
+
* If persistence is enabled, this method deregisters the object from PERSIST_STATE events
|
|
71
|
+
* and persists the current state one last time.
|
|
72
|
+
*/
|
|
73
|
+
async teardown() {
|
|
74
|
+
if (!this.persistenceEnabled || !this.persistState) {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
const eventManager = this.config.getEventManager();
|
|
78
|
+
eventManager.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
|
|
79
|
+
await this.persistState();
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Get the current state.
|
|
83
|
+
*/
|
|
84
|
+
get currentValue() {
|
|
85
|
+
if (this.state === null) {
|
|
86
|
+
throw new Error('Recoverable state has not yet been loaded');
|
|
87
|
+
}
|
|
88
|
+
return this.state;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Reset the state to the default values and clear any persisted state.
|
|
92
|
+
*
|
|
93
|
+
* Resets the current state to the default state and, if persistence is enabled,
|
|
94
|
+
* clears the persisted state from the KeyValueStore.
|
|
95
|
+
*/
|
|
96
|
+
async reset() {
|
|
97
|
+
this.state = this.deserialize(this.serialize(this.defaultState));
|
|
98
|
+
if (this.persistenceEnabled) {
|
|
99
|
+
if (this.keyValueStore === null) {
|
|
100
|
+
throw new Error('Recoverable state has not yet been initialized');
|
|
101
|
+
}
|
|
102
|
+
await this.keyValueStore.setValue(this.persistStateKey, null);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Persist the current state to the KeyValueStore.
|
|
107
|
+
*
|
|
108
|
+
* This method is typically called in response to a PERSIST_STATE event, but can also be called
|
|
109
|
+
* directly when needed.
|
|
110
|
+
*
|
|
111
|
+
* @param eventData Optional data associated with a PERSIST_STATE event
|
|
112
|
+
*/
|
|
113
|
+
async persistState(eventData) {
|
|
114
|
+
this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
|
|
115
|
+
if (this.keyValueStore === null || this.state === null) {
|
|
116
|
+
throw new Error('Recoverable state has not yet been initialized');
|
|
117
|
+
}
|
|
118
|
+
if (this.persistenceEnabled) {
|
|
119
|
+
await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
|
|
120
|
+
contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Load the saved state from the KeyValueStore
|
|
126
|
+
*/
|
|
127
|
+
async loadSavedState() {
|
|
128
|
+
if (this.keyValueStore === null) {
|
|
129
|
+
throw new Error('Recoverable state has not yet been initialized');
|
|
130
|
+
}
|
|
131
|
+
const storedState = await this.keyValueStore.getValue(this.persistStateKey);
|
|
132
|
+
if (storedState === null || storedState === undefined) {
|
|
133
|
+
this.state = this.deserialize(this.serialize(this.defaultState));
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
this.state = this.deserialize(storedState);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=recoverable_state.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recoverable_state.js","sourceRoot":"","sources":["../src/recoverable_state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAa,aAAa,EAAE,MAAM,eAAe,CAAC;AAGxE,OAAO,GAAG,MAAM,YAAY,CAAC;AA6D7B;;;;;;;;;GASG;AACH,MAAM,OAAO,gBAAgB;IACR,YAAY,CAAc;IACnC,KAAK,GAAuB,IAAI,CAAC;IACxB,kBAAkB,CAAU;IAC5B,eAAe,CAAS;IACxB,mBAAmB,CAAU;IAC7B,iBAAiB,CAAU;IACpC,aAAa,GAAyB,IAAI,CAAC;IAClC,GAAG,CAAM;IACT,MAAM,CAAgB;IACtB,SAAS,CAAiC;IAC1C,WAAW,CAA2C;IAEvE;;;;OAIG;IACH,YAAY,OAA6C;QACrD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,KAAK,CAAC;QAC9D,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE,CAAC;QAChE,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC9F,MAAM,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,oCAAoC;QACpC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACnD,YAAY,CAAC,EAAE,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAE5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACjD,OAAO;QACX,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACnD,YAAY,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,YAAY,CAAC,SAAoC;QACnD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uDAAuD,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAChF,WAAW,EAAE,YAAY,EAAE,wHAAwH;aACtJ,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5E,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YACpD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,WAAqB,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;CACJ"}
|
package/request.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BinaryLike } from 'node:crypto';
|
|
2
2
|
import type { Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
|
|
4
|
+
import type { SkippedRequestReason } from './enqueue_links/shared.js';
|
|
4
5
|
import type { AllowedHttpMethods } from './typedefs.js';
|
|
5
6
|
export declare enum RequestState {
|
|
6
7
|
UNPROCESSED = 0,
|
|
@@ -43,7 +44,7 @@ export declare enum RequestState {
|
|
|
43
44
|
* ```
|
|
44
45
|
* @category Sources
|
|
45
46
|
*/
|
|
46
|
-
|
|
47
|
+
declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
47
48
|
/** Request ID */
|
|
48
49
|
id?: string;
|
|
49
50
|
/** URL of the web page to crawl. */
|
|
@@ -87,10 +88,25 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
87
88
|
* `Request` parameters including the URL, HTTP method and headers, and others.
|
|
88
89
|
*/
|
|
89
90
|
constructor(options: RequestOptions<UserData>);
|
|
91
|
+
/**
|
|
92
|
+
* Converts the Crawlee Request object to a `fetch` API Request object.
|
|
93
|
+
* @returns The native `fetch` API Request object.
|
|
94
|
+
*/
|
|
95
|
+
intoFetchAPIRequest(): Request;
|
|
90
96
|
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
91
97
|
get skipNavigation(): boolean;
|
|
92
98
|
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
93
99
|
set skipNavigation(value: boolean);
|
|
100
|
+
/**
|
|
101
|
+
* Depth of the request in the current crawl tree.
|
|
102
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
103
|
+
*/
|
|
104
|
+
get crawlDepth(): number;
|
|
105
|
+
/**
|
|
106
|
+
* Depth of the request in the current crawl tree.
|
|
107
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
108
|
+
*/
|
|
109
|
+
set crawlDepth(value: number);
|
|
94
110
|
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
|
|
95
111
|
get sessionRotationCount(): number;
|
|
96
112
|
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
|
|
@@ -123,8 +139,6 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
123
139
|
* @param [options]
|
|
124
140
|
*/
|
|
125
141
|
pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
|
|
126
|
-
protected _computeUniqueKey(options: ComputeUniqueKeyOptions): string;
|
|
127
|
-
protected _hashPayload(payload: BinaryLike): string;
|
|
128
142
|
/** @internal */
|
|
129
143
|
static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
|
|
130
144
|
/** @internal */
|
|
@@ -203,6 +217,18 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
203
217
|
* @default false
|
|
204
218
|
*/
|
|
205
219
|
skipNavigation?: boolean;
|
|
220
|
+
/**
|
|
221
|
+
* Depth of the request in the current crawl tree.
|
|
222
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
223
|
+
* @default 0
|
|
224
|
+
*/
|
|
225
|
+
crawlDepth?: number;
|
|
226
|
+
/**
|
|
227
|
+
* Reason for skipping this request.
|
|
228
|
+
* This is used to provide more information about why the request was skipped.
|
|
229
|
+
* @internal
|
|
230
|
+
*/
|
|
231
|
+
skippedReason?: SkippedRequestReason;
|
|
206
232
|
/**
|
|
207
233
|
* Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
|
|
208
234
|
*/
|
|
@@ -233,11 +259,11 @@ interface ComputeUniqueKeyOptions {
|
|
|
233
259
|
export type Source = (Partial<RequestOptions> & {
|
|
234
260
|
requestsFromUrl?: string;
|
|
235
261
|
regex?: RegExp;
|
|
236
|
-
}) |
|
|
262
|
+
}) | CrawleeRequest;
|
|
237
263
|
/** @internal */
|
|
238
264
|
export interface InternalSource {
|
|
239
265
|
requestsFromUrl: string;
|
|
240
266
|
regex?: RegExp;
|
|
241
267
|
}
|
|
242
|
-
export {};
|
|
268
|
+
export { CrawleeRequest as Request };
|
|
243
269
|
//# sourceMappingURL=request.d.ts.map
|
package/request.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA4BxD,oBAAY,YAAY;IACpB,WAAW,IAAA;IACX,UAAU,IAAA;IACV,SAAS,IAAA;IACT,eAAe,IAAA;IACf,IAAI,IAAA;IACJ,aAAa,IAAA;IACb,KAAK,IAAA;IACL,OAAO,IAAA;CACV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,cAAM,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;IA2H7C;;;OAGG;IACI,mBAAmB,IAAI,OAAO;IAQrC,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAMhC;IAED;;;OAGG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,EAG3B;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAMrC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED,8HAA8H;IAC9H,IAAI,UAAU,IAAI,MAAM,GAAG,SAAS,CAEnC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAMvC;IAED,uDAAuD;IACvD,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,EAM5B;IAED,OAAO,KAAK,eAAe,GAE1B;IAED,OAAO,KAAK,eAAe,QAM1B;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAc,EACd,OAAO,EACP,eAAuB,EACvB,oBAA4B,GAC/B,EAAE,uBAAuB;IAiB1B,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAGlD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,oBAAoB,CAAC;IAErC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gBAAgB;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IAErB,gBAAgB;IAChB,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,cAAc,CAAC;AAE/G,gBAAgB;AAChB,MAAM,WAAW,cAAc;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
package/request.js
CHANGED
|
@@ -24,6 +24,7 @@ const requestOptionalPredicates = {
|
|
|
24
24
|
keepUrlFragment: ow.optional.boolean,
|
|
25
25
|
useExtendedUniqueKey: ow.optional.boolean,
|
|
26
26
|
skipNavigation: ow.optional.boolean,
|
|
27
|
+
crawlDepth: ow.optional.number.greaterThanOrEqual(0),
|
|
27
28
|
state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
|
|
28
29
|
};
|
|
29
30
|
export var RequestState;
|
|
@@ -68,7 +69,7 @@ export var RequestState;
|
|
|
68
69
|
* ```
|
|
69
70
|
* @category Sources
|
|
70
71
|
*/
|
|
71
|
-
|
|
72
|
+
class CrawleeRequest {
|
|
72
73
|
/** Request ID */
|
|
73
74
|
id;
|
|
74
75
|
/** URL of the web page to crawl. */
|
|
@@ -130,7 +131,7 @@ export class Request {
|
|
|
130
131
|
ow(value, `RequestOptions.${prop}`, predicate);
|
|
131
132
|
}
|
|
132
133
|
});
|
|
133
|
-
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, } = options;
|
|
134
|
+
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
|
|
134
135
|
let { method = 'GET' } = options;
|
|
135
136
|
method = method.toUpperCase();
|
|
136
137
|
if (method === 'GET' && payload)
|
|
@@ -139,7 +140,8 @@ export class Request {
|
|
|
139
140
|
this.url = url;
|
|
140
141
|
this.loadedUrl = loadedUrl;
|
|
141
142
|
this.uniqueKey =
|
|
142
|
-
uniqueKey ||
|
|
143
|
+
uniqueKey ||
|
|
144
|
+
CrawleeRequest.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
|
|
143
145
|
this.method = method;
|
|
144
146
|
this.payload = payload;
|
|
145
147
|
this.noRetry = noRetry;
|
|
@@ -191,11 +193,24 @@ export class Request {
|
|
|
191
193
|
this.skipNavigation = skipNavigation;
|
|
192
194
|
if (maxRetries != null)
|
|
193
195
|
this.maxRetries = maxRetries;
|
|
196
|
+
if (crawlDepth != null)
|
|
197
|
+
this.userData.__crawlee.crawlDepth ??= crawlDepth;
|
|
194
198
|
// If it's already set, don't override it (for instance when fetching from storage)
|
|
195
199
|
if (enqueueStrategy) {
|
|
196
200
|
this.enqueueStrategy ??= enqueueStrategy;
|
|
197
201
|
}
|
|
198
202
|
}
|
|
203
|
+
/**
|
|
204
|
+
* Converts the Crawlee Request object to a `fetch` API Request object.
|
|
205
|
+
* @returns The native `fetch` API Request object.
|
|
206
|
+
*/
|
|
207
|
+
intoFetchAPIRequest() {
|
|
208
|
+
return new Request(this.url, {
|
|
209
|
+
method: this.method,
|
|
210
|
+
headers: this.headers,
|
|
211
|
+
body: this.payload,
|
|
212
|
+
});
|
|
213
|
+
}
|
|
199
214
|
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
200
215
|
get skipNavigation() {
|
|
201
216
|
return this.userData.__crawlee?.skipNavigation ?? false;
|
|
@@ -209,6 +224,21 @@ export class Request {
|
|
|
209
224
|
this.userData.__crawlee.skipNavigation = value;
|
|
210
225
|
}
|
|
211
226
|
}
|
|
227
|
+
/**
|
|
228
|
+
* Depth of the request in the current crawl tree.
|
|
229
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
230
|
+
*/
|
|
231
|
+
get crawlDepth() {
|
|
232
|
+
return this.userData.__crawlee?.crawlDepth ?? 0;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Depth of the request in the current crawl tree.
|
|
236
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
237
|
+
*/
|
|
238
|
+
set crawlDepth(value) {
|
|
239
|
+
this.userData.__crawlee ??= {};
|
|
240
|
+
this.userData.__crawlee.crawlDepth = value;
|
|
241
|
+
}
|
|
212
242
|
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
|
|
213
243
|
get sessionRotationCount() {
|
|
214
244
|
return this.userData.__crawlee?.sessionRotationCount ?? 0;
|
|
@@ -317,14 +347,6 @@ export class Request {
|
|
|
317
347
|
}
|
|
318
348
|
this.errorMessages.push(message);
|
|
319
349
|
}
|
|
320
|
-
// TODO: only for better BC, remove in v4
|
|
321
|
-
_computeUniqueKey(options) {
|
|
322
|
-
return Request.computeUniqueKey(options);
|
|
323
|
-
}
|
|
324
|
-
// TODO: only for better BC, remove in v4
|
|
325
|
-
_hashPayload(payload) {
|
|
326
|
-
return Request.hashPayload(payload);
|
|
327
|
-
}
|
|
328
350
|
/** @internal */
|
|
329
351
|
static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, }) {
|
|
330
352
|
const normalizedMethod = method.toUpperCase();
|
|
@@ -337,7 +359,7 @@ export class Request {
|
|
|
337
359
|
}
|
|
338
360
|
return normalizedUrl;
|
|
339
361
|
}
|
|
340
|
-
const payloadHash = payload ?
|
|
362
|
+
const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
|
|
341
363
|
return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
|
|
342
364
|
}
|
|
343
365
|
/** @internal */
|
|
@@ -345,4 +367,5 @@ export class Request {
|
|
|
345
367
|
return crypto.createHash('sha256').update(payload).digest('base64').replace(/[+/=]/g, '').substring(0, 8);
|
|
346
368
|
}
|
|
347
369
|
}
|
|
370
|
+
export { CrawleeRequest as Request };
|
|
348
371
|
//# sourceMappingURL=request.js.map
|
package/request.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,UAAU,CAAC;AAE7C,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAErC,4DAA4D;AAC5D,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAEpD,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACxC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACzB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACzC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACnC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACpD,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;CACrE,CAAC;AAEF,MAAM,CAAN,IAAY,YASX;AATD,WAAY,YAAY;IACpB,6DAAW,CAAA;IACX,2DAAU,CAAA;IACV,yDAAS,CAAA;IACT,qEAAe,CAAA;IACf,+CAAI,CAAA;IACJ,iEAAa,CAAA;IACb,iDAAK,CAAA;IACL,qDAAO,CAAA;AACX,CAAC,EATW,YAAY,KAAZ,YAAY,QASvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,cAAc;IAChB,iBAAiB;IACjB,EAAE,CAAU;IAEZ,oCAAoC;IACpC,GAAG,CAAS;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAU;IAEnB;;;OAGG;IACH,SAAS,CAAS;IAElB,yCAAyC;IACzC,MAAM,CAAqB;IAE3B,oDAAoD;IACpD,OAAO,CAAU;IAEjB,8FAA8F;IAC9F,OAAO,CAAU;IAEjB,2FAA2F;IAC3F,UAAU,CAAS;IAEnB,0DAA0D;IAC1D,aAAa,CAAW;IAExB,wEAAwE;IACxE,OAAO,CAA0B;IAEjC,sEAAsE;IAC9D,SAAS,GAAwB,EAAE,CAAC;IAE5C,gDAAgD;IAChD,QAAQ,GAAa,EAAc,CAAC;IAEpC;;;OAGG;IACH,SAAS,CAAU;IAEnB;;OAEG;IACH,YAAY,OAAiC;QACzC,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACzC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,0CAA0C;YAC1C,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACjB,OAAO;YACX,CAAC;YAED,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE,CAAC;gBACZ,EAAE,CAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;YACpE,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,oBAAoB,GAAG,CAAC,EACxB,UAAU,EACV,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,KAAK,EACL,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,cAAc,EACd,eAAe,EACf,UAAU,GACb,GAAG,OAMH,CAAC;QAEF,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEjC,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS;YACV,SAAS;gBACT,cAAc,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAI,SAAqB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;QAEzG,IAAI,KAAK,EAAE,CAAC;YACR,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC1B,SAAS,EAAE;gBACP,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,QAAQ,EAAE;gBACrC,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACjB;YACD,QAAQ,EAAE;gBACN,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;gBACzB,GAAG,EAAE,CAAC,KAA0B,EAAE,EAAE;oBAChC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE;wBAC3B,SAAS,EAAE;4BACP,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;4BAC/B,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;wBACD,MAAM,EAAE;4BACJ,KAAK,EAAE,GAAG,EAAE;gCACR,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACnD,OAAO;wCACH,GAAG,IAAI,CAAC,SAAS;wCACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;qCACtC,CAAC;gCACN,CAAC;gCAED,OAAO,IAAI,CAAC,SAAS,CAAC;4BAC1B,CAAC;4BACD,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;qBACJ,CAAC,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,UAAU,EAAE,IAAI;aACnB;SACJ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,cAAc,IAAI,IAAI;YAAE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACjE,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QACrD,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,KAAK,UAAU,CAAC;QAE1E,mFAAmF;QACnF,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,mBAAmB;QACtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,OAAO;SACrB,CAAC,CAAC;IACP,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,IAAI,KAAK,CAAC;IAC5D,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAc;QAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QACxE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;QACnD,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAa;QACvB,IAAI,CAAC,QAAuB,CAAC,SAAS,KAAK,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;IAC/C,CAAC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,oBAAoB,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAa;QAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;QAC9E,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,oBAAoB,GAAG,KAAK,CAAC;QACzD,CAAC;IACL,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAyB;QAC9B,IAAI,CAAC,QAAuB,CAAC,KAAK,GAAG,KAAK,CAAC;IAChD,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC;IAC/C,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAyB;QACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACpE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,IAAI,YAAY,CAAC,WAAW,CAAC;IACtE,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAmB;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAC/D,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAED,IAAY,eAAe;QACvB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;IACpD,CAAC;IAED,IAAY,eAAe,CAAC,KAAsC;QAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;QACzE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,eAAe,GAAG,KAAK,CAAC;QACpD,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClB,OAAO,GAAG,MAAM,CAAC;YACrB,CAAC;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE,CAAC;gBACzC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,CAAC,CAAC,8BAA8B;wBAC9B,cAAc,CAAC,KAAK,CAAC;YAC/B,CAAC;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;gBACxD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;YAC7D,CAAC;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE,CAAC;gBACrE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;gBAC3C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,GAAG,yDAAyD,CAAC;gBACxE,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,GAAG,WAAW,CAAC;QAC1B,CAAC;aAAM,CAAC;YACJ,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAM,GAAG,KAAK,EACd,OAAO,EACP,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,GACN;QACtB,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACxB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE,CAAC;gBACxC,GAAG,CAAC,WAAW,CACX,uBAAuB,gBAAgB,2BAA2B;oBAC9D,kFAAkF;oBAClF,iHAAiH,CACxH,CAAC;YACN,CAAC;YACD,OAAO,aAAa,CAAC;QACzB,CAAC;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAmB;QAClC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9G,CAAC;CACJ;AA8ID,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
|
@@ -1,23 +1,7 @@
|
|
|
1
|
-
import type { Cookie as CookieObject, Dictionary } from '@crawlee/types';
|
|
2
|
-
import type { Cookie
|
|
1
|
+
import type { Cookie as CookieObject, Dictionary, ISession, ProxyInfo, SessionState } from '@crawlee/types';
|
|
2
|
+
import type { Cookie } from 'tough-cookie';
|
|
3
3
|
import { CookieJar } from 'tough-cookie';
|
|
4
4
|
import type { Log } from '@apify/log';
|
|
5
|
-
import type { ResponseLike } from '../cookie_utils.js';
|
|
6
|
-
/**
|
|
7
|
-
* Persistable {@link Session} state.
|
|
8
|
-
*/
|
|
9
|
-
export interface SessionState {
|
|
10
|
-
id: string;
|
|
11
|
-
cookieJar: SerializedCookieJar;
|
|
12
|
-
userData: object;
|
|
13
|
-
errorScore: number;
|
|
14
|
-
maxErrorScore: number;
|
|
15
|
-
errorScoreDecrement: number;
|
|
16
|
-
usageCount: number;
|
|
17
|
-
maxUsageCount: number;
|
|
18
|
-
expiresAt: string;
|
|
19
|
-
createdAt: string;
|
|
20
|
-
}
|
|
21
5
|
export interface SessionOptions {
|
|
22
6
|
/** Id of session used for generating fingerprints. It is used as proxy session name. */
|
|
23
7
|
id?: string;
|
|
@@ -63,6 +47,7 @@ export interface SessionOptions {
|
|
|
63
47
|
log?: Log;
|
|
64
48
|
errorScore?: number;
|
|
65
49
|
cookieJar?: CookieJar;
|
|
50
|
+
proxyInfo?: ProxyInfo;
|
|
66
51
|
}
|
|
67
52
|
/**
|
|
68
53
|
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
@@ -70,7 +55,7 @@ export interface SessionOptions {
|
|
|
70
55
|
* Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
|
|
71
56
|
* @category Scaling
|
|
72
57
|
*/
|
|
73
|
-
export declare class Session {
|
|
58
|
+
export declare class Session implements ISession {
|
|
74
59
|
readonly id: string;
|
|
75
60
|
private maxAgeSecs;
|
|
76
61
|
userData: Dictionary;
|
|
@@ -82,6 +67,7 @@ export declare class Session {
|
|
|
82
67
|
private _maxUsageCount;
|
|
83
68
|
private sessionPool;
|
|
84
69
|
private _errorScore;
|
|
70
|
+
private _proxyInfo?;
|
|
85
71
|
private _cookieJar;
|
|
86
72
|
private log;
|
|
87
73
|
get errorScore(): number;
|
|
@@ -92,6 +78,7 @@ export declare class Session {
|
|
|
92
78
|
get createdAt(): Date;
|
|
93
79
|
get maxUsageCount(): number;
|
|
94
80
|
get cookieJar(): CookieJar;
|
|
81
|
+
get proxyInfo(): ProxyInfo | undefined;
|
|
95
82
|
/**
|
|
96
83
|
* Session configuration.
|
|
97
84
|
*/
|
|
@@ -156,7 +143,7 @@ export declare class Session {
|
|
|
156
143
|
*
|
|
157
144
|
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
158
145
|
*/
|
|
159
|
-
setCookiesFromResponse(response:
|
|
146
|
+
setCookiesFromResponse(response: Response): void;
|
|
160
147
|
/**
|
|
161
148
|
* Saves an array with cookie objects to be used with the session.
|
|
162
149
|
* The objects should be in the format that
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,UAAU,EAAE,
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE5G,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAYtC,MAAM,WAAW,cAAc;IAC3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,OAAO,mBAAmB,EAAE,WAAW,CAAC;IAEtD,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,OAAQ,YAAW,QAAQ;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAC,CAAY;IAC/B,OAAO,CAAC,UAAU,CAAY;IAC9B,OAAO,CAAC,GAAG,CAAM;IAEjB,IAAI,UAAU,WAEb;IAED,IAAI,UAAU,WAEb;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,mBAAmB,WAEtB;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,SAAS,0BAEZ;IAED;;OAEG;gBACS,OAAO,EAAE,cAAc;IA0DnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAgBxB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IASvD;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,QAAQ;IAWzC;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,MAAM;IAK/C;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,EAAE;IAKvC;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAI/C;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|