@crawlee/core 4.0.0-beta.7 → 4.0.0-beta.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/autoscaling/autoscaled_pool.d.ts +3 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -9
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -13
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +18 -29
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +88 -227
- package/configuration.d.ts.map +1 -1
- package/configuration.js +162 -223
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +4 -2
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +18 -12
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +71 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +123 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +20 -29
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +12 -20
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/crawler_utils.d.ts +2 -2
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -1
- package/crawlers/crawler_utils.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +16 -16
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +22 -25
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +36 -21
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +47 -26
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +25 -8
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +69 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +33 -3
- package/errors.d.ts.map +1 -1
- package/errors.js +48 -4
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +34 -7
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +8 -9
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +33 -39
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +4 -2
- package/index.d.ts.map +1 -1
- package/index.js +3 -1
- package/index.js.map +1 -1
- package/log.d.ts +82 -2
- package/log.d.ts.map +1 -1
- package/log.js +102 -0
- package/log.js.map +1 -1
- package/memory-storage/consts.d.ts +5 -0
- package/memory-storage/consts.d.ts.map +1 -0
- package/memory-storage/consts.js +5 -0
- package/memory-storage/consts.js.map +1 -0
- package/memory-storage/index.d.ts +2 -0
- package/memory-storage/index.d.ts.map +1 -0
- package/memory-storage/index.js +2 -0
- package/memory-storage/index.js.map +1 -0
- package/memory-storage/memory-storage.d.ts +47 -0
- package/memory-storage/memory-storage.d.ts.map +1 -0
- package/memory-storage/memory-storage.js +137 -0
- package/memory-storage/memory-storage.js.map +1 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +5 -0
- package/memory-storage/resource-clients/common/base-client.d.ts.map +1 -0
- package/memory-storage/resource-clients/common/base-client.js +7 -0
- package/memory-storage/resource-clients/common/base-client.js.map +1 -0
- package/memory-storage/resource-clients/dataset.d.ts +41 -0
- package/memory-storage/resource-clients/dataset.d.ts.map +1 -0
- package/memory-storage/resource-clients/dataset.js +114 -0
- package/memory-storage/resource-clients/dataset.js.map +1 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +64 -0
- package/memory-storage/resource-clients/key-value-store.d.ts.map +1 -0
- package/memory-storage/resource-clients/key-value-store.js +204 -0
- package/memory-storage/resource-clients/key-value-store.js.map +1 -0
- package/memory-storage/resource-clients/request-queue.d.ts +97 -0
- package/memory-storage/resource-clients/request-queue.d.ts.map +1 -0
- package/memory-storage/resource-clients/request-queue.js +422 -0
- package/memory-storage/resource-clients/request-queue.js.map +1 -0
- package/memory-storage/utils.d.ts +12 -0
- package/memory-storage/utils.d.ts.map +1 -0
- package/memory-storage/utils.js +28 -0
- package/memory-storage/utils.js.map +1 -0
- package/package.json +12 -10
- package/proxy_configuration.d.ts +14 -148
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +19 -167
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +121 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +142 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +84 -12
- package/request.d.ts.map +1 -1
- package/request.js +107 -27
- package/request.js.map +1 -1
- package/router.d.ts.map +1 -1
- package/router.js.map +1 -1
- package/serialization.js +1 -1
- package/serialization.js.map +1 -1
- package/service_locator.d.ts +157 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +239 -0
- package/service_locator.js.map +1 -0
- package/session_pool/consts.d.ts +1 -1
- package/session_pool/consts.d.ts.map +1 -1
- package/session_pool/consts.js +1 -1
- package/session_pool/consts.js.map +1 -1
- package/session_pool/fingerprint.d.ts +10 -0
- package/session_pool/fingerprint.d.ts.map +1 -0
- package/session_pool/fingerprint.js +31 -0
- package/session_pool/fingerprint.js.map +1 -0
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +0 -1
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +35 -72
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +46 -98
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +71 -71
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +111 -99
- package/session_pool/session_pool.js.map +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/access_checking.js +5 -1
- package/storages/access_checking.js.map +1 -1
- package/storages/dataset.d.ts +98 -47
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +166 -123
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +8 -6
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +6 -6
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +163 -33
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +267 -119
- package/storages/key_value_store.js.map +1 -1
- package/storages/key_value_store_codec.d.ts +33 -0
- package/storages/key_value_store_codec.d.ts.map +1 -0
- package/storages/key_value_store_codec.js +114 -0
- package/storages/key_value_store_codec.js.map +1 -0
- package/storages/request_list.d.ts +32 -81
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +49 -39
- package/storages/request_list.js.map +1 -1
- package/storages/request_loader.d.ts +97 -0
- package/storages/request_loader.d.ts.map +1 -0
- package/storages/request_loader.js +2 -0
- package/storages/request_loader.js.map +1 -0
- package/storages/request_manager.d.ts +34 -0
- package/storages/request_manager.d.ts.map +1 -0
- package/storages/request_manager.js +2 -0
- package/storages/request_manager.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +107 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +198 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_queue.d.ts +282 -45
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +592 -213
- package/storages/request_queue.js.map +1 -1
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +29 -24
- package/storages/sitemap_request_loader.d.ts.map +1 -0
- package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +50 -46
- package/storages/sitemap_request_loader.js.map +1 -0
- package/storages/storage_instance_manager.d.ts +92 -0
- package/storages/storage_instance_manager.d.ts.map +1 -0
- package/storages/storage_instance_manager.js +261 -0
- package/storages/storage_instance_manager.js.map +1 -0
- package/storages/storage_stats.d.ts +49 -0
- package/storages/storage_stats.d.ts.map +1 -0
- package/storages/storage_stats.js +30 -0
- package/storages/storage_stats.js.map +1 -0
- package/storages/utils.d.ts +51 -5
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +60 -8
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/validators.d.ts +8 -0
- package/validators.d.ts.map +1 -1
- package/validators.js +8 -0
- package/validators.js.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/storages/request_provider.d.ts +0 -307
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -555
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { KeyValueStore, serviceLocator } from '@crawlee/core';
|
|
2
|
+
/**
|
|
3
|
+
* A class for managing persistent recoverable state using a plain JavaScript object.
|
|
4
|
+
*
|
|
5
|
+
* This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
|
|
6
|
+
* across migrations or restarts. It manages the loading, saving, and resetting of state data,
|
|
7
|
+
* with optional persistence capabilities.
|
|
8
|
+
*
|
|
9
|
+
* The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
|
|
10
|
+
* The class automatically hooks into the event system to persist state when needed.
|
|
11
|
+
*/
|
|
12
|
+
export class RecoverableState {
|
|
13
|
+
defaultState;
|
|
14
|
+
state = null;
|
|
15
|
+
persistenceEnabled;
|
|
16
|
+
persistStateKey;
|
|
17
|
+
persistStateKvsName;
|
|
18
|
+
persistStateKvsId;
|
|
19
|
+
keyValueStore = null;
|
|
20
|
+
log;
|
|
21
|
+
serialize;
|
|
22
|
+
deserialize;
|
|
23
|
+
/**
|
|
24
|
+
* Initialize a new recoverable state object.
|
|
25
|
+
*
|
|
26
|
+
* @param options Configuration options for the recoverable state
|
|
27
|
+
*/
|
|
28
|
+
constructor(options) {
|
|
29
|
+
this.defaultState = options.defaultState;
|
|
30
|
+
this.persistStateKey = options.persistStateKey;
|
|
31
|
+
this.persistenceEnabled = options.persistenceEnabled ?? false;
|
|
32
|
+
this.persistStateKvsName = options.persistStateKvsName;
|
|
33
|
+
this.persistStateKvsId = options.persistStateKvsId;
|
|
34
|
+
this.log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
|
|
35
|
+
this.serialize = options.serialize ?? JSON.stringify;
|
|
36
|
+
this.deserialize = options.deserialize ?? JSON.parse;
|
|
37
|
+
this.persistState = this.persistState.bind(this);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Initialize the recoverable state.
|
|
41
|
+
*
|
|
42
|
+
* This method must be called before using the recoverable state. It loads the saved state
|
|
43
|
+
* if persistence is enabled and registers the object to listen for PERSIST_STATE events.
|
|
44
|
+
*
|
|
45
|
+
* @returns The loaded state object
|
|
46
|
+
*/
|
|
47
|
+
async initialize() {
|
|
48
|
+
if (this.state !== null && this.state !== undefined) {
|
|
49
|
+
return this.currentValue;
|
|
50
|
+
}
|
|
51
|
+
if (!this.persistenceEnabled) {
|
|
52
|
+
this.state = this.deserialize(this.serialize(this.defaultState));
|
|
53
|
+
return this.currentValue;
|
|
54
|
+
}
|
|
55
|
+
let kvsIdentifier = null;
|
|
56
|
+
if (this.persistStateKvsName) {
|
|
57
|
+
kvsIdentifier = { name: this.persistStateKvsName };
|
|
58
|
+
}
|
|
59
|
+
else if (this.persistStateKvsId) {
|
|
60
|
+
kvsIdentifier = { id: this.persistStateKvsId };
|
|
61
|
+
}
|
|
62
|
+
this.keyValueStore = await KeyValueStore.open(kvsIdentifier, { config: serviceLocator.getConfiguration() });
|
|
63
|
+
await this.loadSavedState();
|
|
64
|
+
// Register for persist state events
|
|
65
|
+
const eventManager = serviceLocator.getEventManager();
|
|
66
|
+
eventManager.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
|
|
67
|
+
return this.currentValue;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Clean up resources used by the recoverable state.
|
|
71
|
+
*
|
|
72
|
+
* If persistence is enabled, this method deregisters the object from PERSIST_STATE events
|
|
73
|
+
* and persists the current state one last time.
|
|
74
|
+
*/
|
|
75
|
+
async teardown() {
|
|
76
|
+
if (!this.persistenceEnabled || !this.persistState) {
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
const eventManager = serviceLocator.getEventManager();
|
|
80
|
+
eventManager.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
|
|
81
|
+
await this.persistState();
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Get the current state.
|
|
85
|
+
*/
|
|
86
|
+
get currentValue() {
|
|
87
|
+
if (this.state === null) {
|
|
88
|
+
throw new Error('Recoverable state has not yet been loaded');
|
|
89
|
+
}
|
|
90
|
+
return this.state;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Reset the state to the default values and clear any persisted state.
|
|
94
|
+
*
|
|
95
|
+
* Resets the current state to the default state and, if persistence is enabled,
|
|
96
|
+
* clears the persisted state from the KeyValueStore.
|
|
97
|
+
*/
|
|
98
|
+
async reset() {
|
|
99
|
+
this.state = this.deserialize(this.serialize(this.defaultState));
|
|
100
|
+
if (this.persistenceEnabled) {
|
|
101
|
+
if (this.keyValueStore === null) {
|
|
102
|
+
throw new Error('Recoverable state has not yet been initialized');
|
|
103
|
+
}
|
|
104
|
+
await this.keyValueStore.setValue(this.persistStateKey, null);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Persist the current state to the KeyValueStore.
|
|
109
|
+
*
|
|
110
|
+
* This method is typically called in response to a PERSIST_STATE event, but can also be called
|
|
111
|
+
* directly when needed.
|
|
112
|
+
*
|
|
113
|
+
* @param eventData Optional data associated with a PERSIST_STATE event
|
|
114
|
+
*/
|
|
115
|
+
async persistState(eventData) {
|
|
116
|
+
this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
|
|
117
|
+
if (this.keyValueStore === null || this.state === null) {
|
|
118
|
+
throw new Error('Recoverable state has not yet been initialized');
|
|
119
|
+
}
|
|
120
|
+
if (this.persistenceEnabled) {
|
|
121
|
+
await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
|
|
122
|
+
contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Load the saved state from the KeyValueStore
|
|
128
|
+
*/
|
|
129
|
+
async loadSavedState() {
|
|
130
|
+
if (this.keyValueStore === null) {
|
|
131
|
+
throw new Error('Recoverable state has not yet been initialized');
|
|
132
|
+
}
|
|
133
|
+
const storedState = await this.keyValueStore.getValue(this.persistStateKey);
|
|
134
|
+
if (storedState === null || storedState === undefined) {
|
|
135
|
+
this.state = this.deserialize(this.serialize(this.defaultState));
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
this.state = this.deserialize(storedState);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
//# sourceMappingURL=recoverable_state.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recoverable_state.js","sourceRoot":"","sources":["../src/recoverable_state.ts"],"names":[],"mappings":"AACA,OAAO,EAAa,aAAa,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AA8DzE;;;;;;;;;GASG;AACH,MAAM,OAAO,gBAAgB;IACR,YAAY,CAAc;IACnC,KAAK,GAAuB,IAAI,CAAC;IACxB,kBAAkB,CAAU;IAC5B,eAAe,CAAS;IACxB,mBAAmB,CAAU;IAC7B,iBAAiB,CAAU;IACpC,aAAa,GAAyB,IAAI,CAAC;IAClC,GAAG,CAAgB;IACnB,SAAS,CAAiC;IAC1C,WAAW,CAA2C;IAEvE;;;;OAIG;IACH,YAAY,OAA6C;QACrD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,KAAK,CAAC;QAC9D,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAC9F,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,aAAa,GAA6C,IAAI,CAAC;QAEnE,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,aAAa,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACvD,CAAC;aAAM,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAChC,aAAa,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACnD,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAE5G,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,oCAAoC;QACpC,MAAM,YAAY,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QACtD,YAAY,CAAC,EAAE,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAE5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACjD,OAAO;QACX,CAAC;QAED,MAAM,YAAY,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QACtD,YAAY,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,YAAY,CAAC,SAAoC;QACnD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uDAAuD,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAChF,WAAW,EAAE,YAAY,EAAE,wHAAwH;aACtJ,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5E,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YACpD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,WAAqB,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;CACJ"}
|
package/request.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BinaryLike } from 'node:crypto';
|
|
2
2
|
import type { Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
|
|
4
|
+
import type { SkippedRequestReason } from './enqueue_links/shared.js';
|
|
4
5
|
import type { AllowedHttpMethods } from './typedefs.js';
|
|
5
6
|
export declare enum RequestState {
|
|
6
7
|
UNPROCESSED = 0,
|
|
@@ -43,7 +44,7 @@ export declare enum RequestState {
|
|
|
43
44
|
* ```
|
|
44
45
|
* @category Sources
|
|
45
46
|
*/
|
|
46
|
-
|
|
47
|
+
declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
47
48
|
/** Request ID */
|
|
48
49
|
id?: string;
|
|
49
50
|
/** URL of the web page to crawl. */
|
|
@@ -76,7 +77,12 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
76
77
|
headers?: Record<string, string>;
|
|
77
78
|
/** Private store for the custom user data assigned to the request. */
|
|
78
79
|
private _userData;
|
|
79
|
-
/**
|
|
80
|
+
/**
|
|
81
|
+
* Custom user data assigned to the request.
|
|
82
|
+
*
|
|
83
|
+
* All data stored in `userData` must be JSON-serializable.
|
|
84
|
+
* Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
|
|
85
|
+
*/
|
|
80
86
|
userData: UserData;
|
|
81
87
|
/**
|
|
82
88
|
* ISO datetime string that indicates the time when the request has been processed.
|
|
@@ -87,14 +93,40 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
87
93
|
* `Request` parameters including the URL, HTTP method and headers, and others.
|
|
88
94
|
*/
|
|
89
95
|
constructor(options: RequestOptions<UserData>);
|
|
90
|
-
/**
|
|
96
|
+
/**
|
|
97
|
+
* Converts the Crawlee Request object to a `fetch` API Request object.
|
|
98
|
+
* @returns The native `fetch` API Request object.
|
|
99
|
+
*/
|
|
100
|
+
intoFetchAPIRequest(): Request;
|
|
101
|
+
/**
|
|
102
|
+
* Tells the crawler processing this request to skip the navigation and process the request directly.
|
|
103
|
+
*
|
|
104
|
+
* When this is set to `true`, the crawling context will not contain the results of the navigation
|
|
105
|
+
* (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
|
|
106
|
+
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
107
|
+
*/
|
|
91
108
|
get skipNavigation(): boolean;
|
|
92
|
-
/**
|
|
109
|
+
/**
|
|
110
|
+
* Tells the crawler processing this request to skip the navigation and process the request directly.
|
|
111
|
+
*
|
|
112
|
+
* When this is set to `true`, the crawling context will not contain the results of the navigation
|
|
113
|
+
* (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
|
|
114
|
+
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
115
|
+
*/
|
|
93
116
|
set skipNavigation(value: boolean);
|
|
94
|
-
/**
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
117
|
+
/**
|
|
118
|
+
* Depth of the request in the current crawl tree.
|
|
119
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
120
|
+
*/
|
|
121
|
+
get crawlDepth(): number;
|
|
122
|
+
/**
|
|
123
|
+
* Depth of the request in the current crawl tree.
|
|
124
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
125
|
+
*/
|
|
126
|
+
set crawlDepth(value: number);
|
|
127
|
+
/** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
|
|
128
|
+
get sessionId(): string | undefined;
|
|
129
|
+
set sessionId(value: string | undefined);
|
|
98
130
|
/** shortcut for getting `request.userData.label` */
|
|
99
131
|
get label(): string | undefined;
|
|
100
132
|
/** shortcut for setting `request.userData.label` */
|
|
@@ -107,6 +139,14 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
107
139
|
get state(): RequestState;
|
|
108
140
|
/** Describes the request's current lifecycle state. */
|
|
109
141
|
set state(value: RequestState);
|
|
142
|
+
/**
|
|
143
|
+
* Reason for skipping this request.
|
|
144
|
+
*/
|
|
145
|
+
get skippedReason(): SkippedRequestReason | undefined;
|
|
146
|
+
/**
|
|
147
|
+
* Reason for skipping this request.
|
|
148
|
+
*/
|
|
149
|
+
set skippedReason(value: SkippedRequestReason | undefined);
|
|
110
150
|
private get enqueueStrategy();
|
|
111
151
|
private set enqueueStrategy(value);
|
|
112
152
|
/**
|
|
@@ -124,7 +164,7 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
124
164
|
*/
|
|
125
165
|
pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
|
|
126
166
|
/** @internal */
|
|
127
|
-
static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, }: ComputeUniqueKeyOptions): string;
|
|
167
|
+
static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
|
|
128
168
|
/** @internal */
|
|
129
169
|
static hashPayload(payload: BinaryLike): string;
|
|
130
170
|
}
|
|
@@ -145,7 +185,7 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
145
185
|
* The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
|
|
146
186
|
*
|
|
147
187
|
* The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
|
|
148
|
-
* producing a `uniqueKey` in the following format: `METHOD
|
|
188
|
+
* producing a `uniqueKey` in the following format: `METHOD|payloadHash|normalizedUrl`. This is useful
|
|
149
189
|
* when requests point to the same URL, but with different methods and payloads. For example: form submits.
|
|
150
190
|
*
|
|
151
191
|
* Pass an arbitrary non-empty text value to the `uniqueKey` property
|
|
@@ -169,6 +209,9 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
169
209
|
/**
|
|
170
210
|
* Custom user data assigned to the request. Use this to save any request related data to the
|
|
171
211
|
* request's scope, keeping them accessible on retries, failures etc.
|
|
212
|
+
*
|
|
213
|
+
* All data stored in `userData` must be JSON-serializable.
|
|
214
|
+
* Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
|
|
172
215
|
*/
|
|
173
216
|
userData?: UserData;
|
|
174
217
|
/**
|
|
@@ -190,17 +233,45 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
190
233
|
* @default false
|
|
191
234
|
*/
|
|
192
235
|
useExtendedUniqueKey?: boolean;
|
|
236
|
+
/**
|
|
237
|
+
* If `true` then a random value is included in the `uniqueKey` computation, ensuring the request
|
|
238
|
+
* is always enqueued even if a request with the same URL (and method/payload) is already present
|
|
239
|
+
* in the queue. Cannot be used together with a custom `uniqueKey`.
|
|
240
|
+
* @default false
|
|
241
|
+
*/
|
|
242
|
+
alwaysEnqueue?: boolean;
|
|
193
243
|
/**
|
|
194
244
|
* The `true` value indicates that the request will not be automatically retried on error.
|
|
195
245
|
* @default false
|
|
196
246
|
*/
|
|
197
247
|
noRetry?: boolean;
|
|
248
|
+
/**
|
|
249
|
+
* ID of a session from the crawler's `SessionPool` to use for this request.
|
|
250
|
+
* When set, the crawler will fetch this session from the pool instead of creating a new one.
|
|
251
|
+
*/
|
|
252
|
+
sessionId?: string;
|
|
198
253
|
/**
|
|
199
254
|
* If set to `true` then the crawler processing this request evaluates
|
|
200
255
|
* the `requestHandler` immediately without prior browser navigation.
|
|
256
|
+
*
|
|
257
|
+
* When enabled, the crawling context will not contain the results of the navigation
|
|
258
|
+
* (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
|
|
259
|
+
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
201
260
|
* @default false
|
|
202
261
|
*/
|
|
203
262
|
skipNavigation?: boolean;
|
|
263
|
+
/**
|
|
264
|
+
* Depth of the request in the current crawl tree.
|
|
265
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
266
|
+
* @default 0
|
|
267
|
+
*/
|
|
268
|
+
crawlDepth?: number;
|
|
269
|
+
/**
|
|
270
|
+
* Reason for skipping this request.
|
|
271
|
+
* This is used to provide more information about why the request was skipped.
|
|
272
|
+
* @internal
|
|
273
|
+
*/
|
|
274
|
+
skippedReason?: SkippedRequestReason;
|
|
204
275
|
/**
|
|
205
276
|
* Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
|
|
206
277
|
*/
|
|
@@ -227,15 +298,16 @@ interface ComputeUniqueKeyOptions {
|
|
|
227
298
|
payload?: string | Buffer;
|
|
228
299
|
keepUrlFragment?: boolean;
|
|
229
300
|
useExtendedUniqueKey?: boolean;
|
|
301
|
+
alwaysEnqueue?: boolean;
|
|
230
302
|
}
|
|
231
303
|
export type Source = (Partial<RequestOptions> & {
|
|
232
304
|
requestsFromUrl?: string;
|
|
233
305
|
regex?: RegExp;
|
|
234
|
-
}) |
|
|
306
|
+
}) | CrawleeRequest;
|
|
235
307
|
/** @internal */
|
|
236
308
|
export interface InternalSource {
|
|
237
309
|
requestsFromUrl: string;
|
|
238
310
|
regex?: RegExp;
|
|
239
311
|
}
|
|
240
|
-
export {};
|
|
312
|
+
export { CrawleeRequest as Request };
|
|
241
313
|
//# sourceMappingURL=request.d.ts.map
|
package/request.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA2BxD,oBAAY,YAAY;IACpB,WAAW,IAAA;IACX,UAAU,IAAA;IACV,SAAS,IAAA;IACT,eAAe,IAAA;IACf,IAAI,IAAA;IACJ,aAAa,IAAA;IACb,KAAK,IAAA;IACL,OAAO,IAAA;CACV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,cAAM,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C;;;;;OAKG;IACH,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;IAuI7C;;;OAGG;IACI,mBAAmB,IAAI,OAAO;IAQrC;;;;;;OAMG;IACH,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED;;;;;;OAMG;IACH,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAMhC;IAED;;;OAGG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,EAG3B;IAED,kJAAkJ;IAClJ,IAAI,SAAS,IAAI,MAAM,GAAG,SAAS,CAElC;IAED,IAAI,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAGtC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED,8HAA8H;IAC9H,IAAI,UAAU,IAAI,MAAM,GAAG,SAAS,CAEnC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAMvC;IAED,uDAAuD;IACvD,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,EAM5B;IAED;;OAEG;IACH,IAAI,aAAa,IAAI,oBAAoB,GAAG,SAAS,CAEpD;IAED;;OAEG;IACH,IAAI,aAAa,CAAC,KAAK,EAAE,oBAAoB,GAAG,SAAS,EAMxD;IAED,OAAO,KAAK,eAAe,GAE1B;IAED,OAAO,KAAK,eAAe,QAM1B;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAc,EACd,OAAO,EACP,eAAuB,EACvB,oBAA4B,EAC5B,aAAqB,GACxB,EAAE,uBAAuB;IA6B1B,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAGlD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IAExB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,oBAAoB,CAAC;IAErC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gBAAgB;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IAErB,gBAAgB;IAChB,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,cAAc,CAAC;AAE/G,gBAAgB;AAChB,MAAM,WAAW,cAAc;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
package/request.js
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import crypto from 'node:crypto';
|
|
2
2
|
import util from 'node:util';
|
|
3
3
|
import ow from 'ow';
|
|
4
|
-
import { normalizeUrl } from '@apify/utilities';
|
|
5
|
-
import {
|
|
4
|
+
import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
|
|
5
|
+
import { serviceLocator } from './service_locator.js';
|
|
6
6
|
import { keys } from './typedefs.js';
|
|
7
7
|
// new properties on the Request object breaks serialization
|
|
8
|
-
const log = defaultLog.child({ prefix: 'Request' });
|
|
9
8
|
const requestOptionalPredicates = {
|
|
10
9
|
id: ow.optional.string,
|
|
11
10
|
loadedUrl: ow.optional.string.url,
|
|
@@ -14,7 +13,7 @@ const requestOptionalPredicates = {
|
|
|
14
13
|
payload: ow.optional.any(ow.string, ow.uint8Array),
|
|
15
14
|
noRetry: ow.optional.boolean,
|
|
16
15
|
retryCount: ow.optional.number,
|
|
17
|
-
|
|
16
|
+
sessionId: ow.optional.string,
|
|
18
17
|
maxRetries: ow.optional.number,
|
|
19
18
|
errorMessages: ow.optional.array.ofType(ow.string),
|
|
20
19
|
headers: ow.optional.object,
|
|
@@ -23,7 +22,9 @@ const requestOptionalPredicates = {
|
|
|
23
22
|
handledAt: ow.optional.any(ow.string.date, ow.date),
|
|
24
23
|
keepUrlFragment: ow.optional.boolean,
|
|
25
24
|
useExtendedUniqueKey: ow.optional.boolean,
|
|
25
|
+
alwaysEnqueue: ow.optional.boolean,
|
|
26
26
|
skipNavigation: ow.optional.boolean,
|
|
27
|
+
crawlDepth: ow.optional.number.greaterThanOrEqual(0),
|
|
27
28
|
state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
|
|
28
29
|
};
|
|
29
30
|
export var RequestState;
|
|
@@ -68,7 +69,7 @@ export var RequestState;
|
|
|
68
69
|
* ```
|
|
69
70
|
* @category Sources
|
|
70
71
|
*/
|
|
71
|
-
|
|
72
|
+
class CrawleeRequest {
|
|
72
73
|
/** Request ID */
|
|
73
74
|
id;
|
|
74
75
|
/** URL of the web page to crawl. */
|
|
@@ -101,7 +102,12 @@ export class Request {
|
|
|
101
102
|
headers;
|
|
102
103
|
/** Private store for the custom user data assigned to the request. */
|
|
103
104
|
_userData = {};
|
|
104
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* Custom user data assigned to the request.
|
|
107
|
+
*
|
|
108
|
+
* All data stored in `userData` must be JSON-serializable.
|
|
109
|
+
* Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
|
|
110
|
+
*/
|
|
105
111
|
userData = {};
|
|
106
112
|
/**
|
|
107
113
|
* ISO datetime string that indicates the time when the request has been processed.
|
|
@@ -130,21 +136,31 @@ export class Request {
|
|
|
130
136
|
ow(value, `RequestOptions.${prop}`, predicate);
|
|
131
137
|
}
|
|
132
138
|
});
|
|
133
|
-
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0,
|
|
139
|
+
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
|
|
134
140
|
let { method = 'GET' } = options;
|
|
135
141
|
method = method.toUpperCase();
|
|
136
142
|
if (method === 'GET' && payload)
|
|
137
143
|
throw new Error('Request with GET method cannot have a payload.');
|
|
144
|
+
if (uniqueKey && alwaysEnqueue) {
|
|
145
|
+
throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
|
|
146
|
+
}
|
|
138
147
|
this.id = id;
|
|
139
148
|
this.url = url;
|
|
140
149
|
this.loadedUrl = loadedUrl;
|
|
141
150
|
this.uniqueKey =
|
|
142
|
-
uniqueKey ||
|
|
151
|
+
uniqueKey ||
|
|
152
|
+
CrawleeRequest.computeUniqueKey({
|
|
153
|
+
url,
|
|
154
|
+
method,
|
|
155
|
+
payload,
|
|
156
|
+
keepUrlFragment,
|
|
157
|
+
useExtendedUniqueKey,
|
|
158
|
+
alwaysEnqueue,
|
|
159
|
+
});
|
|
143
160
|
this.method = method;
|
|
144
161
|
this.payload = payload;
|
|
145
162
|
this.noRetry = noRetry;
|
|
146
163
|
this.retryCount = retryCount;
|
|
147
|
-
this.sessionRotationCount = sessionRotationCount;
|
|
148
164
|
this.errorMessages = [...errorMessages];
|
|
149
165
|
this.headers = { ...headers };
|
|
150
166
|
this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
|
|
@@ -191,16 +207,43 @@ export class Request {
|
|
|
191
207
|
this.skipNavigation = skipNavigation;
|
|
192
208
|
if (maxRetries != null)
|
|
193
209
|
this.maxRetries = maxRetries;
|
|
210
|
+
if (crawlDepth != null)
|
|
211
|
+
this.userData.__crawlee.crawlDepth ??= crawlDepth;
|
|
212
|
+
if (sessionId)
|
|
213
|
+
this.sessionId = sessionId;
|
|
194
214
|
// If it's already set, don't override it (for instance when fetching from storage)
|
|
195
215
|
if (enqueueStrategy) {
|
|
196
216
|
this.enqueueStrategy ??= enqueueStrategy;
|
|
197
217
|
}
|
|
198
218
|
}
|
|
199
|
-
/**
|
|
219
|
+
/**
|
|
220
|
+
* Converts the Crawlee Request object to a `fetch` API Request object.
|
|
221
|
+
* @returns The native `fetch` API Request object.
|
|
222
|
+
*/
|
|
223
|
+
intoFetchAPIRequest() {
|
|
224
|
+
return new Request(this.url, {
|
|
225
|
+
method: this.method,
|
|
226
|
+
headers: this.headers,
|
|
227
|
+
body: this.payload,
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Tells the crawler processing this request to skip the navigation and process the request directly.
|
|
232
|
+
*
|
|
233
|
+
* When this is set to `true`, the crawling context will not contain the results of the navigation
|
|
234
|
+
* (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
|
|
235
|
+
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
236
|
+
*/
|
|
200
237
|
get skipNavigation() {
|
|
201
238
|
return this.userData.__crawlee?.skipNavigation ?? false;
|
|
202
239
|
}
|
|
203
|
-
/**
|
|
240
|
+
/**
|
|
241
|
+
* Tells the crawler processing this request to skip the navigation and process the request directly.
|
|
242
|
+
*
|
|
243
|
+
* When this is set to `true`, the crawling context will not contain the results of the navigation
|
|
244
|
+
* (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
|
|
245
|
+
* Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
|
|
246
|
+
*/
|
|
204
247
|
set skipNavigation(value) {
|
|
205
248
|
if (!this.userData.__crawlee) {
|
|
206
249
|
this.userData.__crawlee = { skipNavigation: value };
|
|
@@ -209,18 +252,28 @@ export class Request {
|
|
|
209
252
|
this.userData.__crawlee.skipNavigation = value;
|
|
210
253
|
}
|
|
211
254
|
}
|
|
212
|
-
/**
|
|
213
|
-
|
|
214
|
-
|
|
255
|
+
/**
|
|
256
|
+
* Depth of the request in the current crawl tree.
|
|
257
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
258
|
+
*/
|
|
259
|
+
get crawlDepth() {
|
|
260
|
+
return this.userData.__crawlee?.crawlDepth ?? 0;
|
|
215
261
|
}
|
|
216
|
-
/**
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
262
|
+
/**
|
|
263
|
+
* Depth of the request in the current crawl tree.
|
|
264
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
265
|
+
*/
|
|
266
|
+
set crawlDepth(value) {
|
|
267
|
+
this.userData.__crawlee ??= {};
|
|
268
|
+
this.userData.__crawlee.crawlDepth = value;
|
|
269
|
+
}
|
|
270
|
+
/** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
|
|
271
|
+
get sessionId() {
|
|
272
|
+
return this.userData.__crawlee?.sessionId;
|
|
273
|
+
}
|
|
274
|
+
set sessionId(value) {
|
|
275
|
+
this.userData.__crawlee ??= {};
|
|
276
|
+
this.userData.__crawlee.sessionId = value;
|
|
224
277
|
}
|
|
225
278
|
/** shortcut for getting `request.userData.label` */
|
|
226
279
|
get label() {
|
|
@@ -256,6 +309,23 @@ export class Request {
|
|
|
256
309
|
this.userData.__crawlee.state = value;
|
|
257
310
|
}
|
|
258
311
|
}
|
|
312
|
+
/**
|
|
313
|
+
* Reason for skipping this request.
|
|
314
|
+
*/
|
|
315
|
+
get skippedReason() {
|
|
316
|
+
return this.userData.__crawlee?.skippedReason;
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Reason for skipping this request.
|
|
320
|
+
*/
|
|
321
|
+
set skippedReason(value) {
|
|
322
|
+
if (!this.userData.__crawlee) {
|
|
323
|
+
this.userData.__crawlee = { skippedReason: value };
|
|
324
|
+
}
|
|
325
|
+
else {
|
|
326
|
+
this.userData.__crawlee.skippedReason = value;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
259
329
|
get enqueueStrategy() {
|
|
260
330
|
return this.userData.__crawlee?.enqueueStrategy;
|
|
261
331
|
}
|
|
@@ -318,23 +388,33 @@ export class Request {
|
|
|
318
388
|
this.errorMessages.push(message);
|
|
319
389
|
}
|
|
320
390
|
/** @internal */
|
|
321
|
-
static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, }) {
|
|
391
|
+
static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, }) {
|
|
322
392
|
const normalizedMethod = method.toUpperCase();
|
|
323
393
|
const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
|
|
394
|
+
let uniqueKey;
|
|
324
395
|
if (!useExtendedUniqueKey) {
|
|
325
396
|
if (normalizedMethod !== 'GET' && payload) {
|
|
326
|
-
|
|
397
|
+
serviceLocator
|
|
398
|
+
.getLogger()
|
|
399
|
+
.warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
|
|
327
400
|
'This is fine. Just letting you know that if your requests point to the same URL ' +
|
|
328
401
|
'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
|
|
329
402
|
}
|
|
330
|
-
|
|
403
|
+
uniqueKey = normalizedUrl;
|
|
404
|
+
}
|
|
405
|
+
else {
|
|
406
|
+
const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
|
|
407
|
+
uniqueKey = `${normalizedMethod}|${payloadHash}|${normalizedUrl}`;
|
|
408
|
+
}
|
|
409
|
+
if (alwaysEnqueue) {
|
|
410
|
+
uniqueKey = `${cryptoRandomObjectId(17)}|${uniqueKey}`;
|
|
331
411
|
}
|
|
332
|
-
|
|
333
|
-
return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
|
|
412
|
+
return uniqueKey;
|
|
334
413
|
}
|
|
335
414
|
/** @internal */
|
|
336
415
|
static hashPayload(payload) {
|
|
337
416
|
return crypto.createHash('sha256').update(payload).digest('base64').replace(/[+/=]/g, '').substring(0, 8);
|
|
338
417
|
}
|
|
339
418
|
}
|
|
419
|
+
export { CrawleeRequest as Request };
|
|
340
420
|
//# sourceMappingURL=request.js.map
|
package/request.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAItE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAErC,4DAA4D;AAC5D,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACzB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACzC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IAClC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACnC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACpD,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;CACrE,CAAC;AAEF,MAAM,CAAN,IAAY,YASX;AATD,WAAY,YAAY;IACpB,6DAAW,CAAA;IACX,2DAAU,CAAA;IACV,yDAAS,CAAA;IACT,qEAAe,CAAA;IACf,+CAAI,CAAA;IACJ,iEAAa,CAAA;IACb,iDAAK,CAAA;IACL,qDAAO,CAAA;AACX,CAAC,EATW,YAAY,KAAZ,YAAY,QASvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,cAAc;IAChB,iBAAiB;IACjB,EAAE,CAAU;IAEZ,oCAAoC;IACpC,GAAG,CAAS;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAU;IAEnB;;;OAGG;IACH,SAAS,CAAS;IAElB,yCAAyC;IACzC,MAAM,CAAqB;IAE3B,oDAAoD;IACpD,OAAO,CAAU;IAEjB,8FAA8F;IAC9F,OAAO,CAAU;IAEjB,2FAA2F;IAC3F,UAAU,CAAS;IAEnB,0DAA0D;IAC1D,aAAa,CAAW;IAExB,wEAAwE;IACxE,OAAO,CAA0B;IAEjC,sEAAsE;IAC9D,SAAS,GAAwB,EAAE,CAAC;IAE5C;;;;;OAKG;IACH,QAAQ,GAAa,EAAc,CAAC;IAEpC;;;OAGG;IACH,SAAS,CAAU;IAEnB;;OAEG;IACH,YAAY,OAAiC;QACzC,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACzC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,0CAA0C;YAC1C,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACjB,OAAO;YACX,CAAC;YAED,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE,CAAC;gBACZ,EAAE,CAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;YACpE,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,SAAS,EACT,UAAU,EACV,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,KAAK,EACL,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,KAAK,EACrB,cAAc,EACd,eAAe,EACf,UAAU,GACb,GAAG,OAMH,CAAC;QAEF,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEjC,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,SAAS,IAAI,aAAa,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS;YACV,SAAS;gBACT,cAAc,CAAC,gBAAgB,CAAC;oBAC5B,GAAG;oBACH,MAAM;oBACN,OAAO;oBACP,eAAe;oBACf,oBAAoB;oBACpB,aAAa;iBAChB,CAAC,CAAC;QACP,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAI,SAAqB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;QAEzG,IAAI,KAAK,EAAE,CAAC;YACR,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC1B,SAAS,EAAE;gBACP,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,QAAQ,EAAE;gBACrC,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACjB;YACD,QAAQ,EAAE;gBACN,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;gBACzB,GAAG,EAAE,CAAC,KAA0B,EAAE,EAAE;oBAChC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE;wBAC3B,SAAS,EAAE;4BACP,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;4BAC/B,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;wBACD,MAAM,EAAE;4BACJ,KAAK,EAAE,GAAG,EAAE;gCACR,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACnD,OAAO;wCACH,GAAG,IAAI,CAAC,SAAS;wCACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;qCACtC,CAAC;gCACN,CAAC;gCAED,OAAO,IAAI,CAAC,SAAS,CAAC;4BAC1B,CAAC;4BACD,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;qBACJ,CAAC,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,UAAU,EAAE,IAAI;aACnB;SACJ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,cAAc,IAAI,IAAI;YAAE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACjE,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QACrD,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,KAAK,UAAU,CAAC;QAC1E,IAAI,SAAS;YAAE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE1C,mFAAmF;QACnF,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,mBAAmB;QACtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,OAAO;SACrB,CAAC,CAAC;IACP,CAAC;IAED;;;;;;OAMG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,IAAI,KAAK,CAAC;IAC5D,CAAC;IAED;;;;;;OAMG;IACH,IAAI,cAAc,CAAC,KAAc;QAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QACxE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;QACnD,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAa;QACvB,IAAI,CAAC,QAAuB,CAAC,SAAS,KAAK,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;IAC/C,CAAC;IAED,kJAAkJ;IAClJ,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9C,CAAC;IAED,IAAI,SAAS,CAAC,KAAyB;QAClC,IAAI,CAAC,QAAuB,CAAC,SAAS,KAAK,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,GAAG,KAAK,CAAC;IAC9C,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAyB;QAC9B,IAAI,CAAC,QAAuB,CAAC,KAAK,GAAG,KAAK,CAAC;IAChD,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC;IAC/C,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAyB;QACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACpE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,IAAI,YAAY,CAAC,WAAW,CAAC;IACtE,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAmB;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAC/D,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,IAAI,aAAa,CAAC,KAAuC;QACrD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACvE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,GAAG,KAAK,CAAC;QAClD,CAAC;IACL,CAAC;IAED,IAAY,eAAe;QACvB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;IACpD,CAAC;IAED,IAAY,eAAe,CAAC,KAAsC;QAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;QACzE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,eAAe,GAAG,KAAK,CAAC;QACpD,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClB,OAAO,GAAG,MAAM,CAAC;YACrB,CAAC;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE,CAAC;gBACzC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,CAAC,CAAC,8BAA8B;wBAC9B,cAAc,CAAC,KAAK,CAAC;YAC/B,CAAC;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;gBACxD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;YAC7D,CAAC;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE,CAAC;gBACrE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;gBAC3C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,GAAG,yDAAyD,CAAC;gBACxE,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,GAAG,WAAW,CAAC;QAC1B,CAAC;aAAM,CAAC;YACJ,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAM,GAAG,KAAK,EACd,OAAO,EACP,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,KAAK,GACC;QACtB,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAE9H,IAAI,SAAiB,CAAC;QAEtB,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACxB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE,CAAC;gBACxC,cAAc;qBACT,SAAS,EAAE;qBACX,WAAW,CACR,uBAAuB,gBAAgB,2BAA2B;oBAC9D,kFAAkF;oBAClF,iHAAiH,CACxH,CAAC;YACV,CAAC;YACD,SAAS,GAAG,aAAa,CAAC;QAC9B,CAAC;aAAM,CAAC;YACJ,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,SAAS,GAAG,GAAG,gBAAgB,IAAI,WAAW,IAAI,aAAa,EAAE,CAAC;QACtE,CAAC;QAED,IAAI,aAAa,EAAE,CAAC;YAChB,SAAS,GAAG,GAAG,oBAAoB,CAAC,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QAC3D,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAmB;QAClC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9G,CAAC;CACJ;AAoKD,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
package/router.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAE/G,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAI/C,MAAM,WAAW,aAAa,
|
|
1
|
+
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAE/G,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAI/C,MAAM,WAAW,aAAa,CAC1B,OAAO,SAAS,IAAI,CAAC,yBAAyB,EAAE,cAAc,CAAC,GAAG,eAAe,CACnF,SAAQ,MAAM,CAAC,OAAO,CAAC;IACrB,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACnC;AAED,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;AAE/E,MAAM,MAAM,YAAY,CAAC,OAAO,EAAE,QAAQ,SAAS,UAAU,IAAI;KAC5D,KAAK,IAAI,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG;QAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAA;KAAE,KAAK,SAAS,CAAC,IAAI,CAAC;CAClH,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,MAAM,CAAC,OAAO,SAAS,IAAI,CAAC,yBAAyB,EAAE,cAAc,CAAC;IAC/E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkE;IACzF,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6C;IAEzE;;;OAGG;IACH,SAAS;IAET;;OAEG;IACH,UAAU,CAAC,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC/E,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,OAAO,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG;QAAE,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAA;KAAE,KAAK,SAAS,CAAC,IAAI,CAAC;IAM/G;;OAEG;IACH,iBAAiB,CAAC,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EACtF,OAAO,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG;QAAE,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAA;KAAE,KAAK,SAAS,CAAC,IAAI,CAAC;IAM/G;;;OAGG;IACH,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAIjD;;OAEG;IACH,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAgBtE;;OAEG;IACH,OAAO,CAAC,QAAQ;IAUhB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM,CACT,OAAO,SAAS,IAAI,CAAC,yBAAyB,EAAE,cAAc,CAAC,GAAG,eAAe,EACjF,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,aAAa,CAAC,OAAO,CAAC;CA4BtE"}
|