@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +296 -0
- package/autoscaling/autoscaled_pool.d.ts.map +1 -0
- package/autoscaling/autoscaled_pool.js +673 -0
- package/autoscaling/autoscaled_pool.js.map +1 -0
- package/autoscaling/index.d.ts +4 -0
- package/autoscaling/index.d.ts.map +1 -0
- package/autoscaling/index.js +7 -0
- package/autoscaling/index.js.map +1 -0
- package/autoscaling/snapshotter.d.ts +193 -0
- package/autoscaling/snapshotter.d.ts.map +1 -0
- package/autoscaling/snapshotter.js +377 -0
- package/autoscaling/snapshotter.js.map +1 -0
- package/autoscaling/system_status.d.ts +181 -0
- package/autoscaling/system_status.d.ts.map +1 -0
- package/autoscaling/system_status.js +194 -0
- package/autoscaling/system_status.js.map +1 -0
- package/cache_container.d.ts +18 -0
- package/cache_container.d.ts.map +1 -0
- package/cache_container.js +42 -0
- package/cache_container.js.map +1 -0
- package/configuration.d.ts +150 -0
- package/configuration.d.ts.map +1 -0
- package/configuration.js +284 -0
- package/configuration.js.map +1 -0
- package/constants.d.ts +3 -0
- package/constants.d.ts.map +1 -0
- package/constants.js +6 -0
- package/constants.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +25 -0
- package/crawlers/crawler_commons.d.ts.map +1 -0
- package/crawlers/crawler_commons.js +3 -0
- package/crawlers/crawler_commons.js.map +1 -0
- package/crawlers/crawler_extension.d.ts +11 -0
- package/crawlers/crawler_extension.d.ts.map +1 -0
- package/crawlers/crawler_extension.js +30 -0
- package/crawlers/crawler_extension.js.map +1 -0
- package/crawlers/crawler_utils.d.ts +20 -0
- package/crawlers/crawler_utils.d.ts.map +1 -0
- package/crawlers/crawler_utils.js +61 -0
- package/crawlers/crawler_utils.js.map +1 -0
- package/crawlers/index.d.ts +5 -0
- package/crawlers/index.d.ts.map +1 -0
- package/crawlers/index.js +8 -0
- package/crawlers/index.js.map +1 -0
- package/crawlers/statistics.d.ts +147 -0
- package/crawlers/statistics.d.ts.map +1 -0
- package/crawlers/statistics.js +360 -0
- package/crawlers/statistics.js.map +1 -0
- package/enqueue_links/enqueue_links.d.ts +151 -0
- package/enqueue_links/enqueue_links.d.ts.map +1 -0
- package/enqueue_links/enqueue_links.js +152 -0
- package/enqueue_links/enqueue_links.js.map +1 -0
- package/enqueue_links/index.d.ts +3 -0
- package/enqueue_links/index.d.ts.map +1 -0
- package/enqueue_links/index.js +6 -0
- package/enqueue_links/index.js.map +1 -0
- package/enqueue_links/shared.d.ts +60 -0
- package/enqueue_links/shared.d.ts.map +1 -0
- package/enqueue_links/shared.js +148 -0
- package/enqueue_links/shared.js.map +1 -0
- package/errors.d.ts +8 -0
- package/errors.d.ts.map +1 -0
- package/errors.js +12 -0
- package/errors.js.map +1 -0
- package/events/event_manager.d.ts +41 -0
- package/events/event_manager.d.ts.map +1 -0
- package/events/event_manager.js +103 -0
- package/events/event_manager.js.map +1 -0
- package/events/index.d.ts +3 -0
- package/events/index.d.ts.map +1 -0
- package/events/index.js +6 -0
- package/events/index.js.map +1 -0
- package/events/local_event_manager.d.ts +32 -0
- package/events/local_event_manager.d.ts.map +1 -0
- package/events/local_event_manager.js +114 -0
- package/events/local_event_manager.js.map +1 -0
- package/index.d.ts +14 -0
- package/index.d.ts.map +1 -0
- package/index.js +17 -0
- package/index.js.map +1 -0
- package/index.mjs +61 -0
- package/log.d.ts +3 -0
- package/log.d.ts.map +1 -0
- package/log.js +12 -0
- package/log.js.map +1 -0
- package/package.json +4 -8
- package/proxy_configuration.d.ts +181 -0
- package/proxy_configuration.d.ts.map +1 -0
- package/proxy_configuration.js +209 -0
- package/proxy_configuration.js.map +1 -0
- package/request.d.ts +185 -0
- package/request.d.ts.map +1 -0
- package/request.js +330 -0
- package/request.js.map +1 -0
- package/serialization.d.ts +33 -0
- package/serialization.d.ts.map +1 -0
- package/serialization.js +146 -0
- package/serialization.js.map +1 -0
- package/session_pool/errors.d.ts +8 -0
- package/session_pool/errors.d.ts.map +1 -0
- package/session_pool/errors.js +20 -0
- package/session_pool/errors.js.map +1 -0
- package/session_pool/events.d.ts +3 -0
- package/session_pool/events.d.ts.map +1 -0
- package/session_pool/events.js +6 -0
- package/session_pool/events.js.map +1 -0
- package/session_pool/index.d.ts +6 -0
- package/session_pool/index.d.ts.map +1 -0
- package/session_pool/index.js +9 -0
- package/session_pool/index.js.map +1 -0
- package/session_pool/session.d.ts +212 -0
- package/session_pool/session.d.ts.map +1 -0
- package/session_pool/session.js +360 -0
- package/session_pool/session.js.map +1 -0
- package/session_pool/session_pool.d.ts +222 -0
- package/session_pool/session_pool.d.ts.map +1 -0
- package/session_pool/session_pool.js +389 -0
- package/session_pool/session_pool.js.map +1 -0
- package/session_pool/session_utils.d.ts +9 -0
- package/session_pool/session_utils.d.ts.map +1 -0
- package/session_pool/session_utils.js +19 -0
- package/session_pool/session_utils.js.map +1 -0
- package/storages/dataset.d.ts +317 -0
- package/storages/dataset.d.ts.map +1 -0
- package/storages/dataset.js +367 -0
- package/storages/dataset.js.map +1 -0
- package/storages/index.d.ts +7 -0
- package/storages/index.d.ts.map +1 -0
- package/storages/index.js +11 -0
- package/storages/index.js.map +1 -0
- package/storages/key_value_store.d.ts +242 -0
- package/storages/key_value_store.d.ts.map +1 -0
- package/storages/key_value_store.js +320 -0
- package/storages/key_value_store.js.map +1 -0
- package/storages/request_list.d.ts +469 -0
- package/storages/request_list.d.ts.map +1 -0
- package/storages/request_list.js +724 -0
- package/storages/request_list.js.map +1 -0
- package/storages/request_queue.d.ts +315 -0
- package/storages/request_queue.d.ts.map +1 -0
- package/storages/request_queue.js +730 -0
- package/storages/request_queue.js.map +1 -0
- package/storages/storage.d.ts +260 -0
- package/storages/storage.d.ts.map +1 -0
- package/storages/storage.js +3 -0
- package/storages/storage.js.map +1 -0
- package/storages/storage_manager.d.ts +40 -0
- package/storages/storage_manager.d.ts.map +1 -0
- package/storages/storage_manager.js +119 -0
- package/storages/storage_manager.js.map +1 -0
- package/tsconfig.build.tsbuildinfo +1 -0
- package/typedefs.d.ts +12 -0
- package/typedefs.d.ts.map +1 -0
- package/typedefs.js +15 -0
- package/typedefs.js.map +1 -0
- package/validators.d.ts +21 -0
- package/validators.d.ts.map +1 -0
- package/validators.js +26 -0
- package/validators.js.map +1 -0
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ProxyConfiguration = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
|
+
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
7
|
+
const configuration_1 = require("./configuration");
|
|
8
|
+
/**
|
|
9
|
+
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
10
|
+
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
11
|
+
* them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
|
|
12
|
+
* the {@link ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
|
|
13
|
+
*
|
|
14
|
+
* If you want to use your own proxies, use the {@link ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
|
|
15
|
+
* be rotated by the configuration if this option is provided.
|
|
16
|
+
*
|
|
17
|
+
* **Example usage:**
|
|
18
|
+
*
|
|
19
|
+
* ```javascript
|
|
20
|
+
*
|
|
21
|
+
* const proxyConfiguration = new ProxyConfiguration({
|
|
22
|
+
* proxyUrls: ['...', '...'],
|
|
23
|
+
* });
|
|
24
|
+
*
|
|
25
|
+
* const crawler = new CheerioCrawler({
|
|
26
|
+
* // ...
|
|
27
|
+
* proxyConfiguration,
|
|
28
|
+
* handlePageFunction: ({ proxyInfo }) => {
|
|
29
|
+
* const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
|
|
30
|
+
* }
|
|
31
|
+
* })
|
|
32
|
+
*
|
|
33
|
+
* ```
|
|
34
|
+
* @category Scaling
|
|
35
|
+
*/
|
|
36
|
+
class ProxyConfiguration {
|
|
37
|
+
/**
|
|
38
|
+
* Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
|
|
39
|
+
* blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
40
|
+
* them to use the selected proxies for all connections.
|
|
41
|
+
*
|
|
42
|
+
* ```javascript
|
|
43
|
+
* const proxyConfiguration = new ProxyConfiguration({
|
|
44
|
+
* proxyUrls: ['http://user:pass@proxy-1.com', 'http://user:pass@proxy-2.com'],
|
|
45
|
+
* });
|
|
46
|
+
*
|
|
47
|
+
* const crawler = new CheerioCrawler({
|
|
48
|
+
* // ...
|
|
49
|
+
* proxyConfiguration,
|
|
50
|
+
* handlePageFunction: ({ proxyInfo }) => {
|
|
51
|
+
* const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
|
|
52
|
+
* }
|
|
53
|
+
* })
|
|
54
|
+
*
|
|
55
|
+
* ```
|
|
56
|
+
* @param options
|
|
57
|
+
* @param config
|
|
58
|
+
* @param validateRequired internal flag to disable validation that requires one of `proxyUrls` or `newUrlFunction`, defaults to true
|
|
59
|
+
*/
|
|
60
|
+
constructor(options = {}, config = configuration_1.Configuration.getGlobalConfig(), validateRequired = true) {
|
|
61
|
+
Object.defineProperty(this, "config", {
|
|
62
|
+
enumerable: true,
|
|
63
|
+
configurable: true,
|
|
64
|
+
writable: true,
|
|
65
|
+
value: config
|
|
66
|
+
});
|
|
67
|
+
Object.defineProperty(this, "isManInTheMiddle", {
|
|
68
|
+
enumerable: true,
|
|
69
|
+
configurable: true,
|
|
70
|
+
writable: true,
|
|
71
|
+
value: false
|
|
72
|
+
});
|
|
73
|
+
Object.defineProperty(this, "nextCustomUrlIndex", {
|
|
74
|
+
enumerable: true,
|
|
75
|
+
configurable: true,
|
|
76
|
+
writable: true,
|
|
77
|
+
value: 0
|
|
78
|
+
});
|
|
79
|
+
Object.defineProperty(this, "proxyUrls", {
|
|
80
|
+
enumerable: true,
|
|
81
|
+
configurable: true,
|
|
82
|
+
writable: true,
|
|
83
|
+
value: void 0
|
|
84
|
+
});
|
|
85
|
+
Object.defineProperty(this, "usedProxyUrls", {
|
|
86
|
+
enumerable: true,
|
|
87
|
+
configurable: true,
|
|
88
|
+
writable: true,
|
|
89
|
+
value: new Map()
|
|
90
|
+
});
|
|
91
|
+
Object.defineProperty(this, "newUrlFunction", {
|
|
92
|
+
enumerable: true,
|
|
93
|
+
configurable: true,
|
|
94
|
+
writable: true,
|
|
95
|
+
value: void 0
|
|
96
|
+
});
|
|
97
|
+
Object.defineProperty(this, "log", {
|
|
98
|
+
enumerable: true,
|
|
99
|
+
configurable: true,
|
|
100
|
+
writable: true,
|
|
101
|
+
value: log_1.default.child({ prefix: 'ProxyConfiguration' })
|
|
102
|
+
});
|
|
103
|
+
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
104
|
+
proxyUrls: ow_1.default.optional.array.nonEmpty.ofType(ow_1.default.string.url),
|
|
105
|
+
newUrlFunction: ow_1.default.optional.function,
|
|
106
|
+
}));
|
|
107
|
+
const { proxyUrls, newUrlFunction } = options;
|
|
108
|
+
if (proxyUrls && newUrlFunction)
|
|
109
|
+
this._throwCannotCombineCustomMethods();
|
|
110
|
+
if (!proxyUrls && !newUrlFunction && validateRequired)
|
|
111
|
+
this._throwNoOptionsProvided();
|
|
112
|
+
this.proxyUrls = proxyUrls;
|
|
113
|
+
this.newUrlFunction = newUrlFunction;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* This function creates a new {@link ProxyInfo} info object.
|
|
117
|
+
* It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
|
|
118
|
+
* the currently used proxy via the handlePageFunction parameter: proxyInfo.
|
|
119
|
+
* Use it if you want to work with a rich representation of a proxy URL.
|
|
120
|
+
* If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
|
|
121
|
+
* @param [sessionId]
|
|
122
|
+
* Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
|
|
123
|
+
* you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
|
|
124
|
+
* When the provided sessionId is a number, it's converted to a string. Property sessionId of
|
|
125
|
+
* {@link ProxyInfo} is always returned as a type string.
|
|
126
|
+
*
|
|
127
|
+
* All the HTTP requests going through the proxy with the same session identifier
|
|
128
|
+
* will use the same target proxy server (i.e. the same IP address).
|
|
129
|
+
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
|
|
130
|
+
* @return Represents information about used proxy and its configuration.
|
|
131
|
+
*/
|
|
132
|
+
async newProxyInfo(sessionId) {
|
|
133
|
+
if (typeof sessionId === 'number')
|
|
134
|
+
sessionId = `${sessionId}`;
|
|
135
|
+
const url = await this.newUrl(sessionId);
|
|
136
|
+
const { username, password, port, hostname } = new URL(url);
|
|
137
|
+
return {
|
|
138
|
+
sessionId,
|
|
139
|
+
url,
|
|
140
|
+
username,
|
|
141
|
+
password,
|
|
142
|
+
hostname,
|
|
143
|
+
port: port,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
|
|
148
|
+
* @param [sessionId]
|
|
149
|
+
* Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
|
|
150
|
+
* you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
|
|
151
|
+
* When the provided sessionId is a number, it's converted to a string.
|
|
152
|
+
*
|
|
153
|
+
* All the HTTP requests going through the proxy with the same session identifier
|
|
154
|
+
* will use the same target proxy server (i.e. the same IP address).
|
|
155
|
+
* The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
|
|
156
|
+
* @return A string with a proxy URL, including authentication credentials and port number.
|
|
157
|
+
* For example, `http://bob:password123@proxy.example.com:8000`
|
|
158
|
+
*/
|
|
159
|
+
async newUrl(sessionId) {
|
|
160
|
+
if (typeof sessionId === 'number')
|
|
161
|
+
sessionId = `${sessionId}`;
|
|
162
|
+
if (this.newUrlFunction) {
|
|
163
|
+
return this._callNewUrlFunction(sessionId);
|
|
164
|
+
}
|
|
165
|
+
return this._handleCustomUrl(sessionId);
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Handles custom url rotation with session
|
|
169
|
+
*/
|
|
170
|
+
_handleCustomUrl(sessionId) {
|
|
171
|
+
let customUrlToUse;
|
|
172
|
+
if (!sessionId) {
|
|
173
|
+
return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
|
|
174
|
+
}
|
|
175
|
+
if (this.usedProxyUrls.has(sessionId)) {
|
|
176
|
+
customUrlToUse = this.usedProxyUrls.get(sessionId);
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
customUrlToUse = this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
|
|
180
|
+
this.usedProxyUrls.set(sessionId, customUrlToUse);
|
|
181
|
+
}
|
|
182
|
+
return customUrlToUse;
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Calls the custom newUrlFunction and checks format of its return value
|
|
186
|
+
*/
|
|
187
|
+
async _callNewUrlFunction(sessionId) {
|
|
188
|
+
let proxyUrl;
|
|
189
|
+
try {
|
|
190
|
+
proxyUrl = await this.newUrlFunction(sessionId);
|
|
191
|
+
new URL(proxyUrl); // eslint-disable-line no-new
|
|
192
|
+
return proxyUrl;
|
|
193
|
+
}
|
|
194
|
+
catch (err) {
|
|
195
|
+
this._throwNewUrlFunctionInvalid(err);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
_throwNewUrlFunctionInvalid(err) {
|
|
199
|
+
throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
|
|
200
|
+
}
|
|
201
|
+
_throwCannotCombineCustomMethods() {
|
|
202
|
+
throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
|
|
203
|
+
}
|
|
204
|
+
_throwNoOptionsProvided() {
|
|
205
|
+
throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
exports.ProxyConfiguration = ProxyConfiguration;
|
|
209
|
+
//# sourceMappingURL=proxy_configuration.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,6DAA6B;AAC7B,mDAAgD;AAsFhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,kBAAkB;IAQ3B;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,YAAY,UAAqC,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE,EAAE,gBAAgB,GAAG,IAAI;;;;;mBAAjE;;QA9B9D;;;;mBAAmB,KAAK;WAAC;QACzB;;;;mBAA+B,CAAC;WAAC;QACjC;;;;;WAA+B;QAC/B;;;;mBAA0B,IAAI,GAAG,EAAkB;WAAC;QACpD;;;;;WAAsD;QACtD;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;WAAC;QA0BxD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3D,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAE9C,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACzE,IAAI,CAAC,SAAS,IAAI,CAAC,cAAc,IAAI,gBAAgB;YAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAEtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,YAAY,CAAC,SAA2B;QAC1C,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEzC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO;YACH,SAAS;YACT,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,IAAI,EAAE,IAAK;SACd,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,MAAM,CAAC,SAA2B;QACpC,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAE9D,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAE,CAAC;SAC/C;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,gBAAgB,CAAC,SAAkB;QACzC,IAAI,cAAsB,CAAC;QAE3B,IAAI,CAAC,SAAS,EAAE;YACZ,OAAO,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;SAC9E;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACnC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;SACvD;aAAM;YACH,cAAc,GAAG,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;YACrF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;SACrD;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,SAAkB;QAClD,IAAI,QAAgB,CAAC;QAErB,IAAI;YACA,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,SAAU,CAAC,CAAC;YAClD,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;YAChD,OAAO,QAAQ,CAAC;SACnB;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,2BAA2B,CAAC,GAAY,CAAC,CAAC;SAClD;IACL,CAAC;IAES,2BAA2B,CAAC,GAAU;QAC5C,MAAM,IAAI,KAAK,CAAC,mEAAmE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACtG,CAAC;IAES,gCAAgC;QACtC,MAAM,IAAI,KAAK,CAAC,6GAA6G,CAAC,CAAC;IACnI,CAAC;IAES,uBAAuB;QAC7B,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IACpG,CAAC;CACJ;AApJD,gDAoJC"}
|
package/request.d.ts
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { BinaryLike } from 'crypto';
|
|
3
|
+
import { AllowedHttpMethods } from './typedefs';
|
|
4
|
+
/**
|
|
5
|
+
* Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
|
|
6
|
+
* The `Request` object also stores information about errors that occurred during processing of the request.
|
|
7
|
+
*
|
|
8
|
+
* Each `Request` instance has the `uniqueKey` property, which can be either specified
|
|
9
|
+
* manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
|
|
10
|
+
* are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
|
|
11
|
+
* such as {@link RequestList}, {@link RequestQueue}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
|
|
12
|
+
*
|
|
13
|
+
* Example use:
|
|
14
|
+
*
|
|
15
|
+
* ```javascript
|
|
16
|
+
* const request = new Request({
|
|
17
|
+
* url: 'http://www.example.com',
|
|
18
|
+
* headers: { Accept: 'application/json' },
|
|
19
|
+
* });
|
|
20
|
+
*
|
|
21
|
+
* ...
|
|
22
|
+
*
|
|
23
|
+
* request.userData.foo = 'bar';
|
|
24
|
+
* request.pushErrorMessage(new Error('Request failed!'));
|
|
25
|
+
*
|
|
26
|
+
* ...
|
|
27
|
+
*
|
|
28
|
+
* const foo = request.userData.foo;
|
|
29
|
+
* ```
|
|
30
|
+
* @category Sources
|
|
31
|
+
*/
|
|
32
|
+
export declare class Request {
|
|
33
|
+
/** Request ID */
|
|
34
|
+
id?: string;
|
|
35
|
+
/** URL of the web page to crawl. */
|
|
36
|
+
url: string;
|
|
37
|
+
/**
|
|
38
|
+
* An actually loaded URL after redirects, if present. HTTP redirects are guaranteed
|
|
39
|
+
* to be included.
|
|
40
|
+
*
|
|
41
|
+
* When using {@link PuppeteerCrawler} or {@link PlaywrightCrawler}, meta tag and JavaScript redirects may,
|
|
42
|
+
* or may not be included, depending on their nature. This generally means that redirects,
|
|
43
|
+
* which happen immediately will most likely be included, but delayed redirects will not.
|
|
44
|
+
*/
|
|
45
|
+
loadedUrl?: string;
|
|
46
|
+
/**
|
|
47
|
+
* A unique key identifying the request.
|
|
48
|
+
* Two requests with the same `uniqueKey` are considered as pointing to the same URL.
|
|
49
|
+
*/
|
|
50
|
+
uniqueKey: string;
|
|
51
|
+
/** HTTP method, e.g. `GET` or `POST`. */
|
|
52
|
+
method: AllowedHttpMethods;
|
|
53
|
+
/** HTTP request payload, e.g. for POST requests. */
|
|
54
|
+
payload?: string;
|
|
55
|
+
/** The `true` value indicates that the request will not be automatically retried on error. */
|
|
56
|
+
noRetry: boolean;
|
|
57
|
+
/** Indicates the number of times the crawling of the request has been retried on error. */
|
|
58
|
+
retryCount: number;
|
|
59
|
+
/** An array of error messages from request processing. */
|
|
60
|
+
errorMessages: string[];
|
|
61
|
+
/** Object with HTTP headers. Key is header name, value is the value. */
|
|
62
|
+
headers?: Record<string, string>;
|
|
63
|
+
/** Private store for the custom user data assigned to the request. */
|
|
64
|
+
private _userData;
|
|
65
|
+
/** Custom user data assigned to the request. */
|
|
66
|
+
userData: Record<string, any>;
|
|
67
|
+
/**
|
|
68
|
+
* ISO datetime string that indicates the time when the request has been processed.
|
|
69
|
+
* Is `null` if the request has not been crawled yet.
|
|
70
|
+
*/
|
|
71
|
+
handledAt?: string;
|
|
72
|
+
/**
|
|
73
|
+
* `Request` parameters including the URL, HTTP method and headers, and others.
|
|
74
|
+
*/
|
|
75
|
+
constructor(options: RequestOptions);
|
|
76
|
+
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
77
|
+
get skipNavigation(): boolean;
|
|
78
|
+
set skipNavigation(value: boolean);
|
|
79
|
+
/**
|
|
80
|
+
* Stores information about an error that occurred during processing of this request.
|
|
81
|
+
*
|
|
82
|
+
* You should always use Error instances when throwing errors in JavaScript.
|
|
83
|
+
*
|
|
84
|
+
* Nevertheless, to improve the debugging experience when using third party libraries
|
|
85
|
+
* that may not always throw an Error instance, the function performs a type
|
|
86
|
+
* inspection of the passed argument and attempts to extract as much information
|
|
87
|
+
* as possible, since just throwing a bad type error makes any debugging rather difficult.
|
|
88
|
+
*
|
|
89
|
+
* @param errorOrMessage Error object or error message to be stored in the request.
|
|
90
|
+
* @param [options]
|
|
91
|
+
*/
|
|
92
|
+
pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
|
|
93
|
+
protected _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }: ComputeUniqueKeyOptions): string;
|
|
94
|
+
protected _hashPayload(payload: BinaryLike): string;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Specifies required and optional fields for constructing a {@link Request}.
|
|
98
|
+
*/
|
|
99
|
+
export interface RequestOptions {
|
|
100
|
+
/** URL of the web page to crawl. It must be a non-empty string. */
|
|
101
|
+
url: string;
|
|
102
|
+
/**
|
|
103
|
+
* A unique key identifying the request.
|
|
104
|
+
* Two requests with the same `uniqueKey` are considered as pointing to the same URL.
|
|
105
|
+
*
|
|
106
|
+
* If `uniqueKey` is not provided, then it is automatically generated by normalizing the URL.
|
|
107
|
+
* For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `uniqueKey`
|
|
108
|
+
* of `http://www.example.com/something`.
|
|
109
|
+
*
|
|
110
|
+
* The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
|
|
111
|
+
*
|
|
112
|
+
* The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
|
|
113
|
+
* producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
|
|
114
|
+
* when requests point to the same URL, but with different methods and payloads. For example: form submits.
|
|
115
|
+
*
|
|
116
|
+
* Pass an arbitrary non-empty text value to the `uniqueKey` property
|
|
117
|
+
* to override the default behavior and specify which URLs shall be considered equal.
|
|
118
|
+
*/
|
|
119
|
+
uniqueKey?: string;
|
|
120
|
+
/** @default 'GET' */
|
|
121
|
+
method?: AllowedHttpMethods | Lowercase<AllowedHttpMethods>;
|
|
122
|
+
/** HTTP request payload, e.g. for POST requests. */
|
|
123
|
+
payload?: string;
|
|
124
|
+
/**
|
|
125
|
+
* HTTP headers in the following format:
|
|
126
|
+
* ```
|
|
127
|
+
* {
|
|
128
|
+
* Accept: 'text/html',
|
|
129
|
+
* 'Content-Type': 'application/json'
|
|
130
|
+
* }
|
|
131
|
+
* ```
|
|
132
|
+
*/
|
|
133
|
+
headers?: Record<string, string>;
|
|
134
|
+
/**
|
|
135
|
+
* Custom user data assigned to the request. Use this to save any request related data to the
|
|
136
|
+
* request's scope, keeping them accessible on retries, failures etc.
|
|
137
|
+
*/
|
|
138
|
+
userData?: Record<string, any>;
|
|
139
|
+
/**
|
|
140
|
+
* If `false` then the hash part of a URL is removed when computing the `uniqueKey` property.
|
|
141
|
+
* For example, this causes the `http://www.example.com#foo` and `http://www.example.com#bar` URLs
|
|
142
|
+
* to have the same `uniqueKey` of `http://www.example.com` and thus the URLs are considered equal.
|
|
143
|
+
* Note that this option only has an effect if `uniqueKey` is not set.
|
|
144
|
+
* @default false
|
|
145
|
+
*/
|
|
146
|
+
keepUrlFragment?: boolean;
|
|
147
|
+
/**
|
|
148
|
+
* If `true` then the `uniqueKey` is computed not only from the URL, but also from the method and payload
|
|
149
|
+
* properties. This is useful when making requests to the same URL that are differentiated by method
|
|
150
|
+
* or payload, such as form submit navigations in browsers.
|
|
151
|
+
* @default false
|
|
152
|
+
*/
|
|
153
|
+
useExtendedUniqueKey?: boolean;
|
|
154
|
+
/**
|
|
155
|
+
* The `true` value indicates that the request will not be automatically retried on error.
|
|
156
|
+
* @default false
|
|
157
|
+
*/
|
|
158
|
+
noRetry?: boolean;
|
|
159
|
+
/**
|
|
160
|
+
* If set to `true` then the crawler processing this request evaluates
|
|
161
|
+
* the `handleRequestFunction` immediately without prior browser navigation.
|
|
162
|
+
* @default false
|
|
163
|
+
*/
|
|
164
|
+
skipNavigation?: boolean;
|
|
165
|
+
/** @internal */
|
|
166
|
+
id?: string;
|
|
167
|
+
/** @internal */
|
|
168
|
+
handledAt?: string;
|
|
169
|
+
}
|
|
170
|
+
export interface PushErrorMessageOptions {
|
|
171
|
+
/**
|
|
172
|
+
* Only push the error message without stack trace when true.
|
|
173
|
+
* @default false
|
|
174
|
+
*/
|
|
175
|
+
omitStack?: boolean;
|
|
176
|
+
}
|
|
177
|
+
interface ComputeUniqueKeyOptions {
|
|
178
|
+
url: string;
|
|
179
|
+
method: AllowedHttpMethods;
|
|
180
|
+
payload?: string | Buffer;
|
|
181
|
+
keepUrlFragment?: boolean;
|
|
182
|
+
useExtendedUniqueKey?: boolean;
|
|
183
|
+
}
|
|
184
|
+
export {};
|
|
185
|
+
//# sourceMappingURL=request.d.ts.map
|
package/request.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";AACA,OAAe,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAI5C,OAAO,EAAE,kBAAkB,EAAQ,MAAM,YAAY,CAAC;AAsBtD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,OAAO;IAChB,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAM;IAEnC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAqGnC,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAG5B;IAED,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAKhC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,SAAS,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,EAAE,uBAAuB;IAgBpH,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAQtD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAE3B,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE/B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CAEtB;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC"}
|