apify 4.0.0-beta.31 → 4.0.0-beta.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +6 -23
- package/dist/actor.js +55 -48
- package/dist/apify_dataset_backend.d.ts +1 -1
- package/dist/apify_dataset_backend.js +6 -6
- package/dist/apify_key_value_store_backend.d.ts +1 -1
- package/dist/apify_key_value_store_backend.js +10 -10
- package/dist/apify_request_queue_backend.d.ts +1 -1
- package/dist/apify_request_queue_backend.js +3 -3
- package/dist/apify_request_queue_shared_backend.d.ts +1 -14
- package/dist/apify_request_queue_shared_backend.js +38 -38
- package/dist/apify_request_queue_single_backend.d.ts +1 -10
- package/dist/apify_request_queue_single_backend.js +44 -44
- package/dist/apify_storage_backend.d.ts +1 -12
- package/dist/apify_storage_backend.js +43 -43
- package/dist/charging.d.ts +1 -14
- package/dist/charging.js +84 -81
- package/dist/platform_event_manager.d.ts +1 -2
- package/dist/platform_event_manager.js +7 -7
- package/dist/proxy_configuration.d.ts +7 -18
- package/dist/proxy_configuration.js +46 -57
- package/package.json +1 -1
|
@@ -53,13 +53,13 @@ const SESSION_ID_LENGTH = 12;
|
|
|
53
53
|
*/
|
|
54
54
|
export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
55
55
|
configuration;
|
|
56
|
-
groups;
|
|
57
|
-
countryCode;
|
|
58
|
-
subdivisionCode;
|
|
59
|
-
password;
|
|
60
|
-
hostname;
|
|
61
|
-
port;
|
|
62
|
-
usesApifyProxy;
|
|
56
|
+
#groups;
|
|
57
|
+
#countryCode;
|
|
58
|
+
#subdivisionCode;
|
|
59
|
+
#password;
|
|
60
|
+
#hostname;
|
|
61
|
+
#port;
|
|
62
|
+
#usesApifyProxy;
|
|
63
63
|
log = defaultLog.child({ prefix: 'ProxyConfiguration' });
|
|
64
64
|
/**
|
|
65
65
|
* @internal
|
|
@@ -96,17 +96,15 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
96
96
|
}
|
|
97
97
|
// Validation
|
|
98
98
|
if ((proxyUrls || newUrlFunction) && (groupsToUse.length || countryCodeToUse || subdivisionCodeToUse)) {
|
|
99
|
-
this.
|
|
99
|
+
this.throwCannotCombineCustomWithApify();
|
|
100
100
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
this
|
|
104
|
-
this
|
|
105
|
-
this
|
|
106
|
-
this
|
|
107
|
-
this
|
|
108
|
-
this.port = port;
|
|
109
|
-
this.usesApifyProxy = !proxyUrls && !newUrlFunction;
|
|
101
|
+
this.#groups = groupsToUse;
|
|
102
|
+
this.#countryCode = countryCodeToUse;
|
|
103
|
+
this.#subdivisionCode = subdivisionCodeToUse;
|
|
104
|
+
this.#password = password;
|
|
105
|
+
this.#hostname = hostname;
|
|
106
|
+
this.#port = port;
|
|
107
|
+
this.#usesApifyProxy = !proxyUrls && !newUrlFunction;
|
|
110
108
|
if (proxyUrls && proxyUrls.some((url) => url?.includes('apify.com'))) {
|
|
111
109
|
this.log.warning('Some Apify proxy features may work incorrectly. Please consider setting up Apify properties instead of `proxyUrls`.\n' +
|
|
112
110
|
'See https://docs.apify.com/sdk/js/docs/concepts/proxy-management#apify-proxy-configuration');
|
|
@@ -121,11 +119,11 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
121
119
|
* `ProxyConfiguration` instance instead of calling this manually.
|
|
122
120
|
*/
|
|
123
121
|
async initialize(options) {
|
|
124
|
-
if (this
|
|
125
|
-
if (!this
|
|
126
|
-
await this.
|
|
122
|
+
if (this.#usesApifyProxy) {
|
|
123
|
+
if (!this.#password) {
|
|
124
|
+
await this.setPasswordIfToken();
|
|
127
125
|
}
|
|
128
|
-
if (!this
|
|
126
|
+
if (!this.#password) {
|
|
129
127
|
if (Actor.isAtHome()) {
|
|
130
128
|
throw new Error(`Apify Proxy password must be provided using options.password or the "${APIFY_ENV_VARS.PROXY_PASSWORD}" environment variable. ` +
|
|
131
129
|
`You can also provide your Apify token via the "${APIFY_ENV_VARS.TOKEN}" environment variable, ` +
|
|
@@ -139,7 +137,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
139
137
|
}
|
|
140
138
|
}
|
|
141
139
|
if (options?.checkAccess !== false) {
|
|
142
|
-
return this.
|
|
140
|
+
return this.checkAccess();
|
|
143
141
|
}
|
|
144
142
|
}
|
|
145
143
|
return true;
|
|
@@ -161,12 +159,12 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
161
159
|
hostname: parsed.hostname,
|
|
162
160
|
port: parsed.port,
|
|
163
161
|
};
|
|
164
|
-
if (this
|
|
165
|
-
result.groups = this
|
|
166
|
-
if (this
|
|
167
|
-
result.countryCode = this
|
|
168
|
-
if (this
|
|
169
|
-
result.subdivisionCode = this
|
|
162
|
+
if (this.#usesApifyProxy) {
|
|
163
|
+
result.groups = this.#groups;
|
|
164
|
+
if (this.#countryCode !== undefined)
|
|
165
|
+
result.countryCode = this.#countryCode;
|
|
166
|
+
if (this.#subdivisionCode !== undefined)
|
|
167
|
+
result.subdivisionCode = this.#subdivisionCode;
|
|
170
168
|
}
|
|
171
169
|
return result;
|
|
172
170
|
}
|
|
@@ -176,7 +174,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
176
174
|
* `proxyUrls`, the URLs are rotated round-robin.
|
|
177
175
|
*/
|
|
178
176
|
async newUrl(options) {
|
|
179
|
-
if (!this
|
|
177
|
+
if (!this.#usesApifyProxy) {
|
|
180
178
|
return super.newUrl(options);
|
|
181
179
|
}
|
|
182
180
|
return this.composeDefaultUrl(cryptoRandomObjectId(SESSION_ID_LENGTH));
|
|
@@ -184,40 +182,38 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
184
182
|
/**
|
|
185
183
|
* Returns proxy username.
|
|
186
184
|
*/
|
|
187
|
-
|
|
188
|
-
const { groups, countryCode, subdivisionCode } = this;
|
|
185
|
+
getUsername(sessionId) {
|
|
189
186
|
const parts = [];
|
|
190
|
-
if (groups && groups.length) {
|
|
191
|
-
parts.push(`groups-${groups.join('+')}`);
|
|
187
|
+
if (this.#groups && this.#groups.length) {
|
|
188
|
+
parts.push(`groups-${this.#groups.join('+')}`);
|
|
192
189
|
}
|
|
193
190
|
parts.push(`session-${sessionId}`);
|
|
194
|
-
if (subdivisionCode) {
|
|
195
|
-
parts.push(`country-${countryCode}_${subdivisionCode}`);
|
|
191
|
+
if (this.#subdivisionCode) {
|
|
192
|
+
parts.push(`country-${this.#countryCode}_${this.#subdivisionCode}`);
|
|
196
193
|
}
|
|
197
|
-
else if (countryCode) {
|
|
198
|
-
parts.push(`country-${countryCode}`);
|
|
194
|
+
else if (this.#countryCode) {
|
|
195
|
+
parts.push(`country-${this.#countryCode}`);
|
|
199
196
|
}
|
|
200
197
|
return parts.join(',');
|
|
201
198
|
}
|
|
202
199
|
composeDefaultUrl(sessionId) {
|
|
203
|
-
const username = this.
|
|
204
|
-
const url = new URL(`http://${this
|
|
200
|
+
const username = this.getUsername(sessionId);
|
|
201
|
+
const url = new URL(`http://${this.#hostname}:${this.#port}`);
|
|
205
202
|
url.username = `${username}`;
|
|
206
|
-
url.password = `${this
|
|
203
|
+
url.password = `${this.#password}`;
|
|
207
204
|
const urlString = url.toString();
|
|
208
205
|
return urlString.substring(0, urlString.length - 1);
|
|
209
206
|
}
|
|
210
207
|
/**
|
|
211
208
|
* Fetch & set the proxy password from Apify API if an Apify token is provided.
|
|
212
209
|
*/
|
|
213
|
-
|
|
214
|
-
async _setPasswordIfToken() {
|
|
210
|
+
async setPasswordIfToken() {
|
|
215
211
|
const { token } = this.configuration;
|
|
216
212
|
if (!token)
|
|
217
213
|
return;
|
|
218
214
|
try {
|
|
219
215
|
const user = await Actor.apifyClient.user().get();
|
|
220
|
-
this
|
|
216
|
+
this.#password = user.proxy?.password;
|
|
221
217
|
}
|
|
222
218
|
catch (error) {
|
|
223
219
|
if (Actor.isAtHome()) {
|
|
@@ -235,8 +231,8 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
235
231
|
* If the check can not be made, it only prints a warning and allows the program to continue. This is to
|
|
236
232
|
* prevent program crashes caused by short downtimes of Proxy.
|
|
237
233
|
*/
|
|
238
|
-
async
|
|
239
|
-
const status = await this.
|
|
234
|
+
async checkAccess() {
|
|
235
|
+
const status = await this.fetchStatus();
|
|
240
236
|
if (!status) {
|
|
241
237
|
this.log.warning('Apify Proxy access check timed out. Watch out for errors with status code 407. ' +
|
|
242
238
|
"If you see some, it most likely means you don't have access to either all or some of the proxies you're trying to use.");
|
|
@@ -261,7 +257,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
261
257
|
/**
|
|
262
258
|
* Apify Proxy can be down for a second or a minute, but this should not crash processes.
|
|
263
259
|
*/
|
|
264
|
-
async
|
|
260
|
+
async fetchStatus() {
|
|
265
261
|
const { proxyStatusUrl } = this.configuration;
|
|
266
262
|
const statusUrl = `${proxyStatusUrl}/?format=json`;
|
|
267
263
|
const proxyUrl = await this.newUrl();
|
|
@@ -270,7 +266,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
270
266
|
return undefined;
|
|
271
267
|
for (let attempt = 1; attempt <= CHECK_ACCESS_MAX_ATTEMPTS; attempt++) {
|
|
272
268
|
try {
|
|
273
|
-
return await this.
|
|
269
|
+
return await this.requestStatus(statusUrl, proxyUrl);
|
|
274
270
|
}
|
|
275
271
|
catch {
|
|
276
272
|
// retry connection errors
|
|
@@ -286,7 +282,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
286
282
|
* plus a `Proxy-Authorization` header — so no proxy-agent dependency is
|
|
287
283
|
* needed. The status endpoint (`http://proxy.apify.com`) is plain HTTP.
|
|
288
284
|
*/
|
|
289
|
-
async
|
|
285
|
+
async requestStatus(statusUrl, proxyUrl) {
|
|
290
286
|
const target = new URL(statusUrl);
|
|
291
287
|
const proxy = new URL(proxyUrl);
|
|
292
288
|
const headers = { host: target.host };
|
|
@@ -304,7 +300,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
304
300
|
});
|
|
305
301
|
request.end();
|
|
306
302
|
// `once` rejects if the request emits `error` first (connection refused,
|
|
307
|
-
// timeout/abort), so failures propagate to the retry loop in `
|
|
303
|
+
// timeout/abort), so failures propagate to the retry loop in `fetchStatus`.
|
|
308
304
|
const [response] = (await once(request, 'response'));
|
|
309
305
|
const statusCode = response.statusCode ?? 0;
|
|
310
306
|
if (statusCode < 200 || statusCode >= 300) {
|
|
@@ -317,17 +313,10 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
317
313
|
* Throws cannot combine custom proxies with Apify Proxy
|
|
318
314
|
* @internal
|
|
319
315
|
*/
|
|
320
|
-
|
|
316
|
+
throwCannotCombineCustomWithApify() {
|
|
321
317
|
throw new Error('Cannot combine custom proxies with Apify Proxy! ' +
|
|
322
318
|
'It is not allowed to set "options.proxyUrls" or "options.newUrlFunction" combined with ' +
|
|
323
319
|
'"options.groups", "options.apifyProxyGroups", "options.countryCode", "options.apifyProxyCountry", ' +
|
|
324
320
|
'"options.subdivisionCode" or "options.apifyProxySubdivision".');
|
|
325
321
|
}
|
|
326
|
-
/**
|
|
327
|
-
* Throws cannot combine custom proxies with custom generating function
|
|
328
|
-
* @internal
|
|
329
|
-
*/
|
|
330
|
-
_throwCannotCombineCustomMethods() {
|
|
331
|
-
throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
|
|
332
|
-
}
|
|
333
322
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.32",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|