apify 3.4.0 → 3.4.1-beta.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +26 -0
- package/.turbo/turbo-copy.log +4 -0
- package/LICENSE.md +183 -183
- package/README.md +1 -1
- package/dist/LICENSE.md +201 -0
- package/dist/README.md +98 -0
- package/{actor.d.ts → dist/actor.d.ts} +309 -309
- package/dist/actor.d.ts.map +1 -0
- package/{actor.js → dist/actor.js} +168 -127
- package/dist/actor.js.map +1 -0
- package/{charging.d.ts → dist/charging.d.ts} +18 -18
- package/dist/charging.d.ts.map +1 -0
- package/{charging.js → dist/charging.js} +26 -14
- package/dist/charging.js.map +1 -0
- package/{configuration.d.ts → dist/configuration.d.ts} +4 -4
- package/dist/configuration.d.ts.map +1 -0
- package/{configuration.js → dist/configuration.js} +14 -3
- package/dist/configuration.js.map +1 -0
- package/{index.d.ts → dist/index.d.ts} +6 -6
- package/dist/index.d.ts.map +1 -0
- package/{index.js → dist/index.js} +6 -6
- package/dist/index.js.map +1 -0
- package/dist/key_value_store.d.ts.map +1 -0
- package/{key_value_store.js → dist/key_value_store.js} +1 -1
- package/dist/key_value_store.js.map +1 -0
- package/dist/package.json +80 -0
- package/{platform_event_manager.d.ts → dist/platform_event_manager.d.ts} +4 -4
- package/{platform_event_manager.d.ts.map → dist/platform_event_manager.d.ts.map} +1 -1
- package/{platform_event_manager.js → dist/platform_event_manager.js} +13 -10
- package/dist/platform_event_manager.js.map +1 -0
- package/{proxy_configuration.d.ts → dist/proxy_configuration.d.ts} +10 -10
- package/dist/proxy_configuration.d.ts.map +1 -0
- package/{proxy_configuration.js → dist/proxy_configuration.js} +52 -40
- package/dist/proxy_configuration.js.map +1 -0
- package/dist/utils.d.ts.map +1 -0
- package/{utils.js → dist/utils.js} +5 -6
- package/dist/utils.js.map +1 -0
- package/package.json +10 -10
- package/actor.d.ts.map +0 -1
- package/actor.js.map +0 -1
- package/charging.d.ts.map +0 -1
- package/charging.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/key_value_store.d.ts.map +0 -1
- package/key_value_store.js.map +0 -1
- package/platform_event_manager.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
- package/utils.d.ts.map +0 -1
- package/utils.js.map +0 -1
- /package/{index.mjs → dist/index.mjs} +0 -0
- /package/{key_value_store.d.ts → dist/key_value_store.d.ts} +0 -0
- /package/{utils.d.ts → dist/utils.d.ts} +0 -0
|
@@ -1,31 +1,42 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.Actor = exports.EXIT_CODES = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const node_crypto_1 = require("node:crypto");
|
|
6
|
-
const consts_1 = require("@apify/consts");
|
|
7
|
-
const input_secrets_1 = require("@apify/input_secrets");
|
|
8
|
-
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
9
|
-
const timeout_1 = require("@apify/timeout");
|
|
10
6
|
const core_1 = require("@crawlee/core");
|
|
11
7
|
const utils_1 = require("@crawlee/utils");
|
|
12
8
|
const apify_client_1 = require("apify-client");
|
|
13
9
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
const
|
|
19
|
-
const
|
|
10
|
+
const consts_1 = require("@apify/consts");
|
|
11
|
+
const input_secrets_1 = require("@apify/input_secrets");
|
|
12
|
+
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
13
|
+
const timeout_1 = require("@apify/timeout");
|
|
14
|
+
const charging_js_1 = require("./charging.js");
|
|
15
|
+
const configuration_js_1 = require("./configuration.js");
|
|
16
|
+
const key_value_store_js_1 = require("./key_value_store.js");
|
|
17
|
+
const platform_event_manager_js_1 = require("./platform_event_manager.js");
|
|
18
|
+
const proxy_configuration_js_1 = require("./proxy_configuration.js");
|
|
19
|
+
const utils_js_1 = require("./utils.js");
|
|
20
|
+
/**
|
|
21
|
+
* Exit codes for the Actor process.
|
|
22
|
+
* The error codes must be in the range 1-128, to avoid collision with signal exits
|
|
23
|
+
* and to ensure Docker will handle them correctly!
|
|
24
|
+
* @internal should be removed if we decide to remove `Actor.main()`
|
|
25
|
+
*/
|
|
26
|
+
exports.EXIT_CODES = {
|
|
27
|
+
SUCCESS: 0,
|
|
28
|
+
ERROR_USER_FUNCTION_THREW: 91,
|
|
29
|
+
ERROR_UNKNOWN: 92,
|
|
30
|
+
};
|
|
20
31
|
/**
|
|
21
32
|
* `Actor` class serves as an alternative approach to the static helpers exported from the package. It allows to pass configuration
|
|
22
33
|
* that will be used on the instance methods. Environment variables will have precedence over this configuration.
|
|
23
|
-
* See {@
|
|
34
|
+
* See {@link Configuration} for details about what can be configured and what are the default values.
|
|
24
35
|
*/
|
|
25
36
|
class Actor {
|
|
26
37
|
constructor(options = {}) {
|
|
27
38
|
/**
|
|
28
|
-
* Configuration of this SDK instance (provided to its constructor). See {@
|
|
39
|
+
* Configuration of this SDK instance (provided to its constructor). See {@link Configuration} for details.
|
|
29
40
|
* @internal
|
|
30
41
|
*/
|
|
31
42
|
Object.defineProperty(this, "config", {
|
|
@@ -35,7 +46,7 @@ class Actor {
|
|
|
35
46
|
value: void 0
|
|
36
47
|
});
|
|
37
48
|
/**
|
|
38
|
-
* Default {@
|
|
49
|
+
* Default {@link ApifyClient} instance.
|
|
39
50
|
* @internal
|
|
40
51
|
*/
|
|
41
52
|
Object.defineProperty(this, "apifyClient", {
|
|
@@ -45,7 +56,7 @@ class Actor {
|
|
|
45
56
|
value: void 0
|
|
46
57
|
});
|
|
47
58
|
/**
|
|
48
|
-
* Default {@
|
|
59
|
+
* Default {@link EventManager} instance.
|
|
49
60
|
* @internal
|
|
50
61
|
*/
|
|
51
62
|
Object.defineProperty(this, "eventManager", {
|
|
@@ -55,7 +66,7 @@ class Actor {
|
|
|
55
66
|
value: void 0
|
|
56
67
|
});
|
|
57
68
|
/**
|
|
58
|
-
* Whether the Actor instance was initialized. This is set by calling {@
|
|
69
|
+
* Whether the Actor instance was initialized. This is set by calling {@link Actor.init}.
|
|
59
70
|
*/
|
|
60
71
|
Object.defineProperty(this, "initialized", {
|
|
61
72
|
enumerable: true,
|
|
@@ -89,10 +100,13 @@ class Actor {
|
|
|
89
100
|
value: void 0
|
|
90
101
|
});
|
|
91
102
|
// use default configuration object if nothing overridden (it fallbacks to env vars)
|
|
92
|
-
this.config =
|
|
103
|
+
this.config =
|
|
104
|
+
Object.keys(options).length === 0
|
|
105
|
+
? configuration_js_1.Configuration.getGlobalConfig()
|
|
106
|
+
: new configuration_js_1.Configuration(options);
|
|
93
107
|
this.apifyClient = this.newClient();
|
|
94
|
-
this.eventManager = new
|
|
95
|
-
this.chargingManager = new
|
|
108
|
+
this.eventManager = new platform_event_manager_js_1.PlatformEventManager(this.config);
|
|
109
|
+
this.chargingManager = new charging_js_1.ChargingManager(this.config, this.apifyClient);
|
|
96
110
|
}
|
|
97
111
|
/**
|
|
98
112
|
* Runs the main user function that performs the job of the Actor
|
|
@@ -111,7 +125,7 @@ class Actor {
|
|
|
111
125
|
* - When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),
|
|
112
126
|
* it sets up a connection to listen for platform events.
|
|
113
127
|
* For example, to get a notification about an imminent migration to another server.
|
|
114
|
-
* See {@
|
|
128
|
+
* See {@link Actor.events} for details.
|
|
115
129
|
* - It invokes the user function passed as the `userFunc` parameter.
|
|
116
130
|
* - If the user function returned a promise, waits for it to resolve.
|
|
117
131
|
* - If the user function throws an exception or some other error is encountered,
|
|
@@ -164,12 +178,14 @@ class Actor {
|
|
|
164
178
|
await this.init(options);
|
|
165
179
|
let ret;
|
|
166
180
|
try {
|
|
167
|
-
ret = await userFunc();
|
|
181
|
+
ret = (await userFunc());
|
|
168
182
|
await this.exit(options);
|
|
169
183
|
}
|
|
170
184
|
catch (err) {
|
|
171
185
|
log_1.default.exception(err, err.message);
|
|
172
|
-
await this.exit({
|
|
186
|
+
await this.exit({
|
|
187
|
+
exitCode: exports.EXIT_CODES.ERROR_USER_FUNCTION_THREW,
|
|
188
|
+
});
|
|
173
189
|
}
|
|
174
190
|
return ret;
|
|
175
191
|
})();
|
|
@@ -190,11 +206,11 @@ class Actor {
|
|
|
190
206
|
].join('\n'));
|
|
191
207
|
}
|
|
192
208
|
this.initialized = true;
|
|
193
|
-
(0,
|
|
194
|
-
log_1.default.info('System info', (0,
|
|
195
|
-
(0,
|
|
209
|
+
(0, utils_js_1.checkCrawleeVersion)();
|
|
210
|
+
log_1.default.info('System info', (0, utils_js_1.getSystemInfo)());
|
|
211
|
+
(0, utils_js_1.printOutdatedSdkWarning)();
|
|
196
212
|
// reset global config instance to respect APIFY_ prefixed env vars
|
|
197
|
-
core_1.Configuration.globalConfig =
|
|
213
|
+
core_1.Configuration.globalConfig = configuration_js_1.Configuration.getGlobalConfig();
|
|
198
214
|
if (this.isAtHome()) {
|
|
199
215
|
this.config.set('availableMemoryRatio', 1);
|
|
200
216
|
this.config.set('disableBrowserSandbox', true); // for browser launcher, adds `--no-sandbox` to args
|
|
@@ -212,7 +228,7 @@ class Actor {
|
|
|
212
228
|
onlyPurgeOnce: true,
|
|
213
229
|
});
|
|
214
230
|
log_1.default.debug(`Default storages purged`);
|
|
215
|
-
|
|
231
|
+
configuration_js_1.Configuration.storage.enterWith(this.config);
|
|
216
232
|
await this.chargingManager.init();
|
|
217
233
|
log_1.default.debug(`ChargingManager initialized`, this.chargingManager.getPricingInfo());
|
|
218
234
|
}
|
|
@@ -220,7 +236,10 @@ class Actor {
|
|
|
220
236
|
* @ignore
|
|
221
237
|
*/
|
|
222
238
|
async exit(messageOrOptions, options = {}) {
|
|
223
|
-
options =
|
|
239
|
+
options =
|
|
240
|
+
typeof messageOrOptions === 'string'
|
|
241
|
+
? { ...options, statusMessage: messageOrOptions }
|
|
242
|
+
: { ...messageOrOptions, ...options };
|
|
224
243
|
options.exit ?? (options.exit = true);
|
|
225
244
|
options.exitCode ?? (options.exitCode = exports.EXIT_CODES.SUCCESS);
|
|
226
245
|
options.timeoutSecs ?? (options.timeoutSecs = 30);
|
|
@@ -246,7 +265,10 @@ class Actor {
|
|
|
246
265
|
finished = true;
|
|
247
266
|
}
|
|
248
267
|
if (options.statusMessage != null) {
|
|
249
|
-
await this.setStatusMessage(options.statusMessage, {
|
|
268
|
+
await this.setStatusMessage(options.statusMessage, {
|
|
269
|
+
isStatusMessageTerminal: true,
|
|
270
|
+
level: options.exitCode > 0 ? 'ERROR' : 'INFO',
|
|
271
|
+
});
|
|
250
272
|
}
|
|
251
273
|
}, options.timeoutSecs * 1000, `Waiting for all event listeners to complete their execution timed out after ${options.timeoutSecs} seconds`).catch(() => {
|
|
252
274
|
if (options.exit) {
|
|
@@ -279,9 +301,9 @@ class Actor {
|
|
|
279
301
|
/**
|
|
280
302
|
* Runs an Actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
281
303
|
*
|
|
282
|
-
* The result of the function is an {@
|
|
304
|
+
* The result of the function is an {@link ActorRun} object that contains details about the Actor run.
|
|
283
305
|
*
|
|
284
|
-
* If you want to run an Actor task rather than an Actor, please use the {@
|
|
306
|
+
* If you want to run an Actor task rather than an Actor, please use the {@link Actor.callTask} function instead.
|
|
285
307
|
*
|
|
286
308
|
* For more information about Actors, read the [documentation](https://docs.apify.com/actor).
|
|
287
309
|
*
|
|
@@ -309,7 +331,7 @@ class Actor {
|
|
|
309
331
|
* Runs an Actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
|
|
310
332
|
* unlike `Actor.call`, this method just starts the run without waiting for finish.
|
|
311
333
|
*
|
|
312
|
-
* The result of the function is an {@
|
|
334
|
+
* The result of the function is an {@link ActorRun} object that contains details about the Actor run.
|
|
313
335
|
*
|
|
314
336
|
* For more information about Actors, read the
|
|
315
337
|
* [documentation](https://docs.apify.com/actor).
|
|
@@ -337,7 +359,7 @@ class Actor {
|
|
|
337
359
|
/**
|
|
338
360
|
* Aborts given Actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
339
361
|
*
|
|
340
|
-
* The result of the function is an {@
|
|
362
|
+
* The result of the function is an {@link ActorRun} object that contains details about the Actor run.
|
|
341
363
|
*
|
|
342
364
|
* For more information about Actors, read the
|
|
343
365
|
* [documentation](https://docs.apify.com/actor).
|
|
@@ -353,18 +375,20 @@ class Actor {
|
|
|
353
375
|
const { token, statusMessage, ...rest } = options;
|
|
354
376
|
const client = token ? this.newClient({ token }) : this.apifyClient;
|
|
355
377
|
if (statusMessage) {
|
|
356
|
-
await this.setStatusMessage(statusMessage, {
|
|
378
|
+
await this.setStatusMessage(statusMessage, {
|
|
379
|
+
isStatusMessageTerminal: true,
|
|
380
|
+
});
|
|
357
381
|
}
|
|
358
382
|
return client.run(runId).abort(rest);
|
|
359
383
|
}
|
|
360
384
|
/**
|
|
361
385
|
* Runs an actor task on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
362
386
|
*
|
|
363
|
-
* The result of the function is an {@
|
|
387
|
+
* The result of the function is an {@link ActorRun} object that contains details about the Actor run.
|
|
364
388
|
*
|
|
365
389
|
* Note that an Actor task is a saved input configuration and options for an Actor.
|
|
366
390
|
* If you want to run an Actor directly rather than an Actor task, please use the
|
|
367
|
-
* {@
|
|
391
|
+
* {@link Actor.call} function instead.
|
|
368
392
|
*
|
|
369
393
|
* For more information about Actor tasks, read the [documentation](https://docs.apify.com/tasks).
|
|
370
394
|
*
|
|
@@ -409,7 +433,9 @@ class Actor {
|
|
|
409
433
|
}
|
|
410
434
|
const { customAfterSleepMillis = this.config.get('metamorphAfterSleepMillis'), ...metamorphOpts } = options;
|
|
411
435
|
const runId = this.config.get('actorRunId');
|
|
412
|
-
await this.apifyClient
|
|
436
|
+
await this.apifyClient
|
|
437
|
+
.run(runId)
|
|
438
|
+
.metamorph(targetActorId, input, metamorphOpts);
|
|
413
439
|
// Wait some time for container to be stopped.
|
|
414
440
|
await (0, utils_1.sleep)(customAfterSleepMillis);
|
|
415
441
|
}
|
|
@@ -433,14 +459,20 @@ class Actor {
|
|
|
433
459
|
// Waiting for all the listeners to finish, as `.reboot()` kills the container.
|
|
434
460
|
await Promise.all([
|
|
435
461
|
// `persistState` for individual RequestLists, RequestQueue... instances to be persisted
|
|
436
|
-
...this.config
|
|
462
|
+
...this.config
|
|
463
|
+
.getEventManager()
|
|
464
|
+
.listeners("persistState" /* EventType.PERSIST_STATE */)
|
|
465
|
+
.map(async (x) => x()),
|
|
437
466
|
// `migrating` to pause Apify crawlers
|
|
438
|
-
...this.config
|
|
467
|
+
...this.config
|
|
468
|
+
.getEventManager()
|
|
469
|
+
.listeners("migrating" /* EventType.MIGRATING */)
|
|
470
|
+
.map(async (x) => x()),
|
|
439
471
|
]);
|
|
440
472
|
const runId = this.config.get('actorRunId');
|
|
441
473
|
await this.apifyClient.run(runId).reboot();
|
|
442
474
|
// Wait some time for container to be stopped.
|
|
443
|
-
const { customAfterSleepMillis = this.config.get('metamorphAfterSleepMillis') } = options;
|
|
475
|
+
const { customAfterSleepMillis = this.config.get('metamorphAfterSleepMillis'), } = options;
|
|
444
476
|
await (0, utils_1.sleep)(customAfterSleepMillis);
|
|
445
477
|
}
|
|
446
478
|
/**
|
|
@@ -510,7 +542,10 @@ class Actor {
|
|
|
510
542
|
}
|
|
511
543
|
const client = this.config.getStorageClient();
|
|
512
544
|
// just to be sure, this should be fast
|
|
513
|
-
await (0, timeout_1.addTimeoutToPromise)(async () => client.setStatusMessage(statusMessage, {
|
|
545
|
+
await (0, timeout_1.addTimeoutToPromise)(async () => client.setStatusMessage(statusMessage, {
|
|
546
|
+
isStatusMessageTerminal,
|
|
547
|
+
level,
|
|
548
|
+
}), 1000, 'Setting status message timed out after 1s').catch((e) => log_1.default.warning(e.message));
|
|
514
549
|
const runId = this.config.get('actorRunId');
|
|
515
550
|
if (runId) {
|
|
516
551
|
// just to be sure, this should be fast
|
|
@@ -522,9 +557,9 @@ class Actor {
|
|
|
522
557
|
return {};
|
|
523
558
|
}
|
|
524
559
|
/**
|
|
525
|
-
* Stores an object or an array of objects to the default {@
|
|
560
|
+
* Stores an object or an array of objects to the default {@link Dataset} of the current Actor run.
|
|
526
561
|
*
|
|
527
|
-
* This is just a convenient shortcut for {@
|
|
562
|
+
* This is just a convenient shortcut for {@link Dataset.pushData}.
|
|
528
563
|
* For example, calling the following code:
|
|
529
564
|
* ```js
|
|
530
565
|
* await Actor.pushData({ myValue: 123 });
|
|
@@ -536,7 +571,7 @@ class Actor {
|
|
|
536
571
|
* await dataset.pushData({ myValue: 123 });
|
|
537
572
|
* ```
|
|
538
573
|
*
|
|
539
|
-
* For more information, see {@
|
|
574
|
+
* For more information, see {@link Actor.openDataset} and {@link Dataset.pushData}
|
|
540
575
|
*
|
|
541
576
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
542
577
|
* otherwise the Actor process might finish before the data are stored!
|
|
@@ -546,10 +581,13 @@ class Actor {
|
|
|
546
581
|
* @param eventName If provided, the method will attempt to charge for the event for each pushed item.
|
|
547
582
|
* @ignore
|
|
548
583
|
*/
|
|
584
|
+
// eslint-disable-next-line consistent-return -- The `return` is inconsistent by design here (`ChargeResult` with `eventName` parameter)
|
|
549
585
|
async pushData(item, eventName) {
|
|
550
586
|
this._ensureActorInit('pushData');
|
|
551
587
|
const dataset = await this.openDataset();
|
|
552
|
-
const maxChargedCount = eventName !== undefined
|
|
588
|
+
const maxChargedCount = eventName !== undefined
|
|
589
|
+
? this.chargingManager.calculateMaxEventChargeCountWithinLimit(eventName)
|
|
590
|
+
: Infinity;
|
|
553
591
|
const toCharge = Array.isArray(item) ? item.length : 1;
|
|
554
592
|
if (toCharge > maxChargedCount) {
|
|
555
593
|
// Push as many items as we can charge for
|
|
@@ -560,17 +598,20 @@ class Actor {
|
|
|
560
598
|
await dataset.pushData(item);
|
|
561
599
|
}
|
|
562
600
|
if (eventName) {
|
|
563
|
-
return await this.chargingManager.charge({
|
|
601
|
+
return await this.chargingManager.charge({
|
|
602
|
+
eventName,
|
|
603
|
+
count: Math.min(toCharge, maxChargedCount),
|
|
604
|
+
});
|
|
564
605
|
}
|
|
565
606
|
}
|
|
566
607
|
/**
|
|
567
|
-
* Opens a dataset and returns a promise resolving to an instance of the {@
|
|
608
|
+
* Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
|
|
568
609
|
*
|
|
569
610
|
* Datasets are used to store structured data where each object stored has the same attributes,
|
|
570
611
|
* such as online store products or real estate offers.
|
|
571
612
|
* The actual data is stored either on the local filesystem or in the cloud.
|
|
572
613
|
*
|
|
573
|
-
* For more details and code examples, see the {@
|
|
614
|
+
* For more details and code examples, see the {@link Dataset} class.
|
|
574
615
|
*
|
|
575
616
|
* @param [datasetIdOrName]
|
|
576
617
|
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
@@ -587,9 +628,9 @@ class Actor {
|
|
|
587
628
|
return this._openStorage(core_1.Dataset, datasetIdOrName, options);
|
|
588
629
|
}
|
|
589
630
|
/**
|
|
590
|
-
* Gets a value from the default {@
|
|
631
|
+
* Gets a value from the default {@link KeyValueStore} associated with the current Actor run.
|
|
591
632
|
*
|
|
592
|
-
* This is just a convenient shortcut for {@
|
|
633
|
+
* This is just a convenient shortcut for {@link KeyValueStore.getValue}.
|
|
593
634
|
* For example, calling the following code:
|
|
594
635
|
* ```js
|
|
595
636
|
* const value = await Actor.getValue('my-key');
|
|
@@ -601,10 +642,10 @@ class Actor {
|
|
|
601
642
|
* const value = await store.getValue('my-key');
|
|
602
643
|
* ```
|
|
603
644
|
*
|
|
604
|
-
* To store the value to the default key-value store, you can use the {@
|
|
645
|
+
* To store the value to the default key-value store, you can use the {@link Actor.setValue} function.
|
|
605
646
|
*
|
|
606
|
-
* For more information, see {@
|
|
607
|
-
* and {@
|
|
647
|
+
* For more information, see {@link Actor.openKeyValueStore}
|
|
648
|
+
* and {@link KeyValueStore.getValue}.
|
|
608
649
|
*
|
|
609
650
|
* @param key Unique record key.
|
|
610
651
|
* @returns
|
|
@@ -620,9 +661,9 @@ class Actor {
|
|
|
620
661
|
return store.getValue(key);
|
|
621
662
|
}
|
|
622
663
|
/**
|
|
623
|
-
* Stores or deletes a value in the default {@
|
|
664
|
+
* Stores or deletes a value in the default {@link KeyValueStore} associated with the current Actor run.
|
|
624
665
|
*
|
|
625
|
-
* This is just a convenient shortcut for {@
|
|
666
|
+
* This is just a convenient shortcut for {@link KeyValueStore.setValue}.
|
|
626
667
|
* For example, calling the following code:
|
|
627
668
|
* ```js
|
|
628
669
|
* await Actor.setValue('OUTPUT', { foo: "bar" });
|
|
@@ -634,10 +675,10 @@ class Actor {
|
|
|
634
675
|
* await store.setValue('OUTPUT', { foo: "bar" });
|
|
635
676
|
* ```
|
|
636
677
|
*
|
|
637
|
-
* To get a value from the default key-value store, you can use the {@
|
|
678
|
+
* To get a value from the default key-value store, you can use the {@link Actor.getValue} function.
|
|
638
679
|
*
|
|
639
|
-
* For more information, see {@
|
|
640
|
-
* and {@
|
|
680
|
+
* For more information, see {@link Actor.openKeyValueStore}
|
|
681
|
+
* and {@link KeyValueStore.getValue}.
|
|
641
682
|
*
|
|
642
683
|
* @param key
|
|
643
684
|
* Unique record key.
|
|
@@ -656,7 +697,7 @@ class Actor {
|
|
|
656
697
|
return store.setValue(key, value, options);
|
|
657
698
|
}
|
|
658
699
|
/**
|
|
659
|
-
* Gets the Actor input value from the default {@
|
|
700
|
+
* Gets the Actor input value from the default {@link KeyValueStore} associated with the current Actor run.
|
|
660
701
|
*
|
|
661
702
|
* This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](core/class/KeyValueStore#getValue).
|
|
662
703
|
* For example, calling the following code:
|
|
@@ -674,8 +715,8 @@ class Actor {
|
|
|
674
715
|
* If you need to use the input multiple times in your Actor,
|
|
675
716
|
* it is far more efficient to read it once and store it locally.
|
|
676
717
|
*
|
|
677
|
-
* For more information, see {@
|
|
678
|
-
* and {@
|
|
718
|
+
* For more information, see {@link Actor.openKeyValueStore}
|
|
719
|
+
* and {@link KeyValueStore.getValue}.
|
|
679
720
|
*
|
|
680
721
|
* @returns
|
|
681
722
|
* Returns a promise that resolves to an object, string
|
|
@@ -689,7 +730,9 @@ class Actor {
|
|
|
689
730
|
const inputSecretsPrivateKeyFile = this.config.get('inputSecretsPrivateKeyFile');
|
|
690
731
|
const inputSecretsPrivateKeyPassphrase = this.config.get('inputSecretsPrivateKeyPassphrase');
|
|
691
732
|
const input = await this.getValue(this.config.get('inputKey'));
|
|
692
|
-
if (ow_1.default.isValid(input, ow_1.default.object.nonEmpty) &&
|
|
733
|
+
if (ow_1.default.isValid(input, ow_1.default.object.nonEmpty) &&
|
|
734
|
+
inputSecretsPrivateKeyFile &&
|
|
735
|
+
inputSecretsPrivateKeyPassphrase) {
|
|
693
736
|
const privateKey = (0, node_crypto_1.createPrivateKey)({
|
|
694
737
|
key: Buffer.from(inputSecretsPrivateKeyFile, 'base64'),
|
|
695
738
|
passphrase: inputSecretsPrivateKeyPassphrase,
|
|
@@ -699,7 +742,7 @@ class Actor {
|
|
|
699
742
|
return input;
|
|
700
743
|
}
|
|
701
744
|
/**
|
|
702
|
-
* Gets the Actor input value just like the {@
|
|
745
|
+
* Gets the Actor input value just like the {@link Actor.getInput} method,
|
|
703
746
|
* but throws if it is not found.
|
|
704
747
|
*/
|
|
705
748
|
async getInputOrThrow() {
|
|
@@ -710,13 +753,13 @@ class Actor {
|
|
|
710
753
|
return input;
|
|
711
754
|
}
|
|
712
755
|
/**
|
|
713
|
-
* Opens a key-value store and returns a promise resolving to an instance of the {@
|
|
756
|
+
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
714
757
|
*
|
|
715
758
|
* Key-value stores are used to store records or files, along with their MIME content type.
|
|
716
759
|
* The records are stored and retrieved using a unique key.
|
|
717
760
|
* The actual data is stored either on a local filesystem or in the Apify cloud.
|
|
718
761
|
*
|
|
719
|
-
* For more details and code examples, see the {@
|
|
762
|
+
* For more details and code examples, see the {@link KeyValueStore} class.
|
|
720
763
|
*
|
|
721
764
|
* @param [storeIdOrName]
|
|
722
765
|
* ID or name of the key-value store to be opened. If `null` or `undefined`,
|
|
@@ -730,18 +773,18 @@ class Actor {
|
|
|
730
773
|
forceCloud: ow_1.default.optional.boolean,
|
|
731
774
|
}));
|
|
732
775
|
this._ensureActorInit('openKeyValueStore');
|
|
733
|
-
return this._openStorage(
|
|
776
|
+
return this._openStorage(key_value_store_js_1.KeyValueStore, storeIdOrName, options);
|
|
734
777
|
}
|
|
735
778
|
/**
|
|
736
779
|
* Opens a request queue and returns a promise resolving to an instance
|
|
737
|
-
* of the {@
|
|
780
|
+
* of the {@link RequestQueue} class.
|
|
738
781
|
*
|
|
739
|
-
* {@
|
|
782
|
+
* {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
|
|
740
783
|
* The queue is used for deep crawling of websites, where you start with several URLs and then
|
|
741
784
|
* recursively follow links to other pages. The data structure supports both breadth-first
|
|
742
785
|
* and depth-first crawling orders.
|
|
743
786
|
*
|
|
744
|
-
* For more details and code examples, see the {@
|
|
787
|
+
* For more details and code examples, see the {@link RequestQueue} class.
|
|
745
788
|
*
|
|
746
789
|
* @param [queueIdOrName]
|
|
747
790
|
* ID or name of the request queue to be opened. If `null` or `undefined`,
|
|
@@ -757,18 +800,19 @@ class Actor {
|
|
|
757
800
|
this._ensureActorInit('openRequestQueue');
|
|
758
801
|
const queue = await this._openStorage(core_1.RequestQueue, queueIdOrName, options);
|
|
759
802
|
// eslint-disable-next-line dot-notation
|
|
760
|
-
queue['initialCount'] =
|
|
803
|
+
queue['initialCount'] =
|
|
804
|
+
(await queue.client.get())?.totalRequestCount ?? 0;
|
|
761
805
|
return queue;
|
|
762
806
|
}
|
|
763
807
|
/**
|
|
764
808
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
765
|
-
* of the {@
|
|
809
|
+
* of the {@link ProxyConfiguration} class that is already initialized.
|
|
766
810
|
*
|
|
767
811
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
768
812
|
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
769
813
|
* them to use the selected proxies for all connections.
|
|
770
814
|
*
|
|
771
|
-
* For more details and code examples, see the {@
|
|
815
|
+
* For more details and code examples, see the {@link ProxyConfiguration} class.
|
|
772
816
|
*
|
|
773
817
|
* ```js
|
|
774
818
|
*
|
|
@@ -805,7 +849,7 @@ class Actor {
|
|
|
805
849
|
if (dontUseApifyProxy && dontUseCustomProxies) {
|
|
806
850
|
return undefined;
|
|
807
851
|
}
|
|
808
|
-
const proxyConfiguration = new
|
|
852
|
+
const proxyConfiguration = new proxy_configuration_js_1.ProxyConfiguration(options, this.config);
|
|
809
853
|
if (await proxyConfiguration.initialize()) {
|
|
810
854
|
return proxyConfiguration;
|
|
811
855
|
}
|
|
@@ -830,7 +874,7 @@ class Actor {
|
|
|
830
874
|
return this.chargingManager;
|
|
831
875
|
}
|
|
832
876
|
/**
|
|
833
|
-
* Modifies Actor env vars so parsing respects the structure of {@
|
|
877
|
+
* Modifies Actor env vars so parsing respects the structure of {@link ApifyEnv} interface.
|
|
834
878
|
*/
|
|
835
879
|
getModifiedActorEnvVars() {
|
|
836
880
|
const modifiedActorEnvVars = {};
|
|
@@ -846,7 +890,7 @@ class Actor {
|
|
|
846
890
|
return modifiedActorEnvVars;
|
|
847
891
|
}
|
|
848
892
|
/**
|
|
849
|
-
* Returns a new {@
|
|
893
|
+
* Returns a new {@link ApifyEnv} object which contains information parsed from all the Apify environment variables.
|
|
850
894
|
*
|
|
851
895
|
* For the list of the Apify environment variables, see
|
|
852
896
|
* [Actor documentation](https://docs.apify.com/actor/run#environment-variables).
|
|
@@ -857,7 +901,10 @@ class Actor {
|
|
|
857
901
|
// NOTE: Don't throw if env vars are invalid to simplify local development and debugging of Actors
|
|
858
902
|
const env = process.env || {};
|
|
859
903
|
const envVars = {};
|
|
860
|
-
for (const [shortName, fullName] of Object.entries({
|
|
904
|
+
for (const [shortName, fullName] of Object.entries({
|
|
905
|
+
...consts_1.APIFY_ENV_VARS,
|
|
906
|
+
...this.getModifiedActorEnvVars(),
|
|
907
|
+
})) {
|
|
861
908
|
const camelCaseName = (0, utils_1.snakeCaseToCamelCase)(shortName);
|
|
862
909
|
let value = env[fullName];
|
|
863
910
|
// Parse dates and integers.
|
|
@@ -882,11 +929,14 @@ class Actor {
|
|
|
882
929
|
*/
|
|
883
930
|
newClient(options = {}) {
|
|
884
931
|
const { storageDir, ...storageClientOptions } = this.config.get('storageClientOptions');
|
|
885
|
-
const { apifyVersion, crawleeVersion } = (0,
|
|
932
|
+
const { apifyVersion, crawleeVersion } = (0, utils_js_1.getSystemInfo)();
|
|
886
933
|
return new apify_client_1.ApifyClient({
|
|
887
934
|
baseUrl: this.config.get('apiBaseUrl'),
|
|
888
935
|
token: this.config.get('token'),
|
|
889
|
-
userAgentSuffix: [
|
|
936
|
+
userAgentSuffix: [
|
|
937
|
+
`SDK/${apifyVersion}`,
|
|
938
|
+
`Crawlee/${crawleeVersion}`,
|
|
939
|
+
],
|
|
890
940
|
...storageClientOptions,
|
|
891
941
|
...options, // allow overriding the instance configuration
|
|
892
942
|
});
|
|
@@ -908,7 +958,9 @@ class Actor {
|
|
|
908
958
|
* @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
|
|
909
959
|
*/
|
|
910
960
|
async useState(name, defaultValue = {}, options) {
|
|
911
|
-
const kvStore = await
|
|
961
|
+
const kvStore = await key_value_store_js_1.KeyValueStore.open(options?.keyValueStoreName, {
|
|
962
|
+
config: options?.config || configuration_js_1.Configuration.getGlobalConfig(),
|
|
963
|
+
});
|
|
912
964
|
return kvStore.getAutoSavedValue(name || 'APIFY_GLOBAL_STATE', defaultValue);
|
|
913
965
|
}
|
|
914
966
|
/**
|
|
@@ -940,7 +992,7 @@ class Actor {
|
|
|
940
992
|
* - When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),
|
|
941
993
|
* it sets up a connection to listen for platform events.
|
|
942
994
|
* For example, to get a notification about an imminent migration to another server.
|
|
943
|
-
* See {@
|
|
995
|
+
* See {@link Actor.events} for details.
|
|
944
996
|
* - It invokes the user function passed as the `userFunc` parameter.
|
|
945
997
|
* - If the user function returned a promise, waits for it to resolve.
|
|
946
998
|
* - If the user function throws an exception or some other error is encountered,
|
|
@@ -995,7 +1047,7 @@ class Actor {
|
|
|
995
1047
|
* from 25% to 100% on the platform.
|
|
996
1048
|
*
|
|
997
1049
|
* Calling `Actor.exit()` is required if you use the `Actor.init()` method, since it opens websocket connection
|
|
998
|
-
* (see {@
|
|
1050
|
+
* (see {@link Actor.events} for details), which needs to be terminated for the code to finish.
|
|
999
1051
|
*
|
|
1000
1052
|
* ```js
|
|
1001
1053
|
* import { gotScraping } from 'got-scraping';
|
|
@@ -1038,9 +1090,9 @@ class Actor {
|
|
|
1038
1090
|
/**
|
|
1039
1091
|
* Runs an Actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
1040
1092
|
*
|
|
1041
|
-
* The result of the function is an {@
|
|
1093
|
+
* The result of the function is an {@link ActorRun} object that contains details about the Actor run.
|
|
1042
1094
|
*
|
|
1043
|
-
* If you want to run an Actor task rather than an Actor, please use the {@
|
|
1095
|
+
* If you want to run an Actor task rather than an Actor, please use the {@link Actor.callTask} function instead.
|
|
1044
1096
|
*
|
|
1045
1097
|
* For more information about Actors, read the [documentation](https://docs.apify.com/actor).
|
|
1046
1098
|
*
|
|
@@ -1064,11 +1116,11 @@ class Actor {
|
|
|
1064
1116
|
/**
|
|
1065
1117
|
* Runs an Actor task on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
1066
1118
|
*
|
|
1067
|
-
* The result of the function is an {@
|
|
1119
|
+
* The result of the function is an {@link ActorRun} object that contains details about the Actor run.
|
|
1068
1120
|
*
|
|
1069
1121
|
* Note that an Actor task is a saved input configuration and options for an Actor.
|
|
1070
1122
|
* If you want to run an Actor directly rather than an Actor task, please use the
|
|
1071
|
-
* {@
|
|
1123
|
+
* {@link Actor.call} function instead.
|
|
1072
1124
|
*
|
|
1073
1125
|
* For more information about Actor tasks, read the [documentation](https://docs.apify.com/tasks).
|
|
1074
1126
|
*
|
|
@@ -1093,7 +1145,7 @@ class Actor {
|
|
|
1093
1145
|
* Runs an Actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
|
|
1094
1146
|
* unlike `Actor.call`, this method just starts the run without waiting for finish.
|
|
1095
1147
|
*
|
|
1096
|
-
* The result of the function is an {@
|
|
1148
|
+
* The result of the function is an {@link ActorRun} object that contains details about the Actor run.
|
|
1097
1149
|
*
|
|
1098
1150
|
* For more information about Actors, read the
|
|
1099
1151
|
* [documentation](https://docs.apify.com/actor).
|
|
@@ -1118,7 +1170,7 @@ class Actor {
|
|
|
1118
1170
|
/**
|
|
1119
1171
|
* Aborts given Actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
1120
1172
|
*
|
|
1121
|
-
* The result of the function is an {@
|
|
1173
|
+
* The result of the function is an {@link ActorRun} object that contains details about the Actor run.
|
|
1122
1174
|
*
|
|
1123
1175
|
* For more information about Actors, read the
|
|
1124
1176
|
* [documentation](https://docs.apify.com/actor).
|
|
@@ -1184,9 +1236,9 @@ class Actor {
|
|
|
1184
1236
|
return Actor.getDefaultInstance().setStatusMessage(statusMessage, options);
|
|
1185
1237
|
}
|
|
1186
1238
|
/**
|
|
1187
|
-
* Stores an object or an array of objects to the default {@
|
|
1239
|
+
* Stores an object or an array of objects to the default {@link Dataset} of the current Actor run.
|
|
1188
1240
|
*
|
|
1189
|
-
* This is just a convenient shortcut for {@
|
|
1241
|
+
* This is just a convenient shortcut for {@link Dataset.pushData}.
|
|
1190
1242
|
* For example, calling the following code:
|
|
1191
1243
|
* ```js
|
|
1192
1244
|
* await Actor.pushData({ myValue: 123 });
|
|
@@ -1198,7 +1250,7 @@ class Actor {
|
|
|
1198
1250
|
* await dataset.pushData({ myValue: 123 });
|
|
1199
1251
|
* ```
|
|
1200
1252
|
*
|
|
1201
|
-
* For more information, see {@
|
|
1253
|
+
* For more information, see {@link Actor.openDataset} and {@link Dataset.pushData}
|
|
1202
1254
|
*
|
|
1203
1255
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
1204
1256
|
* otherwise the Actor process might finish before the data are stored!
|
|
@@ -1214,13 +1266,13 @@ class Actor {
|
|
|
1214
1266
|
return await Actor.getDefaultInstance().pushData(item, eventName);
|
|
1215
1267
|
}
|
|
1216
1268
|
/**
|
|
1217
|
-
* Opens a dataset and returns a promise resolving to an instance of the {@
|
|
1269
|
+
* Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
|
|
1218
1270
|
*
|
|
1219
1271
|
* Datasets are used to store structured data where each object stored has the same attributes,
|
|
1220
1272
|
* such as online store products or real estate offers.
|
|
1221
1273
|
* The actual data is stored either on the local filesystem or in the cloud.
|
|
1222
1274
|
*
|
|
1223
|
-
* For more details and code examples, see the {@
|
|
1275
|
+
* For more details and code examples, see the {@link Dataset} class.
|
|
1224
1276
|
*
|
|
1225
1277
|
* @param [datasetIdOrName]
|
|
1226
1278
|
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
@@ -1231,9 +1283,9 @@ class Actor {
|
|
|
1231
1283
|
return Actor.getDefaultInstance().openDataset(datasetIdOrName, options);
|
|
1232
1284
|
}
|
|
1233
1285
|
/**
|
|
1234
|
-
* Gets a value from the default {@
|
|
1286
|
+
* Gets a value from the default {@link KeyValueStore} associated with the current Actor run.
|
|
1235
1287
|
*
|
|
1236
|
-
* This is just a convenient shortcut for {@
|
|
1288
|
+
* This is just a convenient shortcut for {@link KeyValueStore.getValue}.
|
|
1237
1289
|
* For example, calling the following code:
|
|
1238
1290
|
* ```js
|
|
1239
1291
|
* const value = await Actor.getValue('my-key');
|
|
@@ -1245,10 +1297,10 @@ class Actor {
|
|
|
1245
1297
|
* const value = await store.getValue('my-key');
|
|
1246
1298
|
* ```
|
|
1247
1299
|
*
|
|
1248
|
-
* To store the value to the default key-value store, you can use the {@
|
|
1300
|
+
* To store the value to the default key-value store, you can use the {@link Actor.setValue} function.
|
|
1249
1301
|
*
|
|
1250
|
-
* For more information, see {@
|
|
1251
|
-
* and {@
|
|
1302
|
+
* For more information, see {@link Actor.openKeyValueStore}
|
|
1303
|
+
* and {@link KeyValueStore.getValue}.
|
|
1252
1304
|
*
|
|
1253
1305
|
* @param key Unique record key.
|
|
1254
1306
|
* @returns
|
|
@@ -1261,9 +1313,9 @@ class Actor {
|
|
|
1261
1313
|
return Actor.getDefaultInstance().getValue(key);
|
|
1262
1314
|
}
|
|
1263
1315
|
/**
|
|
1264
|
-
* Stores or deletes a value in the default {@
|
|
1316
|
+
* Stores or deletes a value in the default {@link KeyValueStore} associated with the current Actor run.
|
|
1265
1317
|
*
|
|
1266
|
-
* This is just a convenient shortcut for {@
|
|
1318
|
+
* This is just a convenient shortcut for {@link KeyValueStore.setValue}.
|
|
1267
1319
|
* For example, calling the following code:
|
|
1268
1320
|
* ```js
|
|
1269
1321
|
* await Actor.setValue('OUTPUT', { foo: "bar" });
|
|
@@ -1275,10 +1327,10 @@ class Actor {
|
|
|
1275
1327
|
* await store.setValue('OUTPUT', { foo: "bar" });
|
|
1276
1328
|
* ```
|
|
1277
1329
|
*
|
|
1278
|
-
* To get a value from the default key-value store, you can use the {@
|
|
1330
|
+
* To get a value from the default key-value store, you can use the {@link Actor.getValue} function.
|
|
1279
1331
|
*
|
|
1280
|
-
* For more information, see {@
|
|
1281
|
-
* and {@
|
|
1332
|
+
* For more information, see {@link Actor.openKeyValueStore}
|
|
1333
|
+
* and {@link KeyValueStore.getValue}.
|
|
1282
1334
|
*
|
|
1283
1335
|
* @param key
|
|
1284
1336
|
* Unique record key.
|
|
@@ -1294,9 +1346,9 @@ class Actor {
|
|
|
1294
1346
|
return Actor.getDefaultInstance().setValue(key, value, options);
|
|
1295
1347
|
}
|
|
1296
1348
|
/**
|
|
1297
|
-
* Gets the Actor input value from the default {@
|
|
1349
|
+
* Gets the Actor input value from the default {@link KeyValueStore} associated with the current Actor run.
|
|
1298
1350
|
*
|
|
1299
|
-
* This is just a convenient shortcut for {@
|
|
1351
|
+
* This is just a convenient shortcut for {@link KeyValueStore.getValue | `keyValueStore.getValue('INPUT')`}.
|
|
1300
1352
|
* For example, calling the following code:
|
|
1301
1353
|
* ```js
|
|
1302
1354
|
* const input = await Actor.getInput();
|
|
@@ -1312,7 +1364,7 @@ class Actor {
|
|
|
1312
1364
|
* If you need to use the input multiple times in your Actor,
|
|
1313
1365
|
* it is far more efficient to read it once and store it locally.
|
|
1314
1366
|
*
|
|
1315
|
-
* For more information, see {@
|
|
1367
|
+
* For more information, see {@link Actor.openKeyValueStore} and {@link KeyValueStore.getValue}.
|
|
1316
1368
|
*
|
|
1317
1369
|
* @returns
|
|
1318
1370
|
* Returns a promise that resolves to an object, string
|
|
@@ -1324,20 +1376,20 @@ class Actor {
|
|
|
1324
1376
|
return Actor.getDefaultInstance().getInput();
|
|
1325
1377
|
}
|
|
1326
1378
|
/**
|
|
1327
|
-
* Gets the Actor input value just like the {@
|
|
1379
|
+
* Gets the Actor input value just like the {@link Actor.getInput} method,
|
|
1328
1380
|
* but throws if it is not found.
|
|
1329
1381
|
*/
|
|
1330
1382
|
static async getInputOrThrow() {
|
|
1331
1383
|
return Actor.getDefaultInstance().getInputOrThrow();
|
|
1332
1384
|
}
|
|
1333
1385
|
/**
|
|
1334
|
-
* Opens a key-value store and returns a promise resolving to an instance of the {@
|
|
1386
|
+
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
1335
1387
|
*
|
|
1336
1388
|
* Key-value stores are used to store records or files, along with their MIME content type.
|
|
1337
1389
|
* The records are stored and retrieved using a unique key.
|
|
1338
1390
|
* The actual data is stored either on a local filesystem or in the Apify cloud.
|
|
1339
1391
|
*
|
|
1340
|
-
* For more details and code examples, see the {@
|
|
1392
|
+
* For more details and code examples, see the {@link KeyValueStore} class.
|
|
1341
1393
|
*
|
|
1342
1394
|
* @param [storeIdOrName]
|
|
1343
1395
|
* ID or name of the key-value store to be opened. If `null` or `undefined`,
|
|
@@ -1349,14 +1401,14 @@ class Actor {
|
|
|
1349
1401
|
}
|
|
1350
1402
|
/**
|
|
1351
1403
|
* Opens a request queue and returns a promise resolving to an instance
|
|
1352
|
-
* of the {@
|
|
1404
|
+
* of the {@link RequestQueue} class.
|
|
1353
1405
|
*
|
|
1354
|
-
* {@
|
|
1406
|
+
* {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
|
|
1355
1407
|
* The queue is used for deep crawling of websites, where you start with several URLs and then
|
|
1356
1408
|
* recursively follow links to other pages. The data structure supports both breadth-first
|
|
1357
1409
|
* and depth-first crawling orders.
|
|
1358
1410
|
*
|
|
1359
|
-
* For more details and code examples, see the {@
|
|
1411
|
+
* For more details and code examples, see the {@link RequestQueue} class.
|
|
1360
1412
|
*
|
|
1361
1413
|
* @param [queueIdOrName]
|
|
1362
1414
|
* ID or name of the request queue to be opened. If `null` or `undefined`,
|
|
@@ -1368,13 +1420,13 @@ class Actor {
|
|
|
1368
1420
|
}
|
|
1369
1421
|
/**
|
|
1370
1422
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
1371
|
-
* of the {@
|
|
1423
|
+
* of the {@link ProxyConfiguration} class that is already initialized.
|
|
1372
1424
|
*
|
|
1373
1425
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
1374
1426
|
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
1375
1427
|
* them to use the selected proxies for all connections.
|
|
1376
1428
|
*
|
|
1377
|
-
* For more details and code examples, see the {@
|
|
1429
|
+
* For more details and code examples, see the {@link ProxyConfiguration} class.
|
|
1378
1430
|
*
|
|
1379
1431
|
* ```js
|
|
1380
1432
|
*
|
|
@@ -1419,7 +1471,7 @@ class Actor {
|
|
|
1419
1471
|
return Actor.getDefaultInstance().getChargingManager();
|
|
1420
1472
|
}
|
|
1421
1473
|
/**
|
|
1422
|
-
* Returns a new {@
|
|
1474
|
+
* Returns a new {@link ApifyEnv} object which contains information parsed from all the Apify environment variables.
|
|
1423
1475
|
*
|
|
1424
1476
|
* For the list of the Apify environment variables, see
|
|
1425
1477
|
* [Actor documentation](https://docs.apify.com/actor/run#environment-variables).
|
|
@@ -1444,11 +1496,11 @@ class Actor {
|
|
|
1444
1496
|
static isAtHome() {
|
|
1445
1497
|
return Actor.getDefaultInstance().isAtHome();
|
|
1446
1498
|
}
|
|
1447
|
-
/** Default {@
|
|
1499
|
+
/** Default {@link ApifyClient} instance. */
|
|
1448
1500
|
static get apifyClient() {
|
|
1449
1501
|
return Actor.getDefaultInstance().apifyClient;
|
|
1450
1502
|
}
|
|
1451
|
-
/** Default {@
|
|
1503
|
+
/** Default {@link Configuration} instance. */
|
|
1452
1504
|
static get config() {
|
|
1453
1505
|
return Actor.getDefaultInstance().config;
|
|
1454
1506
|
}
|
|
@@ -1477,15 +1529,4 @@ class Actor {
|
|
|
1477
1529
|
}
|
|
1478
1530
|
}
|
|
1479
1531
|
exports.Actor = Actor;
|
|
1480
|
-
/**
|
|
1481
|
-
* Exit codes for the Actor process.
|
|
1482
|
-
* The error codes must be in the range 1-128, to avoid collision with signal exits
|
|
1483
|
-
* and to ensure Docker will handle them correctly!
|
|
1484
|
-
* @internal should be removed if we decide to remove `Actor.main()`
|
|
1485
|
-
*/
|
|
1486
|
-
exports.EXIT_CODES = {
|
|
1487
|
-
SUCCESS: 0,
|
|
1488
|
-
ERROR_USER_FUNCTION_THREW: 91,
|
|
1489
|
-
ERROR_UNKNOWN: 92,
|
|
1490
|
-
};
|
|
1491
1532
|
//# sourceMappingURL=actor.js.map
|