apify 3.0.4-beta.45 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{dist/actor.d.ts → actor.d.ts} +102 -126
- package/actor.d.ts.map +1 -0
- package/{dist/actor.js → actor.js} +124 -130
- package/actor.js.map +1 -0
- package/{dist/configuration.d.ts → configuration.d.ts} +4 -0
- package/{dist/configuration.d.ts.map → configuration.d.ts.map} +1 -1
- package/{dist/configuration.js → configuration.js} +2 -0
- package/configuration.js.map +1 -0
- package/{dist/index.d.ts → index.d.ts} +2 -1
- package/{dist/index.d.ts.map → index.d.ts.map} +1 -1
- package/index.js +21 -0
- package/index.js.map +1 -0
- package/{dist/index.mjs → index.mjs} +7 -0
- package/{dist/key_value_store.d.ts → key_value_store.d.ts} +0 -0
- package/{dist/key_value_store.d.ts.map → key_value_store.d.ts.map} +0 -0
- package/{dist/key_value_store.js → key_value_store.js} +0 -0
- package/{dist/key_value_store.js.map → key_value_store.js.map} +0 -0
- package/package.json +19 -17
- package/{dist/platform_event_manager.d.ts → platform_event_manager.d.ts} +3 -3
- package/{dist/platform_event_manager.d.ts.map → platform_event_manager.d.ts.map} +0 -0
- package/{dist/platform_event_manager.js → platform_event_manager.js} +3 -3
- package/{dist/platform_event_manager.js.map → platform_event_manager.js.map} +0 -0
- package/{dist/proxy_configuration.d.ts → proxy_configuration.d.ts} +9 -9
- package/{dist/proxy_configuration.d.ts.map → proxy_configuration.d.ts.map} +0 -0
- package/{dist/proxy_configuration.js → proxy_configuration.js} +8 -8
- package/{dist/proxy_configuration.js.map → proxy_configuration.js.map} +0 -0
- package/tsconfig.build.tsbuildinfo +1 -0
- package/{dist/utils.d.ts → utils.d.ts} +0 -0
- package/{dist/utils.d.ts.map → utils.d.ts.map} +0 -0
- package/{dist/utils.js → utils.js} +0 -0
- package/{dist/utils.js.map → utils.js.map} +0 -0
- package/.turbo/turbo-build.log +0 -21
- package/.turbo/turbo-copy.log +0 -5
- package/dist/LICENSE.md +0 -201
- package/dist/README.md +0 -98
- package/dist/actor.d.ts.map +0 -1
- package/dist/actor.js.map +0 -1
- package/dist/configuration.js.map +0 -1
- package/dist/index.js +0 -13
- package/dist/index.js.map +0 -1
- package/dist/package.json +0 -69
|
@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.EXIT_CODES = exports.Actor = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
|
+
const node_crypto_1 = require("node:crypto");
|
|
7
|
+
const input_secrets_1 = require("@apify/input_secrets");
|
|
6
8
|
const consts_1 = require("@apify/consts");
|
|
7
9
|
const timeout_1 = require("@apify/timeout");
|
|
8
10
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
@@ -15,14 +17,14 @@ const proxy_configuration_1 = require("./proxy_configuration");
|
|
|
15
17
|
const key_value_store_1 = require("./key_value_store");
|
|
16
18
|
const configuration_1 = require("./configuration");
|
|
17
19
|
/**
|
|
18
|
-
* `
|
|
20
|
+
* `Actor` class serves as an alternative approach to the static helpers exported from the package. It allows to pass configuration
|
|
19
21
|
* that will be used on the instance methods. Environment variables will have precedence over this configuration.
|
|
20
|
-
* See {@
|
|
22
|
+
* See {@apilink Configuration} for details about what can be configured and what are the default values.
|
|
21
23
|
*/
|
|
22
24
|
class Actor {
|
|
23
25
|
constructor(options = {}) {
|
|
24
26
|
/**
|
|
25
|
-
* Configuration of this SDK instance (provided to its constructor). See {@
|
|
27
|
+
* Configuration of this SDK instance (provided to its constructor). See {@apilink Configuration} for details.
|
|
26
28
|
* @internal
|
|
27
29
|
*/
|
|
28
30
|
Object.defineProperty(this, "config", {
|
|
@@ -32,7 +34,7 @@ class Actor {
|
|
|
32
34
|
value: void 0
|
|
33
35
|
});
|
|
34
36
|
/**
|
|
35
|
-
* Default {@
|
|
37
|
+
* Default {@apilink ApifyClient} instance.
|
|
36
38
|
* @internal
|
|
37
39
|
*/
|
|
38
40
|
Object.defineProperty(this, "apifyClient", {
|
|
@@ -42,7 +44,7 @@ class Actor {
|
|
|
42
44
|
value: void 0
|
|
43
45
|
});
|
|
44
46
|
/**
|
|
45
|
-
* Default {@
|
|
47
|
+
* Default {@apilink EventManager} instance.
|
|
46
48
|
* @internal
|
|
47
49
|
*/
|
|
48
50
|
Object.defineProperty(this, "eventManager", {
|
|
@@ -73,7 +75,7 @@ class Actor {
|
|
|
73
75
|
* - When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),
|
|
74
76
|
* it sets up a connection to listen for platform events.
|
|
75
77
|
* For example, to get a notification about an imminent migration to another server.
|
|
76
|
-
* See {@
|
|
78
|
+
* See {@apilink Actor.events} for details.
|
|
77
79
|
* - It checks that either `APIFY_TOKEN` or `APIFY_LOCAL_STORAGE_DIR` environment variable
|
|
78
80
|
* is defined. If not, the functions sets `APIFY_LOCAL_STORAGE_DIR` to `./apify_storage`
|
|
79
81
|
* inside the current working directory. This is to simplify running code examples.
|
|
@@ -147,7 +149,6 @@ class Actor {
|
|
|
147
149
|
(0, utils_2.printOutdatedSdkWarning)();
|
|
148
150
|
// reset global config instance to respect APIFY_ prefixed env vars
|
|
149
151
|
core_1.Configuration.globalConfig = configuration_1.Configuration.getGlobalConfig();
|
|
150
|
-
await this.eventManager.init();
|
|
151
152
|
if (this.isAtHome()) {
|
|
152
153
|
this.config.set('availableMemoryRatio', 1);
|
|
153
154
|
this.config.set('disableBrowserSandbox', true); // for browser launcher, adds `--no-sandbox` to args
|
|
@@ -157,6 +158,8 @@ class Actor {
|
|
|
157
158
|
else if (options.storage) {
|
|
158
159
|
this.config.useStorageClient(options.storage);
|
|
159
160
|
}
|
|
161
|
+
// Init the event manager the config uses
|
|
162
|
+
await this.config.getEventManager().init();
|
|
160
163
|
await (0, core_1.purgeDefaultStorages)(this.config);
|
|
161
164
|
configuration_1.Configuration.storage.enterWith(this.config);
|
|
162
165
|
}
|
|
@@ -169,12 +172,12 @@ class Actor {
|
|
|
169
172
|
options.exitCode ?? (options.exitCode = exports.EXIT_CODES.SUCCESS);
|
|
170
173
|
options.timeoutSecs ?? (options.timeoutSecs = 30);
|
|
171
174
|
// Close the event manager and emit the final PERSIST_STATE event
|
|
172
|
-
await this.
|
|
175
|
+
await this.config.getEventManager().close();
|
|
173
176
|
// Emit the exit event
|
|
174
|
-
this.
|
|
177
|
+
this.config.getEventManager().emit("exit" /* EventType.EXIT */, options);
|
|
175
178
|
// Wait for all event listeners to be processed
|
|
176
179
|
log_1.default.debug(`Waiting for all event listeners to complete their execution (with ${options.timeoutSecs} seconds timeout)`);
|
|
177
|
-
await (0, timeout_1.addTimeoutToPromise)(() => this.
|
|
180
|
+
await (0, timeout_1.addTimeoutToPromise)(() => this.config.getEventManager().waitForAllListenersToComplete(), options.timeoutSecs * 1000, `Waiting for all event listeners to complete their execution timed out after ${options.timeoutSecs} seconds`);
|
|
178
181
|
const client = this.config.getStorageClient();
|
|
179
182
|
if (client.teardown) {
|
|
180
183
|
let finished = false;
|
|
@@ -209,41 +212,29 @@ class Actor {
|
|
|
209
212
|
* @ignore
|
|
210
213
|
*/
|
|
211
214
|
on(event, listener) {
|
|
212
|
-
this.
|
|
215
|
+
this.config.getEventManager().on(event, listener);
|
|
213
216
|
}
|
|
214
217
|
/**
|
|
215
218
|
* @ignore
|
|
216
219
|
*/
|
|
217
220
|
off(event, listener) {
|
|
218
|
-
this.
|
|
221
|
+
this.config.getEventManager().off(event, listener);
|
|
219
222
|
}
|
|
220
223
|
/**
|
|
221
|
-
* Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable)
|
|
222
|
-
* waits for the actor to finish and fetches its output.
|
|
223
|
-
*
|
|
224
|
-
* By passing the `waitSecs` option you can reduce the maximum amount of time to wait for the run to finish.
|
|
225
|
-
* If the value is less than or equal to zero, the function returns immediately after the run is started.
|
|
224
|
+
* Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
226
225
|
*
|
|
227
|
-
* The result of the function is an {@
|
|
228
|
-
* that contains details about the actor run and its output (if any).
|
|
226
|
+
* The result of the function is an {@apilink ActorRun} object that contains details about the actor run.
|
|
229
227
|
*
|
|
230
|
-
* If you want to run an actor task rather than an actor, please use the
|
|
231
|
-
* {@link Actor.callTask} function instead.
|
|
228
|
+
* If you want to run an actor task rather than an actor, please use the {@apilink Actor.callTask} function instead.
|
|
232
229
|
*
|
|
233
|
-
* For more information about actors, read the
|
|
234
|
-
* [documentation](https://docs.apify.com/actor).
|
|
230
|
+
* For more information about actors, read the [documentation](https://docs.apify.com/actor).
|
|
235
231
|
*
|
|
236
232
|
* **Example usage:**
|
|
237
233
|
*
|
|
238
234
|
* ```javascript
|
|
239
235
|
* const run = await Actor.call('apify/hello-world', { myInput: 123 });
|
|
240
|
-
* console.log(`Received message: ${run.output.body.message}`);
|
|
241
236
|
* ```
|
|
242
237
|
*
|
|
243
|
-
* Internally, the `call()` function invokes the
|
|
244
|
-
* [Run actor](https://apify.com/docs/api/v2#/reference/actors/run-collection/run-actor)
|
|
245
|
-
* and several other API endpoints to obtain the output.
|
|
246
|
-
*
|
|
247
238
|
* @param actorId
|
|
248
239
|
* Allowed formats are `username/actor-name`, `userId/actor-name` or actor ID.
|
|
249
240
|
* @param [input]
|
|
@@ -262,7 +253,7 @@ class Actor {
|
|
|
262
253
|
* Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
|
|
263
254
|
* unlike `Actor.call`, this method just starts the run without waiting for finish.
|
|
264
255
|
*
|
|
265
|
-
* The result of the function is an {@
|
|
256
|
+
* The result of the function is an {@apilink ActorRun} object that contains details about the actor run.
|
|
266
257
|
*
|
|
267
258
|
* For more information about actors, read the
|
|
268
259
|
* [documentation](https://docs.apify.com/actor).
|
|
@@ -290,7 +281,7 @@ class Actor {
|
|
|
290
281
|
/**
|
|
291
282
|
* Aborts given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
292
283
|
*
|
|
293
|
-
* The result of the function is an {@
|
|
284
|
+
* The result of the function is an {@apilink ActorRun} object that contains details about the actor run.
|
|
294
285
|
*
|
|
295
286
|
* For more information about actors, read the
|
|
296
287
|
* [documentation](https://docs.apify.com/actor).
|
|
@@ -308,18 +299,13 @@ class Actor {
|
|
|
308
299
|
return client.run(runId).abort(rest);
|
|
309
300
|
}
|
|
310
301
|
/**
|
|
311
|
-
* Runs an actor task on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable)
|
|
312
|
-
* waits for the task to finish and fetches its output.
|
|
313
|
-
*
|
|
314
|
-
* By passing the `waitSecs` option you can reduce the maximum amount of time to wait for the run to finish.
|
|
315
|
-
* If the value is less than or equal to zero, the function returns immediately after the run is started.
|
|
302
|
+
* Runs an actor task on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
316
303
|
*
|
|
317
|
-
* The result of the function is an {@
|
|
318
|
-
* that contains details about the actor run and its output (if any).
|
|
304
|
+
* The result of the function is an {@apilink ActorRun} object that contains details about the actor run.
|
|
319
305
|
*
|
|
320
306
|
* Note that an actor task is a saved input configuration and options for an actor.
|
|
321
307
|
* If you want to run an actor directly rather than an actor task, please use the
|
|
322
|
-
* {@
|
|
308
|
+
* {@apilink Actor.call} function instead.
|
|
323
309
|
*
|
|
324
310
|
* For more information about actor tasks, read the [documentation](https://docs.apify.com/tasks).
|
|
325
311
|
*
|
|
@@ -327,13 +313,8 @@ class Actor {
|
|
|
327
313
|
*
|
|
328
314
|
* ```javascript
|
|
329
315
|
* const run = await Actor.callTask('bob/some-task');
|
|
330
|
-
* console.log(`Received message: ${run.output.body.message}`);
|
|
331
316
|
* ```
|
|
332
317
|
*
|
|
333
|
-
* Internally, the `callTask()` function calls the
|
|
334
|
-
* [Run task](https://apify.com/docs/api/v2#/reference/actor-tasks/run-collection/run-task)
|
|
335
|
-
* and several other API endpoints to obtain the output.
|
|
336
|
-
*
|
|
337
318
|
* @param taskId
|
|
338
319
|
* Allowed formats are `username/task-name`, `userId/task-name` or task ID.
|
|
339
320
|
* @param [input]
|
|
@@ -450,9 +431,9 @@ class Actor {
|
|
|
450
431
|
return this.apifyClient.run(runId).update({ statusMessage });
|
|
451
432
|
}
|
|
452
433
|
/**
|
|
453
|
-
* Stores an object or an array of objects to the default {@
|
|
434
|
+
* Stores an object or an array of objects to the default {@apilink Dataset} of the current actor run.
|
|
454
435
|
*
|
|
455
|
-
* This is just a convenient shortcut for {@
|
|
436
|
+
* This is just a convenient shortcut for {@apilink Dataset.pushData}.
|
|
456
437
|
* For example, calling the following code:
|
|
457
438
|
* ```javascript
|
|
458
439
|
* await Actor.pushData({ myValue: 123 });
|
|
@@ -464,7 +445,7 @@ class Actor {
|
|
|
464
445
|
* await dataset.pushData({ myValue: 123 });
|
|
465
446
|
* ```
|
|
466
447
|
*
|
|
467
|
-
* For more information, see {@
|
|
448
|
+
* For more information, see {@apilink Actor.openDataset} and {@apilink Dataset.pushData}
|
|
468
449
|
*
|
|
469
450
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
470
451
|
* otherwise the actor process might finish before the data are stored!
|
|
@@ -478,13 +459,13 @@ class Actor {
|
|
|
478
459
|
return dataset.pushData(item);
|
|
479
460
|
}
|
|
480
461
|
/**
|
|
481
|
-
* Opens a dataset and returns a promise resolving to an instance of the {@
|
|
462
|
+
* Opens a dataset and returns a promise resolving to an instance of the {@apilink Dataset} class.
|
|
482
463
|
*
|
|
483
464
|
* Datasets are used to store structured data where each object stored has the same attributes,
|
|
484
465
|
* such as online store products or real estate offers.
|
|
485
466
|
* The actual data is stored either on the local filesystem or in the cloud.
|
|
486
467
|
*
|
|
487
|
-
* For more details and code examples, see the {@
|
|
468
|
+
* For more details and code examples, see the {@apilink Dataset} class.
|
|
488
469
|
*
|
|
489
470
|
* @param [datasetIdOrName]
|
|
490
471
|
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
@@ -500,9 +481,9 @@ class Actor {
|
|
|
500
481
|
return this._openStorage(core_1.Dataset, datasetIdOrName, options);
|
|
501
482
|
}
|
|
502
483
|
/**
|
|
503
|
-
* Gets a value from the default {@
|
|
484
|
+
* Gets a value from the default {@apilink KeyValueStore} associated with the current actor run.
|
|
504
485
|
*
|
|
505
|
-
* This is just a convenient shortcut for {@
|
|
486
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.getValue}.
|
|
506
487
|
* For example, calling the following code:
|
|
507
488
|
* ```javascript
|
|
508
489
|
* const value = await Actor.getValue('my-key');
|
|
@@ -514,10 +495,10 @@ class Actor {
|
|
|
514
495
|
* const value = await store.getValue('my-key');
|
|
515
496
|
* ```
|
|
516
497
|
*
|
|
517
|
-
* To store the value to the default key-value store, you can use the {@
|
|
498
|
+
* To store the value to the default key-value store, you can use the {@apilink Actor.setValue} function.
|
|
518
499
|
*
|
|
519
|
-
* For more information, see {@
|
|
520
|
-
* and {@
|
|
500
|
+
* For more information, see {@apilink Actor.openKeyValueStore}
|
|
501
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
521
502
|
*
|
|
522
503
|
* @param key Unique record key.
|
|
523
504
|
* @returns
|
|
@@ -532,9 +513,9 @@ class Actor {
|
|
|
532
513
|
return store.getValue(key);
|
|
533
514
|
}
|
|
534
515
|
/**
|
|
535
|
-
* Stores or deletes a value in the default {@
|
|
516
|
+
* Stores or deletes a value in the default {@apilink KeyValueStore} associated with the current actor run.
|
|
536
517
|
*
|
|
537
|
-
* This is just a convenient shortcut for {@
|
|
518
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.setValue}.
|
|
538
519
|
* For example, calling the following code:
|
|
539
520
|
* ```javascript
|
|
540
521
|
* await Actor.setValue('OUTPUT', { foo: "bar" });
|
|
@@ -546,10 +527,10 @@ class Actor {
|
|
|
546
527
|
* await store.setValue('OUTPUT', { foo: "bar" });
|
|
547
528
|
* ```
|
|
548
529
|
*
|
|
549
|
-
* To get a value from the default key-value store, you can use the {@
|
|
530
|
+
* To get a value from the default key-value store, you can use the {@apilink Actor.getValue} function.
|
|
550
531
|
*
|
|
551
|
-
* For more information, see {@
|
|
552
|
-
* and {@
|
|
532
|
+
* For more information, see {@apilink Actor.openKeyValueStore}
|
|
533
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
553
534
|
*
|
|
554
535
|
* @param key
|
|
555
536
|
* Unique record key.
|
|
@@ -567,7 +548,7 @@ class Actor {
|
|
|
567
548
|
return store.setValue(key, value, options);
|
|
568
549
|
}
|
|
569
550
|
/**
|
|
570
|
-
* Gets the actor input value from the default {@
|
|
551
|
+
* Gets the actor input value from the default {@apilink KeyValueStore} associated with the current actor run.
|
|
571
552
|
*
|
|
572
553
|
* This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](core/class/KeyValueStore#getValue).
|
|
573
554
|
* For example, calling the following code:
|
|
@@ -585,8 +566,8 @@ class Actor {
|
|
|
585
566
|
* If you need to use the input multiple times in your actor,
|
|
586
567
|
* it is far more efficient to read it once and store it locally.
|
|
587
568
|
*
|
|
588
|
-
* For more information, see {@
|
|
589
|
-
* and {@
|
|
569
|
+
* For more information, see {@apilink Actor.openKeyValueStore}
|
|
570
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
590
571
|
*
|
|
591
572
|
* @returns
|
|
592
573
|
* Returns a promise that resolves to an object, string
|
|
@@ -596,16 +577,26 @@ class Actor {
|
|
|
596
577
|
* @ignore
|
|
597
578
|
*/
|
|
598
579
|
async getInput() {
|
|
599
|
-
|
|
580
|
+
const inputSecretsPrivateKeyFile = this.config.get('inputSecretsPrivateKeyFile');
|
|
581
|
+
const inputSecretsPrivateKeyPassphrase = this.config.get('inputSecretsPrivateKeyPassphrase');
|
|
582
|
+
const input = await this.getValue(this.config.get('inputKey'));
|
|
583
|
+
if (ow_1.default.isValid(input, ow_1.default.object.nonEmpty) && inputSecretsPrivateKeyFile && inputSecretsPrivateKeyPassphrase) {
|
|
584
|
+
const privateKey = (0, node_crypto_1.createPrivateKey)({
|
|
585
|
+
key: Buffer.from(inputSecretsPrivateKeyFile, 'base64'),
|
|
586
|
+
passphrase: inputSecretsPrivateKeyPassphrase,
|
|
587
|
+
});
|
|
588
|
+
return (0, input_secrets_1.decryptInputSecrets)({ input, privateKey });
|
|
589
|
+
}
|
|
590
|
+
return input;
|
|
600
591
|
}
|
|
601
592
|
/**
|
|
602
|
-
* Opens a key-value store and returns a promise resolving to an instance of the {@
|
|
593
|
+
* Opens a key-value store and returns a promise resolving to an instance of the {@apilink KeyValueStore} class.
|
|
603
594
|
*
|
|
604
595
|
* Key-value stores are used to store records or files, along with their MIME content type.
|
|
605
596
|
* The records are stored and retrieved using a unique key.
|
|
606
597
|
* The actual data is stored either on a local filesystem or in the Apify cloud.
|
|
607
598
|
*
|
|
608
|
-
* For more details and code examples, see the {@
|
|
599
|
+
* For more details and code examples, see the {@apilink KeyValueStore} class.
|
|
609
600
|
*
|
|
610
601
|
* @param [storeIdOrName]
|
|
611
602
|
* ID or name of the key-value store to be opened. If `null` or `undefined`,
|
|
@@ -622,14 +613,14 @@ class Actor {
|
|
|
622
613
|
}
|
|
623
614
|
/**
|
|
624
615
|
* Opens a request queue and returns a promise resolving to an instance
|
|
625
|
-
* of the {@
|
|
616
|
+
* of the {@apilink RequestQueue} class.
|
|
626
617
|
*
|
|
627
|
-
* {@
|
|
618
|
+
* {@apilink RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
|
|
628
619
|
* The queue is used for deep crawling of websites, where you start with several URLs and then
|
|
629
620
|
* recursively follow links to other pages. The data structure supports both breadth-first
|
|
630
621
|
* and depth-first crawling orders.
|
|
631
622
|
*
|
|
632
|
-
* For more details and code examples, see the {@
|
|
623
|
+
* For more details and code examples, see the {@apilink RequestQueue} class.
|
|
633
624
|
*
|
|
634
625
|
* @param [queueIdOrName]
|
|
635
626
|
* ID or name of the request queue to be opened. If `null` or `undefined`,
|
|
@@ -646,13 +637,13 @@ class Actor {
|
|
|
646
637
|
}
|
|
647
638
|
/**
|
|
648
639
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
649
|
-
* of the {@
|
|
640
|
+
* of the {@apilink ProxyConfiguration} class that is already initialized.
|
|
650
641
|
*
|
|
651
642
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
652
643
|
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
653
644
|
* them to use the selected proxies for all connections.
|
|
654
645
|
*
|
|
655
|
-
* For more details and code examples, see the {@
|
|
646
|
+
* For more details and code examples, see the {@apilink ProxyConfiguration} class.
|
|
656
647
|
*
|
|
657
648
|
* ```javascript
|
|
658
649
|
*
|
|
@@ -694,7 +685,7 @@ class Actor {
|
|
|
694
685
|
return proxyConfiguration;
|
|
695
686
|
}
|
|
696
687
|
/**
|
|
697
|
-
* Returns a new {@
|
|
688
|
+
* Returns a new {@apilink ApifyEnv} object which contains information parsed from all the `APIFY_XXX` environment variables.
|
|
698
689
|
*
|
|
699
690
|
* For the list of the `APIFY_XXX` environment variables, see
|
|
700
691
|
* [Actor documentation](https://docs.apify.com/actor/run#environment-variables).
|
|
@@ -744,6 +735,31 @@ class Actor {
|
|
|
744
735
|
isAtHome() {
|
|
745
736
|
return !!process.env[consts_1.ENV_VARS.IS_AT_HOME];
|
|
746
737
|
}
|
|
738
|
+
/**
|
|
739
|
+
* Easily create and manage state values. All state values are automatically persisted.
|
|
740
|
+
*
|
|
741
|
+
* Values can be modified by simply using the assignment operator.
|
|
742
|
+
*
|
|
743
|
+
* @param name The name of the store to use.
|
|
744
|
+
* @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
|
|
745
|
+
* @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
|
|
746
|
+
*/
|
|
747
|
+
async useState(name, defaultValue = {}, options) {
|
|
748
|
+
const kvStore = await key_value_store_1.KeyValueStore.open(options?.keyValueStoreName, { config: options?.config || configuration_1.Configuration.getGlobalConfig() });
|
|
749
|
+
return kvStore.getAutoSavedValue(name || 'APIFY_GLOBAL_STATE', defaultValue);
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
752
|
+
* Easily create and manage state values. All state values are automatically persisted.
|
|
753
|
+
*
|
|
754
|
+
* Values can be modified by simply using the assignment operator.
|
|
755
|
+
*
|
|
756
|
+
* @param name The name of the store to use.
|
|
757
|
+
* @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
|
|
758
|
+
* @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
|
|
759
|
+
*/
|
|
760
|
+
static async useState(name, defaultValue = {}, options) {
|
|
761
|
+
return Actor.getDefaultInstance().useState(name, defaultValue, options);
|
|
762
|
+
}
|
|
747
763
|
/**
|
|
748
764
|
* Runs the main user function that performs the job of the actor
|
|
749
765
|
* and terminates the process when the user function finishes.
|
|
@@ -761,7 +777,7 @@ class Actor {
|
|
|
761
777
|
* - When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),
|
|
762
778
|
* it sets up a connection to listen for platform events.
|
|
763
779
|
* For example, to get a notification about an imminent migration to another server.
|
|
764
|
-
* See {@
|
|
780
|
+
* See {@apilink Actor.events} for details.
|
|
765
781
|
* - It checks that either `APIFY_TOKEN` or `APIFY_LOCAL_STORAGE_DIR` environment variable
|
|
766
782
|
* is defined. If not, the functions sets `APIFY_LOCAL_STORAGE_DIR` to `./apify_storage`
|
|
767
783
|
* inside the current working directory. This is to simplify running code examples.
|
|
@@ -827,32 +843,20 @@ class Actor {
|
|
|
827
843
|
Actor.getDefaultInstance().off(event, listener);
|
|
828
844
|
}
|
|
829
845
|
/**
|
|
830
|
-
* Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable)
|
|
831
|
-
* waits for the actor to finish and fetches its output.
|
|
846
|
+
* Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
832
847
|
*
|
|
833
|
-
*
|
|
834
|
-
* If the value is less than or equal to zero, the function returns immediately after the run is started.
|
|
848
|
+
* The result of the function is an {@apilink ActorRun} object that contains details about the actor run.
|
|
835
849
|
*
|
|
836
|
-
*
|
|
837
|
-
* that contains details about the actor run and its output (if any).
|
|
850
|
+
* If you want to run an actor task rather than an actor, please use the {@apilink Actor.callTask} function instead.
|
|
838
851
|
*
|
|
839
|
-
*
|
|
840
|
-
* {@link Actor.callTask} function instead.
|
|
841
|
-
*
|
|
842
|
-
* For more information about actors, read the
|
|
843
|
-
* [documentation](https://docs.apify.com/actor).
|
|
852
|
+
* For more information about actors, read the [documentation](https://docs.apify.com/actor).
|
|
844
853
|
*
|
|
845
854
|
* **Example usage:**
|
|
846
855
|
*
|
|
847
856
|
* ```javascript
|
|
848
857
|
* const run = await Actor.call('apify/hello-world', { myInput: 123 });
|
|
849
|
-
* console.log(`Received message: ${run.output.body.message}`);
|
|
850
858
|
* ```
|
|
851
859
|
*
|
|
852
|
-
* Internally, the `call()` function invokes the
|
|
853
|
-
* [Run actor](https://apify.com/docs/api/v2#/reference/actors/run-collection/run-actor)
|
|
854
|
-
* and several other API endpoints to obtain the output.
|
|
855
|
-
*
|
|
856
860
|
* @param actorId
|
|
857
861
|
* Allowed formats are `username/actor-name`, `userId/actor-name` or actor ID.
|
|
858
862
|
* @param [input]
|
|
@@ -865,18 +869,13 @@ class Actor {
|
|
|
865
869
|
return Actor.getDefaultInstance().call(actorId, input, options);
|
|
866
870
|
}
|
|
867
871
|
/**
|
|
868
|
-
* Runs an actor task on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable)
|
|
869
|
-
* waits for the task to finish and fetches its output.
|
|
870
|
-
*
|
|
871
|
-
* By passing the `waitSecs` option you can reduce the maximum amount of time to wait for the run to finish.
|
|
872
|
-
* If the value is less than or equal to zero, the function returns immediately after the run is started.
|
|
872
|
+
* Runs an actor task on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
873
873
|
*
|
|
874
|
-
* The result of the function is an {@
|
|
875
|
-
* that contains details about the actor run and its output (if any).
|
|
874
|
+
* The result of the function is an {@apilink ActorRun} object that contains details about the actor run.
|
|
876
875
|
*
|
|
877
876
|
* Note that an actor task is a saved input configuration and options for an actor.
|
|
878
877
|
* If you want to run an actor directly rather than an actor task, please use the
|
|
879
|
-
* {@
|
|
878
|
+
* {@apilink Actor.call} function instead.
|
|
880
879
|
*
|
|
881
880
|
* For more information about actor tasks, read the [documentation](https://docs.apify.com/tasks).
|
|
882
881
|
*
|
|
@@ -884,13 +883,8 @@ class Actor {
|
|
|
884
883
|
*
|
|
885
884
|
* ```javascript
|
|
886
885
|
* const run = await Actor.callTask('bob/some-task');
|
|
887
|
-
* console.log(`Received message: ${run.output.body.message}`);
|
|
888
886
|
* ```
|
|
889
887
|
*
|
|
890
|
-
* Internally, the `callTask()` function calls the
|
|
891
|
-
* [Run task](https://apify.com/docs/api/v2#/reference/actor-tasks/run-collection/run-task)
|
|
892
|
-
* and several other API endpoints to obtain the output.
|
|
893
|
-
*
|
|
894
888
|
* @param taskId
|
|
895
889
|
* Allowed formats are `username/task-name`, `userId/task-name` or task ID.
|
|
896
890
|
* @param [input]
|
|
@@ -906,7 +900,7 @@ class Actor {
|
|
|
906
900
|
* Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
|
|
907
901
|
* unlike `Actor.call`, this method just starts the run without waiting for finish.
|
|
908
902
|
*
|
|
909
|
-
* The result of the function is an {@
|
|
903
|
+
* The result of the function is an {@apilink ActorRun} object that contains details about the actor run.
|
|
910
904
|
*
|
|
911
905
|
* For more information about actors, read the
|
|
912
906
|
* [documentation](https://docs.apify.com/actor).
|
|
@@ -931,7 +925,7 @@ class Actor {
|
|
|
931
925
|
/**
|
|
932
926
|
* Aborts given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
933
927
|
*
|
|
934
|
-
* The result of the function is an {@
|
|
928
|
+
* The result of the function is an {@apilink ActorRun} object that contains details about the actor run.
|
|
935
929
|
*
|
|
936
930
|
* For more information about actors, read the
|
|
937
931
|
* [documentation](https://docs.apify.com/actor).
|
|
@@ -993,9 +987,9 @@ class Actor {
|
|
|
993
987
|
return Actor.getDefaultInstance().setStatusMessage(statusMessage);
|
|
994
988
|
}
|
|
995
989
|
/**
|
|
996
|
-
* Stores an object or an array of objects to the default {@
|
|
990
|
+
* Stores an object or an array of objects to the default {@apilink Dataset} of the current actor run.
|
|
997
991
|
*
|
|
998
|
-
* This is just a convenient shortcut for {@
|
|
992
|
+
* This is just a convenient shortcut for {@apilink Dataset.pushData}.
|
|
999
993
|
* For example, calling the following code:
|
|
1000
994
|
* ```javascript
|
|
1001
995
|
* await Actor.pushData({ myValue: 123 });
|
|
@@ -1007,7 +1001,7 @@ class Actor {
|
|
|
1007
1001
|
* await dataset.pushData({ myValue: 123 });
|
|
1008
1002
|
* ```
|
|
1009
1003
|
*
|
|
1010
|
-
* For more information, see {@
|
|
1004
|
+
* For more information, see {@apilink Actor.openDataset} and {@apilink Dataset.pushData}
|
|
1011
1005
|
*
|
|
1012
1006
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
1013
1007
|
* otherwise the actor process might finish before the data are stored!
|
|
@@ -1019,13 +1013,13 @@ class Actor {
|
|
|
1019
1013
|
return Actor.getDefaultInstance().pushData(item);
|
|
1020
1014
|
}
|
|
1021
1015
|
/**
|
|
1022
|
-
* Opens a dataset and returns a promise resolving to an instance of the {@
|
|
1016
|
+
* Opens a dataset and returns a promise resolving to an instance of the {@apilink Dataset} class.
|
|
1023
1017
|
*
|
|
1024
1018
|
* Datasets are used to store structured data where each object stored has the same attributes,
|
|
1025
1019
|
* such as online store products or real estate offers.
|
|
1026
1020
|
* The actual data is stored either on the local filesystem or in the cloud.
|
|
1027
1021
|
*
|
|
1028
|
-
* For more details and code examples, see the {@
|
|
1022
|
+
* For more details and code examples, see the {@apilink Dataset} class.
|
|
1029
1023
|
*
|
|
1030
1024
|
* @param [datasetIdOrName]
|
|
1031
1025
|
* ID or name of the dataset to be opened. If `null` or `undefined`,
|
|
@@ -1036,9 +1030,9 @@ class Actor {
|
|
|
1036
1030
|
return Actor.getDefaultInstance().openDataset(datasetIdOrName, options);
|
|
1037
1031
|
}
|
|
1038
1032
|
/**
|
|
1039
|
-
* Gets a value from the default {@
|
|
1033
|
+
* Gets a value from the default {@apilink KeyValueStore} associated with the current actor run.
|
|
1040
1034
|
*
|
|
1041
|
-
* This is just a convenient shortcut for {@
|
|
1035
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.getValue}.
|
|
1042
1036
|
* For example, calling the following code:
|
|
1043
1037
|
* ```javascript
|
|
1044
1038
|
* const value = await Actor.getValue('my-key');
|
|
@@ -1050,10 +1044,10 @@ class Actor {
|
|
|
1050
1044
|
* const value = await store.getValue('my-key');
|
|
1051
1045
|
* ```
|
|
1052
1046
|
*
|
|
1053
|
-
* To store the value to the default key-value store, you can use the {@
|
|
1047
|
+
* To store the value to the default key-value store, you can use the {@apilink Actor.setValue} function.
|
|
1054
1048
|
*
|
|
1055
|
-
* For more information, see {@
|
|
1056
|
-
* and {@
|
|
1049
|
+
* For more information, see {@apilink Actor.openKeyValueStore}
|
|
1050
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
1057
1051
|
*
|
|
1058
1052
|
* @param key Unique record key.
|
|
1059
1053
|
* @returns
|
|
@@ -1066,9 +1060,9 @@ class Actor {
|
|
|
1066
1060
|
return Actor.getDefaultInstance().getValue(key);
|
|
1067
1061
|
}
|
|
1068
1062
|
/**
|
|
1069
|
-
* Stores or deletes a value in the default {@
|
|
1063
|
+
* Stores or deletes a value in the default {@apilink KeyValueStore} associated with the current actor run.
|
|
1070
1064
|
*
|
|
1071
|
-
* This is just a convenient shortcut for {@
|
|
1065
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.setValue}.
|
|
1072
1066
|
* For example, calling the following code:
|
|
1073
1067
|
* ```javascript
|
|
1074
1068
|
* await Actor.setValue('OUTPUT', { foo: "bar" });
|
|
@@ -1080,10 +1074,10 @@ class Actor {
|
|
|
1080
1074
|
* await store.setValue('OUTPUT', { foo: "bar" });
|
|
1081
1075
|
* ```
|
|
1082
1076
|
*
|
|
1083
|
-
* To get a value from the default key-value store, you can use the {@
|
|
1077
|
+
* To get a value from the default key-value store, you can use the {@apilink Actor.getValue} function.
|
|
1084
1078
|
*
|
|
1085
|
-
* For more information, see {@
|
|
1086
|
-
* and {@
|
|
1079
|
+
* For more information, see {@apilink Actor.openKeyValueStore}
|
|
1080
|
+
* and {@apilink KeyValueStore.getValue}.
|
|
1087
1081
|
*
|
|
1088
1082
|
* @param key
|
|
1089
1083
|
* Unique record key.
|
|
@@ -1099,9 +1093,9 @@ class Actor {
|
|
|
1099
1093
|
return Actor.getDefaultInstance().setValue(key, value, options);
|
|
1100
1094
|
}
|
|
1101
1095
|
/**
|
|
1102
|
-
* Gets the actor input value from the default {@
|
|
1096
|
+
* Gets the actor input value from the default {@apilink KeyValueStore} associated with the current actor run.
|
|
1103
1097
|
*
|
|
1104
|
-
* This is just a convenient shortcut for {@
|
|
1098
|
+
* This is just a convenient shortcut for {@apilink KeyValueStore.getValue | `keyValueStore.getValue('INPUT')`}.
|
|
1105
1099
|
* For example, calling the following code:
|
|
1106
1100
|
* ```javascript
|
|
1107
1101
|
* const input = await Actor.getInput();
|
|
@@ -1117,7 +1111,7 @@ class Actor {
|
|
|
1117
1111
|
* If you need to use the input multiple times in your actor,
|
|
1118
1112
|
* it is far more efficient to read it once and store it locally.
|
|
1119
1113
|
*
|
|
1120
|
-
* For more information, see {@
|
|
1114
|
+
* For more information, see {@apilink Actor.openKeyValueStore} and {@apilink KeyValueStore.getValue}.
|
|
1121
1115
|
*
|
|
1122
1116
|
* @returns
|
|
1123
1117
|
* Returns a promise that resolves to an object, string
|
|
@@ -1129,13 +1123,13 @@ class Actor {
|
|
|
1129
1123
|
return Actor.getDefaultInstance().getInput();
|
|
1130
1124
|
}
|
|
1131
1125
|
/**
|
|
1132
|
-
* Opens a key-value store and returns a promise resolving to an instance of the {@
|
|
1126
|
+
* Opens a key-value store and returns a promise resolving to an instance of the {@apilink KeyValueStore} class.
|
|
1133
1127
|
*
|
|
1134
1128
|
* Key-value stores are used to store records or files, along with their MIME content type.
|
|
1135
1129
|
* The records are stored and retrieved using a unique key.
|
|
1136
1130
|
* The actual data is stored either on a local filesystem or in the Apify cloud.
|
|
1137
1131
|
*
|
|
1138
|
-
* For more details and code examples, see the {@
|
|
1132
|
+
* For more details and code examples, see the {@apilink KeyValueStore} class.
|
|
1139
1133
|
*
|
|
1140
1134
|
* @param [storeIdOrName]
|
|
1141
1135
|
* ID or name of the key-value store to be opened. If `null` or `undefined`,
|
|
@@ -1147,14 +1141,14 @@ class Actor {
|
|
|
1147
1141
|
}
|
|
1148
1142
|
/**
|
|
1149
1143
|
* Opens a request queue and returns a promise resolving to an instance
|
|
1150
|
-
* of the {@
|
|
1144
|
+
* of the {@apilink RequestQueue} class.
|
|
1151
1145
|
*
|
|
1152
|
-
* {@
|
|
1146
|
+
* {@apilink RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
|
|
1153
1147
|
* The queue is used for deep crawling of websites, where you start with several URLs and then
|
|
1154
1148
|
* recursively follow links to other pages. The data structure supports both breadth-first
|
|
1155
1149
|
* and depth-first crawling orders.
|
|
1156
1150
|
*
|
|
1157
|
-
* For more details and code examples, see the {@
|
|
1151
|
+
* For more details and code examples, see the {@apilink RequestQueue} class.
|
|
1158
1152
|
*
|
|
1159
1153
|
* @param [queueIdOrName]
|
|
1160
1154
|
* ID or name of the request queue to be opened. If `null` or `undefined`,
|
|
@@ -1166,13 +1160,13 @@ class Actor {
|
|
|
1166
1160
|
}
|
|
1167
1161
|
/**
|
|
1168
1162
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
1169
|
-
* of the {@
|
|
1163
|
+
* of the {@apilink ProxyConfiguration} class that is already initialized.
|
|
1170
1164
|
*
|
|
1171
1165
|
* Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
|
|
1172
1166
|
* your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
|
|
1173
1167
|
* them to use the selected proxies for all connections.
|
|
1174
1168
|
*
|
|
1175
|
-
* For more details and code examples, see the {@
|
|
1169
|
+
* For more details and code examples, see the {@apilink ProxyConfiguration} class.
|
|
1176
1170
|
*
|
|
1177
1171
|
* ```javascript
|
|
1178
1172
|
*
|
|
@@ -1203,7 +1197,7 @@ class Actor {
|
|
|
1203
1197
|
return Actor.getDefaultInstance().createProxyConfiguration(proxyConfigurationOptions);
|
|
1204
1198
|
}
|
|
1205
1199
|
/**
|
|
1206
|
-
* Returns a new {@
|
|
1200
|
+
* Returns a new {@apilink ApifyEnv} object which contains information parsed from all the `APIFY_XXX` environment variables.
|
|
1207
1201
|
*
|
|
1208
1202
|
* For the list of the `APIFY_XXX` environment variables, see
|
|
1209
1203
|
* [Actor documentation](https://docs.apify.com/actor/run#environment-variables).
|
|
@@ -1228,11 +1222,11 @@ class Actor {
|
|
|
1228
1222
|
static isAtHome() {
|
|
1229
1223
|
return Actor.getDefaultInstance().isAtHome();
|
|
1230
1224
|
}
|
|
1231
|
-
/** Default {@
|
|
1225
|
+
/** Default {@apilink ApifyClient} instance. */
|
|
1232
1226
|
static get apifyClient() {
|
|
1233
1227
|
return Actor.getDefaultInstance().apifyClient;
|
|
1234
1228
|
}
|
|
1235
|
-
/** Default {@
|
|
1229
|
+
/** Default {@apilink Configuration} instance. */
|
|
1236
1230
|
static get config() {
|
|
1237
1231
|
return Actor.getDefaultInstance().config;
|
|
1238
1232
|
}
|