apify 2.3.1-beta.4 → 3.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. package/README.md +6 -5
  2. package/package.json +69 -128
  3. package/build/actor.d.ts +0 -113
  4. package/build/actor.d.ts.map +0 -1
  5. package/build/actor.js +0 -582
  6. package/build/actor.js.map +0 -1
  7. package/build/apify.d.ts +0 -752
  8. package/build/apify.d.ts.map +0 -1
  9. package/build/apify.js +0 -877
  10. package/build/apify.js.map +0 -1
  11. package/build/autoscaling/autoscaled_pool.d.ts +0 -384
  12. package/build/autoscaling/autoscaled_pool.d.ts.map +0 -1
  13. package/build/autoscaling/autoscaled_pool.js +0 -557
  14. package/build/autoscaling/autoscaled_pool.js.map +0 -1
  15. package/build/autoscaling/snapshotter.d.ts +0 -278
  16. package/build/autoscaling/snapshotter.d.ts.map +0 -1
  17. package/build/autoscaling/snapshotter.js +0 -447
  18. package/build/autoscaling/snapshotter.js.map +0 -1
  19. package/build/autoscaling/system_status.d.ts +0 -224
  20. package/build/autoscaling/system_status.d.ts.map +0 -1
  21. package/build/autoscaling/system_status.js +0 -228
  22. package/build/autoscaling/system_status.js.map +0 -1
  23. package/build/browser_launchers/browser_launcher.d.ts +0 -154
  24. package/build/browser_launchers/browser_launcher.d.ts.map +0 -1
  25. package/build/browser_launchers/browser_launcher.js +0 -160
  26. package/build/browser_launchers/browser_launcher.js.map +0 -1
  27. package/build/browser_launchers/browser_plugin.d.ts +0 -23
  28. package/build/browser_launchers/browser_plugin.d.ts.map +0 -1
  29. package/build/browser_launchers/browser_plugin.js +0 -25
  30. package/build/browser_launchers/browser_plugin.js.map +0 -1
  31. package/build/browser_launchers/playwright_launcher.d.ts +0 -131
  32. package/build/browser_launchers/playwright_launcher.d.ts.map +0 -1
  33. package/build/browser_launchers/playwright_launcher.js +0 -150
  34. package/build/browser_launchers/playwright_launcher.js.map +0 -1
  35. package/build/browser_launchers/puppeteer_launcher.d.ts +0 -153
  36. package/build/browser_launchers/puppeteer_launcher.d.ts.map +0 -1
  37. package/build/browser_launchers/puppeteer_launcher.js +0 -197
  38. package/build/browser_launchers/puppeteer_launcher.js.map +0 -1
  39. package/build/cache_container.d.ts +0 -31
  40. package/build/cache_container.d.ts.map +0 -1
  41. package/build/cache_container.js +0 -48
  42. package/build/cache_container.js.map +0 -1
  43. package/build/configuration.d.ts +0 -226
  44. package/build/configuration.d.ts.map +0 -1
  45. package/build/configuration.js +0 -325
  46. package/build/configuration.js.map +0 -1
  47. package/build/constants.d.ts +0 -37
  48. package/build/constants.d.ts.map +0 -1
  49. package/build/constants.js +0 -41
  50. package/build/constants.js.map +0 -1
  51. package/build/crawlers/basic_crawler.d.ts +0 -443
  52. package/build/crawlers/basic_crawler.d.ts.map +0 -1
  53. package/build/crawlers/basic_crawler.js +0 -664
  54. package/build/crawlers/basic_crawler.js.map +0 -1
  55. package/build/crawlers/browser_crawler.d.ts +0 -512
  56. package/build/crawlers/browser_crawler.d.ts.map +0 -1
  57. package/build/crawlers/browser_crawler.js +0 -540
  58. package/build/crawlers/browser_crawler.js.map +0 -1
  59. package/build/crawlers/cheerio_crawler.d.ts +0 -931
  60. package/build/crawlers/cheerio_crawler.d.ts.map +0 -1
  61. package/build/crawlers/cheerio_crawler.js +0 -913
  62. package/build/crawlers/cheerio_crawler.js.map +0 -1
  63. package/build/crawlers/crawler_extension.d.ts +0 -10
  64. package/build/crawlers/crawler_extension.d.ts.map +0 -1
  65. package/build/crawlers/crawler_extension.js +0 -19
  66. package/build/crawlers/crawler_extension.js.map +0 -1
  67. package/build/crawlers/crawler_utils.d.ts +0 -34
  68. package/build/crawlers/crawler_utils.d.ts.map +0 -1
  69. package/build/crawlers/crawler_utils.js +0 -87
  70. package/build/crawlers/crawler_utils.js.map +0 -1
  71. package/build/crawlers/playwright_crawler.d.ts +0 -448
  72. package/build/crawlers/playwright_crawler.d.ts.map +0 -1
  73. package/build/crawlers/playwright_crawler.js +0 -299
  74. package/build/crawlers/playwright_crawler.js.map +0 -1
  75. package/build/crawlers/puppeteer_crawler.d.ts +0 -425
  76. package/build/crawlers/puppeteer_crawler.d.ts.map +0 -1
  77. package/build/crawlers/puppeteer_crawler.js +0 -299
  78. package/build/crawlers/puppeteer_crawler.js.map +0 -1
  79. package/build/crawlers/statistics.d.ts +0 -185
  80. package/build/crawlers/statistics.d.ts.map +0 -1
  81. package/build/crawlers/statistics.js +0 -331
  82. package/build/crawlers/statistics.js.map +0 -1
  83. package/build/enqueue_links/click_elements.d.ts +0 -179
  84. package/build/enqueue_links/click_elements.d.ts.map +0 -1
  85. package/build/enqueue_links/click_elements.js +0 -434
  86. package/build/enqueue_links/click_elements.js.map +0 -1
  87. package/build/enqueue_links/enqueue_links.d.ts +0 -117
  88. package/build/enqueue_links/enqueue_links.d.ts.map +0 -1
  89. package/build/enqueue_links/enqueue_links.js +0 -163
  90. package/build/enqueue_links/enqueue_links.js.map +0 -1
  91. package/build/enqueue_links/shared.d.ts +0 -42
  92. package/build/enqueue_links/shared.d.ts.map +0 -1
  93. package/build/enqueue_links/shared.js +0 -121
  94. package/build/enqueue_links/shared.js.map +0 -1
  95. package/build/errors.d.ts +0 -29
  96. package/build/errors.d.ts.map +0 -1
  97. package/build/errors.js +0 -38
  98. package/build/errors.js.map +0 -1
  99. package/build/events.d.ts +0 -11
  100. package/build/events.d.ts.map +0 -1
  101. package/build/events.js +0 -147
  102. package/build/events.js.map +0 -1
  103. package/build/index.d.ts +0 -4
  104. package/build/index.d.ts.map +0 -1
  105. package/build/index.js +0 -7
  106. package/build/index.js.map +0 -1
  107. package/build/main.d.ts +0 -179
  108. package/build/main.d.ts.map +0 -1
  109. package/build/main.js +0 -81
  110. package/build/main.js.map +0 -1
  111. package/build/playwright_utils.d.ts +0 -9
  112. package/build/playwright_utils.d.ts.map +0 -1
  113. package/build/playwright_utils.js +0 -90
  114. package/build/playwright_utils.js.map +0 -1
  115. package/build/proxy_configuration.d.ts +0 -411
  116. package/build/proxy_configuration.d.ts.map +0 -1
  117. package/build/proxy_configuration.js +0 -517
  118. package/build/proxy_configuration.js.map +0 -1
  119. package/build/pseudo_url.d.ts +0 -86
  120. package/build/pseudo_url.d.ts.map +0 -1
  121. package/build/pseudo_url.js +0 -153
  122. package/build/pseudo_url.js.map +0 -1
  123. package/build/puppeteer_request_interception.d.ts +0 -8
  124. package/build/puppeteer_request_interception.d.ts.map +0 -1
  125. package/build/puppeteer_request_interception.js +0 -235
  126. package/build/puppeteer_request_interception.js.map +0 -1
  127. package/build/puppeteer_utils.d.ts +0 -250
  128. package/build/puppeteer_utils.d.ts.map +0 -1
  129. package/build/puppeteer_utils.js +0 -551
  130. package/build/puppeteer_utils.js.map +0 -1
  131. package/build/request.d.ts +0 -180
  132. package/build/request.d.ts.map +0 -1
  133. package/build/request.js +0 -261
  134. package/build/request.js.map +0 -1
  135. package/build/request_list.d.ts +0 -581
  136. package/build/request_list.d.ts.map +0 -1
  137. package/build/request_list.js +0 -826
  138. package/build/request_list.js.map +0 -1
  139. package/build/serialization.d.ts +0 -5
  140. package/build/serialization.d.ts.map +0 -1
  141. package/build/serialization.js +0 -139
  142. package/build/serialization.js.map +0 -1
  143. package/build/session_pool/errors.d.ts +0 -11
  144. package/build/session_pool/errors.d.ts.map +0 -1
  145. package/build/session_pool/errors.js +0 -18
  146. package/build/session_pool/errors.js.map +0 -1
  147. package/build/session_pool/events.d.ts +0 -5
  148. package/build/session_pool/events.d.ts.map +0 -1
  149. package/build/session_pool/events.js +0 -6
  150. package/build/session_pool/events.js.map +0 -1
  151. package/build/session_pool/session.d.ts +0 -286
  152. package/build/session_pool/session.d.ts.map +0 -1
  153. package/build/session_pool/session.js +0 -355
  154. package/build/session_pool/session.js.map +0 -1
  155. package/build/session_pool/session_pool.d.ts +0 -280
  156. package/build/session_pool/session_pool.d.ts.map +0 -1
  157. package/build/session_pool/session_pool.js +0 -393
  158. package/build/session_pool/session_pool.js.map +0 -1
  159. package/build/session_pool/session_utils.d.ts +0 -4
  160. package/build/session_pool/session_utils.d.ts.map +0 -1
  161. package/build/session_pool/session_utils.js +0 -24
  162. package/build/session_pool/session_utils.js.map +0 -1
  163. package/build/stealth/hiding_tricks.d.ts +0 -22
  164. package/build/stealth/hiding_tricks.d.ts.map +0 -1
  165. package/build/stealth/hiding_tricks.js +0 -308
  166. package/build/stealth/hiding_tricks.js.map +0 -1
  167. package/build/stealth/stealth.d.ts +0 -56
  168. package/build/stealth/stealth.d.ts.map +0 -1
  169. package/build/stealth/stealth.js +0 -125
  170. package/build/stealth/stealth.js.map +0 -1
  171. package/build/storages/dataset.d.ts +0 -288
  172. package/build/storages/dataset.d.ts.map +0 -1
  173. package/build/storages/dataset.js +0 -480
  174. package/build/storages/dataset.js.map +0 -1
  175. package/build/storages/key_value_store.d.ts +0 -243
  176. package/build/storages/key_value_store.d.ts.map +0 -1
  177. package/build/storages/key_value_store.js +0 -462
  178. package/build/storages/key_value_store.js.map +0 -1
  179. package/build/storages/request_queue.d.ts +0 -318
  180. package/build/storages/request_queue.d.ts.map +0 -1
  181. package/build/storages/request_queue.js +0 -636
  182. package/build/storages/request_queue.js.map +0 -1
  183. package/build/storages/storage_manager.d.ts +0 -87
  184. package/build/storages/storage_manager.d.ts.map +0 -1
  185. package/build/storages/storage_manager.js +0 -150
  186. package/build/storages/storage_manager.js.map +0 -1
  187. package/build/tsconfig.tsbuildinfo +0 -1
  188. package/build/typedefs.d.ts +0 -146
  189. package/build/typedefs.d.ts.map +0 -1
  190. package/build/typedefs.js +0 -88
  191. package/build/typedefs.js.map +0 -1
  192. package/build/utils.d.ts +0 -175
  193. package/build/utils.d.ts.map +0 -1
  194. package/build/utils.js +0 -731
  195. package/build/utils.js.map +0 -1
  196. package/build/utils_log.d.ts +0 -41
  197. package/build/utils_log.d.ts.map +0 -1
  198. package/build/utils_log.js +0 -192
  199. package/build/utils_log.js.map +0 -1
  200. package/build/utils_request.d.ts +0 -77
  201. package/build/utils_request.d.ts.map +0 -1
  202. package/build/utils_request.js +0 -385
  203. package/build/utils_request.js.map +0 -1
  204. package/build/utils_social.d.ts +0 -210
  205. package/build/utils_social.d.ts.map +0 -1
  206. package/build/utils_social.js +0 -787
  207. package/build/utils_social.js.map +0 -1
  208. package/build/validators.d.ts +0 -23
  209. package/build/validators.d.ts.map +0 -1
  210. package/build/validators.js +0 -29
  211. package/build/validators.js.map +0 -1
package/build/apify.js DELETED
@@ -1,877 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.Apify = void 0;
4
- const tslib_1 = require("tslib");
5
- const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
6
- const consts_1 = require("@apify/consts");
7
- // eslint-disable-next-line import/no-duplicates
8
- const actor_1 = require("./actor");
9
- const events_1 = require("./events");
10
- const storage_manager_1 = require("./storages/storage_manager");
11
- const dataset_1 = require("./storages/dataset");
12
- const key_value_store_1 = require("./storages/key_value_store");
13
- const request_list_1 = require("./request_list");
14
- const request_queue_1 = require("./storages/request_queue");
15
- // eslint-disable-next-line import/no-duplicates
16
- const session_pool_1 = require("./session_pool/session_pool");
17
- const proxy_configuration_1 = require("./proxy_configuration");
18
- const utils_1 = require("./utils");
19
- const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
20
- const constants_1 = require("./constants");
21
- const configuration_1 = require("./configuration");
22
- const puppeteer_utils_1 = require("./puppeteer_utils");
23
- const playwright_utils_1 = require("./playwright_utils");
24
- const utils_social_1 = require("./utils_social");
25
- const enqueue_links_1 = require("./enqueue_links/enqueue_links");
26
- const utils_request_1 = require("./utils_request");
27
- const errors_1 = require("./errors");
28
- /* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
29
- /**
30
- * `Apify` class serves as an alternative approach to the static helpers exported from the package. It allows to pass configuration
31
- * that will be used on the instance methods. Environment variables will have precedence over this configuration.
32
- * See {@link Configuration} for details about what can be configured and what are the default values.
33
- *
34
- * @property {Configuration} config Configuration of this SDK instance (provided to its constructor). See {@link Configuration} for details.
35
- * @ignore
36
- */
37
- class Apify {
38
- constructor(options = {}) {
39
- this.config = new configuration_1.Configuration(options);
40
- this._storageManagers = new Map();
41
- }
42
- /**
43
- * Runs the main user function that performs the job of the actor
44
- * and terminates the process when the user function finishes.
45
- *
46
- * **The `Apify.main()` function is optional** and is provided merely for your convenience.
47
- * It is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).
48
- * However, if you want to use Apify SDK tools directly inside your existing projects, e.g.
49
- * running in an [Express](https://expressjs.com/) server, on
50
- * [Google Cloud functions](https://cloud.google.com/functions)
51
- * or [AWS Lambda](https://aws.amazon.com/lambda/), it's better to avoid
52
- * it since the function terminates the main process when it finishes!
53
- *
54
- * The `Apify.main()` function performs the following actions:
55
- *
56
- * - When running on the Apify platform (i.e. <code>APIFY_IS_AT_HOME</code> environment variable is set),
57
- * it sets up a connection to listen for platform events.
58
- * For example, to get a notification about an imminent migration to another server.
59
- * See {@link Apify.events} for details.
60
- * - It checks that either <code>APIFY_TOKEN</code> or <code>APIFY_LOCAL_STORAGE_DIR</code> environment variable
61
- * is defined. If not, the functions sets <code>APIFY_LOCAL_STORAGE_DIR</code> to <code>./apify_storage</code>
62
- * inside the current working directory. This is to simplify running code examples.
63
- * - It invokes the user function passed as the <code>userFunc</code> parameter.
64
- * - If the user function returned a promise, waits for it to resolve.
65
- * - If the user function throws an exception or some other error is encountered,
66
- * prints error details to console so that they are stored to the log.
67
- * - Exits the Node.js process, with zero exit code on success and non-zero on errors.
68
- *
69
- * The user function can be synchronous:
70
- *
71
- * ```javascript
72
- * Apify.main(() => {
73
- * // My synchronous function that returns immediately
74
- * console.log('Hello world from actor!');
75
- * });
76
- * ```
77
- *
78
- * If the user function returns a promise, it is considered asynchronous:
79
- * ```javascript
80
- * const { requestAsBrowser } = require('some-request-library');
81
- *
82
- * Apify.main(() => {
83
- * // My asynchronous function that returns a promise
84
- * return request('http://www.example.com').then((html) => {
85
- * console.log(html);
86
- * });
87
- * });
88
- * ```
89
- *
90
- * To simplify your code, you can take advantage of the `async`/`await` keywords:
91
- *
92
- * ```javascript
93
- * const request = require('some-request-library');
94
- *
95
- * Apify.main(async () => {
96
- * // My asynchronous function
97
- * const html = await request('http://www.example.com');
98
- * console.log(html);
99
- * });
100
- * ```
101
- *
102
- * @param {UserFunc} userFunc User function to be executed. If it returns a promise,
103
- * the promise will be awaited. The user function is called with no arguments.
104
- * @return {Promise<unknown>}
105
- */
106
- main(userFunc) {
107
- if (!userFunc || typeof (userFunc) !== 'function') {
108
- // eslint-disable-next-line max-len
109
- throw new Error(`Apify.main() accepts a single parameter that must be a function (was '${userFunc === null ? 'null' : typeof userFunc}').`);
110
- }
111
- // Logging some basic system info (apify and apify-client version, NodeJS version, ...).
112
- (0, utils_1.logSystemInfo)();
113
- // Log warning if SDK is outdated.
114
- (0, utils_1.printOutdatedSdkWarning)();
115
- // This is to enable unit tests where process.exit() is mocked and doesn't really exit the process
116
- // Note that mocked process.exit() might throw, so set exited flag before calling it to avoid confusion.
117
- let exited = false;
118
- const exitWithError = (err, exitCode) => {
119
- utils_log_1.default.exception(err);
120
- exited = true;
121
- process.exit(exitCode);
122
- };
123
- // Set dummy interval to ensure the process will not be killed while awaiting empty promise:
124
- // await new Promise(() => {})
125
- // Such a construct is used for testing of actor timeouts and aborts.
126
- const intervalId = setInterval((i) => i, 9999999);
127
- // Using async here to have nice stack traces for errors
128
- try {
129
- (0, events_1.initializeEvents)(this.config);
130
- return (async () => {
131
- try {
132
- await configuration_1.Configuration.storage.run(this.config, userFunc);
133
- process.exit(constants_1.EXIT_CODES.SUCCESS);
134
- }
135
- catch (err) {
136
- if (!exited) {
137
- exitWithError(err, constants_1.EXIT_CODES.ERROR_USER_FUNCTION_THREW);
138
- }
139
- }
140
- })();
141
- }
142
- catch (err) {
143
- exitWithError(err, constants_1.EXIT_CODES.ERROR_UNKNOWN);
144
- }
145
- finally {
146
- (0, events_1.stopEvents)();
147
- clearInterval(intervalId);
148
- }
149
- }
150
- /**
151
- * Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
152
- * waits for the actor to finish and fetches its output.
153
- *
154
- * By passing the `waitSecs` option you can reduce the maximum amount of time to wait for the run to finish.
155
- * If the value is less than or equal to zero, the function returns immediately after the run is started.
156
- *
157
- * The result of the function is an {@link ActorRun} object
158
- * that contains details about the actor run and its output (if any).
159
- * If the actor run fails, the function throws the {@link ApifyCallError} exception.
160
- *
161
- * If you want to run an actor task rather than an actor, please use the
162
- * {@link Apify#callTask} function instead.
163
- *
164
- * For more information about actors, read the
165
- * [documentation](https://docs.apify.com/actor).
166
- *
167
- * **Example usage:**
168
- *
169
- * ```javascript
170
- * const run = await Apify.call('apify/hello-world', { myInput: 123 });
171
- * console.log(`Received message: ${run.output.body.message}`);
172
- * ```
173
- *
174
- * Internally, the `call()` function invokes the
175
- * [Run actor](https://apify.com/docs/api/v2#/reference/actors/run-collection/run-actor)
176
- * and several other API endpoints to obtain the output.
177
- *
178
- * @param {string} actId
179
- * Allowed formats are `username/actor-name`, `userId/actor-name` or actor ID.
180
- * @param {Object<string, *>} [input]
181
- * Input for the actor. If it is an object, it will be stringified to
182
- * JSON and its content type set to `application/json; charset=utf-8`.
183
- * Otherwise the `options.contentType` parameter must be provided.
184
- * @param {object} [options]
185
- * Object with the settings below:
186
- * @param {string} [options.contentType]
187
- * Content type for the `input`. If not specified,
188
- * `input` is expected to be an object that will be stringified to JSON and content type set to
189
- * `application/json; charset=utf-8`. If `options.contentType` is specified, then `input` must be a
190
- * `String` or `Buffer`.
191
- * @param {string} [options.token]
192
- * User API token that is used to run the actor. By default, it is taken from the `APIFY_TOKEN` environment variable.
193
- * @param {number} [options.memoryMbytes]
194
- * Memory in megabytes which will be allocated for the new actor run.
195
- * If not provided, the run uses memory of the default actor run configuration.
196
- * @param {number} [options.timeoutSecs]
197
- * Timeout for the actor run in seconds. Zero value means there is no timeout.
198
- * If not provided, the run uses timeout of the default actor run configuration.
199
- * @param {string} [options.build]
200
- * Tag or number of the actor build to run (e.g. `beta` or `1.2.345`).
201
- * If not provided, the run uses build tag or number from the default actor run configuration (typically `latest`).
202
- * @param {number} [options.waitSecs]
203
- * Maximum time to wait for the actor run to finish, in seconds.
204
- * If the limit is reached, the returned promise is resolved to a run object that will have
205
- * status `READY` or `RUNNING` and it will not contain the actor run output.
206
- * If `waitSecs` is null or undefined, the function waits for the actor to finish (default behavior).
207
- * @param {boolean} [options.fetchOutput=true]
208
- * If `false` then the function does not fetch output of the actor.
209
- * @param {boolean} [options.disableBodyParser=false]
210
- * If `true` then the function will not attempt to parse the
211
- * actor's output and will return it in a raw `Buffer`.
212
- * @param {Array<AdhocWebhook>} [options.webhooks] Specifies optional webhooks associated with the actor run, which can be used
213
- * to receive a notification e.g. when the actor finished or failed, see
214
- * [ad hook webhooks documentation](https://docs.apify.com/webhooks/ad-hoc-webhooks) for detailed description.
215
- * @returns {Promise<ActorRun>}
216
- * @throws {ApifyCallError} If the run did not succeed, e.g. if it failed or timed out.
217
- */
218
- async call(actId, input, options = {}) {
219
- (0, ow_1.default)(actId, ow_1.default.string);
220
- // input can be anything, no reason to validate
221
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
222
- contentType: ow_1.default.optional.string.nonEmpty,
223
- token: ow_1.default.optional.string,
224
- memoryMbytes: ow_1.default.optional.number.not.negative,
225
- timeoutSecs: ow_1.default.optional.number.not.negative,
226
- build: ow_1.default.optional.string,
227
- waitSecs: ow_1.default.optional.number.not.negative,
228
- fetchOutput: ow_1.default.optional.boolean,
229
- disableBodyParser: ow_1.default.optional.boolean,
230
- webhooks: ow_1.default.optional.array.ofType(ow_1.default.object),
231
- }));
232
- const { token = this.config.get('token'), fetchOutput = true, disableBodyParser = false, memoryMbytes, timeoutSecs, ...callActorOpts } = options;
233
- callActorOpts.memory = memoryMbytes;
234
- callActorOpts.timeout = timeoutSecs;
235
- callActorOpts.token = token;
236
- if (input) {
237
- callActorOpts.contentType = (0, utils_1.addCharsetToContentType)(callActorOpts.contentType);
238
- input = (0, key_value_store_1.maybeStringify)(input, callActorOpts);
239
- }
240
- const client = this.newClient({ token });
241
- let run;
242
- try {
243
- run = await client.actor(actId).call(input, callActorOpts);
244
- }
245
- catch (err) {
246
- if (err.message.startsWith('Waiting for run to finish')) {
247
- throw new errors_1.ApifyCallError({ id: run.id, actId: run.actId }, 'Apify.call() failed, cannot fetch actor run details from the server');
248
- }
249
- throw err;
250
- }
251
- if (this._isRunUnsuccessful(run.status)) {
252
- // eslint-disable-next-line max-len
253
- const message = `The actor ${actId} invoked by Apify.call() did not succeed. For details, see https://console.apify.com/view/runs/${run.id}`;
254
- throw new errors_1.ApifyCallError(run, message);
255
- }
256
- // Finish if output is not requested or run haven't finished.
257
- if (!fetchOutput || run.status !== consts_1.ACT_JOB_STATUSES.SUCCEEDED)
258
- return run;
259
- // Fetch output.
260
- let getRecordOptions = {};
261
- if (disableBodyParser)
262
- getRecordOptions = { buffer: true };
263
- const { value: body, contentType } = await client.keyValueStore(run.defaultKeyValueStoreId).getRecord('OUTPUT', getRecordOptions);
264
- return { ...run, output: { contentType, body } };
265
- }
266
- /**
267
- * Runs an actor task on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
268
- * waits for the task to finish and fetches its output.
269
- *
270
- * By passing the `waitSecs` option you can reduce the maximum amount of time to wait for the run to finish.
271
- * If the value is less than or equal to zero, the function returns immediately after the run is started.
272
- *
273
- * The result of the function is an {@link ActorRun} object
274
- * that contains details about the actor run and its output (if any).
275
- * If the actor run failed, the function fails with {@link ApifyCallError} exception.
276
- *
277
- * Note that an actor task is a saved input configuration and options for an actor.
278
- * If you want to run an actor directly rather than an actor task, please use the
279
- * {@link Apify#call} function instead.
280
- *
281
- * For more information about actor tasks, read the [documentation](https://docs.apify.com/tasks).
282
- *
283
- * **Example usage:**
284
- *
285
- * ```javascript
286
- * const run = await Apify.callTask('bob/some-task');
287
- * console.log(`Received message: ${run.output.body.message}`);
288
- * ```
289
- *
290
- * Internally, the `callTask()` function calls the
291
- * [Run task](https://apify.com/docs/api/v2#/reference/actor-tasks/run-collection/run-task)
292
- * and several other API endpoints to obtain the output.
293
- *
294
- * @param {string} taskId
295
- * Allowed formats are `username/task-name`, `userId/task-name` or task ID.
296
- * @param {Object<string, *>} [input]
297
- * Input overrides for the actor task. If it is an object, it will be stringified to
298
- * JSON and its content type set to `application/json; charset=utf-8`.
299
- * Provided input will be merged with actor task input.
300
- * @param {object} [options]
301
- * Object with the settings below:
302
- * @param {string} [options.token]
303
- * User API token that is used to run the actor. By default, it is taken from the `APIFY_TOKEN` environment variable.
304
- * @param {number} [options.memoryMbytes]
305
- * Memory in megabytes which will be allocated for the new actor task run.
306
- * If not provided, the run uses memory of the default actor run configuration.
307
- * @param {number} [options.timeoutSecs]
308
- * Timeout for the actor task run in seconds. Zero value means there is no timeout.
309
- * If not provided, the run uses timeout of the default actor run configuration.
310
- * @param {string} [options.build]
311
- * Tag or number of the actor build to run (e.g. `beta` or `1.2.345`).
312
- * If not provided, the run uses build tag or number from the default actor run configuration (typically `latest`).
313
- * @param {number} [options.waitSecs]
314
- * Maximum time to wait for the actor task run to finish, in seconds.
315
- * If the limit is reached, the returned promise is resolved to a run object that will have
316
- * status `READY` or `RUNNING` and it will not contain the actor run output.
317
- * If `waitSecs` is null or undefined, the function waits for the actor task to finish (default behavior).
318
- * @param {Array<AdhocWebhook>} [options.webhooks] Specifies optional webhooks associated with the actor run, which can be used
319
- * to receive a notification e.g. when the actor finished or failed, see
320
- * [ad hook webhooks documentation](https://docs.apify.com/webhooks/ad-hoc-webhooks) for detailed description.
321
- * @returns {Promise<ActorRun>}
322
- * @throws {ApifyCallError} If the run did not succeed, e.g. if it failed or timed out.
323
- */
324
- async callTask(taskId, input, options = {}) {
325
- (0, ow_1.default)(taskId, ow_1.default.string);
326
- (0, ow_1.default)(input, ow_1.default.optional.any(ow_1.default.string, ow_1.default.object));
327
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
328
- token: ow_1.default.optional.string,
329
- memoryMbytes: ow_1.default.optional.number.not.negative,
330
- timeoutSecs: ow_1.default.optional.number.not.negative,
331
- build: ow_1.default.optional.string,
332
- waitSecs: ow_1.default.optional.number.not.negative,
333
- fetchOutput: ow_1.default.optional.boolean,
334
- disableBodyParser: ow_1.default.optional.boolean,
335
- webhooks: ow_1.default.optional.array.ofType(ow_1.default.object),
336
- }));
337
- const { token = this.config.get('token'), fetchOutput = true, disableBodyParser = false, memoryMbytes, timeoutSecs, ...callTaskOpts } = options;
338
- callTaskOpts.memory = memoryMbytes;
339
- callTaskOpts.timeout = timeoutSecs;
340
- callTaskOpts.token = token;
341
- const client = this.newClient({ token });
342
- // Start task and wait for run to finish if waitSecs is provided
343
- let run;
344
- try {
345
- run = await client.task(taskId).call(input, callTaskOpts);
346
- }
347
- catch (err) {
348
- if (err.message.startsWith('Waiting for run to finish')) {
349
- throw new errors_1.ApifyCallError({ id: run.id, actId: run.actId }, 'Apify.call() failed, cannot fetch actor run details from the server');
350
- }
351
- throw err;
352
- }
353
- if (this._isRunUnsuccessful(run.status)) {
354
- // eslint-disable-next-line max-len
355
- const message = `The actor task ${taskId} invoked by Apify.callTask() did not succeed. For details, see https://console.apify.com/view/runs/${run.id}`;
356
- throw new errors_1.ApifyCallError(run, message);
357
- }
358
- // Finish if output is not requested or run haven't finished.
359
- if (!fetchOutput || run.status !== consts_1.ACT_JOB_STATUSES.SUCCEEDED)
360
- return run;
361
- // Fetch output.
362
- let getRecordOptions = {};
363
- if (disableBodyParser)
364
- getRecordOptions = { buffer: true };
365
- const { value: body, contentType } = await client.keyValueStore(run.defaultKeyValueStoreId).getRecord('OUTPUT', getRecordOptions);
366
- return { ...run, output: { contentType, body } };
367
- }
368
- /**
369
- * Transforms this actor run to an actor run of a given actor. The system stops the current container and starts
370
- * the new container instead. All the default storages are preserved and the new input is stored under the `INPUT-METAMORPH-1` key
371
- * in the same default key-value store.
372
- *
373
- * @param {string} targetActorId
374
- * Either `username/actor-name` or actor ID of an actor to which we want to metamorph.
375
- * @param {Object<string, *>} [input]
376
- * Input for the actor. If it is an object, it will be stringified to
377
- * JSON and its content type set to `application/json; charset=utf-8`.
378
- * Otherwise the `options.contentType` parameter must be provided.
379
- * @param {object} [options]
380
- * Object with the settings below:
381
- * @param {string} [options.contentType]
382
- * Content type for the `input`. If not specified,
383
- * `input` is expected to be an object that will be stringified to JSON and content type set to
384
- * `application/json; charset=utf-8`. If `options.contentType` is specified, then `input` must be a
385
- * `String` or `Buffer`.
386
- * @param {string} [options.build]
387
- * Tag or number of the target actor build to metamorph into (e.g. `beta` or `1.2.345`).
388
- * If not provided, the run uses build tag or number from the default actor run configuration (typically `latest`).
389
- * @returns {Promise<void>}
390
- */
391
- async metamorph(targetActorId, input, options = {}) {
392
- (0, ow_1.default)(targetActorId, ow_1.default.string);
393
- // input can be anything, no reason to validate
394
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
395
- contentType: ow_1.default.optional.string.nonEmpty,
396
- build: ow_1.default.optional.string,
397
- customAfterSleepMillis: ow_1.default.optional.number.not.negative,
398
- }));
399
- const { customAfterSleepMillis, ...metamorphOpts } = options;
400
- const actorId = this.config.get('actorId');
401
- const runId = this.config.get('actorRunId');
402
- if (!actorId)
403
- throw new Error(`Environment variable ${consts_1.ENV_VARS.ACTOR_ID} must be provided!`);
404
- if (!runId)
405
- throw new Error(`Environment variable ${consts_1.ENV_VARS.ACTOR_RUN_ID} must be provided!`);
406
- if (input) {
407
- metamorphOpts.contentType = (0, utils_1.addCharsetToContentType)(metamorphOpts.contentType);
408
- input = (0, key_value_store_1.maybeStringify)(input, metamorphOpts);
409
- }
410
- await this.newClient().run(runId).metamorph(targetActorId, input, metamorphOpts);
411
- // Wait some time for container to be stopped.
412
- // NOTE: option.customAfterSleepMillis is used in tests
413
- await (0, utils_1.sleep)(customAfterSleepMillis || this.config.get('metamorphAfterSleepMillis'));
414
- }
415
- /**
416
- *
417
- * Creates an ad-hoc webhook for the current actor run, which lets you receive a notification when the actor run finished or failed.
418
- * For more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).
419
- *
420
- * Note that webhooks are only supported for actors running on the Apify platform.
421
- * In local environment, the function will print a warning and have no effect.
422
- *
423
- * @param {object} options
424
- * @param {EventTypes} options.eventTypes
425
- * Array of event types, which you can set for actor run, see
426
- * the [actor run events](https://docs.apify.com/webhooks/events#actor-run) in the Apify doc.
427
- * @param {string} options.requestUrl
428
- * URL which will be requested using HTTP POST request, when actor run will reach the set event type.
429
- * @param {string} [options.payloadTemplate]
430
- * Payload template is a JSON-like string that describes the structure of the webhook POST request payload.
431
- * It uses JSON syntax, extended with a double curly braces syntax for injecting variables `{{variable}}`.
432
- * Those variables are resolved at the time of the webhook's dispatch, and a list of available variables with their descriptions
433
- * is available in the [Apify webhook documentation](https://docs.apify.com/webhooks).
434
- * If `payloadTemplate` is omitted, the default payload template is used
435
- * ([view docs](https://docs.apify.com/webhooks/actions#payload-template)).
436
- * @param {string} [options.idempotencyKey]
437
- * Idempotency key enables you to ensure that a webhook will not be added multiple times in case of
438
- * an actor restart or other situation that would cause the `addWebhook()` function to be called again.
439
- * We suggest using the actor run ID as the idempotency key. You can get the run ID by calling
440
- * {@link Apify#getEnv} function.
441
- * @return {Promise<WebhookRun|undefined>} The return value is the Webhook object.
442
- * For more information, see the [Get webhook](https://apify.com/docs/api/v2#/reference/webhooks/webhook-object/get-webhook) API endpoint.
443
- */
444
- async addWebhook(options) {
445
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
446
- eventTypes: ow_1.default.array.ofType(ow_1.default.string),
447
- requestUrl: ow_1.default.string,
448
- payloadTemplate: ow_1.default.optional.string,
449
- idempotencyKey: ow_1.default.optional.string,
450
- }));
451
- const { eventTypes, requestUrl, payloadTemplate, idempotencyKey } = options;
452
- if (!this.isAtHome()) {
453
- utils_log_1.default.warning('Apify.addWebhook() is only supported when running on the Apify platform. The webhook will not be invoked.');
454
- return;
455
- }
456
- const runId = this.config.get('actorRunId');
457
- if (!runId) {
458
- throw new Error(`Environment variable ${consts_1.ENV_VARS.ACTOR_RUN_ID} is not set!`);
459
- }
460
- return this.newClient().webhooks().create({
461
- isAdHoc: true,
462
- eventTypes,
463
- condition: {
464
- actorRunId: runId,
465
- },
466
- requestUrl,
467
- payloadTemplate,
468
- idempotencyKey,
469
- });
470
- }
471
- /**
472
- * Stores an object or an array of objects to the default {@link Dataset} of the current actor run.
473
- *
474
- * This is just a convenient shortcut for {@link Dataset#pushData}.
475
- * For example, calling the following code:
476
- * ```javascript
477
- * await Apify.pushData({ myValue: 123 });
478
- * ```
479
- *
480
- * is equivalent to:
481
- * ```javascript
482
- * const dataset = await Apify.openDataset();
483
- * await dataset.pushData({ myValue: 123 });
484
- * ```
485
- *
486
- * For more information, see {@link Apify#openDataset} and {@link Dataset#pushData}
487
- *
488
- * **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
489
- * otherwise the actor process might finish before the data are stored!
490
- *
491
- * @param {object} item Object or array of objects containing data to be stored in the default dataset.
492
- * The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
493
- * @returns {Promise<void>}
494
- */
495
- async pushData(item) {
496
- const dataset = await this.openDataset();
497
- return dataset.pushData(item);
498
- }
499
- /**
500
- * Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
501
- *
502
- * Datasets are used to store structured data where each object stored has the same attributes,
503
- * such as online store products or real estate offers.
504
- * The actual data is stored either on the local filesystem or in the cloud.
505
- *
506
- * For more details and code examples, see the {@link Dataset} class.
507
- *
508
- * @param {string} [datasetIdOrName]
509
- * ID or name of the dataset to be opened. If `null` or `undefined`,
510
- * the function returns the default dataset associated with the actor run.
511
- * @param {Object} [options]
512
- * @param {boolean} [options.forceCloud=false]
513
- * If set to `true` then the function uses cloud storage usage even if the `APIFY_LOCAL_STORAGE_DIR`
514
- * environment variable is set. This way it is possible to combine local and cloud storage.
515
- * @returns {Promise<Dataset>}
516
- */
517
- async openDataset(datasetIdOrName, options = {}) {
518
- (0, ow_1.default)(datasetIdOrName, ow_1.default.optional.string);
519
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
520
- forceCloud: ow_1.default.optional.boolean,
521
- }));
522
- return this._getStorageManager(dataset_1.Dataset).openStorage(datasetIdOrName, options);
523
- }
524
- /**
525
- * Gets a value from the default {@link KeyValueStore} associated with the current actor run.
526
- *
527
- * This is just a convenient shortcut for {@link KeyValueStore#getValue}.
528
- * For example, calling the following code:
529
- * ```javascript
530
- * const value = await Apify.getValue('my-key');
531
- * ```
532
- *
533
- * is equivalent to:
534
- * ```javascript
535
- * const store = await Apify.openKeyValueStore();
536
- * const value = await store.getValue('my-key');
537
- * ```
538
- *
539
- * To store the value to the default key-value store, you can use the {@link Apify#setValue} function.
540
- *
541
- * For more information, see {@link Apify#openKeyValueStore}
542
- * and {@link KeyValueStore#getValue}.
543
- *
544
- * @param {string} key
545
- * Unique record key.
546
- * @returns {Promise<Object<string, *>|string|Buffer|null>}
547
- * Returns a promise that resolves to an object, string
548
- * or [`Buffer`](https://nodejs.org/api/buffer.html), depending
549
- * on the MIME content type of the record, or `null`
550
- * if the record is missing.
551
- */
552
- async getValue(key) {
553
- const store = await this.openKeyValueStore();
554
- return store.getValue(key);
555
- }
556
- /**
557
- * Stores or deletes a value in the default {@link KeyValueStore} associated with the current actor run.
558
- *
559
- * This is just a convenient shortcut for {@link KeyValueStore#setValue}.
560
- * For example, calling the following code:
561
- * ```javascript
562
- * await Apify.setValue('OUTPUT', { foo: "bar" });
563
- * ```
564
- *
565
- * is equivalent to:
566
- * ```javascript
567
- * const store = await Apify.openKeyValueStore();
568
- * await store.setValue('OUTPUT', { foo: "bar" });
569
- * ```
570
- *
571
- * To get a value from the default key-value store, you can use the {@link Apify#getValue} function.
572
- *
573
- * For more information, see {@link Apify#openKeyValueStore}
574
- * and {@link KeyValueStore#getValue}.
575
- *
576
- * @param {string} key
577
- * Unique record key.
578
- * @param {*} value
579
- * Record data, which can be one of the following values:
580
- * - If `null`, the record in the key-value store is deleted.
581
- * - If no `options.contentType` is specified, `value` can be any JavaScript object and it will be stringified to JSON.
582
- * - If `options.contentType` is set, `value` is taken as is and it must be a `String` or [`Buffer`](https://nodejs.org/api/buffer.html).
583
- * For any other value an error will be thrown.
584
- * @param {object} [options]
585
- * @param {string} [options.contentType]
586
- * Specifies a custom MIME content type of the record.
587
- * @return {Promise<void>}
588
- */
589
- async setValue(key, value, options) {
590
- const store = await this.openKeyValueStore();
591
- return store.setValue(key, value, options);
592
- }
593
- /**
594
- * Gets the actor input value from the default {@link KeyValueStore} associated with the current actor run.
595
- *
596
- * This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](key-value-store#getvalue).
597
- * For example, calling the following code:
598
- * ```javascript
599
- * const input = await Apify.getInput();
600
- * ```
601
- *
602
- * is equivalent to:
603
- * ```javascript
604
- * const store = await Apify.openKeyValueStore();
605
- * await store.getValue('INPUT');
606
- * ```
607
- *
608
- * Note that the `getInput()` function does not cache the value read from the key-value store.
609
- * If you need to use the input multiple times in your actor,
610
- * it is far more efficient to read it once and store it locally.
611
- *
612
- * For more information, see {@link Apify#openKeyValueStore}
613
- * and {@link KeyValueStore#getValue}.
614
- *
615
- * @returns {Promise<Object<string, *>|string|Buffer|null>}
616
- * Returns a promise that resolves to an object, string
617
- * or [`Buffer`](https://nodejs.org/api/buffer.html), depending
618
- * on the MIME content type of the record, or `null`
619
- * if the record is missing.
620
- */
621
- async getInput() {
622
- return this.getValue(this.config.get('inputKey'));
623
- }
624
- /**
625
- * Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
626
- *
627
- * Key-value stores are used to store records or files, along with their MIME content type.
628
- * The records are stored and retrieved using a unique key.
629
- * The actual data is stored either on a local filesystem or in the Apify cloud.
630
- *
631
- * For more details and code examples, see the {@link KeyValueStore} class.
632
- *
633
- * @param {string} [storeIdOrName]
634
- * ID or name of the key-value store to be opened. If `null` or `undefined`,
635
- * the function returns the default key-value store associated with the actor run.
636
- * @param {object} [options]
637
- * @param {boolean} [options.forceCloud=false]
638
- * If set to `true` then the function uses cloud storage usage even if the `APIFY_LOCAL_STORAGE_DIR`
639
- * environment variable is set. This way it is possible to combine local and cloud storage.
640
- * @returns {Promise<KeyValueStore>}
641
- */
642
- async openKeyValueStore(storeIdOrName, options = {}) {
643
- (0, ow_1.default)(storeIdOrName, ow_1.default.optional.string);
644
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
645
- forceCloud: ow_1.default.optional.boolean,
646
- }));
647
- return this._getStorageManager(key_value_store_1.KeyValueStore).openStorage(storeIdOrName, options);
648
- }
649
- /**
650
- * Opens a request list and returns a promise resolving to an instance
651
- * of the {@link RequestList} class that is already initialized.
652
- *
653
- * {@link RequestList} represents a list of URLs to crawl, which is always stored in memory.
654
- * To enable picking up where left off after a process restart, the request list sources
655
- * are persisted to the key-value store at initialization of the list. Then, while crawling,
656
- * a small state object is regularly persisted to keep track of the crawling status.
657
- *
658
- * For more details and code examples, see the {@link RequestList} class.
659
- *
660
- * **Example usage:**
661
- *
662
- * ```javascript
663
- * const sources = [
664
- * 'https://www.example.com',
665
- * 'https://www.google.com',
666
- * 'https://www.bing.com'
667
- * ];
668
- *
669
- * const requestList = await Apify.openRequestList('my-name', sources);
670
- * ```
671
- *
672
- * @param {string|null} listName
673
- * Name of the request list to be opened. Setting a name enables the `RequestList`'s state to be persisted
674
- * in the key-value store. This is useful in case of a restart or migration. Since `RequestList` is only
675
- * stored in memory, a restart or migration wipes it clean. Setting a name will enable the `RequestList`'s
676
- * state to survive those situations and continue where it left off.
677
- *
678
- * The name will be used as a prefix in key-value store, producing keys such as `NAME-REQUEST_LIST_STATE`
679
- * and `NAME-REQUEST_LIST_SOURCES`.
680
- *
681
- * If `null`, the list will not be persisted and will only be stored in memory. Process restart
682
- * will then cause the list to be crawled again from the beginning. We suggest always using a name.
683
- * @param {Array<RequestOptions|Request|string>} sources
684
- * An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
685
- * plain objects that define at least the `url` property, or an array of {@link Request} instances.
686
- *
687
- * **IMPORTANT:** The `sources` array will be consumed (left empty) after {@link RequestList} initializes.
688
- * This is a measure to prevent memory leaks in situations when millions of sources are
689
- * added.
690
- *
691
- * Additionally, the `requestsFromUrl` property may be used instead of `url`,
692
- * which will instruct {@link RequestList} to download the source URLs from a given remote location.
693
- * The URLs will be parsed from the received response. In this case you can limit the URLs
694
- * using `regex` parameter containing regular expression pattern for URLs to be included.
695
- *
696
- * For details, see the {@link RequestListOptions.sources}
697
- * @param {RequestListOptions} [options]
698
- * The {@link RequestList} options. Note that the `listName` parameter supersedes
699
- * the {@link RequestListOptions.persistStateKey} and {@link RequestListOptions.persistRequestsKey}
700
- * options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
701
- * @returns {Promise<RequestList>}
702
- */
703
- async openRequestList(listName, sources, options = {}) {
704
- (0, ow_1.default)(listName, ow_1.default.any(ow_1.default.string, ow_1.default.null));
705
- (0, ow_1.default)(sources, ow_1.default.array);
706
- (0, ow_1.default)(options, ow_1.default.object.is((v) => !Array.isArray(v)));
707
- const rl = new request_list_1.RequestList({
708
- ...options,
709
- persistStateKey: listName ? `${listName}-${request_list_1.STATE_PERSISTENCE_KEY}` : undefined,
710
- persistRequestsKey: listName ? `${listName}-${request_list_1.REQUESTS_PERSISTENCE_KEY}` : undefined,
711
- sources,
712
- });
713
- await rl.initialize();
714
- return rl;
715
- }
716
- /**
717
- * Opens a request queue and returns a promise resolving to an instance
718
- * of the {@link RequestQueue} class.
719
- *
720
- * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
721
- * The queue is used for deep crawling of websites, where you start with several URLs and then
722
- * recursively follow links to other pages. The data structure supports both breadth-first
723
- * and depth-first crawling orders.
724
- *
725
- * For more details and code examples, see the {@link RequestQueue} class.
726
- *
727
- * @param {string} [queueIdOrName]
728
- * ID or name of the request queue to be opened. If `null` or `undefined`,
729
- * the function returns the default request queue associated with the actor run.
730
- * @param {object} [options]
731
- * @param {boolean} [options.forceCloud=false]
732
- * If set to `true` then the function uses cloud storage usage even if the `APIFY_LOCAL_STORAGE_DIR`
733
- * environment variable is set. This way it is possible to combine local and cloud storage.
734
- * @returns {Promise<RequestQueue>}
735
- */
736
- async openRequestQueue(queueIdOrName, options = {}) {
737
- (0, ow_1.default)(queueIdOrName, ow_1.default.optional.string);
738
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
739
- forceCloud: ow_1.default.optional.boolean,
740
- }));
741
- return this._getStorageManager(request_queue_1.RequestQueue).openStorage(queueIdOrName, options);
742
- }
743
- /**
744
- * Opens a SessionPool and returns a promise resolving to an instance
745
- * of the {@link SessionPool} class that is already initialized.
746
- *
747
- * For more details and code examples, see the {@link SessionPool} class.
748
- *
749
- * @param {SessionPoolOptions} sessionPoolOptions
750
- * @return {Promise<SessionPool>}
751
- */
752
- async openSessionPool(sessionPoolOptions) {
753
- const sessionPool = new session_pool_1.SessionPool(sessionPoolOptions, this.config);
754
- await sessionPool.initialize();
755
- return sessionPool;
756
- }
757
- /**
758
- * Creates a proxy configuration and returns a promise resolving to an instance
759
- * of the {@link ProxyConfiguration} class that is already initialized.
760
- *
761
- * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
762
- * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
763
- * them to use the selected proxies for all connections.
764
- *
765
- * For more details and code examples, see the {@link ProxyConfiguration} class.
766
- *
767
- * ```javascript
768
- *
769
- * // Returns initialized proxy configuration class
770
- * const proxyConfiguration = await Apify.createProxyConfiguration({
771
- * groups: ['GROUP1', 'GROUP2'] // List of Apify proxy groups
772
- * countryCode: 'US'
773
- * });
774
- *
775
- * const crawler = new Apify.CheerioCrawler({
776
- * // ...
777
- * proxyConfiguration,
778
- * handlePageFunction: ({ proxyInfo }) => {
779
- * const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
780
- * }
781
- * })
782
- *
783
- * ```
784
- *
785
- * For compatibility with existing Actor Input UI (Input Schema), this function
786
- * returns `undefined` when the following object is passed as `proxyConfigurationOptions`.
787
- *
788
- * ```
789
- * { useApifyProxy: false }
790
- * ```
791
- *
792
- * @param {ProxyConfigurationOptions} [proxyConfigurationOptions]
793
- * @returns {Promise<ProxyConfiguration|undefined>}
794
- */
795
- async createProxyConfiguration(proxyConfigurationOptions = {}) {
796
- // Compatibility fix for Input UI where proxy: None returns { useApifyProxy: false }
797
- // Without this, it would cause proxy to use the zero config / auto mode.
798
- const dontUseApifyProxy = proxyConfigurationOptions.useApifyProxy === false;
799
- const dontUseCustomProxies = !proxyConfigurationOptions.proxyUrls;
800
- if (dontUseApifyProxy && dontUseCustomProxies) {
801
- return;
802
- }
803
- const proxyConfiguration = new proxy_configuration_1.ProxyConfiguration(proxyConfigurationOptions, this.config);
804
- await proxyConfiguration.initialize();
805
- return proxyConfiguration;
806
- }
807
- /**
808
- * Returns a new {@link ApifyEnv} object which contains information parsed from all the `APIFY_XXX` environment variables.
809
- *
810
- * For the list of the `APIFY_XXX` environment variables, see
811
- * [Actor documentation](https://docs.apify.com/actor/run#environment-variables).
812
- * If some of the variables are not defined or are invalid, the corresponding value in the resulting object will be null.
813
- * @returns {ApifyEnv}
814
- */
815
- getEnv() {
816
- return (0, actor_1.getEnv)();
817
- }
818
- /**
819
- * Returns a new instance of the Apify API client. The `ApifyClient` class is provided
820
- * by the <a href="https://www.npmjs.com/package/apify-client" target="_blank">apify-client</a>
821
- * NPM package, and it is automatically configured using the `APIFY_API_BASE_URL`, and `APIFY_TOKEN`
822
- * environment variables. You can override the token via the available options. That's useful
823
- * if you want to use the client as a different Apify user than the SDK internals are using.
824
- *
825
- * @param {object} [options]
826
- * @param {string} [options.token]
827
- * @param {string} [options.maxRetries]
828
- * @param {string} [options.minDelayBetweenRetriesMillis]
829
- * @return {ApifyClient}
830
- */
831
- newClient(options = {}) {
832
- return this.config.createClient(options);
833
- }
834
- /**
835
- * Returns `true` when code is running on Apify platform and `false` otherwise (for example locally).
836
- *
837
- * @returns {boolean}
838
- */
839
- isAtHome() {
840
- return !!this.config.get('isAtHome');
841
- }
842
- get utils() {
843
- return {
844
- ...utils_1.publicUtils,
845
- puppeteer: puppeteer_utils_1.puppeteerUtils,
846
- playwright: playwright_utils_1.playwrightUtils,
847
- social: utils_social_1.socialUtils,
848
- log: utils_log_1.default,
849
- enqueueLinks: enqueue_links_1.enqueueLinks,
850
- requestAsBrowser: utils_request_1.requestAsBrowser,
851
- };
852
- }
853
- /**
854
- * @param {Function} storageClass
855
- * @return {StorageManager}
856
- * @private
857
- */
858
- _getStorageManager(storageClass) {
859
- if (!this._storageManagers.has(storageClass)) {
860
- const manager = new storage_manager_1.StorageManager(storageClass, this.config);
861
- this._storageManagers.set(storageClass, manager);
862
- }
863
- return this._storageManagers.get(storageClass);
864
- }
865
- /**
866
- * @param {ACT_JOB_STATUSES} status
867
- * @return {boolean}
868
- * @private
869
- */
870
- _isRunUnsuccessful(status) {
871
- return status !== consts_1.ACT_JOB_STATUSES.SUCCEEDED
872
- && status !== consts_1.ACT_JOB_STATUSES.RUNNING
873
- && status !== consts_1.ACT_JOB_STATUSES.READY;
874
- }
875
- }
876
- exports.Apify = Apify;
877
- //# sourceMappingURL=apify.js.map