apify 2.3.1-beta.4 → 3.0.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. package/README.md +6 -5
  2. package/package.json +69 -128
  3. package/build/actor.d.ts +0 -113
  4. package/build/actor.d.ts.map +0 -1
  5. package/build/actor.js +0 -582
  6. package/build/actor.js.map +0 -1
  7. package/build/apify.d.ts +0 -752
  8. package/build/apify.d.ts.map +0 -1
  9. package/build/apify.js +0 -877
  10. package/build/apify.js.map +0 -1
  11. package/build/autoscaling/autoscaled_pool.d.ts +0 -384
  12. package/build/autoscaling/autoscaled_pool.d.ts.map +0 -1
  13. package/build/autoscaling/autoscaled_pool.js +0 -557
  14. package/build/autoscaling/autoscaled_pool.js.map +0 -1
  15. package/build/autoscaling/snapshotter.d.ts +0 -278
  16. package/build/autoscaling/snapshotter.d.ts.map +0 -1
  17. package/build/autoscaling/snapshotter.js +0 -447
  18. package/build/autoscaling/snapshotter.js.map +0 -1
  19. package/build/autoscaling/system_status.d.ts +0 -224
  20. package/build/autoscaling/system_status.d.ts.map +0 -1
  21. package/build/autoscaling/system_status.js +0 -228
  22. package/build/autoscaling/system_status.js.map +0 -1
  23. package/build/browser_launchers/browser_launcher.d.ts +0 -154
  24. package/build/browser_launchers/browser_launcher.d.ts.map +0 -1
  25. package/build/browser_launchers/browser_launcher.js +0 -160
  26. package/build/browser_launchers/browser_launcher.js.map +0 -1
  27. package/build/browser_launchers/browser_plugin.d.ts +0 -23
  28. package/build/browser_launchers/browser_plugin.d.ts.map +0 -1
  29. package/build/browser_launchers/browser_plugin.js +0 -25
  30. package/build/browser_launchers/browser_plugin.js.map +0 -1
  31. package/build/browser_launchers/playwright_launcher.d.ts +0 -131
  32. package/build/browser_launchers/playwright_launcher.d.ts.map +0 -1
  33. package/build/browser_launchers/playwright_launcher.js +0 -150
  34. package/build/browser_launchers/playwright_launcher.js.map +0 -1
  35. package/build/browser_launchers/puppeteer_launcher.d.ts +0 -153
  36. package/build/browser_launchers/puppeteer_launcher.d.ts.map +0 -1
  37. package/build/browser_launchers/puppeteer_launcher.js +0 -197
  38. package/build/browser_launchers/puppeteer_launcher.js.map +0 -1
  39. package/build/cache_container.d.ts +0 -31
  40. package/build/cache_container.d.ts.map +0 -1
  41. package/build/cache_container.js +0 -48
  42. package/build/cache_container.js.map +0 -1
  43. package/build/configuration.d.ts +0 -226
  44. package/build/configuration.d.ts.map +0 -1
  45. package/build/configuration.js +0 -325
  46. package/build/configuration.js.map +0 -1
  47. package/build/constants.d.ts +0 -37
  48. package/build/constants.d.ts.map +0 -1
  49. package/build/constants.js +0 -41
  50. package/build/constants.js.map +0 -1
  51. package/build/crawlers/basic_crawler.d.ts +0 -443
  52. package/build/crawlers/basic_crawler.d.ts.map +0 -1
  53. package/build/crawlers/basic_crawler.js +0 -664
  54. package/build/crawlers/basic_crawler.js.map +0 -1
  55. package/build/crawlers/browser_crawler.d.ts +0 -512
  56. package/build/crawlers/browser_crawler.d.ts.map +0 -1
  57. package/build/crawlers/browser_crawler.js +0 -540
  58. package/build/crawlers/browser_crawler.js.map +0 -1
  59. package/build/crawlers/cheerio_crawler.d.ts +0 -931
  60. package/build/crawlers/cheerio_crawler.d.ts.map +0 -1
  61. package/build/crawlers/cheerio_crawler.js +0 -913
  62. package/build/crawlers/cheerio_crawler.js.map +0 -1
  63. package/build/crawlers/crawler_extension.d.ts +0 -10
  64. package/build/crawlers/crawler_extension.d.ts.map +0 -1
  65. package/build/crawlers/crawler_extension.js +0 -19
  66. package/build/crawlers/crawler_extension.js.map +0 -1
  67. package/build/crawlers/crawler_utils.d.ts +0 -34
  68. package/build/crawlers/crawler_utils.d.ts.map +0 -1
  69. package/build/crawlers/crawler_utils.js +0 -87
  70. package/build/crawlers/crawler_utils.js.map +0 -1
  71. package/build/crawlers/playwright_crawler.d.ts +0 -448
  72. package/build/crawlers/playwright_crawler.d.ts.map +0 -1
  73. package/build/crawlers/playwright_crawler.js +0 -299
  74. package/build/crawlers/playwright_crawler.js.map +0 -1
  75. package/build/crawlers/puppeteer_crawler.d.ts +0 -425
  76. package/build/crawlers/puppeteer_crawler.d.ts.map +0 -1
  77. package/build/crawlers/puppeteer_crawler.js +0 -299
  78. package/build/crawlers/puppeteer_crawler.js.map +0 -1
  79. package/build/crawlers/statistics.d.ts +0 -185
  80. package/build/crawlers/statistics.d.ts.map +0 -1
  81. package/build/crawlers/statistics.js +0 -331
  82. package/build/crawlers/statistics.js.map +0 -1
  83. package/build/enqueue_links/click_elements.d.ts +0 -179
  84. package/build/enqueue_links/click_elements.d.ts.map +0 -1
  85. package/build/enqueue_links/click_elements.js +0 -434
  86. package/build/enqueue_links/click_elements.js.map +0 -1
  87. package/build/enqueue_links/enqueue_links.d.ts +0 -117
  88. package/build/enqueue_links/enqueue_links.d.ts.map +0 -1
  89. package/build/enqueue_links/enqueue_links.js +0 -163
  90. package/build/enqueue_links/enqueue_links.js.map +0 -1
  91. package/build/enqueue_links/shared.d.ts +0 -42
  92. package/build/enqueue_links/shared.d.ts.map +0 -1
  93. package/build/enqueue_links/shared.js +0 -121
  94. package/build/enqueue_links/shared.js.map +0 -1
  95. package/build/errors.d.ts +0 -29
  96. package/build/errors.d.ts.map +0 -1
  97. package/build/errors.js +0 -38
  98. package/build/errors.js.map +0 -1
  99. package/build/events.d.ts +0 -11
  100. package/build/events.d.ts.map +0 -1
  101. package/build/events.js +0 -147
  102. package/build/events.js.map +0 -1
  103. package/build/index.d.ts +0 -4
  104. package/build/index.d.ts.map +0 -1
  105. package/build/index.js +0 -7
  106. package/build/index.js.map +0 -1
  107. package/build/main.d.ts +0 -179
  108. package/build/main.d.ts.map +0 -1
  109. package/build/main.js +0 -81
  110. package/build/main.js.map +0 -1
  111. package/build/playwright_utils.d.ts +0 -9
  112. package/build/playwright_utils.d.ts.map +0 -1
  113. package/build/playwright_utils.js +0 -90
  114. package/build/playwright_utils.js.map +0 -1
  115. package/build/proxy_configuration.d.ts +0 -411
  116. package/build/proxy_configuration.d.ts.map +0 -1
  117. package/build/proxy_configuration.js +0 -517
  118. package/build/proxy_configuration.js.map +0 -1
  119. package/build/pseudo_url.d.ts +0 -86
  120. package/build/pseudo_url.d.ts.map +0 -1
  121. package/build/pseudo_url.js +0 -153
  122. package/build/pseudo_url.js.map +0 -1
  123. package/build/puppeteer_request_interception.d.ts +0 -8
  124. package/build/puppeteer_request_interception.d.ts.map +0 -1
  125. package/build/puppeteer_request_interception.js +0 -235
  126. package/build/puppeteer_request_interception.js.map +0 -1
  127. package/build/puppeteer_utils.d.ts +0 -250
  128. package/build/puppeteer_utils.d.ts.map +0 -1
  129. package/build/puppeteer_utils.js +0 -551
  130. package/build/puppeteer_utils.js.map +0 -1
  131. package/build/request.d.ts +0 -180
  132. package/build/request.d.ts.map +0 -1
  133. package/build/request.js +0 -261
  134. package/build/request.js.map +0 -1
  135. package/build/request_list.d.ts +0 -581
  136. package/build/request_list.d.ts.map +0 -1
  137. package/build/request_list.js +0 -826
  138. package/build/request_list.js.map +0 -1
  139. package/build/serialization.d.ts +0 -5
  140. package/build/serialization.d.ts.map +0 -1
  141. package/build/serialization.js +0 -139
  142. package/build/serialization.js.map +0 -1
  143. package/build/session_pool/errors.d.ts +0 -11
  144. package/build/session_pool/errors.d.ts.map +0 -1
  145. package/build/session_pool/errors.js +0 -18
  146. package/build/session_pool/errors.js.map +0 -1
  147. package/build/session_pool/events.d.ts +0 -5
  148. package/build/session_pool/events.d.ts.map +0 -1
  149. package/build/session_pool/events.js +0 -6
  150. package/build/session_pool/events.js.map +0 -1
  151. package/build/session_pool/session.d.ts +0 -286
  152. package/build/session_pool/session.d.ts.map +0 -1
  153. package/build/session_pool/session.js +0 -355
  154. package/build/session_pool/session.js.map +0 -1
  155. package/build/session_pool/session_pool.d.ts +0 -280
  156. package/build/session_pool/session_pool.d.ts.map +0 -1
  157. package/build/session_pool/session_pool.js +0 -393
  158. package/build/session_pool/session_pool.js.map +0 -1
  159. package/build/session_pool/session_utils.d.ts +0 -4
  160. package/build/session_pool/session_utils.d.ts.map +0 -1
  161. package/build/session_pool/session_utils.js +0 -24
  162. package/build/session_pool/session_utils.js.map +0 -1
  163. package/build/stealth/hiding_tricks.d.ts +0 -22
  164. package/build/stealth/hiding_tricks.d.ts.map +0 -1
  165. package/build/stealth/hiding_tricks.js +0 -308
  166. package/build/stealth/hiding_tricks.js.map +0 -1
  167. package/build/stealth/stealth.d.ts +0 -56
  168. package/build/stealth/stealth.d.ts.map +0 -1
  169. package/build/stealth/stealth.js +0 -125
  170. package/build/stealth/stealth.js.map +0 -1
  171. package/build/storages/dataset.d.ts +0 -288
  172. package/build/storages/dataset.d.ts.map +0 -1
  173. package/build/storages/dataset.js +0 -480
  174. package/build/storages/dataset.js.map +0 -1
  175. package/build/storages/key_value_store.d.ts +0 -243
  176. package/build/storages/key_value_store.d.ts.map +0 -1
  177. package/build/storages/key_value_store.js +0 -462
  178. package/build/storages/key_value_store.js.map +0 -1
  179. package/build/storages/request_queue.d.ts +0 -318
  180. package/build/storages/request_queue.d.ts.map +0 -1
  181. package/build/storages/request_queue.js +0 -636
  182. package/build/storages/request_queue.js.map +0 -1
  183. package/build/storages/storage_manager.d.ts +0 -87
  184. package/build/storages/storage_manager.d.ts.map +0 -1
  185. package/build/storages/storage_manager.js +0 -150
  186. package/build/storages/storage_manager.js.map +0 -1
  187. package/build/tsconfig.tsbuildinfo +0 -1
  188. package/build/typedefs.d.ts +0 -146
  189. package/build/typedefs.d.ts.map +0 -1
  190. package/build/typedefs.js +0 -88
  191. package/build/typedefs.js.map +0 -1
  192. package/build/utils.d.ts +0 -175
  193. package/build/utils.d.ts.map +0 -1
  194. package/build/utils.js +0 -731
  195. package/build/utils.js.map +0 -1
  196. package/build/utils_log.d.ts +0 -41
  197. package/build/utils_log.d.ts.map +0 -1
  198. package/build/utils_log.js +0 -192
  199. package/build/utils_log.js.map +0 -1
  200. package/build/utils_request.d.ts +0 -77
  201. package/build/utils_request.d.ts.map +0 -1
  202. package/build/utils_request.js +0 -385
  203. package/build/utils_request.js.map +0 -1
  204. package/build/utils_social.d.ts +0 -210
  205. package/build/utils_social.d.ts.map +0 -1
  206. package/build/utils_social.js +0 -787
  207. package/build/utils_social.js.map +0 -1
  208. package/build/validators.d.ts +0 -23
  209. package/build/validators.d.ts.map +0 -1
  210. package/build/validators.js +0 -29
  211. package/build/validators.js.map +0 -1
@@ -1,462 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.getInput = exports.setValue = exports.getValue = exports.openKeyValueStore = exports.KeyValueStore = exports.maybeStringify = void 0;
4
- const tslib_1 = require("tslib");
5
- const consts_1 = require("@apify/consts");
6
- const utilities_1 = require("@apify/utilities");
7
- const ow_1 = (0, tslib_1.__importStar)(require("ow"));
8
- const storage_manager_1 = require("./storage_manager");
9
- const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
10
- const configuration_1 = require("../configuration");
11
- const constants_1 = require("../constants");
12
- /* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
13
- /**
14
- * @typedef {(Object<string, *>|null|Buffer|string)} KeyValueStoreValueTypes
15
- */
16
- /**
17
- * Helper function to possibly stringify value if options.contentType is not set.
18
- *
19
- * @ignore
20
- */
21
- const maybeStringify = (value, options) => {
22
- // If contentType is missing, value will be stringified to JSON
23
- if (options.contentType === null || options.contentType === undefined) {
24
- options.contentType = 'application/json; charset=utf-8';
25
- try {
26
- // Format JSON to simplify debugging, the overheads with compression is negligible
27
- value = (0, utilities_1.jsonStringifyExtended)(value, null, 2);
28
- }
29
- catch (e) {
30
- // Give more meaningful error message
31
- if (e.message && e.message.indexOf('Invalid string length') >= 0) {
32
- e.message = 'Object is too large';
33
- }
34
- throw new Error(`The "value" parameter cannot be stringified to JSON: ${e.message}`);
35
- }
36
- if (value === undefined) {
37
- throw new Error('The "value" parameter was stringified to JSON and returned undefined. '
38
- + 'Make sure you\'re not trying to stringify an undefined value.');
39
- }
40
- }
41
- return value;
42
- };
43
- exports.maybeStringify = maybeStringify;
44
- /**
45
- * The `KeyValueStore` class represents a key-value store, a simple data storage that is used
46
- * for saving and reading data records or files. Each data record is
47
- * represented by a unique key and associated with a MIME content type. Key-value stores are ideal
48
- * for saving screenshots, actor inputs and outputs, web pages, PDFs or to persist the state of crawlers.
49
- *
50
- * Do not instantiate this class directly, use the
51
- * {@link Apify#openKeyValueStore} function instead.
52
- *
53
- * Each actor run is associated with a default key-value store, which is created exclusively
54
- * for the run. By convention, the actor input and output are stored into the
55
- * default key-value store under the `INPUT` and `OUTPUT` key, respectively.
56
- * Typically, input and output are JSON files, although it can be any other format.
57
- * To access the default key-value store directly, you can use the
58
- * {@link Apify#getValue} and {@link Apify#setValue} convenience functions.
59
- *
60
- * To access the input, you can also use the {@link Apify#getInput} convenience function.
61
- *
62
- * `KeyValueStore` stores its data either on local disk or in the Apify cloud,
63
- * depending on whether the [`APIFY_LOCAL_STORAGE_DIR`](../guides/environment-variables#apify_local_storage_dir)
64
- * or [`APIFY_TOKEN`](../guides/environment-variables#apify_token) environment variables are set.
65
- *
66
- * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in
67
- * the local directory in the following files:
68
- * ```
69
- * {APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
70
- * ```
71
- * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,
72
- * unless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
73
- * The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
74
- *
75
- * If the [`APIFY_TOKEN`](../guides/environment-variables#apify_token) environment variable is set but
76
- * [`APIFY_LOCAL_STORAGE_DIR`](../guides/environment-variables#apify_local_storage_dir) not,
77
- * the data is stored in the [Apify Key-value store](https://docs.apify.com/storage/key-value-store)
78
- * cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
79
- * option to {@link Apify#openKeyValueStore} function, even if the
80
- * [`APIFY_LOCAL_STORAGE_DIR`](../guides/environment-variables#apify_local_storage_dir) variable is set.
81
- *
82
- * **Example usage:**
83
- *
84
- * ```javascript
85
- * // Get actor input from the default key-value store.
86
- * const input = await Apify.getInput();
87
- * // Get some value from the default key-value store.
88
- * const otherValue = await Apify.getValue('my-key');
89
- *
90
- * // Write actor output to the default key-value store.
91
- * await Apify.setValue('OUTPUT', { myResult: 123 });
92
- *
93
- * // Open a named key-value store
94
- * const store = await Apify.openKeyValueStore('some-name');
95
- *
96
- * // Write a record. JavaScript object is automatically converted to JSON,
97
- * // strings and binary buffers are stored as they are
98
- * await store.setValue('some-key', { foo: 'bar' });
99
- *
100
- * // Read a record. Note that JSON is automatically parsed to a JavaScript object,
101
- * // text data returned as a string and other data is returned as binary buffer
102
- * const value = await store.getValue('some-key');
103
- *
104
- * // Drop (delete) the store
105
- * await store.drop();
106
- * ```
107
- * @hideconstructor
108
- */
109
- class KeyValueStore {
110
- /**
111
- * @param {object} options
112
- * @param {string} options.id
113
- * @param {string} [options.name]
114
- * @param {ApifyClient|ApifyStorageLocal} options.client
115
- * @param {boolean} options.isLocal
116
- * @param {Configuration} [config]
117
- */
118
- constructor(options, config = configuration_1.Configuration.getGlobalConfig()) {
119
- this.id = options.id;
120
- this.name = options.name;
121
- this.isLocal = options.isLocal;
122
- this.client = options.client.keyValueStore(this.id);
123
- this.log = utils_log_1.default.child({ prefix: 'KeyValueStore' });
124
- this.config = config;
125
- }
126
- /**
127
- * Gets a value from the key-value store.
128
- *
129
- * The function returns a `Promise` that resolves to the record value,
130
- * whose JavaScript type depends on the MIME content type of the record.
131
- * Records with the `application/json`
132
- * content type are automatically parsed and returned as a JavaScript object.
133
- * Similarly, records with `text/plain` content types are returned as a string.
134
- * For all other content types, the value is returned as a raw
135
- * [`Buffer`](https://nodejs.org/api/buffer.html) instance.
136
- *
137
- * If the record does not exist, the function resolves to `null`.
138
- *
139
- * To save or delete a value in the key-value store, use the
140
- * {@link KeyValueStore#setValue} function.
141
- *
142
- * **Example usage:**
143
- *
144
- * ```javascript
145
- * const store = await Apify.openKeyValueStore();
146
- * const buffer = await store.getValue('screenshot1.png');
147
- * ```
148
- * @param {string} key
149
- * Unique key of the record. It can be at most 256 characters long and only consist
150
- * of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
151
- * @returns {Promise<KeyValueStoreValueTypes>}
152
- * Returns a promise that resolves to an object, string
153
- * or [`Buffer`](https://nodejs.org/api/buffer.html), depending
154
- * on the MIME content type of the record.
155
- */
156
- async getValue(key) {
157
- (0, ow_1.default)(key, ow_1.default.string.nonEmpty);
158
- // TODO: Perhaps we should add options.contentType or options.asBuffer/asString
159
- // to enforce the representation of value
160
- const record = await this.client.getRecord(key);
161
- return record ? record.value : null;
162
- }
163
- /**
164
- * Saves or deletes a record in the key-value store.
165
- * The function returns a promise that resolves once the record has been saved or deleted.
166
- *
167
- * **Example usage:**
168
- *
169
- * ```javascript
170
- * const store = await Apify.openKeyValueStore();
171
- * await store.setValue('OUTPUT', { foo: 'bar' });
172
- * ```
173
- *
174
- * Beware that the key can be at most 256 characters long and only contain the following characters: `a-zA-Z0-9!-_.'()`
175
- *
176
- * By default, `value` is converted to JSON and stored with the
177
- * `application/json; charset=utf-8` MIME content type.
178
- * To store the value with another content type, pass it in the options as follows:
179
- * ```javascript
180
- * const store = await Apify.openKeyValueStore('my-text-store');
181
- * await store.setValue('RESULTS', 'my text data', { contentType: 'text/plain' });
182
- * ```
183
- * If you set custom content type, `value` must be either a string or
184
- * [`Buffer`](https://nodejs.org/api/buffer.html), otherwise an error will be thrown.
185
- *
186
- * If `value` is `null`, the record is deleted instead. Note that the `setValue()` function succeeds
187
- * regardless whether the record existed or not.
188
- *
189
- * To retrieve a value from the key-value store, use the
190
- * {@link KeyValueStore#getValue} function.
191
- *
192
- * **IMPORTANT:** Always make sure to use the `await` keyword when calling `setValue()`,
193
- * otherwise the actor process might finish before the value is stored!
194
- *
195
- * @param {string} key
196
- * Unique key of the record. It can be at most 256 characters long and only consist
197
- * of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
198
- * @param {KeyValueStoreValueTypes} value
199
- * Record data, which can be one of the following values:
200
- * - If `null`, the record in the key-value store is deleted.
201
- * - If no `options.contentType` is specified, `value` can be any JavaScript object and it will be stringified to JSON.
202
- * - If `options.contentType` is set, `value` is taken as is and it must be a `String` or [`Buffer`](https://nodejs.org/api/buffer.html).
203
- * For any other value an error will be thrown.
204
- * @param {object} [options]
205
- * @param {string} [options.contentType]
206
- * Specifies a custom MIME content type of the record.
207
- * @returns {Promise<void>}
208
- *
209
- */
210
- setValue(key, value, options = {}) {
211
- (0, ow_1.default)(key, ow_1.default.string.nonEmpty);
212
- (0, ow_1.default)(key, ow_1.default.string.validate((k) => ({
213
- validator: ow_1.default.isValid(k, ow_1.default.string.matches(consts_1.KEY_VALUE_STORE_KEY_REGEX)),
214
- message: 'The "key" argument must be at most 256 characters long and only contain the following characters: a-zA-Z0-9!-_.\'()',
215
- })));
216
- if (options.contentType && !(ow_1.default.isValid(value, ow_1.default.any(ow_1.default.string, ow_1.default.buffer)) || (ow_1.default.isValid(value, ow_1.default.object) && typeof value?.pipe === 'function'))) {
217
- throw new ow_1.ArgumentError('The "value" parameter must be a String, Buffer or Stream when "options.contentType" is specified.', this.setValue);
218
- }
219
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
220
- contentType: ow_1.default.optional.string.nonEmpty,
221
- }));
222
- // Make copy of options, don't update what user passed.
223
- const optionsCopy = { ...options };
224
- // In this case delete the record.
225
- if (value === null)
226
- return this.client.deleteRecord(key);
227
- // TODO the function mutates optionsCopy, but is also used in actor.js
228
- // Remove the mutation when actor.js usages are removed.
229
- value = (0, exports.maybeStringify)(value, optionsCopy);
230
- return this.client.setRecord({
231
- key,
232
- value,
233
- contentType: optionsCopy.contentType,
234
- });
235
- }
236
- /**
237
- * Removes the key-value store either from the Apify cloud storage or from the local directory,
238
- * depending on the mode of operation.
239
- *
240
- * @return {Promise<void>}
241
- */
242
- async drop() {
243
- await this.client.delete();
244
- const manager = new storage_manager_1.StorageManager(KeyValueStore, this.config);
245
- manager.closeStorage(this);
246
- }
247
- /**
248
- * Returns a URL for the given key that may be used to publicly
249
- * access the value in the remote key-value store.
250
- *
251
- * @param {string} key
252
- * @return {string}
253
- */
254
- getPublicUrl(key) {
255
- return `${constants_1.APIFY_API_BASE_URL}/key-value-stores/${this.id}/records/${key}`;
256
- }
257
- /**
258
- * Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
259
- * Each invocation of `iteratee` is called with three arguments: `(key, index, info)`, where `key`
260
- * is the record key, `index` is a zero-based index of the key in the current iteration
261
- * (regardless of `options.exclusiveStartKey`) and `info` is an object that contains a single property `size`
262
- * indicating size of the record in bytes.
263
- *
264
- * If the `iteratee` function returns a Promise then it is awaited before the next call.
265
- * If it throws an error, the iteration is aborted and the `forEachKey` function throws the error.
266
- *
267
- * **Example usage**
268
- * ```javascript
269
- * const keyValueStore = await Apify.openKeyValueStore();
270
- * await keyValueStore.forEachKey(async (key, index, info) => {
271
- * console.log(`Key at ${index}: ${key} has size ${info.size}`);
272
- * });
273
- * ```
274
- *
275
- * @param {KeyConsumer} iteratee A function that is called for every key in the key-value store.
276
- * @param {object} [options] All `forEachKey()` parameters are passed
277
- * via an options object with the following keys:
278
- * @param {string} [options.exclusiveStartKey] All keys up to this one (including) are skipped from the result.
279
- * @return {Promise<void>}
280
- */
281
- async forEachKey(iteratee, options = {}) {
282
- return this._forEachKey(iteratee, options);
283
- }
284
- /**
285
- * @param {KeyConsumer} iteratee
286
- * @param {Record<string, any>} [options]
287
- * @param {number} [index=0]
288
- * @return {Promise<Promise<void> | undefined>}
289
- * @private
290
- */
291
- async _forEachKey(iteratee, options = {}, index = 0) {
292
- const { exclusiveStartKey } = options;
293
- (0, ow_1.default)(iteratee, ow_1.default.function);
294
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
295
- exclusiveStartKey: ow_1.default.optional.string,
296
- }));
297
- const response = await this.client.listKeys({ exclusiveStartKey });
298
- const { nextExclusiveStartKey, isTruncated, items } = response;
299
- for (const item of items) {
300
- await iteratee(item.key, index++, { size: item.size });
301
- }
302
- return isTruncated
303
- ? this._forEachKey(iteratee, { exclusiveStartKey: nextExclusiveStartKey }, index)
304
- : undefined; // [].forEach() returns undefined.
305
- }
306
- }
307
- exports.KeyValueStore = KeyValueStore;
308
- /**
309
- * Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
310
- *
311
- * Key-value stores are used to store records or files, along with their MIME content type.
312
- * The records are stored and retrieved using a unique key.
313
- * The actual data is stored either on a local filesystem or in the Apify cloud.
314
- *
315
- * For more details and code examples, see the {@link KeyValueStore} class.
316
- *
317
- * @param {string} [storeIdOrName]
318
- * ID or name of the key-value store to be opened. If `null` or `undefined`,
319
- * the function returns the default key-value store associated with the actor run.
320
- * @param {object} [options]
321
- * @param {boolean} [options.forceCloud=false]
322
- * If set to `true` then the function uses cloud storage usage even if the `APIFY_LOCAL_STORAGE_DIR`
323
- * environment variable is set. This way it is possible to combine local and cloud storage.
324
- * @param {Configuration} [options.config] SDK configuration instance, defaults to the static register
325
- * @returns {Promise<KeyValueStore>}
326
- * @memberof module:Apify
327
- * @name openKeyValueStore
328
- * @function
329
- */
330
- const openKeyValueStore = async (storeIdOrName, options = {}) => {
331
- (0, ow_1.default)(storeIdOrName, ow_1.default.optional.string);
332
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
333
- forceCloud: ow_1.default.optional.boolean,
334
- config: ow_1.default.optional.object.instanceOf(configuration_1.Configuration),
335
- }));
336
- const manager = new storage_manager_1.StorageManager(KeyValueStore, options.config);
337
- return manager.openStorage(storeIdOrName, options);
338
- };
339
- exports.openKeyValueStore = openKeyValueStore;
340
- /**
341
- * Gets a value from the default {@link KeyValueStore} associated with the current actor run.
342
- *
343
- * This is just a convenient shortcut for {@link KeyValueStore#getValue}.
344
- * For example, calling the following code:
345
- * ```javascript
346
- * const value = await Apify.getValue('my-key');
347
- * ```
348
- *
349
- * is equivalent to:
350
- * ```javascript
351
- * const store = await Apify.openKeyValueStore();
352
- * const value = await store.getValue('my-key');
353
- * ```
354
- *
355
- * To store the value to the default key-value store, you can use the {@link Apify#setValue} function.
356
- *
357
- * For more information, see {@link Apify#openKeyValueStore}
358
- * and {@link KeyValueStore#getValue}.
359
- *
360
- * @param {string} key
361
- * Unique record key.
362
- * @returns {Promise<Object<string, *>|string|Buffer|null>}
363
- * Returns a promise that resolves to an object, string
364
- * or [`Buffer`](https://nodejs.org/api/buffer.html), depending
365
- * on the MIME content type of the record, or `null`
366
- * if the record is missing.
367
- * @memberof module:Apify
368
- * @name getValue
369
- * @function
370
- */
371
- const getValue = async (key) => {
372
- const store = await (0, exports.openKeyValueStore)();
373
- return store.getValue(key);
374
- };
375
- exports.getValue = getValue;
376
- /**
377
- * Stores or deletes a value in the default {@link KeyValueStore} associated with the current actor run.
378
- *
379
- * This is just a convenient shortcut for {@link KeyValueStore#setValue}.
380
- * For example, calling the following code:
381
- * ```javascript
382
- * await Apify.setValue('OUTPUT', { foo: "bar" });
383
- * ```
384
- *
385
- * is equivalent to:
386
- * ```javascript
387
- * const store = await Apify.openKeyValueStore();
388
- * await store.setValue('OUTPUT', { foo: "bar" });
389
- * ```
390
- *
391
- * To get a value from the default key-value store, you can use the {@link Apify#getValue} function.
392
- *
393
- * For more information, see {@link Apify#openKeyValueStore}
394
- * and {@link KeyValueStore#getValue}.
395
- *
396
- * @param {string} key
397
- * Unique record key.
398
- * @param {*} value
399
- * Record data, which can be one of the following values:
400
- * - If `null`, the record in the key-value store is deleted.
401
- * - If no `options.contentType` is specified, `value` can be any JavaScript object and it will be stringified to JSON.
402
- * - If `options.contentType` is set, `value` is taken as is and it must be a `String` or [`Buffer`](https://nodejs.org/api/buffer.html).
403
- * For any other value an error will be thrown.
404
- * @param {object} [options]
405
- * @param {string} [options.contentType]
406
- * Specifies a custom MIME content type of the record.
407
- * @return {Promise<void>}
408
- * @memberof module:Apify
409
- * @name setValue
410
- * @function
411
- */
412
- const setValue = async (key, value, options) => {
413
- const store = await (0, exports.openKeyValueStore)();
414
- return store.setValue(key, value, options);
415
- };
416
- exports.setValue = setValue;
417
- /**
418
- * Gets the actor input value from the default {@link KeyValueStore} associated with the current actor run.
419
- *
420
- * This is just a convenient shortcut for [`keyValueStore.getValue('INPUT')`](key-value-store#getvalue).
421
- * For example, calling the following code:
422
- * ```javascript
423
- * const input = await Apify.getInput();
424
- * ```
425
- *
426
- * is equivalent to:
427
- * ```javascript
428
- * const store = await Apify.openKeyValueStore();
429
- * await store.getValue('INPUT');
430
- * ```
431
- *
432
- * Note that the `getInput()` function does not cache the value read from the key-value store.
433
- * If you need to use the input multiple times in your actor,
434
- * it is far more efficient to read it once and store it locally.
435
- *
436
- * For more information, see {@link Apify#openKeyValueStore}
437
- * and {@link KeyValueStore#getValue}.
438
- *
439
- * @returns {Promise<Object<string, *>|string|Buffer|null>}
440
- * Returns a promise that resolves to an object, string
441
- * or [`Buffer`](https://nodejs.org/api/buffer.html), depending
442
- * on the MIME content type of the record, or `null`
443
- * if the record is missing.
444
- * @memberof module:Apify
445
- * @name getInput
446
- * @function
447
- */
448
- const getInput = async () => (0, exports.getValue)(process.env[consts_1.ENV_VARS.INPUT_KEY] || consts_1.KEY_VALUE_STORE_KEYS.INPUT);
449
- exports.getInput = getInput;
450
- /**
451
- * User-function used in the {@link KeyValueStore#forEachKey} method.
452
- * @callback KeyConsumer
453
- * @param {string} key
454
- * Current {KeyValue} key being processed.
455
- * @param {number} index
456
- * Position of the current key in {@link KeyValueStore}.
457
- * @param {*} info
458
- * Information about the current {@link KeyValueStore} entry.
459
- * @param {number} info.size
460
- * Size of the value associated with the current key in bytes.
461
- */
462
- //# sourceMappingURL=key_value_store.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.js"],"names":[],"mappings":";;;;AAAA,0CAA0F;AAC1F,gDAAyD;AACzD,sDAAuC;AACvC,uDAAmD;AACnD,0EAA+B;AAO/B,oDAAiD;AACjD,4CAAkD;AAClD,iFAAiF;AAEjF;;GAEG;AAEH;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IAC7C,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;SACjD;QAAC,OAAO,CAAC,EAAE;YACR,qCAAqC;YACrC,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBAC9D,CAAC,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACrC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACxF;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAvBW,QAAA,cAAc,kBAuBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAa,aAAa;IACtB;;;;;;;OAOG;IACH,YAAY,OAAO,EAAE,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE;QACzD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,GAAG,GAAG,mBAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAG;QACd,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAE5B,+EAA+E;QAC/E,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8CG;IACH,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,GAAG,EAAE;QAC7B,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAO,KAAK,EAAE,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAClJ,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,sEAAsE;QACtE,wDAAwD;QACxD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/D,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;OAMG;IACH,YAAY,CAAC,GAAG;QACZ,OAAO,GAAG,8BAAkB,qBAAqB,IAAI,CAAC,EAAE,YAAY,GAAG,EAAE,CAAC;IAC9E,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,GAAG,EAAE;QACnC,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,KAAK,GAAG,CAAC;QAC/C,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;CACJ;AAjND,sCAiNC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACI,MAAM,iBAAiB,GAAG,KAAK,EAAE,aAAa,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IACnE,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACtC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC/B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;KACvD,CAAC,CAAC,CAAC;IAEJ,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAClE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;AACvD,CAAC,CAAC;AATW,QAAA,iBAAiB,qBAS5B;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACI,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAG,EAAE,EAAE;IAClC,MAAM,KAAK,GAAG,MAAM,IAAA,yBAAiB,GAAE,CAAC;IAExC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AAC/B,CAAC,CAAC;AAJW,QAAA,QAAQ,YAInB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACI,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAClD,MAAM,KAAK,GAAG,MAAM,IAAA,yBAAiB,GAAE,CAAC;IAExC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;AAC/C,CAAC,CAAC;AAJW,QAAA,QAAQ,YAInB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACI,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE,CAAC,IAAA,gBAAQ,EAAC,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,SAAS,CAAC,IAAI,6BAAoB,CAAC,KAAK,CAAC,CAAC;AAA/F,QAAA,QAAQ,YAAuF;AAE5G;;;;;;;;;;;GAWG"}