apify 2.3.1-beta.4 → 3.0.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/README.md +6 -5
  2. package/actor.d.ts +1092 -0
  3. package/actor.d.ts.map +1 -0
  4. package/actor.js +1221 -0
  5. package/actor.js.map +1 -0
  6. package/index.d.ts +4 -0
  7. package/index.d.ts.map +1 -0
  8. package/index.js +7 -0
  9. package/index.js.map +1 -0
  10. package/index.mjs +7 -0
  11. package/package.json +54 -128
  12. package/platform_event_manager.d.ts +55 -0
  13. package/platform_event_manager.d.ts.map +1 -0
  14. package/platform_event_manager.js +116 -0
  15. package/platform_event_manager.js.map +1 -0
  16. package/proxy_configuration.d.ts +210 -0
  17. package/proxy_configuration.d.ts.map +1 -0
  18. package/proxy_configuration.js +297 -0
  19. package/proxy_configuration.js.map +1 -0
  20. package/tsconfig.build.tsbuildinfo +1 -0
  21. package/utils.d.ts +11 -0
  22. package/utils.d.ts.map +1 -0
  23. package/utils.js +40 -0
  24. package/utils.js.map +1 -0
  25. package/build/actor.d.ts +0 -113
  26. package/build/actor.d.ts.map +0 -1
  27. package/build/actor.js +0 -582
  28. package/build/actor.js.map +0 -1
  29. package/build/apify.d.ts +0 -752
  30. package/build/apify.d.ts.map +0 -1
  31. package/build/apify.js +0 -877
  32. package/build/apify.js.map +0 -1
  33. package/build/autoscaling/autoscaled_pool.d.ts +0 -384
  34. package/build/autoscaling/autoscaled_pool.d.ts.map +0 -1
  35. package/build/autoscaling/autoscaled_pool.js +0 -557
  36. package/build/autoscaling/autoscaled_pool.js.map +0 -1
  37. package/build/autoscaling/snapshotter.d.ts +0 -278
  38. package/build/autoscaling/snapshotter.d.ts.map +0 -1
  39. package/build/autoscaling/snapshotter.js +0 -447
  40. package/build/autoscaling/snapshotter.js.map +0 -1
  41. package/build/autoscaling/system_status.d.ts +0 -224
  42. package/build/autoscaling/system_status.d.ts.map +0 -1
  43. package/build/autoscaling/system_status.js +0 -228
  44. package/build/autoscaling/system_status.js.map +0 -1
  45. package/build/browser_launchers/browser_launcher.d.ts +0 -154
  46. package/build/browser_launchers/browser_launcher.d.ts.map +0 -1
  47. package/build/browser_launchers/browser_launcher.js +0 -160
  48. package/build/browser_launchers/browser_launcher.js.map +0 -1
  49. package/build/browser_launchers/browser_plugin.d.ts +0 -23
  50. package/build/browser_launchers/browser_plugin.d.ts.map +0 -1
  51. package/build/browser_launchers/browser_plugin.js +0 -25
  52. package/build/browser_launchers/browser_plugin.js.map +0 -1
  53. package/build/browser_launchers/playwright_launcher.d.ts +0 -131
  54. package/build/browser_launchers/playwright_launcher.d.ts.map +0 -1
  55. package/build/browser_launchers/playwright_launcher.js +0 -150
  56. package/build/browser_launchers/playwright_launcher.js.map +0 -1
  57. package/build/browser_launchers/puppeteer_launcher.d.ts +0 -153
  58. package/build/browser_launchers/puppeteer_launcher.d.ts.map +0 -1
  59. package/build/browser_launchers/puppeteer_launcher.js +0 -197
  60. package/build/browser_launchers/puppeteer_launcher.js.map +0 -1
  61. package/build/cache_container.d.ts +0 -31
  62. package/build/cache_container.d.ts.map +0 -1
  63. package/build/cache_container.js +0 -48
  64. package/build/cache_container.js.map +0 -1
  65. package/build/configuration.d.ts +0 -226
  66. package/build/configuration.d.ts.map +0 -1
  67. package/build/configuration.js +0 -325
  68. package/build/configuration.js.map +0 -1
  69. package/build/constants.d.ts +0 -37
  70. package/build/constants.d.ts.map +0 -1
  71. package/build/constants.js +0 -41
  72. package/build/constants.js.map +0 -1
  73. package/build/crawlers/basic_crawler.d.ts +0 -443
  74. package/build/crawlers/basic_crawler.d.ts.map +0 -1
  75. package/build/crawlers/basic_crawler.js +0 -664
  76. package/build/crawlers/basic_crawler.js.map +0 -1
  77. package/build/crawlers/browser_crawler.d.ts +0 -512
  78. package/build/crawlers/browser_crawler.d.ts.map +0 -1
  79. package/build/crawlers/browser_crawler.js +0 -540
  80. package/build/crawlers/browser_crawler.js.map +0 -1
  81. package/build/crawlers/cheerio_crawler.d.ts +0 -931
  82. package/build/crawlers/cheerio_crawler.d.ts.map +0 -1
  83. package/build/crawlers/cheerio_crawler.js +0 -913
  84. package/build/crawlers/cheerio_crawler.js.map +0 -1
  85. package/build/crawlers/crawler_extension.d.ts +0 -10
  86. package/build/crawlers/crawler_extension.d.ts.map +0 -1
  87. package/build/crawlers/crawler_extension.js +0 -19
  88. package/build/crawlers/crawler_extension.js.map +0 -1
  89. package/build/crawlers/crawler_utils.d.ts +0 -34
  90. package/build/crawlers/crawler_utils.d.ts.map +0 -1
  91. package/build/crawlers/crawler_utils.js +0 -87
  92. package/build/crawlers/crawler_utils.js.map +0 -1
  93. package/build/crawlers/playwright_crawler.d.ts +0 -448
  94. package/build/crawlers/playwright_crawler.d.ts.map +0 -1
  95. package/build/crawlers/playwright_crawler.js +0 -299
  96. package/build/crawlers/playwright_crawler.js.map +0 -1
  97. package/build/crawlers/puppeteer_crawler.d.ts +0 -425
  98. package/build/crawlers/puppeteer_crawler.d.ts.map +0 -1
  99. package/build/crawlers/puppeteer_crawler.js +0 -299
  100. package/build/crawlers/puppeteer_crawler.js.map +0 -1
  101. package/build/crawlers/statistics.d.ts +0 -185
  102. package/build/crawlers/statistics.d.ts.map +0 -1
  103. package/build/crawlers/statistics.js +0 -331
  104. package/build/crawlers/statistics.js.map +0 -1
  105. package/build/enqueue_links/click_elements.d.ts +0 -179
  106. package/build/enqueue_links/click_elements.d.ts.map +0 -1
  107. package/build/enqueue_links/click_elements.js +0 -434
  108. package/build/enqueue_links/click_elements.js.map +0 -1
  109. package/build/enqueue_links/enqueue_links.d.ts +0 -117
  110. package/build/enqueue_links/enqueue_links.d.ts.map +0 -1
  111. package/build/enqueue_links/enqueue_links.js +0 -163
  112. package/build/enqueue_links/enqueue_links.js.map +0 -1
  113. package/build/enqueue_links/shared.d.ts +0 -42
  114. package/build/enqueue_links/shared.d.ts.map +0 -1
  115. package/build/enqueue_links/shared.js +0 -121
  116. package/build/enqueue_links/shared.js.map +0 -1
  117. package/build/errors.d.ts +0 -29
  118. package/build/errors.d.ts.map +0 -1
  119. package/build/errors.js +0 -38
  120. package/build/errors.js.map +0 -1
  121. package/build/events.d.ts +0 -11
  122. package/build/events.d.ts.map +0 -1
  123. package/build/events.js +0 -147
  124. package/build/events.js.map +0 -1
  125. package/build/index.d.ts +0 -4
  126. package/build/index.d.ts.map +0 -1
  127. package/build/index.js +0 -7
  128. package/build/index.js.map +0 -1
  129. package/build/main.d.ts +0 -179
  130. package/build/main.d.ts.map +0 -1
  131. package/build/main.js +0 -81
  132. package/build/main.js.map +0 -1
  133. package/build/playwright_utils.d.ts +0 -9
  134. package/build/playwright_utils.d.ts.map +0 -1
  135. package/build/playwright_utils.js +0 -90
  136. package/build/playwright_utils.js.map +0 -1
  137. package/build/proxy_configuration.d.ts +0 -411
  138. package/build/proxy_configuration.d.ts.map +0 -1
  139. package/build/proxy_configuration.js +0 -517
  140. package/build/proxy_configuration.js.map +0 -1
  141. package/build/pseudo_url.d.ts +0 -86
  142. package/build/pseudo_url.d.ts.map +0 -1
  143. package/build/pseudo_url.js +0 -153
  144. package/build/pseudo_url.js.map +0 -1
  145. package/build/puppeteer_request_interception.d.ts +0 -8
  146. package/build/puppeteer_request_interception.d.ts.map +0 -1
  147. package/build/puppeteer_request_interception.js +0 -235
  148. package/build/puppeteer_request_interception.js.map +0 -1
  149. package/build/puppeteer_utils.d.ts +0 -250
  150. package/build/puppeteer_utils.d.ts.map +0 -1
  151. package/build/puppeteer_utils.js +0 -551
  152. package/build/puppeteer_utils.js.map +0 -1
  153. package/build/request.d.ts +0 -180
  154. package/build/request.d.ts.map +0 -1
  155. package/build/request.js +0 -261
  156. package/build/request.js.map +0 -1
  157. package/build/request_list.d.ts +0 -581
  158. package/build/request_list.d.ts.map +0 -1
  159. package/build/request_list.js +0 -826
  160. package/build/request_list.js.map +0 -1
  161. package/build/serialization.d.ts +0 -5
  162. package/build/serialization.d.ts.map +0 -1
  163. package/build/serialization.js +0 -139
  164. package/build/serialization.js.map +0 -1
  165. package/build/session_pool/errors.d.ts +0 -11
  166. package/build/session_pool/errors.d.ts.map +0 -1
  167. package/build/session_pool/errors.js +0 -18
  168. package/build/session_pool/errors.js.map +0 -1
  169. package/build/session_pool/events.d.ts +0 -5
  170. package/build/session_pool/events.d.ts.map +0 -1
  171. package/build/session_pool/events.js +0 -6
  172. package/build/session_pool/events.js.map +0 -1
  173. package/build/session_pool/session.d.ts +0 -286
  174. package/build/session_pool/session.d.ts.map +0 -1
  175. package/build/session_pool/session.js +0 -355
  176. package/build/session_pool/session.js.map +0 -1
  177. package/build/session_pool/session_pool.d.ts +0 -280
  178. package/build/session_pool/session_pool.d.ts.map +0 -1
  179. package/build/session_pool/session_pool.js +0 -393
  180. package/build/session_pool/session_pool.js.map +0 -1
  181. package/build/session_pool/session_utils.d.ts +0 -4
  182. package/build/session_pool/session_utils.d.ts.map +0 -1
  183. package/build/session_pool/session_utils.js +0 -24
  184. package/build/session_pool/session_utils.js.map +0 -1
  185. package/build/stealth/hiding_tricks.d.ts +0 -22
  186. package/build/stealth/hiding_tricks.d.ts.map +0 -1
  187. package/build/stealth/hiding_tricks.js +0 -308
  188. package/build/stealth/hiding_tricks.js.map +0 -1
  189. package/build/stealth/stealth.d.ts +0 -56
  190. package/build/stealth/stealth.d.ts.map +0 -1
  191. package/build/stealth/stealth.js +0 -125
  192. package/build/stealth/stealth.js.map +0 -1
  193. package/build/storages/dataset.d.ts +0 -288
  194. package/build/storages/dataset.d.ts.map +0 -1
  195. package/build/storages/dataset.js +0 -480
  196. package/build/storages/dataset.js.map +0 -1
  197. package/build/storages/key_value_store.d.ts +0 -243
  198. package/build/storages/key_value_store.d.ts.map +0 -1
  199. package/build/storages/key_value_store.js +0 -462
  200. package/build/storages/key_value_store.js.map +0 -1
  201. package/build/storages/request_queue.d.ts +0 -318
  202. package/build/storages/request_queue.d.ts.map +0 -1
  203. package/build/storages/request_queue.js +0 -636
  204. package/build/storages/request_queue.js.map +0 -1
  205. package/build/storages/storage_manager.d.ts +0 -87
  206. package/build/storages/storage_manager.d.ts.map +0 -1
  207. package/build/storages/storage_manager.js +0 -150
  208. package/build/storages/storage_manager.js.map +0 -1
  209. package/build/tsconfig.tsbuildinfo +0 -1
  210. package/build/typedefs.d.ts +0 -146
  211. package/build/typedefs.d.ts.map +0 -1
  212. package/build/typedefs.js +0 -88
  213. package/build/typedefs.js.map +0 -1
  214. package/build/utils.d.ts +0 -175
  215. package/build/utils.d.ts.map +0 -1
  216. package/build/utils.js +0 -731
  217. package/build/utils.js.map +0 -1
  218. package/build/utils_log.d.ts +0 -41
  219. package/build/utils_log.d.ts.map +0 -1
  220. package/build/utils_log.js +0 -192
  221. package/build/utils_log.js.map +0 -1
  222. package/build/utils_request.d.ts +0 -77
  223. package/build/utils_request.d.ts.map +0 -1
  224. package/build/utils_request.js +0 -385
  225. package/build/utils_request.js.map +0 -1
  226. package/build/utils_social.d.ts +0 -210
  227. package/build/utils_social.d.ts.map +0 -1
  228. package/build/utils_social.js +0 -787
  229. package/build/utils_social.js.map +0 -1
  230. package/build/validators.d.ts +0 -23
  231. package/build/validators.d.ts.map +0 -1
  232. package/build/validators.js +0 -29
  233. package/build/validators.js.map +0 -1
@@ -1,180 +0,0 @@
1
- export function hashPayload(payload: any): string;
2
- export default Request;
3
- /**
4
- * Specifies required and optional fields for constructing a {@link Request }.
5
- */
6
- export type RequestOptions = {
7
- /**
8
- * URL of the web page to crawl. It must be a non-empty string.
9
- */
10
- url: string;
11
- /**
12
- * A unique key identifying the request.
13
- * Two requests with the same `uniqueKey` are considered as pointing to the same URL.
14
- *
15
- * If `uniqueKey` is not provided, then it is automatically generated by normalizing the URL.
16
- * For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `uniqueKey`
17
- * of `http://www.example.com/something`.
18
- *
19
- * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
20
- *
21
- * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
22
- * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
23
- * when requests point to the same URL, but with different methods and payloads. For example: form submits.
24
- *
25
- * Pass an arbitrary non-empty text value to the `uniqueKey` property
26
- * to override the default behavior and specify which URLs shall be considered equal.
27
- */
28
- uniqueKey?: string | undefined;
29
- method?: string | undefined;
30
- /**
31
- * HTTP request payload, e.g. for POST requests.
32
- */
33
- payload?: string | Buffer | undefined;
34
- /**
35
- * HTTP headers in the following format:
36
- * ```
37
- * {
38
- * Accept: 'text/html',
39
- * 'Content-Type': 'application/json'
40
- * }
41
- * ```
42
- */
43
- headers?: {
44
- [x: string]: string;
45
- } | undefined;
46
- /**
47
- * Custom user data assigned to the request. Use this to save any request related data to the
48
- * request's scope, keeping them accessible on retries, failures etc.
49
- */
50
- userData?: {
51
- [x: string]: any;
52
- } | undefined;
53
- /**
54
- * If `false` then the hash part of a URL is removed when computing the `uniqueKey` property.
55
- * For example, this causes the `http://www.example.com#foo` and `http://www.example.com#bar` URLs
56
- * to have the same `uniqueKey` of `http://www.example.com` and thus the URLs are considered equal.
57
- * Note that this option only has an effect if `uniqueKey` is not set.
58
- */
59
- keepUrlFragment?: boolean | undefined;
60
- /**
61
- * If `true` then the `uniqueKey` is computed not only from the URL, but also from the method and payload
62
- * properties. This is useful when making requests to the same URL that are differentiated by method
63
- * or payload, such as form submit navigations in browsers.
64
- */
65
- useExtendedUniqueKey?: boolean | undefined;
66
- };
67
- /**
68
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
69
- * The `Request` object also stores information about errors that occurred during processing of the request.
70
- *
71
- * Each `Request` instance has the `uniqueKey` property, which can be either specified
72
- * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
73
- * are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
74
- * such as {@link RequestList}, {@link RequestQueue}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
75
- *
76
- * Example use:
77
- *
78
- * ```javascript
79
- * const request = new Apify.Request({
80
- * url: 'http://www.example.com',
81
- * headers: { Accept: 'application/json' },
82
- * });
83
- *
84
- * ...
85
- *
86
- * request.userData.foo = 'bar';
87
- * request.pushErrorMessage(new Error('Request failed!'));
88
- *
89
- * ...
90
- *
91
- * const foo = request.userData.foo;
92
- * ```
93
- *
94
- * @property {string} id
95
- * Request ID
96
- * @property {string} url
97
- * URL of the web page to crawl.
98
- * @property {string} loadedUrl
99
- * An actually loaded URL after redirects, if present. HTTP redirects are guaranteed
100
- * to be included.
101
- *
102
- * When using {@link PuppeteerCrawler} or {@link PlaywrightCrawler}, meta tag and JavaScript redirects may,
103
- * or may not be included, depending on their nature. This generally means that redirects,
104
- * which happen immediately will most likely be included, but delayed redirects will not.
105
- * @property {string} uniqueKey
106
- * A unique key identifying the request.
107
- * Two requests with the same `uniqueKey` are considered as pointing to the same URL.
108
- * @property {string} method
109
- * HTTP method, e.g. `GET` or `POST`.
110
- * @property {(string|Buffer)} payload
111
- * HTTP request payload, e.g. for POST requests.
112
- * @property {boolean} noRetry
113
- * The `true` value indicates that the request will not be automatically retried on error.
114
- * @property {number} retryCount
115
- * Indicates the number of times the crawling of the request has been retried on error.
116
- * @property {string[]} errorMessages
117
- * An array of error messages from request processing.
118
- * @property {Object<string, string>} headers
119
- * Object with HTTP headers. Key is header name, value is the value.
120
- * @property {Object<string, *>} userData
121
- * Custom user data assigned to the request.
122
- * @property {Date} handledAt
123
- * Indicates the time when the request has been processed.
124
- * Is `null` if the request has not been crawled yet.
125
- */
126
- declare class Request {
127
- /**
128
- * @param {RequestOptions} options
129
- * `Request` parameters including the URL, HTTP method and headers, and others.
130
- */
131
- constructor(options: RequestOptions);
132
- id: any;
133
- url: string;
134
- loadedUrl: any;
135
- uniqueKey: any;
136
- method: string;
137
- payload: string | Buffer | undefined;
138
- noRetry: any;
139
- retryCount: any;
140
- errorMessages: any[];
141
- /** @type {Object<string, string>} */
142
- headers: {
143
- [x: string]: string;
144
- };
145
- /** @type {Object<string, any>} */
146
- userData: {
147
- [x: string]: any;
148
- };
149
- handledAt: any;
150
- /**
151
- * Stores information about an error that occurred during processing of this request.
152
- *
153
- * You should always use Error instances when throwing errors in JavaScript.
154
- *
155
- * Nevertheless, to improve the debugging experience when using third party libraries
156
- * that may not always throw an Error instance, the function performs a type
157
- * inspection of the passed argument and attempts to extract as much information
158
- * as possible, since just throwing a bad type error makes any debugging rather difficult.
159
- *
160
- * @param {(Error|string)} errorOrMessage Error object or error message to be stored in the request.
161
- * @param {object} [options]
162
- * @param {boolean} [options.omitStack=false] Only push the error message without stack trace when true.
163
- */
164
- pushErrorMessage(errorOrMessage: (Error | string), options?: {
165
- omitStack?: boolean | undefined;
166
- } | undefined): void;
167
- /**
168
- * @ignore
169
- * @protected
170
- * @internal
171
- */
172
- protected _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }: {
173
- url: any;
174
- method: any;
175
- payload: any;
176
- keepUrlFragment: any;
177
- useExtendedUniqueKey: any;
178
- }): any;
179
- }
180
- //# sourceMappingURL=request.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.js"],"names":[],"mappings":"AASA,kDAOC;;;;;;;;;SAyNa,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAvMpB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH;IACI;;;OAGG;IACH,qBAHW,cAAc,EA8DxB;IApBG,QAAY;IACZ,YAAc;IACd,eAA0B;IAC1B,eAAqH;IACrH,eAAoB;IACpB,qCAAsB;IACtB,aAAsB;IACtB,gBAA4B;IAC5B,qBAAuC;IAGvC,qCAAqC;IACrC;YADkB,MAAM,GAAE,MAAM;MACH;IAC7B,kCAAkC;IAClC;YADkB,MAAM,GAAE,GAAG;MACE;IAG/B,eAEe;IAGnB;;;;;;;;;;;;;OAaG;IACH,iCAJW,CAAC,KAAK,GAAC,MAAM,CAAC;;yBAkCxB;IAED;;;;OAIG;IACH;;;;;;YAcC;CACJ"}
package/build/request.js DELETED
@@ -1,261 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.hashPayload = void 0;
4
- const tslib_1 = require("tslib");
5
- const utilities_1 = require("@apify/utilities");
6
- const crypto_1 = (0, tslib_1.__importDefault)(require("crypto"));
7
- const ow_1 = (0, tslib_1.__importStar)(require("ow"));
8
- const util_1 = (0, tslib_1.__importDefault)(require("util"));
9
- const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
10
- // new properties on the Request object breaks serialization
11
- const log = utils_log_1.default.child({ prefix: 'Request' });
12
- function hashPayload(payload) {
13
- return crypto_1.default
14
- .createHash('sha256')
15
- .update(payload)
16
- .digest('base64')
17
- .replace(/(\+|\/|=)/g, '')
18
- .substr(0, 8);
19
- }
20
- exports.hashPayload = hashPayload;
21
- const requestOptionalPredicates = {
22
- id: ow_1.default.optional.string,
23
- loadedUrl: ow_1.default.optional.string.url,
24
- uniqueKey: ow_1.default.optional.string,
25
- method: ow_1.default.optional.string,
26
- payload: ow_1.default.optional.any(ow_1.default.string, ow_1.default.buffer),
27
- noRetry: ow_1.default.optional.boolean,
28
- retryCount: ow_1.default.optional.number,
29
- errorMessages: ow_1.default.optional.array.ofType(ow_1.default.string),
30
- headers: ow_1.default.optional.object,
31
- userData: ow_1.default.optional.object,
32
- handledAt: ow_1.default.optional.any(ow_1.default.string.date, ow_1.default.date),
33
- keepUrlFragment: ow_1.default.optional.boolean,
34
- useExtendedUniqueKey: ow_1.default.optional.boolean,
35
- };
36
- /**
37
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
38
- * The `Request` object also stores information about errors that occurred during processing of the request.
39
- *
40
- * Each `Request` instance has the `uniqueKey` property, which can be either specified
41
- * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
42
- * are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
43
- * such as {@link RequestList}, {@link RequestQueue}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
44
- *
45
- * Example use:
46
- *
47
- * ```javascript
48
- * const request = new Apify.Request({
49
- * url: 'http://www.example.com',
50
- * headers: { Accept: 'application/json' },
51
- * });
52
- *
53
- * ...
54
- *
55
- * request.userData.foo = 'bar';
56
- * request.pushErrorMessage(new Error('Request failed!'));
57
- *
58
- * ...
59
- *
60
- * const foo = request.userData.foo;
61
- * ```
62
- *
63
- * @property {string} id
64
- * Request ID
65
- * @property {string} url
66
- * URL of the web page to crawl.
67
- * @property {string} loadedUrl
68
- * An actually loaded URL after redirects, if present. HTTP redirects are guaranteed
69
- * to be included.
70
- *
71
- * When using {@link PuppeteerCrawler} or {@link PlaywrightCrawler}, meta tag and JavaScript redirects may,
72
- * or may not be included, depending on their nature. This generally means that redirects,
73
- * which happen immediately will most likely be included, but delayed redirects will not.
74
- * @property {string} uniqueKey
75
- * A unique key identifying the request.
76
- * Two requests with the same `uniqueKey` are considered as pointing to the same URL.
77
- * @property {string} method
78
- * HTTP method, e.g. `GET` or `POST`.
79
- * @property {(string|Buffer)} payload
80
- * HTTP request payload, e.g. for POST requests.
81
- * @property {boolean} noRetry
82
- * The `true` value indicates that the request will not be automatically retried on error.
83
- * @property {number} retryCount
84
- * Indicates the number of times the crawling of the request has been retried on error.
85
- * @property {string[]} errorMessages
86
- * An array of error messages from request processing.
87
- * @property {Object<string, string>} headers
88
- * Object with HTTP headers. Key is header name, value is the value.
89
- * @property {Object<string, *>} userData
90
- * Custom user data assigned to the request.
91
- * @property {Date} handledAt
92
- * Indicates the time when the request has been processed.
93
- * Is `null` if the request has not been crawled yet.
94
- */
95
- class Request {
96
- /**
97
- * @param {RequestOptions} options
98
- * `Request` parameters including the URL, HTTP method and headers, and others.
99
- */
100
- constructor(options) {
101
- (0, ow_1.default)(options, 'RequestOptions', ow_1.default.object);
102
- (0, ow_1.default)(options.url, 'RequestOptions.url', ow_1.default.string.url);
103
- // 'ow' validation is slow, because it checks all predicates
104
- // even if the validated object has only 1 property.
105
- // This custom validation loop iterates only over existing
106
- // properties and speeds up the validation cca 3-fold.
107
- // See https://github.com/sindresorhus/ow/issues/193
108
- Object.keys(options).forEach((prop) => {
109
- const predicate = requestOptionalPredicates[prop];
110
- const value = options[prop];
111
- if (predicate) {
112
- (0, ow_1.default)(value, `RequestOptions.${prop}`, predicate);
113
- // 'url' is checked above because it's not optional
114
- }
115
- else if (prop !== 'url') {
116
- const msg = `Did not expect property \`${prop}\` to exist, got \`${value}\` in object \`RequestOptions\``;
117
- throw new ow_1.ArgumentError(msg, this.constructor);
118
- }
119
- });
120
- const { id, url, loadedUrl, uniqueKey, method = 'GET', payload, noRetry = false, retryCount = 0, errorMessages = [], headers = {}, userData = {}, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, } = options;
121
- if (method === 'GET' && payload)
122
- throw new Error('Request with GET method cannot have a payload.');
123
- this.id = id;
124
- this.url = url;
125
- this.loadedUrl = loadedUrl;
126
- this.uniqueKey = uniqueKey || this._computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
127
- this.method = method;
128
- this.payload = payload;
129
- this.noRetry = noRetry;
130
- this.retryCount = retryCount;
131
- this.errorMessages = [...errorMessages];
132
- // @property are ignored when reassigning, needs to enforced set again,
133
- // otherwise the type will be {}
134
- /** @type {Object<string, string>} */
135
- this.headers = { ...headers };
136
- /** @type {Object<string, any>} */
137
- this.userData = { ...userData };
138
- // Requests received from API will have ISOString dates,
139
- // but we want to have a Date instance.
140
- this.handledAt = typeof handledAt === 'string'
141
- ? new Date(handledAt)
142
- : handledAt;
143
- }
144
- /**
145
- * Stores information about an error that occurred during processing of this request.
146
- *
147
- * You should always use Error instances when throwing errors in JavaScript.
148
- *
149
- * Nevertheless, to improve the debugging experience when using third party libraries
150
- * that may not always throw an Error instance, the function performs a type
151
- * inspection of the passed argument and attempts to extract as much information
152
- * as possible, since just throwing a bad type error makes any debugging rather difficult.
153
- *
154
- * @param {(Error|string)} errorOrMessage Error object or error message to be stored in the request.
155
- * @param {object} [options]
156
- * @param {boolean} [options.omitStack=false] Only push the error message without stack trace when true.
157
- */
158
- pushErrorMessage(errorOrMessage, options = {}) {
159
- const { omitStack } = options;
160
- let message;
161
- const type = typeof errorOrMessage;
162
- if (type === 'object') {
163
- if (!errorOrMessage) {
164
- message = 'null';
165
- }
166
- else if (errorOrMessage instanceof Error) {
167
- message = omitStack
168
- ? errorOrMessage.message
169
- // .stack includes the message
170
- : errorOrMessage.stack;
171
- }
172
- else if (errorOrMessage.message) {
173
- message = errorOrMessage.message; // eslint-disable-line prefer-destructuring
174
- }
175
- else if (errorOrMessage.toString() !== '[object Object]') {
176
- message = errorOrMessage.toString();
177
- }
178
- else {
179
- try {
180
- message = util_1.default.inspect(errorOrMessage);
181
- }
182
- catch (err) {
183
- message = 'Unable to extract any message from the received object.';
184
- }
185
- }
186
- }
187
- else if (type === 'undefined') {
188
- message = 'undefined';
189
- }
190
- else {
191
- message = errorOrMessage.toString();
192
- }
193
- this.errorMessages.push(message);
194
- }
195
- /**
196
- * @ignore
197
- * @protected
198
- * @internal
199
- */
200
- _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }) {
201
- const normalizedMethod = method.toUpperCase();
202
- const normalizedUrl = (0, utilities_1.normalizeUrl)(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
203
- if (!useExtendedUniqueKey) {
204
- if (normalizedMethod !== 'GET' && payload) {
205
- // Using log.deprecated to log only once. We should add log.once or some such.
206
- log.deprecated(`We've encountered a ${normalizedMethod} Request with a payload. `
207
- + 'This is fine. Just letting you know that if your requests point to the same URL '
208
- + 'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
209
- }
210
- return normalizedUrl;
211
- }
212
- const payloadHash = payload ? hashPayload(payload) : '';
213
- return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
214
- }
215
- }
216
- exports.default = Request;
217
- /**
218
- * Specifies required and optional fields for constructing a {@link Request}.
219
- *
220
- * @typedef RequestOptions
221
- * @property {string} url URL of the web page to crawl. It must be a non-empty string.
222
- * @property {string} [uniqueKey] A unique key identifying the request.
223
- * Two requests with the same `uniqueKey` are considered as pointing to the same URL.
224
- *
225
- * If `uniqueKey` is not provided, then it is automatically generated by normalizing the URL.
226
- * For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `uniqueKey`
227
- * of `http://www.example.com/something`.
228
- *
229
- * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
230
- *
231
- * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
232
- * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
233
- * when requests point to the same URL, but with different methods and payloads. For example: form submits.
234
- *
235
- * Pass an arbitrary non-empty text value to the `uniqueKey` property
236
- * to override the default behavior and specify which URLs shall be considered equal.
237
- * @property {string} [method='GET']
238
- * @property {(string|Buffer)} [payload]
239
- * HTTP request payload, e.g. for POST requests.
240
- * @property {Object<string,string>} [headers]
241
- * HTTP headers in the following format:
242
- * ```
243
- * {
244
- * Accept: 'text/html',
245
- * 'Content-Type': 'application/json'
246
- * }
247
- * ```
248
- * @property {Object<string,*>} [userData]
249
- * Custom user data assigned to the request. Use this to save any request related data to the
250
- * request's scope, keeping them accessible on retries, failures etc.
251
- * @property {boolean} [keepUrlFragment=false]
252
- * If `false` then the hash part of a URL is removed when computing the `uniqueKey` property.
253
- * For example, this causes the `http://www.example.com#foo` and `http://www.example.com#bar` URLs
254
- * to have the same `uniqueKey` of `http://www.example.com` and thus the URLs are considered equal.
255
- * Note that this option only has an effect if `uniqueKey` is not set.
256
- * @property {boolean} [useExtendedUniqueKey=false]
257
- * If `true` then the `uniqueKey` is computed not only from the URL, but also from the method and payload
258
- * properties. This is useful when making requests to the same URL that are differentiated by method
259
- * or payload, such as form submit navigations in browsers.
260
- */
261
- //# sourceMappingURL=request.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.js"],"names":[],"mappings":";;;;AAAA,gDAAgD;AAChD,iEAA4B;AAC5B,sDAAuC;AACvC,6DAAwB;AACxB,yEAAqC;AAErC,4DAA4D;AAC5D,MAAM,GAAG,GAAG,mBAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAEpD,SAAgB,WAAW,CAAC,OAAO;IAC/B,OAAO,gBAAM;SACR,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC;SACf,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;SACzB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtB,CAAC;AAPD,kCAOC;AAED,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,CAAC,IAAI,EAAE,YAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;CAC5C,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AACH,MAAM,OAAO;IACT;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,gBAAgB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACzC,IAAA,YAAE,EAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACrD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAClC,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE;gBACX,IAAA,YAAE,EAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAAS,CAAC,CAAC;gBAC/C,mDAAmD;aACtD;iBAAM,IAAI,IAAI,KAAK,KAAK,EAAE;gBACvB,MAAM,GAAG,GAAG,6BAA6B,IAAI,sBAAsB,KAAK,iCAAiC,CAAC;gBAC1G,MAAM,IAAI,kBAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;aAClD;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,MAAM,GAAG,KAAK,EACd,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,GAC/B,GAAG,OAAO,CAAC;QAEZ,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACtH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,uEAAuE;QACvE,gCAAgC;QAChC,qCAAqC;QACrC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,kCAAkC;QAClC,IAAI,CAAC,QAAQ,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAChC,wDAAwD;QACxD,uCAAuC;QACvC,IAAI,CAAC,SAAS,GAAG,OAAO,SAAS,KAAK,QAAQ;YAC1C,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC;YACrB,CAAC,CAAC,SAAS,CAAC;IACpB,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,GAAG,EAAE;QACzC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,IAAI,CAAC,cAAc,EAAE;gBACjB,OAAO,GAAG,MAAM,CAAC;aACpB;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE;gBACxC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,8BAA8B;oBAC9B,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC;aAC9B;iBAAM,IAAI,cAAc,CAAC,OAAO,EAAE;gBAC/B,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,2CAA2C;aAChF;iBAAM,IAAI,cAAc,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE;gBACxD,OAAO,GAAG,cAAc,CAAC,QAAQ,EAAE,CAAC;aACvC;iBAAM;gBACH,IAAI;oBACA,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;iBAC1C;gBAAC,OAAO,GAAG,EAAE;oBACV,OAAO,GAAG,yDAAyD,CAAC;iBACvE;aACJ;SACJ;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE;YAC7B,OAAO,GAAG,WAAW,CAAC;SACzB;aAAM;YACH,OAAO,GAAG,cAAc,CAAC,QAAQ,EAAE,CAAC;SACvC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE;QAC7E,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAA,wBAAY,EAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE;YACvB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE;gBACvC,8EAA8E;gBAC9E,GAAG,CAAC,UAAU,CAAC,uBAAuB,gBAAgB,2BAA2B;sBAC3E,kFAAkF;sBAClF,iHAAiH,CAAC,CAAC;aAC5H;YACD,OAAO,aAAa,CAAC;SACxB;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;CACJ;AAED,kBAAe,OAAO,CAAC;AAEvB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG"}