@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/autoscaling/autoscaled_pool.d.ts +296 -0
  2. package/autoscaling/autoscaled_pool.d.ts.map +1 -0
  3. package/autoscaling/autoscaled_pool.js +673 -0
  4. package/autoscaling/autoscaled_pool.js.map +1 -0
  5. package/autoscaling/index.d.ts +4 -0
  6. package/autoscaling/index.d.ts.map +1 -0
  7. package/autoscaling/index.js +7 -0
  8. package/autoscaling/index.js.map +1 -0
  9. package/autoscaling/snapshotter.d.ts +193 -0
  10. package/autoscaling/snapshotter.d.ts.map +1 -0
  11. package/autoscaling/snapshotter.js +377 -0
  12. package/autoscaling/snapshotter.js.map +1 -0
  13. package/autoscaling/system_status.d.ts +181 -0
  14. package/autoscaling/system_status.d.ts.map +1 -0
  15. package/autoscaling/system_status.js +194 -0
  16. package/autoscaling/system_status.js.map +1 -0
  17. package/cache_container.d.ts +18 -0
  18. package/cache_container.d.ts.map +1 -0
  19. package/cache_container.js +42 -0
  20. package/cache_container.js.map +1 -0
  21. package/configuration.d.ts +150 -0
  22. package/configuration.d.ts.map +1 -0
  23. package/configuration.js +284 -0
  24. package/configuration.js.map +1 -0
  25. package/constants.d.ts +3 -0
  26. package/constants.d.ts.map +1 -0
  27. package/constants.js +6 -0
  28. package/constants.js.map +1 -0
  29. package/crawlers/crawler_commons.d.ts +25 -0
  30. package/crawlers/crawler_commons.d.ts.map +1 -0
  31. package/crawlers/crawler_commons.js +3 -0
  32. package/crawlers/crawler_commons.js.map +1 -0
  33. package/crawlers/crawler_extension.d.ts +11 -0
  34. package/crawlers/crawler_extension.d.ts.map +1 -0
  35. package/crawlers/crawler_extension.js +30 -0
  36. package/crawlers/crawler_extension.js.map +1 -0
  37. package/crawlers/crawler_utils.d.ts +20 -0
  38. package/crawlers/crawler_utils.d.ts.map +1 -0
  39. package/crawlers/crawler_utils.js +61 -0
  40. package/crawlers/crawler_utils.js.map +1 -0
  41. package/crawlers/index.d.ts +5 -0
  42. package/crawlers/index.d.ts.map +1 -0
  43. package/crawlers/index.js +8 -0
  44. package/crawlers/index.js.map +1 -0
  45. package/crawlers/statistics.d.ts +147 -0
  46. package/crawlers/statistics.d.ts.map +1 -0
  47. package/crawlers/statistics.js +360 -0
  48. package/crawlers/statistics.js.map +1 -0
  49. package/enqueue_links/enqueue_links.d.ts +151 -0
  50. package/enqueue_links/enqueue_links.d.ts.map +1 -0
  51. package/enqueue_links/enqueue_links.js +152 -0
  52. package/enqueue_links/enqueue_links.js.map +1 -0
  53. package/enqueue_links/index.d.ts +3 -0
  54. package/enqueue_links/index.d.ts.map +1 -0
  55. package/enqueue_links/index.js +6 -0
  56. package/enqueue_links/index.js.map +1 -0
  57. package/enqueue_links/shared.d.ts +60 -0
  58. package/enqueue_links/shared.d.ts.map +1 -0
  59. package/enqueue_links/shared.js +148 -0
  60. package/enqueue_links/shared.js.map +1 -0
  61. package/errors.d.ts +8 -0
  62. package/errors.d.ts.map +1 -0
  63. package/errors.js +12 -0
  64. package/errors.js.map +1 -0
  65. package/events/event_manager.d.ts +41 -0
  66. package/events/event_manager.d.ts.map +1 -0
  67. package/events/event_manager.js +103 -0
  68. package/events/event_manager.js.map +1 -0
  69. package/events/index.d.ts +3 -0
  70. package/events/index.d.ts.map +1 -0
  71. package/events/index.js +6 -0
  72. package/events/index.js.map +1 -0
  73. package/events/local_event_manager.d.ts +32 -0
  74. package/events/local_event_manager.d.ts.map +1 -0
  75. package/events/local_event_manager.js +114 -0
  76. package/events/local_event_manager.js.map +1 -0
  77. package/index.d.ts +14 -0
  78. package/index.d.ts.map +1 -0
  79. package/index.js +17 -0
  80. package/index.js.map +1 -0
  81. package/index.mjs +61 -0
  82. package/log.d.ts +3 -0
  83. package/log.d.ts.map +1 -0
  84. package/log.js +12 -0
  85. package/log.js.map +1 -0
  86. package/package.json +4 -8
  87. package/proxy_configuration.d.ts +181 -0
  88. package/proxy_configuration.d.ts.map +1 -0
  89. package/proxy_configuration.js +209 -0
  90. package/proxy_configuration.js.map +1 -0
  91. package/request.d.ts +185 -0
  92. package/request.d.ts.map +1 -0
  93. package/request.js +330 -0
  94. package/request.js.map +1 -0
  95. package/serialization.d.ts +33 -0
  96. package/serialization.d.ts.map +1 -0
  97. package/serialization.js +146 -0
  98. package/serialization.js.map +1 -0
  99. package/session_pool/errors.d.ts +8 -0
  100. package/session_pool/errors.d.ts.map +1 -0
  101. package/session_pool/errors.js +20 -0
  102. package/session_pool/errors.js.map +1 -0
  103. package/session_pool/events.d.ts +3 -0
  104. package/session_pool/events.d.ts.map +1 -0
  105. package/session_pool/events.js +6 -0
  106. package/session_pool/events.js.map +1 -0
  107. package/session_pool/index.d.ts +6 -0
  108. package/session_pool/index.d.ts.map +1 -0
  109. package/session_pool/index.js +9 -0
  110. package/session_pool/index.js.map +1 -0
  111. package/session_pool/session.d.ts +212 -0
  112. package/session_pool/session.d.ts.map +1 -0
  113. package/session_pool/session.js +360 -0
  114. package/session_pool/session.js.map +1 -0
  115. package/session_pool/session_pool.d.ts +222 -0
  116. package/session_pool/session_pool.d.ts.map +1 -0
  117. package/session_pool/session_pool.js +389 -0
  118. package/session_pool/session_pool.js.map +1 -0
  119. package/session_pool/session_utils.d.ts +9 -0
  120. package/session_pool/session_utils.d.ts.map +1 -0
  121. package/session_pool/session_utils.js +19 -0
  122. package/session_pool/session_utils.js.map +1 -0
  123. package/storages/dataset.d.ts +317 -0
  124. package/storages/dataset.d.ts.map +1 -0
  125. package/storages/dataset.js +367 -0
  126. package/storages/dataset.js.map +1 -0
  127. package/storages/index.d.ts +7 -0
  128. package/storages/index.d.ts.map +1 -0
  129. package/storages/index.js +11 -0
  130. package/storages/index.js.map +1 -0
  131. package/storages/key_value_store.d.ts +242 -0
  132. package/storages/key_value_store.d.ts.map +1 -0
  133. package/storages/key_value_store.js +320 -0
  134. package/storages/key_value_store.js.map +1 -0
  135. package/storages/request_list.d.ts +469 -0
  136. package/storages/request_list.d.ts.map +1 -0
  137. package/storages/request_list.js +724 -0
  138. package/storages/request_list.js.map +1 -0
  139. package/storages/request_queue.d.ts +315 -0
  140. package/storages/request_queue.d.ts.map +1 -0
  141. package/storages/request_queue.js +730 -0
  142. package/storages/request_queue.js.map +1 -0
  143. package/storages/storage.d.ts +260 -0
  144. package/storages/storage.d.ts.map +1 -0
  145. package/storages/storage.js +3 -0
  146. package/storages/storage.js.map +1 -0
  147. package/storages/storage_manager.d.ts +40 -0
  148. package/storages/storage_manager.d.ts.map +1 -0
  149. package/storages/storage_manager.js +119 -0
  150. package/storages/storage_manager.js.map +1 -0
  151. package/tsconfig.build.tsbuildinfo +1 -0
  152. package/typedefs.d.ts +12 -0
  153. package/typedefs.d.ts.map +1 -0
  154. package/typedefs.js +15 -0
  155. package/typedefs.js.map +1 -0
  156. package/validators.d.ts +21 -0
  157. package/validators.d.ts.map +1 -0
  158. package/validators.js +26 -0
  159. package/validators.js.map +1 -0
@@ -0,0 +1,209 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ProxyConfiguration = void 0;
4
+ const tslib_1 = require("tslib");
5
+ const ow_1 = tslib_1.__importDefault(require("ow"));
6
+ const log_1 = tslib_1.__importDefault(require("@apify/log"));
7
+ const configuration_1 = require("./configuration");
8
+ /**
9
+ * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
10
+ * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
11
+ * them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
12
+ * the {@link ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
13
+ *
14
+ * If you want to use your own proxies, use the {@link ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
15
+ * be rotated by the configuration if this option is provided.
16
+ *
17
+ * **Example usage:**
18
+ *
19
+ * ```javascript
20
+ *
21
+ * const proxyConfiguration = new ProxyConfiguration({
22
+ * proxyUrls: ['...', '...'],
23
+ * });
24
+ *
25
+ * const crawler = new CheerioCrawler({
26
+ * // ...
27
+ * proxyConfiguration,
28
+ * handlePageFunction: ({ proxyInfo }) => {
29
+ * const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
30
+ * }
31
+ * })
32
+ *
33
+ * ```
34
+ * @category Scaling
35
+ */
36
+ class ProxyConfiguration {
37
+ /**
38
+ * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
39
+ * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
40
+ * them to use the selected proxies for all connections.
41
+ *
42
+ * ```javascript
43
+ * const proxyConfiguration = new ProxyConfiguration({
44
+ * proxyUrls: ['http://user:pass@proxy-1.com', 'http://user:pass@proxy-2.com'],
45
+ * });
46
+ *
47
+ * const crawler = new CheerioCrawler({
48
+ * // ...
49
+ * proxyConfiguration,
50
+ * handlePageFunction: ({ proxyInfo }) => {
51
+ * const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
52
+ * }
53
+ * })
54
+ *
55
+ * ```
56
+ * @param options
57
+ * @param config
58
+ * @param validateRequired internal flag to disable validation that requires one of `proxyUrls` or `newUrlFunction`, defaults to true
59
+ */
60
+ constructor(options = {}, config = configuration_1.Configuration.getGlobalConfig(), validateRequired = true) {
61
+ Object.defineProperty(this, "config", {
62
+ enumerable: true,
63
+ configurable: true,
64
+ writable: true,
65
+ value: config
66
+ });
67
+ Object.defineProperty(this, "isManInTheMiddle", {
68
+ enumerable: true,
69
+ configurable: true,
70
+ writable: true,
71
+ value: false
72
+ });
73
+ Object.defineProperty(this, "nextCustomUrlIndex", {
74
+ enumerable: true,
75
+ configurable: true,
76
+ writable: true,
77
+ value: 0
78
+ });
79
+ Object.defineProperty(this, "proxyUrls", {
80
+ enumerable: true,
81
+ configurable: true,
82
+ writable: true,
83
+ value: void 0
84
+ });
85
+ Object.defineProperty(this, "usedProxyUrls", {
86
+ enumerable: true,
87
+ configurable: true,
88
+ writable: true,
89
+ value: new Map()
90
+ });
91
+ Object.defineProperty(this, "newUrlFunction", {
92
+ enumerable: true,
93
+ configurable: true,
94
+ writable: true,
95
+ value: void 0
96
+ });
97
+ Object.defineProperty(this, "log", {
98
+ enumerable: true,
99
+ configurable: true,
100
+ writable: true,
101
+ value: log_1.default.child({ prefix: 'ProxyConfiguration' })
102
+ });
103
+ (0, ow_1.default)(options, ow_1.default.object.exactShape({
104
+ proxyUrls: ow_1.default.optional.array.nonEmpty.ofType(ow_1.default.string.url),
105
+ newUrlFunction: ow_1.default.optional.function,
106
+ }));
107
+ const { proxyUrls, newUrlFunction } = options;
108
+ if (proxyUrls && newUrlFunction)
109
+ this._throwCannotCombineCustomMethods();
110
+ if (!proxyUrls && !newUrlFunction && validateRequired)
111
+ this._throwNoOptionsProvided();
112
+ this.proxyUrls = proxyUrls;
113
+ this.newUrlFunction = newUrlFunction;
114
+ }
115
+ /**
116
+ * This function creates a new {@link ProxyInfo} info object.
117
+ * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
118
+ * the currently used proxy via the handlePageFunction parameter: proxyInfo.
119
+ * Use it if you want to work with a rich representation of a proxy URL.
120
+ * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
121
+ * @param [sessionId]
122
+ * Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
123
+ * you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
124
+ * When the provided sessionId is a number, it's converted to a string. Property sessionId of
125
+ * {@link ProxyInfo} is always returned as a type string.
126
+ *
127
+ * All the HTTP requests going through the proxy with the same session identifier
128
+ * will use the same target proxy server (i.e. the same IP address).
129
+ * The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
130
+ * @return Represents information about used proxy and its configuration.
131
+ */
132
+ async newProxyInfo(sessionId) {
133
+ if (typeof sessionId === 'number')
134
+ sessionId = `${sessionId}`;
135
+ const url = await this.newUrl(sessionId);
136
+ const { username, password, port, hostname } = new URL(url);
137
+ return {
138
+ sessionId,
139
+ url,
140
+ username,
141
+ password,
142
+ hostname,
143
+ port: port,
144
+ };
145
+ }
146
+ /**
147
+ * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
148
+ * @param [sessionId]
149
+ * Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
150
+ * you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
151
+ * When the provided sessionId is a number, it's converted to a string.
152
+ *
153
+ * All the HTTP requests going through the proxy with the same session identifier
154
+ * will use the same target proxy server (i.e. the same IP address).
155
+ * The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
156
+ * @return A string with a proxy URL, including authentication credentials and port number.
157
+ * For example, `http://bob:password123@proxy.example.com:8000`
158
+ */
159
+ async newUrl(sessionId) {
160
+ if (typeof sessionId === 'number')
161
+ sessionId = `${sessionId}`;
162
+ if (this.newUrlFunction) {
163
+ return this._callNewUrlFunction(sessionId);
164
+ }
165
+ return this._handleCustomUrl(sessionId);
166
+ }
167
+ /**
168
+ * Handles custom url rotation with session
169
+ */
170
+ _handleCustomUrl(sessionId) {
171
+ let customUrlToUse;
172
+ if (!sessionId) {
173
+ return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
174
+ }
175
+ if (this.usedProxyUrls.has(sessionId)) {
176
+ customUrlToUse = this.usedProxyUrls.get(sessionId);
177
+ }
178
+ else {
179
+ customUrlToUse = this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
180
+ this.usedProxyUrls.set(sessionId, customUrlToUse);
181
+ }
182
+ return customUrlToUse;
183
+ }
184
+ /**
185
+ * Calls the custom newUrlFunction and checks format of its return value
186
+ */
187
+ async _callNewUrlFunction(sessionId) {
188
+ let proxyUrl;
189
+ try {
190
+ proxyUrl = await this.newUrlFunction(sessionId);
191
+ new URL(proxyUrl); // eslint-disable-line no-new
192
+ return proxyUrl;
193
+ }
194
+ catch (err) {
195
+ this._throwNewUrlFunctionInvalid(err);
196
+ }
197
+ }
198
+ _throwNewUrlFunctionInvalid(err) {
199
+ throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
200
+ }
201
+ _throwCannotCombineCustomMethods() {
202
+ throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
203
+ }
204
+ _throwNoOptionsProvided() {
205
+ throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
206
+ }
207
+ }
208
+ exports.ProxyConfiguration = ProxyConfiguration;
209
+ //# sourceMappingURL=proxy_configuration.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,6DAA6B;AAC7B,mDAAgD;AAsFhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,kBAAkB;IAQ3B;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,YAAY,UAAqC,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE,EAAE,gBAAgB,GAAG,IAAI;;;;;mBAAjE;;QA9B9D;;;;mBAAmB,KAAK;WAAC;QACzB;;;;mBAA+B,CAAC;WAAC;QACjC;;;;;WAA+B;QAC/B;;;;mBAA0B,IAAI,GAAG,EAAkB;WAAC;QACpD;;;;;WAAsD;QACtD;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;WAAC;QA0BxD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3D,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAE9C,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACzE,IAAI,CAAC,SAAS,IAAI,CAAC,cAAc,IAAI,gBAAgB;YAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAEtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,YAAY,CAAC,SAA2B;QAC1C,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEzC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO;YACH,SAAS;YACT,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,IAAI,EAAE,IAAK;SACd,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,MAAM,CAAC,SAA2B;QACpC,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAE9D,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAE,CAAC;SAC/C;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,gBAAgB,CAAC,SAAkB;QACzC,IAAI,cAAsB,CAAC;QAE3B,IAAI,CAAC,SAAS,EAAE;YACZ,OAAO,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;SAC9E;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACnC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;SACvD;aAAM;YACH,cAAc,GAAG,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;YACrF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;SACrD;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,SAAkB;QAClD,IAAI,QAAgB,CAAC;QAErB,IAAI;YACA,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,SAAU,CAAC,CAAC;YAClD,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;YAChD,OAAO,QAAQ,CAAC;SACnB;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,2BAA2B,CAAC,GAAY,CAAC,CAAC;SAClD;IACL,CAAC;IAES,2BAA2B,CAAC,GAAU;QAC5C,MAAM,IAAI,KAAK,CAAC,mEAAmE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACtG,CAAC;IAES,gCAAgC;QACtC,MAAM,IAAI,KAAK,CAAC,6GAA6G,CAAC,CAAC;IACnI,CAAC;IAES,uBAAuB;QAC7B,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IACpG,CAAC;CACJ;AApJD,gDAoJC"}
package/request.d.ts ADDED
@@ -0,0 +1,185 @@
1
+ /// <reference types="node" />
2
+ import { BinaryLike } from 'crypto';
3
+ import { AllowedHttpMethods } from './typedefs';
4
+ /**
5
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
6
+ * The `Request` object also stores information about errors that occurred during processing of the request.
7
+ *
8
+ * Each `Request` instance has the `uniqueKey` property, which can be either specified
9
+ * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
10
+ * are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
11
+ * such as {@link RequestList}, {@link RequestQueue}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
12
+ *
13
+ * Example use:
14
+ *
15
+ * ```javascript
16
+ * const request = new Request({
17
+ * url: 'http://www.example.com',
18
+ * headers: { Accept: 'application/json' },
19
+ * });
20
+ *
21
+ * ...
22
+ *
23
+ * request.userData.foo = 'bar';
24
+ * request.pushErrorMessage(new Error('Request failed!'));
25
+ *
26
+ * ...
27
+ *
28
+ * const foo = request.userData.foo;
29
+ * ```
30
+ * @category Sources
31
+ */
32
+ export declare class Request {
33
+ /** Request ID */
34
+ id?: string;
35
+ /** URL of the web page to crawl. */
36
+ url: string;
37
+ /**
38
+ * An actually loaded URL after redirects, if present. HTTP redirects are guaranteed
39
+ * to be included.
40
+ *
41
+ * When using {@link PuppeteerCrawler} or {@link PlaywrightCrawler}, meta tag and JavaScript redirects may,
42
+ * or may not be included, depending on their nature. This generally means that redirects,
43
+ * which happen immediately will most likely be included, but delayed redirects will not.
44
+ */
45
+ loadedUrl?: string;
46
+ /**
47
+ * A unique key identifying the request.
48
+ * Two requests with the same `uniqueKey` are considered as pointing to the same URL.
49
+ */
50
+ uniqueKey: string;
51
+ /** HTTP method, e.g. `GET` or `POST`. */
52
+ method: AllowedHttpMethods;
53
+ /** HTTP request payload, e.g. for POST requests. */
54
+ payload?: string;
55
+ /** The `true` value indicates that the request will not be automatically retried on error. */
56
+ noRetry: boolean;
57
+ /** Indicates the number of times the crawling of the request has been retried on error. */
58
+ retryCount: number;
59
+ /** An array of error messages from request processing. */
60
+ errorMessages: string[];
61
+ /** Object with HTTP headers. Key is header name, value is the value. */
62
+ headers?: Record<string, string>;
63
+ /** Private store for the custom user data assigned to the request. */
64
+ private _userData;
65
+ /** Custom user data assigned to the request. */
66
+ userData: Record<string, any>;
67
+ /**
68
+ * ISO datetime string that indicates the time when the request has been processed.
69
+ * Is `null` if the request has not been crawled yet.
70
+ */
71
+ handledAt?: string;
72
+ /**
73
+ * `Request` parameters including the URL, HTTP method and headers, and others.
74
+ */
75
+ constructor(options: RequestOptions);
76
+ /** Tells the crawler processing this request to skip the navigation and process the request directly. */
77
+ get skipNavigation(): boolean;
78
+ set skipNavigation(value: boolean);
79
+ /**
80
+ * Stores information about an error that occurred during processing of this request.
81
+ *
82
+ * You should always use Error instances when throwing errors in JavaScript.
83
+ *
84
+ * Nevertheless, to improve the debugging experience when using third party libraries
85
+ * that may not always throw an Error instance, the function performs a type
86
+ * inspection of the passed argument and attempts to extract as much information
87
+ * as possible, since just throwing a bad type error makes any debugging rather difficult.
88
+ *
89
+ * @param errorOrMessage Error object or error message to be stored in the request.
90
+ * @param [options]
91
+ */
92
+ pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
93
+ protected _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }: ComputeUniqueKeyOptions): string;
94
+ protected _hashPayload(payload: BinaryLike): string;
95
+ }
96
+ /**
97
+ * Specifies required and optional fields for constructing a {@link Request}.
98
+ */
99
+ export interface RequestOptions {
100
+ /** URL of the web page to crawl. It must be a non-empty string. */
101
+ url: string;
102
+ /**
103
+ * A unique key identifying the request.
104
+ * Two requests with the same `uniqueKey` are considered as pointing to the same URL.
105
+ *
106
+ * If `uniqueKey` is not provided, then it is automatically generated by normalizing the URL.
107
+ * For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `uniqueKey`
108
+ * of `http://www.example.com/something`.
109
+ *
110
+ * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
111
+ *
112
+ * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
113
+ * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
114
+ * when requests point to the same URL, but with different methods and payloads. For example: form submits.
115
+ *
116
+ * Pass an arbitrary non-empty text value to the `uniqueKey` property
117
+ * to override the default behavior and specify which URLs shall be considered equal.
118
+ */
119
+ uniqueKey?: string;
120
+ /** @default 'GET' */
121
+ method?: AllowedHttpMethods | Lowercase<AllowedHttpMethods>;
122
+ /** HTTP request payload, e.g. for POST requests. */
123
+ payload?: string;
124
+ /**
125
+ * HTTP headers in the following format:
126
+ * ```
127
+ * {
128
+ * Accept: 'text/html',
129
+ * 'Content-Type': 'application/json'
130
+ * }
131
+ * ```
132
+ */
133
+ headers?: Record<string, string>;
134
+ /**
135
+ * Custom user data assigned to the request. Use this to save any request related data to the
136
+ * request's scope, keeping them accessible on retries, failures etc.
137
+ */
138
+ userData?: Record<string, any>;
139
+ /**
140
+ * If `false` then the hash part of a URL is removed when computing the `uniqueKey` property.
141
+ * For example, this causes the `http://www.example.com#foo` and `http://www.example.com#bar` URLs
142
+ * to have the same `uniqueKey` of `http://www.example.com` and thus the URLs are considered equal.
143
+ * Note that this option only has an effect if `uniqueKey` is not set.
144
+ * @default false
145
+ */
146
+ keepUrlFragment?: boolean;
147
+ /**
148
+ * If `true` then the `uniqueKey` is computed not only from the URL, but also from the method and payload
149
+ * properties. This is useful when making requests to the same URL that are differentiated by method
150
+ * or payload, such as form submit navigations in browsers.
151
+ * @default false
152
+ */
153
+ useExtendedUniqueKey?: boolean;
154
+ /**
155
+ * The `true` value indicates that the request will not be automatically retried on error.
156
+ * @default false
157
+ */
158
+ noRetry?: boolean;
159
+ /**
160
+ * If set to `true` then the crawler processing this request evaluates
161
+ * the `handleRequestFunction` immediately without prior browser navigation.
162
+ * @default false
163
+ */
164
+ skipNavigation?: boolean;
165
+ /** @internal */
166
+ id?: string;
167
+ /** @internal */
168
+ handledAt?: string;
169
+ }
170
+ export interface PushErrorMessageOptions {
171
+ /**
172
+ * Only push the error message without stack trace when true.
173
+ * @default false
174
+ */
175
+ omitStack?: boolean;
176
+ }
177
+ interface ComputeUniqueKeyOptions {
178
+ url: string;
179
+ method: AllowedHttpMethods;
180
+ payload?: string | Buffer;
181
+ keepUrlFragment?: boolean;
182
+ useExtendedUniqueKey?: boolean;
183
+ }
184
+ export {};
185
+ //# sourceMappingURL=request.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";AACA,OAAe,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAI5C,OAAO,EAAE,kBAAkB,EAAQ,MAAM,YAAY,CAAC;AAsBtD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,OAAO;IAChB,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAM;IAEnC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAqGnC,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAG5B;IAED,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAKhC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,SAAS,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,EAAE,uBAAuB;IAgBpH,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAQtD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAE3B,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE/B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CAEtB;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC"}