apify 2.3.1-beta.4 → 3.0.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. package/README.md +6 -5
  2. package/package.json +69 -128
  3. package/build/actor.d.ts +0 -113
  4. package/build/actor.d.ts.map +0 -1
  5. package/build/actor.js +0 -582
  6. package/build/actor.js.map +0 -1
  7. package/build/apify.d.ts +0 -752
  8. package/build/apify.d.ts.map +0 -1
  9. package/build/apify.js +0 -877
  10. package/build/apify.js.map +0 -1
  11. package/build/autoscaling/autoscaled_pool.d.ts +0 -384
  12. package/build/autoscaling/autoscaled_pool.d.ts.map +0 -1
  13. package/build/autoscaling/autoscaled_pool.js +0 -557
  14. package/build/autoscaling/autoscaled_pool.js.map +0 -1
  15. package/build/autoscaling/snapshotter.d.ts +0 -278
  16. package/build/autoscaling/snapshotter.d.ts.map +0 -1
  17. package/build/autoscaling/snapshotter.js +0 -447
  18. package/build/autoscaling/snapshotter.js.map +0 -1
  19. package/build/autoscaling/system_status.d.ts +0 -224
  20. package/build/autoscaling/system_status.d.ts.map +0 -1
  21. package/build/autoscaling/system_status.js +0 -228
  22. package/build/autoscaling/system_status.js.map +0 -1
  23. package/build/browser_launchers/browser_launcher.d.ts +0 -154
  24. package/build/browser_launchers/browser_launcher.d.ts.map +0 -1
  25. package/build/browser_launchers/browser_launcher.js +0 -160
  26. package/build/browser_launchers/browser_launcher.js.map +0 -1
  27. package/build/browser_launchers/browser_plugin.d.ts +0 -23
  28. package/build/browser_launchers/browser_plugin.d.ts.map +0 -1
  29. package/build/browser_launchers/browser_plugin.js +0 -25
  30. package/build/browser_launchers/browser_plugin.js.map +0 -1
  31. package/build/browser_launchers/playwright_launcher.d.ts +0 -131
  32. package/build/browser_launchers/playwright_launcher.d.ts.map +0 -1
  33. package/build/browser_launchers/playwright_launcher.js +0 -150
  34. package/build/browser_launchers/playwright_launcher.js.map +0 -1
  35. package/build/browser_launchers/puppeteer_launcher.d.ts +0 -153
  36. package/build/browser_launchers/puppeteer_launcher.d.ts.map +0 -1
  37. package/build/browser_launchers/puppeteer_launcher.js +0 -197
  38. package/build/browser_launchers/puppeteer_launcher.js.map +0 -1
  39. package/build/cache_container.d.ts +0 -31
  40. package/build/cache_container.d.ts.map +0 -1
  41. package/build/cache_container.js +0 -48
  42. package/build/cache_container.js.map +0 -1
  43. package/build/configuration.d.ts +0 -226
  44. package/build/configuration.d.ts.map +0 -1
  45. package/build/configuration.js +0 -325
  46. package/build/configuration.js.map +0 -1
  47. package/build/constants.d.ts +0 -37
  48. package/build/constants.d.ts.map +0 -1
  49. package/build/constants.js +0 -41
  50. package/build/constants.js.map +0 -1
  51. package/build/crawlers/basic_crawler.d.ts +0 -443
  52. package/build/crawlers/basic_crawler.d.ts.map +0 -1
  53. package/build/crawlers/basic_crawler.js +0 -664
  54. package/build/crawlers/basic_crawler.js.map +0 -1
  55. package/build/crawlers/browser_crawler.d.ts +0 -512
  56. package/build/crawlers/browser_crawler.d.ts.map +0 -1
  57. package/build/crawlers/browser_crawler.js +0 -540
  58. package/build/crawlers/browser_crawler.js.map +0 -1
  59. package/build/crawlers/cheerio_crawler.d.ts +0 -931
  60. package/build/crawlers/cheerio_crawler.d.ts.map +0 -1
  61. package/build/crawlers/cheerio_crawler.js +0 -913
  62. package/build/crawlers/cheerio_crawler.js.map +0 -1
  63. package/build/crawlers/crawler_extension.d.ts +0 -10
  64. package/build/crawlers/crawler_extension.d.ts.map +0 -1
  65. package/build/crawlers/crawler_extension.js +0 -19
  66. package/build/crawlers/crawler_extension.js.map +0 -1
  67. package/build/crawlers/crawler_utils.d.ts +0 -34
  68. package/build/crawlers/crawler_utils.d.ts.map +0 -1
  69. package/build/crawlers/crawler_utils.js +0 -87
  70. package/build/crawlers/crawler_utils.js.map +0 -1
  71. package/build/crawlers/playwright_crawler.d.ts +0 -448
  72. package/build/crawlers/playwright_crawler.d.ts.map +0 -1
  73. package/build/crawlers/playwright_crawler.js +0 -299
  74. package/build/crawlers/playwright_crawler.js.map +0 -1
  75. package/build/crawlers/puppeteer_crawler.d.ts +0 -425
  76. package/build/crawlers/puppeteer_crawler.d.ts.map +0 -1
  77. package/build/crawlers/puppeteer_crawler.js +0 -299
  78. package/build/crawlers/puppeteer_crawler.js.map +0 -1
  79. package/build/crawlers/statistics.d.ts +0 -185
  80. package/build/crawlers/statistics.d.ts.map +0 -1
  81. package/build/crawlers/statistics.js +0 -331
  82. package/build/crawlers/statistics.js.map +0 -1
  83. package/build/enqueue_links/click_elements.d.ts +0 -179
  84. package/build/enqueue_links/click_elements.d.ts.map +0 -1
  85. package/build/enqueue_links/click_elements.js +0 -434
  86. package/build/enqueue_links/click_elements.js.map +0 -1
  87. package/build/enqueue_links/enqueue_links.d.ts +0 -117
  88. package/build/enqueue_links/enqueue_links.d.ts.map +0 -1
  89. package/build/enqueue_links/enqueue_links.js +0 -163
  90. package/build/enqueue_links/enqueue_links.js.map +0 -1
  91. package/build/enqueue_links/shared.d.ts +0 -42
  92. package/build/enqueue_links/shared.d.ts.map +0 -1
  93. package/build/enqueue_links/shared.js +0 -121
  94. package/build/enqueue_links/shared.js.map +0 -1
  95. package/build/errors.d.ts +0 -29
  96. package/build/errors.d.ts.map +0 -1
  97. package/build/errors.js +0 -38
  98. package/build/errors.js.map +0 -1
  99. package/build/events.d.ts +0 -11
  100. package/build/events.d.ts.map +0 -1
  101. package/build/events.js +0 -147
  102. package/build/events.js.map +0 -1
  103. package/build/index.d.ts +0 -4
  104. package/build/index.d.ts.map +0 -1
  105. package/build/index.js +0 -7
  106. package/build/index.js.map +0 -1
  107. package/build/main.d.ts +0 -179
  108. package/build/main.d.ts.map +0 -1
  109. package/build/main.js +0 -81
  110. package/build/main.js.map +0 -1
  111. package/build/playwright_utils.d.ts +0 -9
  112. package/build/playwright_utils.d.ts.map +0 -1
  113. package/build/playwright_utils.js +0 -90
  114. package/build/playwright_utils.js.map +0 -1
  115. package/build/proxy_configuration.d.ts +0 -411
  116. package/build/proxy_configuration.d.ts.map +0 -1
  117. package/build/proxy_configuration.js +0 -517
  118. package/build/proxy_configuration.js.map +0 -1
  119. package/build/pseudo_url.d.ts +0 -86
  120. package/build/pseudo_url.d.ts.map +0 -1
  121. package/build/pseudo_url.js +0 -153
  122. package/build/pseudo_url.js.map +0 -1
  123. package/build/puppeteer_request_interception.d.ts +0 -8
  124. package/build/puppeteer_request_interception.d.ts.map +0 -1
  125. package/build/puppeteer_request_interception.js +0 -235
  126. package/build/puppeteer_request_interception.js.map +0 -1
  127. package/build/puppeteer_utils.d.ts +0 -250
  128. package/build/puppeteer_utils.d.ts.map +0 -1
  129. package/build/puppeteer_utils.js +0 -551
  130. package/build/puppeteer_utils.js.map +0 -1
  131. package/build/request.d.ts +0 -180
  132. package/build/request.d.ts.map +0 -1
  133. package/build/request.js +0 -261
  134. package/build/request.js.map +0 -1
  135. package/build/request_list.d.ts +0 -581
  136. package/build/request_list.d.ts.map +0 -1
  137. package/build/request_list.js +0 -826
  138. package/build/request_list.js.map +0 -1
  139. package/build/serialization.d.ts +0 -5
  140. package/build/serialization.d.ts.map +0 -1
  141. package/build/serialization.js +0 -139
  142. package/build/serialization.js.map +0 -1
  143. package/build/session_pool/errors.d.ts +0 -11
  144. package/build/session_pool/errors.d.ts.map +0 -1
  145. package/build/session_pool/errors.js +0 -18
  146. package/build/session_pool/errors.js.map +0 -1
  147. package/build/session_pool/events.d.ts +0 -5
  148. package/build/session_pool/events.d.ts.map +0 -1
  149. package/build/session_pool/events.js +0 -6
  150. package/build/session_pool/events.js.map +0 -1
  151. package/build/session_pool/session.d.ts +0 -286
  152. package/build/session_pool/session.d.ts.map +0 -1
  153. package/build/session_pool/session.js +0 -355
  154. package/build/session_pool/session.js.map +0 -1
  155. package/build/session_pool/session_pool.d.ts +0 -280
  156. package/build/session_pool/session_pool.d.ts.map +0 -1
  157. package/build/session_pool/session_pool.js +0 -393
  158. package/build/session_pool/session_pool.js.map +0 -1
  159. package/build/session_pool/session_utils.d.ts +0 -4
  160. package/build/session_pool/session_utils.d.ts.map +0 -1
  161. package/build/session_pool/session_utils.js +0 -24
  162. package/build/session_pool/session_utils.js.map +0 -1
  163. package/build/stealth/hiding_tricks.d.ts +0 -22
  164. package/build/stealth/hiding_tricks.d.ts.map +0 -1
  165. package/build/stealth/hiding_tricks.js +0 -308
  166. package/build/stealth/hiding_tricks.js.map +0 -1
  167. package/build/stealth/stealth.d.ts +0 -56
  168. package/build/stealth/stealth.d.ts.map +0 -1
  169. package/build/stealth/stealth.js +0 -125
  170. package/build/stealth/stealth.js.map +0 -1
  171. package/build/storages/dataset.d.ts +0 -288
  172. package/build/storages/dataset.d.ts.map +0 -1
  173. package/build/storages/dataset.js +0 -480
  174. package/build/storages/dataset.js.map +0 -1
  175. package/build/storages/key_value_store.d.ts +0 -243
  176. package/build/storages/key_value_store.d.ts.map +0 -1
  177. package/build/storages/key_value_store.js +0 -462
  178. package/build/storages/key_value_store.js.map +0 -1
  179. package/build/storages/request_queue.d.ts +0 -318
  180. package/build/storages/request_queue.d.ts.map +0 -1
  181. package/build/storages/request_queue.js +0 -636
  182. package/build/storages/request_queue.js.map +0 -1
  183. package/build/storages/storage_manager.d.ts +0 -87
  184. package/build/storages/storage_manager.d.ts.map +0 -1
  185. package/build/storages/storage_manager.js +0 -150
  186. package/build/storages/storage_manager.js.map +0 -1
  187. package/build/tsconfig.tsbuildinfo +0 -1
  188. package/build/typedefs.d.ts +0 -146
  189. package/build/typedefs.d.ts.map +0 -1
  190. package/build/typedefs.js +0 -88
  191. package/build/typedefs.js.map +0 -1
  192. package/build/utils.d.ts +0 -175
  193. package/build/utils.d.ts.map +0 -1
  194. package/build/utils.js +0 -731
  195. package/build/utils.js.map +0 -1
  196. package/build/utils_log.d.ts +0 -41
  197. package/build/utils_log.d.ts.map +0 -1
  198. package/build/utils_log.js +0 -192
  199. package/build/utils_log.js.map +0 -1
  200. package/build/utils_request.d.ts +0 -77
  201. package/build/utils_request.d.ts.map +0 -1
  202. package/build/utils_request.js +0 -385
  203. package/build/utils_request.js.map +0 -1
  204. package/build/utils_social.d.ts +0 -210
  205. package/build/utils_social.d.ts.map +0 -1
  206. package/build/utils_social.js +0 -787
  207. package/build/utils_social.js.map +0 -1
  208. package/build/validators.d.ts +0 -23
  209. package/build/validators.d.ts.map +0 -1
  210. package/build/validators.js +0 -29
  211. package/build/validators.js.map +0 -1
@@ -1,551 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.puppeteerUtils = exports.infiniteScroll = exports.gotoExtended = void 0;
4
- const tslib_1 = require("tslib");
5
- const fs_1 = (0, tslib_1.__importDefault)(require("fs"));
6
- const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
7
- const vm_1 = (0, tslib_1.__importDefault)(require("vm"));
8
- const util_1 = (0, tslib_1.__importDefault)(require("util"));
9
- const underscore_1 = (0, tslib_1.__importDefault)(require("underscore"));
10
- const datastructures_1 = require("@apify/datastructures");
11
- const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
12
- const validators_1 = require("./validators");
13
- const click_elements_1 = require("./enqueue_links/click_elements");
14
- const puppeteer_request_interception_1 = require("./puppeteer_request_interception");
15
- const key_value_store_1 = require("./storages/key_value_store");
16
- const jqueryPath = require.resolve('jquery');
17
- const underscorePath = require.resolve('underscore');
18
- const readFilePromised = util_1.default.promisify(fs_1.default.readFile);
19
- const MAX_INJECT_FILE_CACHE_SIZE = 10;
20
- const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
21
- /**
22
- * @typedef {object} CompiledScriptParams
23
- * @property {Page} params.page
24
- * @property {Request} params.request
25
- */
26
- /**
27
- * @callback CompiledScriptFunction
28
- * @param {CompiledScriptParams} params
29
- * @returns {Promise<*>}
30
- */
31
- /**
32
- * Cache contents of previously injected files to limit file system access.
33
- */
34
- const injectedFilesCache = new datastructures_1.LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE });
35
- /**
36
- * Injects a JavaScript file into a Puppeteer page.
37
- * Unlike Puppeteer's `addScriptTag` function, this function works on pages
38
- * with arbitrary Cross-Origin Resource Sharing (CORS) policies.
39
- *
40
- * File contents are cached for up to 10 files to limit file system access.
41
- *
42
- * @param {Page} page
43
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
44
- * @param {string} filePath File path
45
- * @param {object} [options]
46
- * @param {boolean} [options.surviveNavigations]
47
- * Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
48
- * This does not mean, however, that internal state will be preserved. Just that it will be automatically
49
- * re-injected on each navigation before any other scripts get the chance to execute.
50
- * @return {Promise<*>}
51
- * @memberOf puppeteer
52
- */
53
- const injectFile = async (page, filePath, options = {}) => {
54
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
55
- (0, ow_1.default)(filePath, ow_1.default.string);
56
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
57
- surviveNavigations: ow_1.default.optional.boolean,
58
- }));
59
- let contents = injectedFilesCache.get(filePath);
60
- if (!contents) {
61
- contents = await readFilePromised(filePath, 'utf8');
62
- injectedFilesCache.add(filePath, contents);
63
- }
64
- const evalP = page.evaluate(contents);
65
- return options.surviveNavigations
66
- ? Promise.all([page.evaluateOnNewDocument(contents), evalP])
67
- : evalP;
68
- };
69
- /**
70
- * Injects the [jQuery](https://jquery.com/) library into a Puppeteer page.
71
- * jQuery is often useful for various web scraping and crawling tasks.
72
- * For example, it can help extract text from HTML elements using CSS selectors.
73
- *
74
- * Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with
75
- * other libraries included by the page that use the same variable name (e.g. another version of jQuery).
76
- * This can affect functionality of page's scripts.
77
- *
78
- * The injected jQuery will survive page navigations and reloads.
79
- *
80
- * **Example usage:**
81
- * ```javascript
82
- * await Apify.utils.puppeteer.injectJQuery(page);
83
- * const title = await page.evaluate(() => {
84
- * return $('head title').text();
85
- * });
86
- * ```
87
- *
88
- * Note that `injectJQuery()` does not affect the Puppeteer's
89
- * [`page.$()`](https://pptr.dev/#?product=Puppeteer&show=api-pageselector)
90
- * function in any way.
91
- *
92
- * @param {Page} page
93
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
94
- * @return {Promise<*>}
95
- * @memberOf puppeteer
96
- */
97
- const injectJQuery = (page) => {
98
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
99
- return injectFile(page, jqueryPath, { surviveNavigations: true });
100
- };
101
- /**
102
- * Injects the [Underscore](https://underscorejs.org/) library into a Puppeteer page.
103
- *
104
- * Beware that the injected Underscore object will be set to the `window._` variable and thus it might cause conflicts with
105
- * libraries included by the page that use the same variable name.
106
- * This can affect functionality of page's scripts.
107
- *
108
- * The injected Underscore will survive page navigations and reloads.
109
- *
110
- * **Example usage:**
111
- * ```javascript
112
- * await Apify.utils.puppeteer.injectUnderscore(page);
113
- * const escapedHtml = await page.evaluate(() => {
114
- * return _.escape('<h1>Hello</h1>');
115
- * });
116
- * ```
117
- *
118
- * @param {Page} page Puppeteer [Page](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#class-page) object.
119
- * @return {Promise<*>}
120
- * @memberOf puppeteer
121
- */
122
- const injectUnderscore = (page) => {
123
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
124
- return injectFile(page, underscorePath, { surviveNavigations: true });
125
- };
126
- /**
127
- * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
128
- * This is useful to speed up crawling of websites, since it reduces the amount
129
- * of data that needs to be downloaded from the web, but it may break some websites
130
- * or unexpectedly prevent loading of resources.
131
- *
132
- * By default, the function will block all URLs including the following patterns:
133
- *
134
- * ```json
135
- * [".css", ".jpg", ".jpeg", ".png", ".svg", ".gif", ".woff", ".pdf", ".zip"]
136
- * ```
137
- *
138
- * If you want to extend this list further, use the `extraUrlPatterns` option,
139
- * which will keep blocking the default patterns, as well as add your custom ones.
140
- * If you would like to block only specific patterns, use the `urlPatterns` option,
141
- * which will override the defaults and block only URLs with your custom patterns.
142
- *
143
- * This function does not use Puppeteer's request interception and therefore does not interfere
144
- * with browser cache. It's also faster than blocking requests using interception,
145
- * because the blocking happens directly in the browser without the round-trip to Node.js,
146
- * but it does not provide the extra benefits of request interception.
147
- *
148
- * The function will never block main document loads and their respective redirects.
149
- *
150
- * **Example usage**
151
- * ```javascript
152
- * const Apify = require('apify');
153
- *
154
- * const browser = await Apify.launchPuppeteer();
155
- * const page = await browser.newPage();
156
- *
157
- * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
158
- * await Apify.utils.puppeteer.blockRequests(page, {
159
- * extraUrlPatterns: ['adsbygoogle.js'],
160
- * });
161
- *
162
- * await page.goto('https://cnn.com');
163
- * ```
164
- *
165
- * @param {Page} page
166
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
167
- * @param {object} [options]
168
- * @param {string[]} [options.urlPatterns]
169
- * The patterns of URLs to block from being loaded by the browser.
170
- * Only `*` can be used as a wildcard. It is also automatically added to the beginning
171
- * and end of the pattern. This limitation is enforced by the DevTools protocol.
172
- * `.png` is the same as `*.png*`.
173
- * @param {string[]} [options.extraUrlPatterns]
174
- * If you just want to append to the default blocked patterns, use this property.
175
- * @return {Promise<void>}
176
- * @memberOf puppeteer
177
- */
178
- const blockRequests = async (page, options = {}) => {
179
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
180
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
181
- urlPatterns: ow_1.default.optional.array.ofType(ow_1.default.string),
182
- extraUrlPatterns: ow_1.default.optional.array.ofType(ow_1.default.string),
183
- }));
184
- const { urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS, extraUrlPatterns = [], } = options;
185
- const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
186
- await page._client.send('Network.setBlockedURLs', { urls: patternsToBlock }); // eslint-disable-line no-underscore-dangle
187
- };
188
- /**
189
- * `blockResources()` has a high impact on performance in recent versions of Puppeteer.
190
- * 'Until this resolves, please use `Apify.utils.puppeteer.blockRequests()`.
191
- * @deprecated
192
- */
193
- const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'image', 'media']) => {
194
- utils_log_1.default.deprecated('Apify.utils.puppeteer.blockResources() has a high impact on performance in recent versions of Puppeteer. '
195
- + 'Until this resolves, please use Apify.utils.puppeteer.blockRequests()');
196
- await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, async (request) => {
197
- const type = request.resourceType();
198
- if (resourceTypes.includes(type))
199
- await request.abort();
200
- else
201
- await request.continue();
202
- });
203
- };
204
- /**
205
- * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
206
- * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
207
- *
208
- * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
209
- * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
210
- * This issue should be resolved or atleast mitigated in future iterations of this feature.
211
- * @param {Page} page
212
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
213
- * @param {Object<string, *>} cache
214
- * Object in which responses are stored
215
- * @param {Array<(string|RegExp)>} responseUrlRules
216
- * List of rules that are used to check if the response should be cached.
217
- * String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
218
- * @return {Promise<void>}
219
- * @memberOf puppeteer
220
- * @deprecated
221
- */
222
- const cacheResponses = async (page, cache, responseUrlRules) => {
223
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
224
- (0, ow_1.default)(cache, ow_1.default.object);
225
- (0, ow_1.default)(responseUrlRules, ow_1.default.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.regExp)));
226
- utils_log_1.default.deprecated('Apify.utils.puppeteer.cacheResponses() has a high impact on performance '
227
- + 'in recent versions of Puppeteer so it\'s use is discouraged until this issue resolves.');
228
- await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, async (request) => {
229
- const url = request.url();
230
- if (cache[url]) {
231
- await request.respond(cache[url]);
232
- return;
233
- }
234
- await request.continue();
235
- });
236
- page.on('response', async (response) => {
237
- const url = response.url();
238
- // Response is already cached, do nothing
239
- if (cache[url])
240
- return;
241
- const shouldCache = responseUrlRules.some((rule) => {
242
- if (typeof rule === 'string')
243
- return url.includes(rule);
244
- if (rule instanceof RegExp)
245
- return rule.test(url);
246
- });
247
- try {
248
- if (shouldCache) {
249
- const buffer = await response.buffer();
250
- cache[url] = {
251
- status: response.status(),
252
- headers: response.headers(),
253
- body: buffer,
254
- };
255
- }
256
- }
257
- catch (e) {
258
- // ignore errors, usualy means that buffer is empty or broken connection
259
- }
260
- });
261
- };
262
- /**
263
- * Compiles a Puppeteer script into an async function that may be executed at any time
264
- * by providing it with the following object:
265
- * ```
266
- * {
267
- * page: Page,
268
- * request: Request,
269
- * }
270
- * ```
271
- * Where `page` is a Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page)
272
- * and `request` is a {@link Request}.
273
- *
274
- * The function is compiled by using the `scriptString` parameter as the function's body,
275
- * so any limitations to function bodies apply. Return value of the compiled function
276
- * is the return value of the function body = the `scriptString` parameter.
277
- *
278
- * As a security measure, no globals such as `process` or `require` are accessible
279
- * from within the function body. Note that the function does not provide a safe
280
- * sandbox and even though globals are not easily accessible, malicious code may
281
- * still execute in the main process via prototype manipulation. Therefore you
282
- * should only use this function to execute sanitized or safe code.
283
- *
284
- * Custom context may also be provided using the `context` parameter. To improve security,
285
- * make sure to only pass the really necessary objects to the context. Preferably making
286
- * secured copies beforehand.
287
- *
288
- * @param {string} scriptString
289
- * @param {Object<string, *>} context
290
- * @return {CompiledScriptFunction}
291
- * @memberOf puppeteer
292
- */
293
- const compileScript = (scriptString, context = Object.create(null)) => {
294
- const funcString = `async ({ page, request }) => {${scriptString}}`;
295
- let func;
296
- try {
297
- func = vm_1.default.runInNewContext(funcString, context); // "Secure" the context by removing prototypes, unless custom context is provided.
298
- }
299
- catch (err) {
300
- utils_log_1.default.exception(err, 'Cannot compile script!');
301
- throw err;
302
- }
303
- if (!underscore_1.default.isFunction(func))
304
- throw new Error('Compilation result is not a function!'); // This should not happen...
305
- return func;
306
- };
307
- /**
308
- * Extended version of Puppeteer's `page.goto()` allowing to perform requests with HTTP method other than GET,
309
- * with custom headers and POST payload. URL, method, headers and payload are taken from
310
- * request parameter that must be an instance of Apify.Request class.
311
- *
312
- * *NOTE:* In recent versions of Puppeteer using requests other than GET, overriding headers and adding payloads disables
313
- * browser cache which degrades performance.
314
- *
315
- * @param {Page} page
316
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
317
- * @param {Request} request
318
- * @param {DirectNavigationOptions} [gotoOptions] Custom options for `page.goto()`.
319
- * @return {Promise<(Response|null)>}
320
- *
321
- * @memberOf puppeteer
322
- * @name gotoExtended
323
- */
324
- const gotoExtended = async (page, request, gotoOptions = {}) => {
325
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
326
- (0, ow_1.default)(request, ow_1.default.object.partialShape({
327
- url: ow_1.default.string.url,
328
- method: ow_1.default.optional.string,
329
- headers: ow_1.default.optional.object,
330
- payload: ow_1.default.optional.any(ow_1.default.string, ow_1.default.buffer),
331
- }));
332
- (0, ow_1.default)(gotoOptions, ow_1.default.object);
333
- const { url, method, headers, payload } = request;
334
- if (method !== 'GET' || payload || !underscore_1.default.isEmpty(headers)) {
335
- // This is not deprecated, we use it to log only once.
336
- utils_log_1.default.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance '
337
- + 'in recent versions of Puppeteer. Use only when necessary.');
338
- let wasCalled = false;
339
- const interceptRequestHandler = async (interceptedRequest) => {
340
- // We want to ensure that this won't get executed again in a case that there is a subsequent request
341
- // for example for some asset file link from main HTML.
342
- if (wasCalled) {
343
- return interceptedRequest.continue();
344
- }
345
- wasCalled = true;
346
- const overrides = {};
347
- if (method !== 'GET')
348
- overrides.method = method;
349
- if (payload)
350
- overrides.postData = payload;
351
- if (!underscore_1.default.isEmpty(headers))
352
- overrides.headers = headers;
353
- await (0, puppeteer_request_interception_1.removeInterceptRequestHandler)(page, interceptRequestHandler);
354
- interceptedRequest.continue(overrides);
355
- };
356
- await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, interceptRequestHandler);
357
- }
358
- return page.goto(url, gotoOptions);
359
- };
360
- exports.gotoExtended = gotoExtended;
361
- /**
362
- * Scrolls to the bottom of a page, or until it times out.
363
- * Loads dynamic content when it hits the bottom of a page, and then continues scrolling.
364
- * @param {Page} page
365
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
366
- * @param {object} [options]
367
- * @param {number} [options.timeoutSecs=0]
368
- * How many seconds to scroll for. If 0, will scroll until bottom of page.
369
- * @param {number} [options.waitForSecs=4]
370
- * How many seconds to wait for no new content to load before exit.
371
- * @param {boolean} [options.scrollDownAndUp=false]
372
- * If true, it will scroll up a bit after each scroll down. This is required on some websites for the scroll to work.
373
- * @param {string} [options.buttonSelector]
374
- * Optionally checks and clicks a button if it appears while scrolling. This is required on some websites for the scroll to work.
375
- * @param {function} [options.stopScrollCallback]
376
- * This function is called after every scroll and stops the scrolling process if it returns `true`. The function can be `async`.
377
- * @returns {Promise<void>}
378
- * @memberOf puppeteer
379
- * @name infiniteScroll
380
- */
381
- const infiniteScroll = async (page, options = {}) => {
382
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
383
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
384
- timeoutSecs: ow_1.default.optional.number,
385
- waitForSecs: ow_1.default.optional.number,
386
- scrollDownAndUp: ow_1.default.optional.boolean,
387
- buttonSelector: ow_1.default.optional.string,
388
- stopScrollCallback: ow_1.default.optional.function,
389
- }));
390
- const { timeoutSecs = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;
391
- let finished;
392
- const startTime = Date.now();
393
- const CHECK_INTERVAL_MILLIS = 1000;
394
- const SCROLL_HEIGHT_IF_ZERO = 10000;
395
- const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];
396
- const resourcesStats = {
397
- newRequested: 0,
398
- oldRequested: 0,
399
- matchNumber: 0,
400
- };
401
- page.on('request', (msg) => {
402
- if (maybeResourceTypesInfiniteScroll.includes(msg.resourceType())) {
403
- resourcesStats.newRequested++;
404
- }
405
- });
406
- // Move mouse to the center of the page, so we can scroll up-down
407
- let body = await page.$('body');
408
- let retry = 0;
409
- while (!body && retry < 10) {
410
- await page.waitForTimeout(100);
411
- body = await page.$('body');
412
- retry++;
413
- }
414
- if (!body) {
415
- return;
416
- }
417
- const boundingBox = await body.boundingBox();
418
- await page.mouse.move(boundingBox.x + boundingBox.width / 2, // x
419
- boundingBox.y + boundingBox.height / 2);
420
- const checkFinished = setInterval(() => {
421
- if (resourcesStats.oldRequested === resourcesStats.newRequested) {
422
- resourcesStats.matchNumber++;
423
- if (resourcesStats.matchNumber >= waitForSecs) {
424
- clearInterval(checkFinished);
425
- finished = true;
426
- return;
427
- }
428
- }
429
- else {
430
- resourcesStats.matchNumber = 0;
431
- resourcesStats.oldRequested = resourcesStats.newRequested;
432
- }
433
- // check if timeout has been reached
434
- if (timeoutSecs !== 0 && (Date.now() - startTime) / 1000 > timeoutSecs) {
435
- clearInterval(checkFinished);
436
- finished = true;
437
- }
438
- }, CHECK_INTERVAL_MILLIS);
439
- const doScroll = async () => {
440
- /* istanbul ignore next */
441
- const bodyScrollHeight = await page.evaluate(() => document.body.scrollHeight);
442
- const delta = bodyScrollHeight === 0 ? SCROLL_HEIGHT_IF_ZERO : bodyScrollHeight;
443
- await page.mouse.wheel({ deltaY: delta });
444
- };
445
- const maybeClickButton = async () => {
446
- const button = await page.$(buttonSelector);
447
- // Box model returns null if the button is not visible
448
- if (button && await button.boxModel()) {
449
- await button.click({ delay: 10 });
450
- }
451
- };
452
- while (!finished) {
453
- await doScroll();
454
- await page.waitForTimeout(250);
455
- if (scrollDownAndUp) {
456
- await page.mouse.wheel({ deltaY: -1000 });
457
- }
458
- if (buttonSelector) {
459
- await maybeClickButton();
460
- }
461
- if (stopScrollCallback) {
462
- if (await stopScrollCallback()) {
463
- clearInterval(checkFinished);
464
- break;
465
- }
466
- }
467
- }
468
- };
469
- exports.infiniteScroll = infiniteScroll;
470
- /**
471
- * Saves a full screenshot and HTML of the current page into a Key-Value store.
472
- * @param {Page} page
473
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
474
- * @param {object} [options]
475
- * @param {string} [options.key=SNAPSHOT]
476
- * Key under which the screenshot and HTML will be saved. `.jpg` will be appended for screenshot and `.html` for HTML.
477
- * @param {number} [options.screenshotQuality=50]
478
- * The quality of the image, between 0-100. Higher quality images have bigger size and require more storage.
479
- * @param {boolean} [options.saveScreenshot=true]
480
- * If true, it will save a full screenshot of the current page as a record with `key` appended by `.jpg`.
481
- * @param {boolean} [options.saveHtml=true]
482
- * If true, it will save a full HTML of the current page as a record with `key` appended by `.html`.
483
- * @param {string|null} [options.keyValueStoreName=null]
484
- * Name or id of the Key-Value store where snapshot is saved. By default it is saved to default Key-Value store.
485
- * @returns {Promise<void>}
486
- * @memberOf puppeteer
487
- * @name saveSnapshot
488
- */
489
- const saveSnapshot = async (page, options = {}) => {
490
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
491
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
492
- key: ow_1.default.optional.string.nonEmpty,
493
- screenshotQuality: ow_1.default.optional.number,
494
- saveScreenshot: ow_1.default.optional.boolean,
495
- saveHtml: ow_1.default.optional.boolean,
496
- keyValueStoreName: ow_1.default.optional.string,
497
- }));
498
- const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, } = options;
499
- try {
500
- const store = await (0, key_value_store_1.openKeyValueStore)(keyValueStoreName);
501
- if (saveScreenshot) {
502
- const screenshotName = `${key}.jpg`;
503
- const screenshotBuffer = await page.screenshot({ fullPage: true, screenshotQuality, type: 'jpeg' });
504
- await store.setValue(screenshotName, screenshotBuffer, { contentType: 'image/jpeg' });
505
- }
506
- if (saveHtml) {
507
- const htmlName = `${key}.html`;
508
- const html = await page.content();
509
- await store.setValue(htmlName, html, { contentType: 'text/html' });
510
- }
511
- }
512
- catch (err) {
513
- throw new Error(`saveSnapshot with key ${key} failed.\nCause:${err.message}`);
514
- }
515
- };
516
- /**
517
- * A namespace that contains various utilities for
518
- * [Puppeteer](https://github.com/puppeteer/puppeteer) - the headless Chrome Node API.
519
- *
520
- * **Example usage:**
521
- *
522
- * ```javascript
523
- * const Apify = require('apify');
524
- * const { puppeteer } = Apify.utils;
525
- *
526
- * // Open https://www.example.com in Puppeteer
527
- * const browser = await Apify.launchPuppeteer();
528
- * const page = await browser.newPage();
529
- * await page.goto('https://www.example.com');
530
- *
531
- * // Inject jQuery into a page
532
- * await puppeteer.injectJQuery(page);
533
- * ```
534
- * @namespace puppeteer
535
- */
536
- exports.puppeteerUtils = {
537
- injectFile,
538
- injectJQuery,
539
- injectUnderscore,
540
- enqueueLinksByClickingElements: click_elements_1.enqueueLinksByClickingElements,
541
- blockRequests,
542
- blockResources,
543
- cacheResponses,
544
- compileScript,
545
- gotoExtended: exports.gotoExtended,
546
- addInterceptRequestHandler: puppeteer_request_interception_1.addInterceptRequestHandler,
547
- removeInterceptRequestHandler: puppeteer_request_interception_1.removeInterceptRequestHandler,
548
- infiniteScroll: exports.infiniteScroll,
549
- saveSnapshot,
550
- };
551
- //# sourceMappingURL=puppeteer_utils.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"puppeteer_utils.js","sourceRoot":"","sources":["../src/puppeteer_utils.js"],"names":[],"mappings":";;;;AAAA,yDAAoB;AACpB,yDAAoB;AACpB,yDAAoB;AACpB,6DAAwB;AACxB,yEAA2B;AAC3B,0DAAiD;AAEjD,yEAA8B;AAC9B,6CAA0C;AAE1C,mEAAgF;AAChF,qFAA6G;AAC7G,gEAA+D;AAE/D,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAC7C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AACrD,MAAM,gBAAgB,GAAG,cAAI,CAAC,SAAS,CAAC,YAAE,CAAC,QAAQ,CAAC,CAAC;AAErD,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,MAAM,kCAAkC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAEtH;;;;GAIG;AACH;;;;GAIG;AACH;;GAEG;AACH,MAAM,kBAAkB,GAAG,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAEnF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,GAAG,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IACtD,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACxB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;KAC1C,CAAC,CAAC,CAAC;IAEJ,IAAI,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,EAAE;QACX,QAAQ,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpD,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;KAC9C;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtC,OAAO,OAAO,CAAC,kBAAkB;QAC7B,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,KAAK,CAAC;AAChB,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,EAAE;IAC1B,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,EAAE;IAC9B,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,IAAI,EAAE,cAAc,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,MAAM,aAAa,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IAC/C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChD,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;KACxD,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,WAAW,GAAG,kCAAkC,EAChD,gBAAgB,GAAG,EAAE,GACxB,GAAG,OAAO,CAAC;IAEZ,MAAM,eAAe,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,gBAAgB,CAAC,CAAC;IAE9D,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,2CAA2C;AAC7H,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,cAAc,GAAG,KAAK,EAAE,IAAI,EAAE,aAAa,GAAG,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE;IAC5F,mBAAG,CAAC,UAAU,CAAC,2GAA2G;UACpH,uEAAuE,CAAC,CAAC;IAC/E,MAAM,IAAA,2DAA0B,EAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACrD,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;QACpC,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;;YACnD,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,cAAc,GAAG,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAAE;IAC3D,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACrB,IAAA,YAAE,EAAC,gBAAgB,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAEpE,mBAAG,CAAC,UAAU,CAAC,0EAA0E;UACnF,wFAAwF,CAAC,CAAC;IAEhG,MAAM,IAAA,2DAA0B,EAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACrD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAE1B,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE;YACZ,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,OAAO;SACV;QAED,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC;QAE3B,yCAAyC;QACzC,IAAI,KAAK,CAAC,GAAG,CAAC;YAAE,OAAO;QAEvB,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YAC/C,IAAI,OAAO,IAAI,KAAK,QAAQ;gBAAE,OAAO,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACxD,IAAI,IAAI,YAAY,MAAM;gBAAE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,IAAI;YACA,IAAI,WAAW,EAAE;gBACb,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACvC,KAAK,CAAC,GAAG,CAAC,GAAG;oBACT,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE;oBACzB,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE;oBAC3B,IAAI,EAAE,MAAM;iBACf,CAAC;aACL;SACJ;QAAC,OAAO,CAAC,EAAE;YACR,wEAAwE;SAC3E;IACL,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,aAAa,GAAG,CAAC,YAAY,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE;IAClE,MAAM,UAAU,GAAG,iCAAiC,YAAY,GAAG,CAAC;IAEpE,IAAI,IAAI,CAAC;IACT,IAAI;QACA,IAAI,GAAG,YAAE,CAAC,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,kFAAkF;KACrI;IAAC,OAAO,GAAG,EAAE;QACV,mBAAG,CAAC,SAAS,CAAC,GAAG,EAAE,wBAAwB,CAAC,CAAC;QAC7C,MAAM,GAAG,CAAC;KACb;IAED,IAAI,CAAC,oBAAC,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC,4BAA4B;IAE/G,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;GAgBG;AACI,MAAM,YAAY,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,GAAG,EAAE,EAAE,EAAE;IAClE,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAC/B,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;KACjD,CAAC,CAAC,CAAC;IACJ,IAAA,YAAE,EAAC,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAE3B,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAElD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO,IAAI,CAAC,oBAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QACpD,sDAAsD;QACtD,mBAAG,CAAC,UAAU,CAAC,+GAA+G;cACxH,2DAA2D,CAAC,CAAC;QACnE,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,uBAAuB,GAAG,KAAK,EAAE,kBAAkB,EAAE,EAAE;YACzD,oGAAoG;YACpG,uDAAuD;YACvD,IAAI,SAAS,EAAE;gBACX,OAAO,kBAAkB,CAAC,QAAQ,EAAE,CAAC;aACxC;YAED,SAAS,GAAG,IAAI,CAAC;YACjB,MAAM,SAAS,GAAG,EAAE,CAAC;YAErB,IAAI,MAAM,KAAK,KAAK;gBAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC;YAChD,IAAI,OAAO;gBAAE,SAAS,CAAC,QAAQ,GAAG,OAAO,CAAC;YAC1C,IAAI,CAAC,oBAAC,CAAC,OAAO,CAAC,OAAO,CAAC;gBAAE,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC;YACrD,MAAM,IAAA,8DAA6B,EAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;YACnE,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC,CAAC;QAEF,MAAM,IAAA,2DAA0B,EAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;KACnE;IAED,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;AACvC,CAAC,CAAC;AAtCW,QAAA,YAAY,gBAsCvB;AAEF;;;;;;;;;;;;;;;;;;;GAmBG;AACI,MAAM,cAAc,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IACvD,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;KAC3C,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,EAAE,eAAe,GAAG,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC;IAElH,IAAI,QAAQ,CAAC;IACb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,qBAAqB,GAAG,IAAI,CAAC;IACnC,MAAM,qBAAqB,GAAG,KAAK,CAAC;IACpC,MAAM,gCAAgC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG;QACnB,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;KACjB,CAAC;IAEF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,IAAI,gCAAgC,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,EAAE;YAC/D,cAAc,CAAC,YAAY,EAAE,CAAC;SACjC;IACL,CAAC,CAAC,CAAC;IAEH,iEAAiE;IACjE,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,CAAC,IAAI,IAAI,KAAK,GAAG,EAAE,EAAE;QACxB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAC5B,KAAK,EAAE,CAAC;KACX;IAED,IAAI,CAAC,IAAI,EAAE;QACP,OAAO;KACV;IAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACjB,WAAW,CAAC,CAAC,GAAG,WAAW,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI;IAC3C,WAAW,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CACzC,CAAC;IAEF,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,IAAI,cAAc,CAAC,YAAY,KAAK,cAAc,CAAC,YAAY,EAAE;YAC7D,cAAc,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,cAAc,CAAC,WAAW,IAAI,WAAW,EAAE;gBAC3C,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,QAAQ,GAAG,IAAI,CAAC;gBAChB,OAAO;aACV;SACJ;aAAM;YACH,cAAc,CAAC,WAAW,GAAG,CAAC,CAAC;YAC/B,cAAc,CAAC,YAAY,GAAG,cAAc,CAAC,YAAY,CAAC;SAC7D;QACD,oCAAoC;QACpC,IAAI,WAAW,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG,WAAW,EAAE;YACpE,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;SACnB;IACL,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAE1B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QACxB,0BAA0B;QAC1B,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE/E,MAAM,KAAK,GAAG,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEhF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC;IAEF,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QAC5C,sDAAsD;QACtD,IAAI,MAAM,IAAI,MAAM,MAAM,CAAC,QAAQ,EAAE,EAAE;YACnC,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;SACrC;IACL,CAAC,CAAC;IAEF,OAAO,CAAC,QAAQ,EAAE;QACd,MAAM,QAAQ,EAAE,CAAC;QACjB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,eAAe,EAAE;YACjB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;SAC7C;QACD,IAAI,cAAc,EAAE;YAChB,MAAM,gBAAgB,EAAE,CAAC;SAC5B;QACD,IAAI,kBAAkB,EAAE;YACpB,IAAI,MAAM,kBAAkB,EAAE,EAAE;gBAC5B,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,MAAM;aACT;SACJ;KACJ;AACL,CAAC,CAAC;AArGW,QAAA,cAAc,kBAqGzB;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,YAAY,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IAC9C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;QAChC,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACxC,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,GAAG,GAAG,UAAU,EAChB,iBAAiB,GAAG,EAAE,EACtB,cAAc,GAAG,IAAI,EACrB,QAAQ,GAAG,IAAI,EACf,iBAAiB,GACpB,GAAG,OAAO,CAAC;IAEZ,IAAI;QACA,MAAM,KAAK,GAAG,MAAM,IAAA,mCAAiB,EAAC,iBAAiB,CAAC,CAAC;QAEzD,IAAI,cAAc,EAAE;YAChB,MAAM,cAAc,GAAG,GAAG,GAAG,MAAM,CAAC;YACpC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YACpG,MAAM,KAAK,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC;SACzF;QACD,IAAI,QAAQ,EAAE;YACV,MAAM,QAAQ,GAAG,GAAG,GAAG,OAAO,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;SACtE;KACJ;IAAC,OAAO,GAAG,EAAE;QACV,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,mBAAmB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;KACjF;AACL,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;GAmBG;AACU,QAAA,cAAc,GAAG;IAC1B,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,8BAA8B,EAA9B,+CAA8B;IAC9B,aAAa;IACb,cAAc;IACd,cAAc;IACd,aAAa;IACb,YAAY,EAAZ,oBAAY;IACZ,0BAA0B,EAA1B,2DAA0B;IAC1B,6BAA6B,EAA7B,8DAA6B;IAC7B,cAAc,EAAd,sBAAc;IACd,YAAY;CACf,CAAC"}