apify 2.3.1-beta.4 → 3.0.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/README.md +6 -5
  2. package/actor.d.ts +1092 -0
  3. package/actor.d.ts.map +1 -0
  4. package/actor.js +1221 -0
  5. package/actor.js.map +1 -0
  6. package/index.d.ts +4 -0
  7. package/index.d.ts.map +1 -0
  8. package/index.js +7 -0
  9. package/index.js.map +1 -0
  10. package/index.mjs +7 -0
  11. package/package.json +54 -128
  12. package/platform_event_manager.d.ts +55 -0
  13. package/platform_event_manager.d.ts.map +1 -0
  14. package/platform_event_manager.js +116 -0
  15. package/platform_event_manager.js.map +1 -0
  16. package/proxy_configuration.d.ts +210 -0
  17. package/proxy_configuration.d.ts.map +1 -0
  18. package/proxy_configuration.js +297 -0
  19. package/proxy_configuration.js.map +1 -0
  20. package/tsconfig.build.tsbuildinfo +1 -0
  21. package/utils.d.ts +11 -0
  22. package/utils.d.ts.map +1 -0
  23. package/utils.js +40 -0
  24. package/utils.js.map +1 -0
  25. package/build/actor.d.ts +0 -113
  26. package/build/actor.d.ts.map +0 -1
  27. package/build/actor.js +0 -582
  28. package/build/actor.js.map +0 -1
  29. package/build/apify.d.ts +0 -752
  30. package/build/apify.d.ts.map +0 -1
  31. package/build/apify.js +0 -877
  32. package/build/apify.js.map +0 -1
  33. package/build/autoscaling/autoscaled_pool.d.ts +0 -384
  34. package/build/autoscaling/autoscaled_pool.d.ts.map +0 -1
  35. package/build/autoscaling/autoscaled_pool.js +0 -557
  36. package/build/autoscaling/autoscaled_pool.js.map +0 -1
  37. package/build/autoscaling/snapshotter.d.ts +0 -278
  38. package/build/autoscaling/snapshotter.d.ts.map +0 -1
  39. package/build/autoscaling/snapshotter.js +0 -447
  40. package/build/autoscaling/snapshotter.js.map +0 -1
  41. package/build/autoscaling/system_status.d.ts +0 -224
  42. package/build/autoscaling/system_status.d.ts.map +0 -1
  43. package/build/autoscaling/system_status.js +0 -228
  44. package/build/autoscaling/system_status.js.map +0 -1
  45. package/build/browser_launchers/browser_launcher.d.ts +0 -154
  46. package/build/browser_launchers/browser_launcher.d.ts.map +0 -1
  47. package/build/browser_launchers/browser_launcher.js +0 -160
  48. package/build/browser_launchers/browser_launcher.js.map +0 -1
  49. package/build/browser_launchers/browser_plugin.d.ts +0 -23
  50. package/build/browser_launchers/browser_plugin.d.ts.map +0 -1
  51. package/build/browser_launchers/browser_plugin.js +0 -25
  52. package/build/browser_launchers/browser_plugin.js.map +0 -1
  53. package/build/browser_launchers/playwright_launcher.d.ts +0 -131
  54. package/build/browser_launchers/playwright_launcher.d.ts.map +0 -1
  55. package/build/browser_launchers/playwright_launcher.js +0 -150
  56. package/build/browser_launchers/playwright_launcher.js.map +0 -1
  57. package/build/browser_launchers/puppeteer_launcher.d.ts +0 -153
  58. package/build/browser_launchers/puppeteer_launcher.d.ts.map +0 -1
  59. package/build/browser_launchers/puppeteer_launcher.js +0 -197
  60. package/build/browser_launchers/puppeteer_launcher.js.map +0 -1
  61. package/build/cache_container.d.ts +0 -31
  62. package/build/cache_container.d.ts.map +0 -1
  63. package/build/cache_container.js +0 -48
  64. package/build/cache_container.js.map +0 -1
  65. package/build/configuration.d.ts +0 -226
  66. package/build/configuration.d.ts.map +0 -1
  67. package/build/configuration.js +0 -325
  68. package/build/configuration.js.map +0 -1
  69. package/build/constants.d.ts +0 -37
  70. package/build/constants.d.ts.map +0 -1
  71. package/build/constants.js +0 -41
  72. package/build/constants.js.map +0 -1
  73. package/build/crawlers/basic_crawler.d.ts +0 -443
  74. package/build/crawlers/basic_crawler.d.ts.map +0 -1
  75. package/build/crawlers/basic_crawler.js +0 -664
  76. package/build/crawlers/basic_crawler.js.map +0 -1
  77. package/build/crawlers/browser_crawler.d.ts +0 -512
  78. package/build/crawlers/browser_crawler.d.ts.map +0 -1
  79. package/build/crawlers/browser_crawler.js +0 -540
  80. package/build/crawlers/browser_crawler.js.map +0 -1
  81. package/build/crawlers/cheerio_crawler.d.ts +0 -931
  82. package/build/crawlers/cheerio_crawler.d.ts.map +0 -1
  83. package/build/crawlers/cheerio_crawler.js +0 -913
  84. package/build/crawlers/cheerio_crawler.js.map +0 -1
  85. package/build/crawlers/crawler_extension.d.ts +0 -10
  86. package/build/crawlers/crawler_extension.d.ts.map +0 -1
  87. package/build/crawlers/crawler_extension.js +0 -19
  88. package/build/crawlers/crawler_extension.js.map +0 -1
  89. package/build/crawlers/crawler_utils.d.ts +0 -34
  90. package/build/crawlers/crawler_utils.d.ts.map +0 -1
  91. package/build/crawlers/crawler_utils.js +0 -87
  92. package/build/crawlers/crawler_utils.js.map +0 -1
  93. package/build/crawlers/playwright_crawler.d.ts +0 -448
  94. package/build/crawlers/playwright_crawler.d.ts.map +0 -1
  95. package/build/crawlers/playwright_crawler.js +0 -299
  96. package/build/crawlers/playwright_crawler.js.map +0 -1
  97. package/build/crawlers/puppeteer_crawler.d.ts +0 -425
  98. package/build/crawlers/puppeteer_crawler.d.ts.map +0 -1
  99. package/build/crawlers/puppeteer_crawler.js +0 -299
  100. package/build/crawlers/puppeteer_crawler.js.map +0 -1
  101. package/build/crawlers/statistics.d.ts +0 -185
  102. package/build/crawlers/statistics.d.ts.map +0 -1
  103. package/build/crawlers/statistics.js +0 -331
  104. package/build/crawlers/statistics.js.map +0 -1
  105. package/build/enqueue_links/click_elements.d.ts +0 -179
  106. package/build/enqueue_links/click_elements.d.ts.map +0 -1
  107. package/build/enqueue_links/click_elements.js +0 -434
  108. package/build/enqueue_links/click_elements.js.map +0 -1
  109. package/build/enqueue_links/enqueue_links.d.ts +0 -117
  110. package/build/enqueue_links/enqueue_links.d.ts.map +0 -1
  111. package/build/enqueue_links/enqueue_links.js +0 -163
  112. package/build/enqueue_links/enqueue_links.js.map +0 -1
  113. package/build/enqueue_links/shared.d.ts +0 -42
  114. package/build/enqueue_links/shared.d.ts.map +0 -1
  115. package/build/enqueue_links/shared.js +0 -121
  116. package/build/enqueue_links/shared.js.map +0 -1
  117. package/build/errors.d.ts +0 -29
  118. package/build/errors.d.ts.map +0 -1
  119. package/build/errors.js +0 -38
  120. package/build/errors.js.map +0 -1
  121. package/build/events.d.ts +0 -11
  122. package/build/events.d.ts.map +0 -1
  123. package/build/events.js +0 -147
  124. package/build/events.js.map +0 -1
  125. package/build/index.d.ts +0 -4
  126. package/build/index.d.ts.map +0 -1
  127. package/build/index.js +0 -7
  128. package/build/index.js.map +0 -1
  129. package/build/main.d.ts +0 -179
  130. package/build/main.d.ts.map +0 -1
  131. package/build/main.js +0 -81
  132. package/build/main.js.map +0 -1
  133. package/build/playwright_utils.d.ts +0 -9
  134. package/build/playwright_utils.d.ts.map +0 -1
  135. package/build/playwright_utils.js +0 -90
  136. package/build/playwright_utils.js.map +0 -1
  137. package/build/proxy_configuration.d.ts +0 -411
  138. package/build/proxy_configuration.d.ts.map +0 -1
  139. package/build/proxy_configuration.js +0 -517
  140. package/build/proxy_configuration.js.map +0 -1
  141. package/build/pseudo_url.d.ts +0 -86
  142. package/build/pseudo_url.d.ts.map +0 -1
  143. package/build/pseudo_url.js +0 -153
  144. package/build/pseudo_url.js.map +0 -1
  145. package/build/puppeteer_request_interception.d.ts +0 -8
  146. package/build/puppeteer_request_interception.d.ts.map +0 -1
  147. package/build/puppeteer_request_interception.js +0 -235
  148. package/build/puppeteer_request_interception.js.map +0 -1
  149. package/build/puppeteer_utils.d.ts +0 -250
  150. package/build/puppeteer_utils.d.ts.map +0 -1
  151. package/build/puppeteer_utils.js +0 -551
  152. package/build/puppeteer_utils.js.map +0 -1
  153. package/build/request.d.ts +0 -180
  154. package/build/request.d.ts.map +0 -1
  155. package/build/request.js +0 -261
  156. package/build/request.js.map +0 -1
  157. package/build/request_list.d.ts +0 -581
  158. package/build/request_list.d.ts.map +0 -1
  159. package/build/request_list.js +0 -826
  160. package/build/request_list.js.map +0 -1
  161. package/build/serialization.d.ts +0 -5
  162. package/build/serialization.d.ts.map +0 -1
  163. package/build/serialization.js +0 -139
  164. package/build/serialization.js.map +0 -1
  165. package/build/session_pool/errors.d.ts +0 -11
  166. package/build/session_pool/errors.d.ts.map +0 -1
  167. package/build/session_pool/errors.js +0 -18
  168. package/build/session_pool/errors.js.map +0 -1
  169. package/build/session_pool/events.d.ts +0 -5
  170. package/build/session_pool/events.d.ts.map +0 -1
  171. package/build/session_pool/events.js +0 -6
  172. package/build/session_pool/events.js.map +0 -1
  173. package/build/session_pool/session.d.ts +0 -286
  174. package/build/session_pool/session.d.ts.map +0 -1
  175. package/build/session_pool/session.js +0 -355
  176. package/build/session_pool/session.js.map +0 -1
  177. package/build/session_pool/session_pool.d.ts +0 -280
  178. package/build/session_pool/session_pool.d.ts.map +0 -1
  179. package/build/session_pool/session_pool.js +0 -393
  180. package/build/session_pool/session_pool.js.map +0 -1
  181. package/build/session_pool/session_utils.d.ts +0 -4
  182. package/build/session_pool/session_utils.d.ts.map +0 -1
  183. package/build/session_pool/session_utils.js +0 -24
  184. package/build/session_pool/session_utils.js.map +0 -1
  185. package/build/stealth/hiding_tricks.d.ts +0 -22
  186. package/build/stealth/hiding_tricks.d.ts.map +0 -1
  187. package/build/stealth/hiding_tricks.js +0 -308
  188. package/build/stealth/hiding_tricks.js.map +0 -1
  189. package/build/stealth/stealth.d.ts +0 -56
  190. package/build/stealth/stealth.d.ts.map +0 -1
  191. package/build/stealth/stealth.js +0 -125
  192. package/build/stealth/stealth.js.map +0 -1
  193. package/build/storages/dataset.d.ts +0 -288
  194. package/build/storages/dataset.d.ts.map +0 -1
  195. package/build/storages/dataset.js +0 -480
  196. package/build/storages/dataset.js.map +0 -1
  197. package/build/storages/key_value_store.d.ts +0 -243
  198. package/build/storages/key_value_store.d.ts.map +0 -1
  199. package/build/storages/key_value_store.js +0 -462
  200. package/build/storages/key_value_store.js.map +0 -1
  201. package/build/storages/request_queue.d.ts +0 -318
  202. package/build/storages/request_queue.d.ts.map +0 -1
  203. package/build/storages/request_queue.js +0 -636
  204. package/build/storages/request_queue.js.map +0 -1
  205. package/build/storages/storage_manager.d.ts +0 -87
  206. package/build/storages/storage_manager.d.ts.map +0 -1
  207. package/build/storages/storage_manager.js +0 -150
  208. package/build/storages/storage_manager.js.map +0 -1
  209. package/build/tsconfig.tsbuildinfo +0 -1
  210. package/build/typedefs.d.ts +0 -146
  211. package/build/typedefs.d.ts.map +0 -1
  212. package/build/typedefs.js +0 -88
  213. package/build/typedefs.js.map +0 -1
  214. package/build/utils.d.ts +0 -175
  215. package/build/utils.d.ts.map +0 -1
  216. package/build/utils.js +0 -731
  217. package/build/utils.js.map +0 -1
  218. package/build/utils_log.d.ts +0 -41
  219. package/build/utils_log.d.ts.map +0 -1
  220. package/build/utils_log.js +0 -192
  221. package/build/utils_log.js.map +0 -1
  222. package/build/utils_request.d.ts +0 -77
  223. package/build/utils_request.d.ts.map +0 -1
  224. package/build/utils_request.js +0 -385
  225. package/build/utils_request.js.map +0 -1
  226. package/build/utils_social.d.ts +0 -210
  227. package/build/utils_social.d.ts.map +0 -1
  228. package/build/utils_social.js +0 -787
  229. package/build/utils_social.js.map +0 -1
  230. package/build/validators.d.ts +0 -23
  231. package/build/validators.d.ts.map +0 -1
  232. package/build/validators.js +0 -29
  233. package/build/validators.js.map +0 -1
@@ -1,551 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.puppeteerUtils = exports.infiniteScroll = exports.gotoExtended = void 0;
4
- const tslib_1 = require("tslib");
5
- const fs_1 = (0, tslib_1.__importDefault)(require("fs"));
6
- const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
7
- const vm_1 = (0, tslib_1.__importDefault)(require("vm"));
8
- const util_1 = (0, tslib_1.__importDefault)(require("util"));
9
- const underscore_1 = (0, tslib_1.__importDefault)(require("underscore"));
10
- const datastructures_1 = require("@apify/datastructures");
11
- const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
12
- const validators_1 = require("./validators");
13
- const click_elements_1 = require("./enqueue_links/click_elements");
14
- const puppeteer_request_interception_1 = require("./puppeteer_request_interception");
15
- const key_value_store_1 = require("./storages/key_value_store");
16
- const jqueryPath = require.resolve('jquery');
17
- const underscorePath = require.resolve('underscore');
18
- const readFilePromised = util_1.default.promisify(fs_1.default.readFile);
19
- const MAX_INJECT_FILE_CACHE_SIZE = 10;
20
- const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
21
- /**
22
- * @typedef {object} CompiledScriptParams
23
- * @property {Page} params.page
24
- * @property {Request} params.request
25
- */
26
- /**
27
- * @callback CompiledScriptFunction
28
- * @param {CompiledScriptParams} params
29
- * @returns {Promise<*>}
30
- */
31
- /**
32
- * Cache contents of previously injected files to limit file system access.
33
- */
34
- const injectedFilesCache = new datastructures_1.LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE });
35
- /**
36
- * Injects a JavaScript file into a Puppeteer page.
37
- * Unlike Puppeteer's `addScriptTag` function, this function works on pages
38
- * with arbitrary Cross-Origin Resource Sharing (CORS) policies.
39
- *
40
- * File contents are cached for up to 10 files to limit file system access.
41
- *
42
- * @param {Page} page
43
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
44
- * @param {string} filePath File path
45
- * @param {object} [options]
46
- * @param {boolean} [options.surviveNavigations]
47
- * Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
48
- * This does not mean, however, that internal state will be preserved. Just that it will be automatically
49
- * re-injected on each navigation before any other scripts get the chance to execute.
50
- * @return {Promise<*>}
51
- * @memberOf puppeteer
52
- */
53
- const injectFile = async (page, filePath, options = {}) => {
54
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
55
- (0, ow_1.default)(filePath, ow_1.default.string);
56
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
57
- surviveNavigations: ow_1.default.optional.boolean,
58
- }));
59
- let contents = injectedFilesCache.get(filePath);
60
- if (!contents) {
61
- contents = await readFilePromised(filePath, 'utf8');
62
- injectedFilesCache.add(filePath, contents);
63
- }
64
- const evalP = page.evaluate(contents);
65
- return options.surviveNavigations
66
- ? Promise.all([page.evaluateOnNewDocument(contents), evalP])
67
- : evalP;
68
- };
69
- /**
70
- * Injects the [jQuery](https://jquery.com/) library into a Puppeteer page.
71
- * jQuery is often useful for various web scraping and crawling tasks.
72
- * For example, it can help extract text from HTML elements using CSS selectors.
73
- *
74
- * Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with
75
- * other libraries included by the page that use the same variable name (e.g. another version of jQuery).
76
- * This can affect functionality of page's scripts.
77
- *
78
- * The injected jQuery will survive page navigations and reloads.
79
- *
80
- * **Example usage:**
81
- * ```javascript
82
- * await Apify.utils.puppeteer.injectJQuery(page);
83
- * const title = await page.evaluate(() => {
84
- * return $('head title').text();
85
- * });
86
- * ```
87
- *
88
- * Note that `injectJQuery()` does not affect the Puppeteer's
89
- * [`page.$()`](https://pptr.dev/#?product=Puppeteer&show=api-pageselector)
90
- * function in any way.
91
- *
92
- * @param {Page} page
93
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
94
- * @return {Promise<*>}
95
- * @memberOf puppeteer
96
- */
97
- const injectJQuery = (page) => {
98
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
99
- return injectFile(page, jqueryPath, { surviveNavigations: true });
100
- };
101
- /**
102
- * Injects the [Underscore](https://underscorejs.org/) library into a Puppeteer page.
103
- *
104
- * Beware that the injected Underscore object will be set to the `window._` variable and thus it might cause conflicts with
105
- * libraries included by the page that use the same variable name.
106
- * This can affect functionality of page's scripts.
107
- *
108
- * The injected Underscore will survive page navigations and reloads.
109
- *
110
- * **Example usage:**
111
- * ```javascript
112
- * await Apify.utils.puppeteer.injectUnderscore(page);
113
- * const escapedHtml = await page.evaluate(() => {
114
- * return _.escape('<h1>Hello</h1>');
115
- * });
116
- * ```
117
- *
118
- * @param {Page} page Puppeteer [Page](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#class-page) object.
119
- * @return {Promise<*>}
120
- * @memberOf puppeteer
121
- */
122
- const injectUnderscore = (page) => {
123
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
124
- return injectFile(page, underscorePath, { surviveNavigations: true });
125
- };
126
- /**
127
- * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
128
- * This is useful to speed up crawling of websites, since it reduces the amount
129
- * of data that needs to be downloaded from the web, but it may break some websites
130
- * or unexpectedly prevent loading of resources.
131
- *
132
- * By default, the function will block all URLs including the following patterns:
133
- *
134
- * ```json
135
- * [".css", ".jpg", ".jpeg", ".png", ".svg", ".gif", ".woff", ".pdf", ".zip"]
136
- * ```
137
- *
138
- * If you want to extend this list further, use the `extraUrlPatterns` option,
139
- * which will keep blocking the default patterns, as well as add your custom ones.
140
- * If you would like to block only specific patterns, use the `urlPatterns` option,
141
- * which will override the defaults and block only URLs with your custom patterns.
142
- *
143
- * This function does not use Puppeteer's request interception and therefore does not interfere
144
- * with browser cache. It's also faster than blocking requests using interception,
145
- * because the blocking happens directly in the browser without the round-trip to Node.js,
146
- * but it does not provide the extra benefits of request interception.
147
- *
148
- * The function will never block main document loads and their respective redirects.
149
- *
150
- * **Example usage**
151
- * ```javascript
152
- * const Apify = require('apify');
153
- *
154
- * const browser = await Apify.launchPuppeteer();
155
- * const page = await browser.newPage();
156
- *
157
- * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
158
- * await Apify.utils.puppeteer.blockRequests(page, {
159
- * extraUrlPatterns: ['adsbygoogle.js'],
160
- * });
161
- *
162
- * await page.goto('https://cnn.com');
163
- * ```
164
- *
165
- * @param {Page} page
166
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
167
- * @param {object} [options]
168
- * @param {string[]} [options.urlPatterns]
169
- * The patterns of URLs to block from being loaded by the browser.
170
- * Only `*` can be used as a wildcard. It is also automatically added to the beginning
171
- * and end of the pattern. This limitation is enforced by the DevTools protocol.
172
- * `.png` is the same as `*.png*`.
173
- * @param {string[]} [options.extraUrlPatterns]
174
- * If you just want to append to the default blocked patterns, use this property.
175
- * @return {Promise<void>}
176
- * @memberOf puppeteer
177
- */
178
- const blockRequests = async (page, options = {}) => {
179
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
180
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
181
- urlPatterns: ow_1.default.optional.array.ofType(ow_1.default.string),
182
- extraUrlPatterns: ow_1.default.optional.array.ofType(ow_1.default.string),
183
- }));
184
- const { urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS, extraUrlPatterns = [], } = options;
185
- const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
186
- await page._client.send('Network.setBlockedURLs', { urls: patternsToBlock }); // eslint-disable-line no-underscore-dangle
187
- };
188
- /**
189
- * `blockResources()` has a high impact on performance in recent versions of Puppeteer.
190
- * 'Until this resolves, please use `Apify.utils.puppeteer.blockRequests()`.
191
- * @deprecated
192
- */
193
- const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'image', 'media']) => {
194
- utils_log_1.default.deprecated('Apify.utils.puppeteer.blockResources() has a high impact on performance in recent versions of Puppeteer. '
195
- + 'Until this resolves, please use Apify.utils.puppeteer.blockRequests()');
196
- await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, async (request) => {
197
- const type = request.resourceType();
198
- if (resourceTypes.includes(type))
199
- await request.abort();
200
- else
201
- await request.continue();
202
- });
203
- };
204
- /**
205
- * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
206
- * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
207
- *
208
- * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
209
- * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
210
- * This issue should be resolved or atleast mitigated in future iterations of this feature.
211
- * @param {Page} page
212
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
213
- * @param {Object<string, *>} cache
214
- * Object in which responses are stored
215
- * @param {Array<(string|RegExp)>} responseUrlRules
216
- * List of rules that are used to check if the response should be cached.
217
- * String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
218
- * @return {Promise<void>}
219
- * @memberOf puppeteer
220
- * @deprecated
221
- */
222
- const cacheResponses = async (page, cache, responseUrlRules) => {
223
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
224
- (0, ow_1.default)(cache, ow_1.default.object);
225
- (0, ow_1.default)(responseUrlRules, ow_1.default.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.regExp)));
226
- utils_log_1.default.deprecated('Apify.utils.puppeteer.cacheResponses() has a high impact on performance '
227
- + 'in recent versions of Puppeteer so it\'s use is discouraged until this issue resolves.');
228
- await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, async (request) => {
229
- const url = request.url();
230
- if (cache[url]) {
231
- await request.respond(cache[url]);
232
- return;
233
- }
234
- await request.continue();
235
- });
236
- page.on('response', async (response) => {
237
- const url = response.url();
238
- // Response is already cached, do nothing
239
- if (cache[url])
240
- return;
241
- const shouldCache = responseUrlRules.some((rule) => {
242
- if (typeof rule === 'string')
243
- return url.includes(rule);
244
- if (rule instanceof RegExp)
245
- return rule.test(url);
246
- });
247
- try {
248
- if (shouldCache) {
249
- const buffer = await response.buffer();
250
- cache[url] = {
251
- status: response.status(),
252
- headers: response.headers(),
253
- body: buffer,
254
- };
255
- }
256
- }
257
- catch (e) {
258
- // ignore errors, usualy means that buffer is empty or broken connection
259
- }
260
- });
261
- };
262
- /**
263
- * Compiles a Puppeteer script into an async function that may be executed at any time
264
- * by providing it with the following object:
265
- * ```
266
- * {
267
- * page: Page,
268
- * request: Request,
269
- * }
270
- * ```
271
- * Where `page` is a Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page)
272
- * and `request` is a {@link Request}.
273
- *
274
- * The function is compiled by using the `scriptString` parameter as the function's body,
275
- * so any limitations to function bodies apply. Return value of the compiled function
276
- * is the return value of the function body = the `scriptString` parameter.
277
- *
278
- * As a security measure, no globals such as `process` or `require` are accessible
279
- * from within the function body. Note that the function does not provide a safe
280
- * sandbox and even though globals are not easily accessible, malicious code may
281
- * still execute in the main process via prototype manipulation. Therefore you
282
- * should only use this function to execute sanitized or safe code.
283
- *
284
- * Custom context may also be provided using the `context` parameter. To improve security,
285
- * make sure to only pass the really necessary objects to the context. Preferably making
286
- * secured copies beforehand.
287
- *
288
- * @param {string} scriptString
289
- * @param {Object<string, *>} context
290
- * @return {CompiledScriptFunction}
291
- * @memberOf puppeteer
292
- */
293
- const compileScript = (scriptString, context = Object.create(null)) => {
294
- const funcString = `async ({ page, request }) => {${scriptString}}`;
295
- let func;
296
- try {
297
- func = vm_1.default.runInNewContext(funcString, context); // "Secure" the context by removing prototypes, unless custom context is provided.
298
- }
299
- catch (err) {
300
- utils_log_1.default.exception(err, 'Cannot compile script!');
301
- throw err;
302
- }
303
- if (!underscore_1.default.isFunction(func))
304
- throw new Error('Compilation result is not a function!'); // This should not happen...
305
- return func;
306
- };
307
- /**
308
- * Extended version of Puppeteer's `page.goto()` allowing to perform requests with HTTP method other than GET,
309
- * with custom headers and POST payload. URL, method, headers and payload are taken from
310
- * request parameter that must be an instance of Apify.Request class.
311
- *
312
- * *NOTE:* In recent versions of Puppeteer using requests other than GET, overriding headers and adding payloads disables
313
- * browser cache which degrades performance.
314
- *
315
- * @param {Page} page
316
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
317
- * @param {Request} request
318
- * @param {DirectNavigationOptions} [gotoOptions] Custom options for `page.goto()`.
319
- * @return {Promise<(Response|null)>}
320
- *
321
- * @memberOf puppeteer
322
- * @name gotoExtended
323
- */
324
- const gotoExtended = async (page, request, gotoOptions = {}) => {
325
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
326
- (0, ow_1.default)(request, ow_1.default.object.partialShape({
327
- url: ow_1.default.string.url,
328
- method: ow_1.default.optional.string,
329
- headers: ow_1.default.optional.object,
330
- payload: ow_1.default.optional.any(ow_1.default.string, ow_1.default.buffer),
331
- }));
332
- (0, ow_1.default)(gotoOptions, ow_1.default.object);
333
- const { url, method, headers, payload } = request;
334
- if (method !== 'GET' || payload || !underscore_1.default.isEmpty(headers)) {
335
- // This is not deprecated, we use it to log only once.
336
- utils_log_1.default.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance '
337
- + 'in recent versions of Puppeteer. Use only when necessary.');
338
- let wasCalled = false;
339
- const interceptRequestHandler = async (interceptedRequest) => {
340
- // We want to ensure that this won't get executed again in a case that there is a subsequent request
341
- // for example for some asset file link from main HTML.
342
- if (wasCalled) {
343
- return interceptedRequest.continue();
344
- }
345
- wasCalled = true;
346
- const overrides = {};
347
- if (method !== 'GET')
348
- overrides.method = method;
349
- if (payload)
350
- overrides.postData = payload;
351
- if (!underscore_1.default.isEmpty(headers))
352
- overrides.headers = headers;
353
- await (0, puppeteer_request_interception_1.removeInterceptRequestHandler)(page, interceptRequestHandler);
354
- interceptedRequest.continue(overrides);
355
- };
356
- await (0, puppeteer_request_interception_1.addInterceptRequestHandler)(page, interceptRequestHandler);
357
- }
358
- return page.goto(url, gotoOptions);
359
- };
360
- exports.gotoExtended = gotoExtended;
361
- /**
362
- * Scrolls to the bottom of a page, or until it times out.
363
- * Loads dynamic content when it hits the bottom of a page, and then continues scrolling.
364
- * @param {Page} page
365
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
366
- * @param {object} [options]
367
- * @param {number} [options.timeoutSecs=0]
368
- * How many seconds to scroll for. If 0, will scroll until bottom of page.
369
- * @param {number} [options.waitForSecs=4]
370
- * How many seconds to wait for no new content to load before exit.
371
- * @param {boolean} [options.scrollDownAndUp=false]
372
- * If true, it will scroll up a bit after each scroll down. This is required on some websites for the scroll to work.
373
- * @param {string} [options.buttonSelector]
374
- * Optionally checks and clicks a button if it appears while scrolling. This is required on some websites for the scroll to work.
375
- * @param {function} [options.stopScrollCallback]
376
- * This function is called after every scroll and stops the scrolling process if it returns `true`. The function can be `async`.
377
- * @returns {Promise<void>}
378
- * @memberOf puppeteer
379
- * @name infiniteScroll
380
- */
381
- const infiniteScroll = async (page, options = {}) => {
382
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
383
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
384
- timeoutSecs: ow_1.default.optional.number,
385
- waitForSecs: ow_1.default.optional.number,
386
- scrollDownAndUp: ow_1.default.optional.boolean,
387
- buttonSelector: ow_1.default.optional.string,
388
- stopScrollCallback: ow_1.default.optional.function,
389
- }));
390
- const { timeoutSecs = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback } = options;
391
- let finished;
392
- const startTime = Date.now();
393
- const CHECK_INTERVAL_MILLIS = 1000;
394
- const SCROLL_HEIGHT_IF_ZERO = 10000;
395
- const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];
396
- const resourcesStats = {
397
- newRequested: 0,
398
- oldRequested: 0,
399
- matchNumber: 0,
400
- };
401
- page.on('request', (msg) => {
402
- if (maybeResourceTypesInfiniteScroll.includes(msg.resourceType())) {
403
- resourcesStats.newRequested++;
404
- }
405
- });
406
- // Move mouse to the center of the page, so we can scroll up-down
407
- let body = await page.$('body');
408
- let retry = 0;
409
- while (!body && retry < 10) {
410
- await page.waitForTimeout(100);
411
- body = await page.$('body');
412
- retry++;
413
- }
414
- if (!body) {
415
- return;
416
- }
417
- const boundingBox = await body.boundingBox();
418
- await page.mouse.move(boundingBox.x + boundingBox.width / 2, // x
419
- boundingBox.y + boundingBox.height / 2);
420
- const checkFinished = setInterval(() => {
421
- if (resourcesStats.oldRequested === resourcesStats.newRequested) {
422
- resourcesStats.matchNumber++;
423
- if (resourcesStats.matchNumber >= waitForSecs) {
424
- clearInterval(checkFinished);
425
- finished = true;
426
- return;
427
- }
428
- }
429
- else {
430
- resourcesStats.matchNumber = 0;
431
- resourcesStats.oldRequested = resourcesStats.newRequested;
432
- }
433
- // check if timeout has been reached
434
- if (timeoutSecs !== 0 && (Date.now() - startTime) / 1000 > timeoutSecs) {
435
- clearInterval(checkFinished);
436
- finished = true;
437
- }
438
- }, CHECK_INTERVAL_MILLIS);
439
- const doScroll = async () => {
440
- /* istanbul ignore next */
441
- const bodyScrollHeight = await page.evaluate(() => document.body.scrollHeight);
442
- const delta = bodyScrollHeight === 0 ? SCROLL_HEIGHT_IF_ZERO : bodyScrollHeight;
443
- await page.mouse.wheel({ deltaY: delta });
444
- };
445
- const maybeClickButton = async () => {
446
- const button = await page.$(buttonSelector);
447
- // Box model returns null if the button is not visible
448
- if (button && await button.boxModel()) {
449
- await button.click({ delay: 10 });
450
- }
451
- };
452
- while (!finished) {
453
- await doScroll();
454
- await page.waitForTimeout(250);
455
- if (scrollDownAndUp) {
456
- await page.mouse.wheel({ deltaY: -1000 });
457
- }
458
- if (buttonSelector) {
459
- await maybeClickButton();
460
- }
461
- if (stopScrollCallback) {
462
- if (await stopScrollCallback()) {
463
- clearInterval(checkFinished);
464
- break;
465
- }
466
- }
467
- }
468
- };
469
- exports.infiniteScroll = infiniteScroll;
470
- /**
471
- * Saves a full screenshot and HTML of the current page into a Key-Value store.
472
- * @param {Page} page
473
- * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
474
- * @param {object} [options]
475
- * @param {string} [options.key=SNAPSHOT]
476
- * Key under which the screenshot and HTML will be saved. `.jpg` will be appended for screenshot and `.html` for HTML.
477
- * @param {number} [options.screenshotQuality=50]
478
- * The quality of the image, between 0-100. Higher quality images have bigger size and require more storage.
479
- * @param {boolean} [options.saveScreenshot=true]
480
- * If true, it will save a full screenshot of the current page as a record with `key` appended by `.jpg`.
481
- * @param {boolean} [options.saveHtml=true]
482
- * If true, it will save a full HTML of the current page as a record with `key` appended by `.html`.
483
- * @param {string|null} [options.keyValueStoreName=null]
484
- * Name or id of the Key-Value store where snapshot is saved. By default it is saved to default Key-Value store.
485
- * @returns {Promise<void>}
486
- * @memberOf puppeteer
487
- * @name saveSnapshot
488
- */
489
- const saveSnapshot = async (page, options = {}) => {
490
- (0, ow_1.default)(page, ow_1.default.object.validate(validators_1.validators.browserPage));
491
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
492
- key: ow_1.default.optional.string.nonEmpty,
493
- screenshotQuality: ow_1.default.optional.number,
494
- saveScreenshot: ow_1.default.optional.boolean,
495
- saveHtml: ow_1.default.optional.boolean,
496
- keyValueStoreName: ow_1.default.optional.string,
497
- }));
498
- const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, } = options;
499
- try {
500
- const store = await (0, key_value_store_1.openKeyValueStore)(keyValueStoreName);
501
- if (saveScreenshot) {
502
- const screenshotName = `${key}.jpg`;
503
- const screenshotBuffer = await page.screenshot({ fullPage: true, screenshotQuality, type: 'jpeg' });
504
- await store.setValue(screenshotName, screenshotBuffer, { contentType: 'image/jpeg' });
505
- }
506
- if (saveHtml) {
507
- const htmlName = `${key}.html`;
508
- const html = await page.content();
509
- await store.setValue(htmlName, html, { contentType: 'text/html' });
510
- }
511
- }
512
- catch (err) {
513
- throw new Error(`saveSnapshot with key ${key} failed.\nCause:${err.message}`);
514
- }
515
- };
516
- /**
517
- * A namespace that contains various utilities for
518
- * [Puppeteer](https://github.com/puppeteer/puppeteer) - the headless Chrome Node API.
519
- *
520
- * **Example usage:**
521
- *
522
- * ```javascript
523
- * const Apify = require('apify');
524
- * const { puppeteer } = Apify.utils;
525
- *
526
- * // Open https://www.example.com in Puppeteer
527
- * const browser = await Apify.launchPuppeteer();
528
- * const page = await browser.newPage();
529
- * await page.goto('https://www.example.com');
530
- *
531
- * // Inject jQuery into a page
532
- * await puppeteer.injectJQuery(page);
533
- * ```
534
- * @namespace puppeteer
535
- */
536
- exports.puppeteerUtils = {
537
- injectFile,
538
- injectJQuery,
539
- injectUnderscore,
540
- enqueueLinksByClickingElements: click_elements_1.enqueueLinksByClickingElements,
541
- blockRequests,
542
- blockResources,
543
- cacheResponses,
544
- compileScript,
545
- gotoExtended: exports.gotoExtended,
546
- addInterceptRequestHandler: puppeteer_request_interception_1.addInterceptRequestHandler,
547
- removeInterceptRequestHandler: puppeteer_request_interception_1.removeInterceptRequestHandler,
548
- infiniteScroll: exports.infiniteScroll,
549
- saveSnapshot,
550
- };
551
- //# sourceMappingURL=puppeteer_utils.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"puppeteer_utils.js","sourceRoot":"","sources":["../src/puppeteer_utils.js"],"names":[],"mappings":";;;;AAAA,yDAAoB;AACpB,yDAAoB;AACpB,yDAAoB;AACpB,6DAAwB;AACxB,yEAA2B;AAC3B,0DAAiD;AAEjD,yEAA8B;AAC9B,6CAA0C;AAE1C,mEAAgF;AAChF,qFAA6G;AAC7G,gEAA+D;AAE/D,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAC7C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AACrD,MAAM,gBAAgB,GAAG,cAAI,CAAC,SAAS,CAAC,YAAE,CAAC,QAAQ,CAAC,CAAC;AAErD,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,MAAM,kCAAkC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAEtH;;;;GAIG;AACH;;;;GAIG;AACH;;GAEG;AACH,MAAM,kBAAkB,GAAG,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAEnF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,GAAG,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IACtD,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACxB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;KAC1C,CAAC,CAAC,CAAC;IAEJ,IAAI,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,EAAE;QACX,QAAQ,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpD,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;KAC9C;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtC,OAAO,OAAO,CAAC,kBAAkB;QAC7B,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,KAAK,CAAC;AAChB,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,EAAE;IAC1B,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;AACtE,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,EAAE;IAC9B,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,IAAI,EAAE,cAAc,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,MAAM,aAAa,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IAC/C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChD,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;KACxD,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,WAAW,GAAG,kCAAkC,EAChD,gBAAgB,GAAG,EAAE,GACxB,GAAG,OAAO,CAAC;IAEZ,MAAM,eAAe,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,gBAAgB,CAAC,CAAC;IAE9D,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,2CAA2C;AAC7H,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,cAAc,GAAG,KAAK,EAAE,IAAI,EAAE,aAAa,GAAG,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE;IAC5F,mBAAG,CAAC,UAAU,CAAC,2GAA2G;UACpH,uEAAuE,CAAC,CAAC;IAC/E,MAAM,IAAA,2DAA0B,EAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACrD,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;QACpC,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;;YACnD,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,cAAc,GAAG,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAAE;IAC3D,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACrB,IAAA,YAAE,EAAC,gBAAgB,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAEpE,mBAAG,CAAC,UAAU,CAAC,0EAA0E;UACnF,wFAAwF,CAAC,CAAC;IAEhG,MAAM,IAAA,2DAA0B,EAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACrD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAE1B,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE;YACZ,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,OAAO;SACV;QAED,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC;QAE3B,yCAAyC;QACzC,IAAI,KAAK,CAAC,GAAG,CAAC;YAAE,OAAO;QAEvB,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YAC/C,IAAI,OAAO,IAAI,KAAK,QAAQ;gBAAE,OAAO,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACxD,IAAI,IAAI,YAAY,MAAM;gBAAE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,IAAI;YACA,IAAI,WAAW,EAAE;gBACb,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACvC,KAAK,CAAC,GAAG,CAAC,GAAG;oBACT,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE;oBACzB,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE;oBAC3B,IAAI,EAAE,MAAM;iBACf,CAAC;aACL;SACJ;QAAC,OAAO,CAAC,EAAE;YACR,wEAAwE;SAC3E;IACL,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,aAAa,GAAG,CAAC,YAAY,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE;IAClE,MAAM,UAAU,GAAG,iCAAiC,YAAY,GAAG,CAAC;IAEpE,IAAI,IAAI,CAAC;IACT,IAAI;QACA,IAAI,GAAG,YAAE,CAAC,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,kFAAkF;KACrI;IAAC,OAAO,GAAG,EAAE;QACV,mBAAG,CAAC,SAAS,CAAC,GAAG,EAAE,wBAAwB,CAAC,CAAC;QAC7C,MAAM,GAAG,CAAC;KACb;IAED,IAAI,CAAC,oBAAC,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC,4BAA4B;IAE/G,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;GAgBG;AACI,MAAM,YAAY,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,GAAG,EAAE,EAAE,EAAE;IAClE,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAC/B,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;KACjD,CAAC,CAAC,CAAC;IACJ,IAAA,YAAE,EAAC,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAE3B,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAElD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO,IAAI,CAAC,oBAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QACpD,sDAAsD;QACtD,mBAAG,CAAC,UAAU,CAAC,+GAA+G;cACxH,2DAA2D,CAAC,CAAC;QACnE,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,uBAAuB,GAAG,KAAK,EAAE,kBAAkB,EAAE,EAAE;YACzD,oGAAoG;YACpG,uDAAuD;YACvD,IAAI,SAAS,EAAE;gBACX,OAAO,kBAAkB,CAAC,QAAQ,EAAE,CAAC;aACxC;YAED,SAAS,GAAG,IAAI,CAAC;YACjB,MAAM,SAAS,GAAG,EAAE,CAAC;YAErB,IAAI,MAAM,KAAK,KAAK;gBAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC;YAChD,IAAI,OAAO;gBAAE,SAAS,CAAC,QAAQ,GAAG,OAAO,CAAC;YAC1C,IAAI,CAAC,oBAAC,CAAC,OAAO,CAAC,OAAO,CAAC;gBAAE,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC;YACrD,MAAM,IAAA,8DAA6B,EAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;YACnE,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC,CAAC;QAEF,MAAM,IAAA,2DAA0B,EAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;KACnE;IAED,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;AACvC,CAAC,CAAC;AAtCW,QAAA,YAAY,gBAsCvB;AAEF;;;;;;;;;;;;;;;;;;;GAmBG;AACI,MAAM,cAAc,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IACvD,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;KAC3C,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,EAAE,eAAe,GAAG,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC;IAElH,IAAI,QAAQ,CAAC;IACb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,qBAAqB,GAAG,IAAI,CAAC;IACnC,MAAM,qBAAqB,GAAG,KAAK,CAAC;IACpC,MAAM,gCAAgC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG;QACnB,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;KACjB,CAAC;IAEF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,IAAI,gCAAgC,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,EAAE;YAC/D,cAAc,CAAC,YAAY,EAAE,CAAC;SACjC;IACL,CAAC,CAAC,CAAC;IAEH,iEAAiE;IACjE,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,CAAC,IAAI,IAAI,KAAK,GAAG,EAAE,EAAE;QACxB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAC5B,KAAK,EAAE,CAAC;KACX;IAED,IAAI,CAAC,IAAI,EAAE;QACP,OAAO;KACV;IAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACjB,WAAW,CAAC,CAAC,GAAG,WAAW,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI;IAC3C,WAAW,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CACzC,CAAC;IAEF,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,IAAI,cAAc,CAAC,YAAY,KAAK,cAAc,CAAC,YAAY,EAAE;YAC7D,cAAc,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,cAAc,CAAC,WAAW,IAAI,WAAW,EAAE;gBAC3C,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,QAAQ,GAAG,IAAI,CAAC;gBAChB,OAAO;aACV;SACJ;aAAM;YACH,cAAc,CAAC,WAAW,GAAG,CAAC,CAAC;YAC/B,cAAc,CAAC,YAAY,GAAG,cAAc,CAAC,YAAY,CAAC;SAC7D;QACD,oCAAoC;QACpC,IAAI,WAAW,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG,WAAW,EAAE;YACpE,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;SACnB;IACL,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAE1B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QACxB,0BAA0B;QAC1B,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE/E,MAAM,KAAK,GAAG,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEhF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC;IAEF,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QAC5C,sDAAsD;QACtD,IAAI,MAAM,IAAI,MAAM,MAAM,CAAC,QAAQ,EAAE,EAAE;YACnC,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;SACrC;IACL,CAAC,CAAC;IAEF,OAAO,CAAC,QAAQ,EAAE;QACd,MAAM,QAAQ,EAAE,CAAC;QACjB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,eAAe,EAAE;YACjB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;SAC7C;QACD,IAAI,cAAc,EAAE;YAChB,MAAM,gBAAgB,EAAE,CAAC;SAC5B;QACD,IAAI,kBAAkB,EAAE;YACpB,IAAI,MAAM,kBAAkB,EAAE,EAAE;gBAC5B,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,MAAM;aACT;SACJ;KACJ;AACL,CAAC,CAAC;AArGW,QAAA,cAAc,kBAqGzB;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,YAAY,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IAC9C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;QAChC,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACxC,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,GAAG,GAAG,UAAU,EAChB,iBAAiB,GAAG,EAAE,EACtB,cAAc,GAAG,IAAI,EACrB,QAAQ,GAAG,IAAI,EACf,iBAAiB,GACpB,GAAG,OAAO,CAAC;IAEZ,IAAI;QACA,MAAM,KAAK,GAAG,MAAM,IAAA,mCAAiB,EAAC,iBAAiB,CAAC,CAAC;QAEzD,IAAI,cAAc,EAAE;YAChB,MAAM,cAAc,GAAG,GAAG,GAAG,MAAM,CAAC;YACpC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YACpG,MAAM,KAAK,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC;SACzF;QACD,IAAI,QAAQ,EAAE;YACV,MAAM,QAAQ,GAAG,GAAG,GAAG,OAAO,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;SACtE;KACJ;IAAC,OAAO,GAAG,EAAE;QACV,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,mBAAmB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;KACjF;AACL,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;GAmBG;AACU,QAAA,cAAc,GAAG;IAC1B,UAAU;IACV,YAAY;IACZ,gBAAgB;IAChB,8BAA8B,EAA9B,+CAA8B;IAC9B,aAAa;IACb,cAAc;IACd,cAAc;IACd,aAAa;IACb,YAAY,EAAZ,oBAAY;IACZ,0BAA0B,EAA1B,2DAA0B;IAC1B,6BAA6B,EAA7B,8DAA6B;IAC7B,cAAc,EAAd,sBAAc;IACd,YAAY;CACf,CAAC"}