@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.200

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. package/README.md +17 -13
  2. package/byte_utils.d.ts +17 -0
  3. package/byte_utils.js +42 -0
  4. package/configuration.d.ts +88 -234
  5. package/configuration.js +162 -230
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +43 -10
  9. package/errors.js +57 -12
  10. package/events/event_manager.d.ts +35 -9
  11. package/events/event_manager.js +12 -13
  12. package/events/index.d.ts +1 -1
  13. package/events/index.js +0 -1
  14. package/events/local_event_manager.d.ts +15 -5
  15. package/events/local_event_manager.js +39 -42
  16. package/events/system_info.d.ts +38 -0
  17. package/events/system_info.js +1 -0
  18. package/index.d.ts +8 -11
  19. package/index.js +9 -10
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +81 -3
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +43 -0
  31. package/memory-storage/memory-storage.js +158 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  35. package/memory-storage/resource-clients/dataset.js +106 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +56 -0
  37. package/memory-storage/resource-clients/key-value-store.js +180 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  39. package/memory-storage/resource-clients/request-queue.js +414 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned_or_injected.d.ts +58 -0
  43. package/owned_or_injected.js +98 -0
  44. package/package.json +19 -20
  45. package/proxy_configuration.d.ts +29 -155
  46. package/proxy_configuration.js +38 -183
  47. package/recoverable_state.d.ts +177 -0
  48. package/recoverable_state.js +268 -0
  49. package/request.d.ts +62 -81
  50. package/request.js +125 -220
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +18 -18
  53. package/service_locator.d.ts +168 -0
  54. package/service_locator.js +255 -0
  55. package/storages/batched_adds.d.ts +37 -0
  56. package/storages/batched_adds.js +73 -0
  57. package/storages/dataset.d.ts +120 -56
  58. package/storages/dataset.js +299 -148
  59. package/storages/index.d.ts +9 -8
  60. package/storages/index.js +5 -8
  61. package/storages/key_value_store.d.ts +192 -65
  62. package/storages/key_value_store.js +453 -193
  63. package/storages/key_value_store_codec.d.ts +32 -0
  64. package/storages/key_value_store_codec.js +113 -0
  65. package/storages/request_dedup_cache.d.ts +22 -0
  66. package/storages/request_dedup_cache.js +48 -0
  67. package/storages/request_list.d.ts +55 -137
  68. package/storages/request_list.js +272 -245
  69. package/storages/request_loader.d.ts +127 -0
  70. package/storages/request_loader.js +36 -0
  71. package/storages/request_manager.d.ts +119 -0
  72. package/storages/request_manager.js +1 -0
  73. package/storages/request_manager_tandem.d.ts +94 -0
  74. package/storages/request_manager_tandem.js +200 -0
  75. package/storages/request_queue.d.ts +293 -48
  76. package/storages/request_queue.js +766 -216
  77. package/storages/storage_instance_manager.d.ts +86 -0
  78. package/storages/storage_instance_manager.js +256 -0
  79. package/storages/storage_stats.d.ts +48 -0
  80. package/storages/storage_stats.js +29 -0
  81. package/storages/transaction.d.ts +271 -0
  82. package/storages/transaction.js +296 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +75 -15
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -300
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -533
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -189
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -292
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -181
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -169
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -113
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/crawler_commons.d.ts +0 -199
  122. package/crawlers/crawler_commons.d.ts.map +0 -1
  123. package/crawlers/crawler_commons.js +0 -116
  124. package/crawlers/crawler_commons.js.map +0 -1
  125. package/crawlers/crawler_extension.d.ts +0 -12
  126. package/crawlers/crawler_extension.d.ts.map +0 -1
  127. package/crawlers/crawler_extension.js +0 -14
  128. package/crawlers/crawler_extension.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -59
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -54
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/statistics.d.ts +0 -204
  146. package/crawlers/statistics.d.ts.map +0 -1
  147. package/crawlers/statistics.js +0 -346
  148. package/crawlers/statistics.js.map +0 -1
  149. package/enqueue_links/enqueue_links.d.ts +0 -242
  150. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  151. package/enqueue_links/enqueue_links.js +0 -250
  152. package/enqueue_links/enqueue_links.js.map +0 -1
  153. package/enqueue_links/index.d.ts +0 -3
  154. package/enqueue_links/index.d.ts.map +0 -1
  155. package/enqueue_links/index.js +0 -3
  156. package/enqueue_links/index.js.map +0 -1
  157. package/enqueue_links/shared.d.ts +0 -67
  158. package/enqueue_links/shared.d.ts.map +0 -1
  159. package/enqueue_links/shared.js +0 -190
  160. package/enqueue_links/shared.js.map +0 -1
  161. package/errors.d.ts.map +0 -1
  162. package/errors.js.map +0 -1
  163. package/events/event_manager.d.ts.map +0 -1
  164. package/events/event_manager.js.map +0 -1
  165. package/events/index.d.ts.map +0 -1
  166. package/events/index.js.map +0 -1
  167. package/events/local_event_manager.d.ts.map +0 -1
  168. package/events/local_event_manager.js.map +0 -1
  169. package/http_clients/base-http-client.d.ts +0 -134
  170. package/http_clients/base-http-client.d.ts.map +0 -1
  171. package/http_clients/base-http-client.js +0 -33
  172. package/http_clients/base-http-client.js.map +0 -1
  173. package/http_clients/form-data-like.d.ts +0 -67
  174. package/http_clients/form-data-like.d.ts.map +0 -1
  175. package/http_clients/form-data-like.js +0 -5
  176. package/http_clients/form-data-like.js.map +0 -1
  177. package/http_clients/got-scraping-http-client.d.ts +0 -15
  178. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  179. package/http_clients/got-scraping-http-client.js +0 -69
  180. package/http_clients/got-scraping-http-client.js.map +0 -1
  181. package/http_clients/index.d.ts +0 -3
  182. package/http_clients/index.d.ts.map +0 -1
  183. package/http_clients/index.js +0 -3
  184. package/http_clients/index.js.map +0 -1
  185. package/index.d.ts.map +0 -1
  186. package/index.js.map +0 -1
  187. package/log.d.ts.map +0 -1
  188. package/log.js.map +0 -1
  189. package/proxy_configuration.d.ts.map +0 -1
  190. package/proxy_configuration.js.map +0 -1
  191. package/request.d.ts.map +0 -1
  192. package/request.js.map +0 -1
  193. package/router.d.ts +0 -134
  194. package/router.d.ts.map +0 -1
  195. package/router.js +0 -164
  196. package/router.js.map +0 -1
  197. package/serialization.d.ts.map +0 -1
  198. package/serialization.js.map +0 -1
  199. package/session_pool/consts.d.ts +0 -4
  200. package/session_pool/consts.d.ts.map +0 -1
  201. package/session_pool/consts.js +0 -4
  202. package/session_pool/consts.js.map +0 -1
  203. package/session_pool/errors.d.ts +0 -8
  204. package/session_pool/errors.d.ts.map +0 -1
  205. package/session_pool/errors.js +0 -12
  206. package/session_pool/errors.js.map +0 -1
  207. package/session_pool/events.d.ts +0 -3
  208. package/session_pool/events.d.ts.map +0 -1
  209. package/session_pool/events.js +0 -3
  210. package/session_pool/events.js.map +0 -1
  211. package/session_pool/index.d.ts +0 -6
  212. package/session_pool/index.d.ts.map +0 -1
  213. package/session_pool/index.js +0 -6
  214. package/session_pool/index.js.map +0 -1
  215. package/session_pool/session.d.ts +0 -199
  216. package/session_pool/session.d.ts.map +0 -1
  217. package/session_pool/session.js +0 -271
  218. package/session_pool/session.js.map +0 -1
  219. package/session_pool/session_pool.d.ts +0 -251
  220. package/session_pool/session_pool.d.ts.map +0 -1
  221. package/session_pool/session_pool.js +0 -375
  222. package/session_pool/session_pool.js.map +0 -1
  223. package/storages/access_checking.d.ts +0 -13
  224. package/storages/access_checking.d.ts.map +0 -1
  225. package/storages/access_checking.js +0 -14
  226. package/storages/access_checking.js.map +0 -1
  227. package/storages/dataset.d.ts.map +0 -1
  228. package/storages/dataset.js.map +0 -1
  229. package/storages/index.d.ts.map +0 -1
  230. package/storages/index.js.map +0 -1
  231. package/storages/key_value_store.d.ts.map +0 -1
  232. package/storages/key_value_store.js.map +0 -1
  233. package/storages/request_list.d.ts.map +0 -1
  234. package/storages/request_list.js.map +0 -1
  235. package/storages/request_provider.d.ts +0 -307
  236. package/storages/request_provider.d.ts.map +0 -1
  237. package/storages/request_provider.js +0 -555
  238. package/storages/request_provider.js.map +0 -1
  239. package/storages/request_queue.d.ts.map +0 -1
  240. package/storages/request_queue.js.map +0 -1
  241. package/storages/request_queue_v2.d.ts +0 -87
  242. package/storages/request_queue_v2.d.ts.map +0 -1
  243. package/storages/request_queue_v2.js +0 -438
  244. package/storages/request_queue_v2.js.map +0 -1
  245. package/storages/sitemap_request_list.d.ts +0 -245
  246. package/storages/sitemap_request_list.d.ts.map +0 -1
  247. package/storages/sitemap_request_list.js +0 -430
  248. package/storages/sitemap_request_list.js.map +0 -1
  249. package/storages/storage_manager.d.ts +0 -58
  250. package/storages/storage_manager.d.ts.map +0 -1
  251. package/storages/storage_manager.js +0 -105
  252. package/storages/storage_manager.js.map +0 -1
  253. package/storages/utils.d.ts.map +0 -1
  254. package/storages/utils.js.map +0 -1
  255. package/tsconfig.build.tsbuildinfo +0 -1
  256. package/typedefs.d.ts.map +0 -1
  257. package/typedefs.js.map +0 -1
  258. package/validators.d.ts.map +0 -1
  259. package/validators.js.map +0 -1
@@ -1,250 +0,0 @@
1
- import ow from 'ow';
2
- import { getDomain } from 'tldts';
3
- import log from '@apify/log';
4
- import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, createRequests, filterRequestsByPatterns, } from './shared.js';
5
- /**
6
- * The different enqueueing strategies available.
7
- *
8
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
9
- *
10
- * ```md
11
- * Protocol Domain
12
- * ┌────┐ ┌─────────┐
13
- * https://example.crawlee.dev/...
14
- * │ └─────────────────┤
15
- * │ Hostname │
16
- * │ │
17
- * └─────────────────────────┘
18
- * Origin
19
- *```
20
- *
21
- * - The `Protocol` is usually `http` or `https`
22
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
23
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
24
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
25
- */
26
- export var EnqueueStrategy;
27
- (function (EnqueueStrategy) {
28
- /**
29
- * Matches any URLs found
30
- */
31
- EnqueueStrategy["All"] = "all";
32
- /**
33
- * Matches any URLs that have the same hostname.
34
- * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
35
- * `https://example.com/hello` will not be matched.
36
- *
37
- * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
38
- */
39
- EnqueueStrategy["SameHostname"] = "same-hostname";
40
- /**
41
- * Matches any URLs that have the same domain as the base URL.
42
- * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
43
- * `https://example.com`.
44
- *
45
- * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
46
- */
47
- EnqueueStrategy["SameDomain"] = "same-domain";
48
- /**
49
- * Matches any URLs that have the same hostname and protocol.
50
- * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
51
- * `http://wow.example.com/hello` will not be matched.
52
- *
53
- * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
54
- */
55
- EnqueueStrategy["SameOrigin"] = "same-origin";
56
- })(EnqueueStrategy || (EnqueueStrategy = {}));
57
- /**
58
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
59
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
60
- *
61
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
62
- * and override settings of the enqueued {@link Request} objects.
63
- *
64
- * **Example usage**
65
- *
66
- * ```javascript
67
- * await enqueueLinks({
68
- * urls: aListOfFoundUrls,
69
- * requestQueue,
70
- * selector: 'a.product-detail',
71
- * globs: [
72
- * 'https://www.example.com/handbags/*',
73
- * 'https://www.example.com/purses/*'
74
- * ],
75
- * });
76
- * ```
77
- *
78
- * @param options All `enqueueLinks()` parameters are passed via an options object.
79
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
80
- */
81
- export async function enqueueLinks(options) {
82
- if (!options || Object.keys(options).length === 0) {
83
- throw new RangeError([
84
- 'enqueueLinks() was called without the required options. You can only do that when you use the `crawlingContext.enqueueLinks()` method in request handlers.',
85
- 'Check out our guide on how to use enqueueLinks() here: https://crawlee.dev/js/docs/examples/crawl-relative-links',
86
- ].join('\n'));
87
- }
88
- ow(options, ow.object.exactShape({
89
- urls: ow.array.ofType(ow.string),
90
- requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
91
- robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
92
- onSkippedRequest: ow.optional.function,
93
- forefront: ow.optional.boolean,
94
- skipNavigation: ow.optional.boolean,
95
- limit: ow.optional.number,
96
- selector: ow.optional.string,
97
- baseUrl: ow.optional.string,
98
- userData: ow.optional.object,
99
- label: ow.optional.string,
100
- pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
101
- globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
102
- exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
103
- regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
104
- transformRequestFunction: ow.optional.function,
105
- strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
106
- waitForAllRequestsToBeAdded: ow.optional.boolean,
107
- }));
108
- const { requestQueue, limit, urls, pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
109
- const urlExcludePatternObjects = [];
110
- const urlPatternObjects = [];
111
- if (exclude?.length) {
112
- for (const excl of exclude) {
113
- if (typeof excl === 'string' || 'glob' in excl) {
114
- urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
115
- }
116
- else if (excl instanceof RegExp || 'regexp' in excl) {
117
- urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
118
- }
119
- }
120
- }
121
- if (pseudoUrls?.length) {
122
- log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
123
- urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
124
- }
125
- if (globs?.length) {
126
- urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
127
- }
128
- if (regexps?.length) {
129
- urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
130
- }
131
- if (!urlPatternObjects.length) {
132
- options.strategy ??= EnqueueStrategy.SameHostname;
133
- }
134
- const enqueueStrategyPatterns = [];
135
- if (options.baseUrl) {
136
- const url = new URL(options.baseUrl);
137
- switch (options.strategy) {
138
- case EnqueueStrategy.SameHostname:
139
- // We need to get the origin of the passed in domain in the event someone sets baseUrl
140
- // to an url like https://example.com/deep/default/path and one of the found urls is an
141
- // absolute relative path (/path/to/page)
142
- enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
143
- break;
144
- case EnqueueStrategy.SameDomain: {
145
- // Get the actual hostname from the base url
146
- const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
147
- if (baseUrlHostname) {
148
- // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
149
- url.hostname = baseUrlHostname;
150
- enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) }, { glob: ignoreHttpSchema(`${url.origin}/**`) });
151
- }
152
- else {
153
- // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
154
- // as SameDomainAndSubdomain
155
- enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
156
- }
157
- break;
158
- }
159
- case EnqueueStrategy.SameOrigin: {
160
- // The same behavior as SameHostname, but respecting the protocol of the URL
161
- enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
162
- break;
163
- }
164
- case EnqueueStrategy.All:
165
- default:
166
- enqueueStrategyPatterns.push({ glob: `http{s,}://**` });
167
- break;
168
- }
169
- }
170
- let requestOptions = createRequestOptions(urls, options);
171
- if (robotsTxtFile) {
172
- const skippedRequests = [];
173
- requestOptions = requestOptions.filter((request) => {
174
- if (robotsTxtFile.isAllowed(request.url)) {
175
- return true;
176
- }
177
- skippedRequests.push(request);
178
- return false;
179
- });
180
- if (onSkippedRequest && skippedRequests.length > 0) {
181
- await Promise.all(skippedRequests.map((request) => {
182
- return onSkippedRequest({ url: request.url, reason: 'robotsTxt' });
183
- }));
184
- }
185
- }
186
- if (transformRequestFunction) {
187
- requestOptions = requestOptions
188
- .map((request) => transformRequestFunction(request))
189
- .filter((r) => !!r);
190
- }
191
- function createFilteredRequests() {
192
- // No user provided patterns means we can skip an extra filtering step
193
- if (urlPatternObjects.length === 0) {
194
- return createRequests(requestOptions, enqueueStrategyPatterns, urlExcludePatternObjects, options.strategy);
195
- }
196
- // Generate requests based on the user patterns first
197
- const generatedRequestsFromUserFilters = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy);
198
- // ...then filter them by the enqueue links strategy (making this an AND check)
199
- return filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns);
200
- }
201
- let requests = createFilteredRequests();
202
- if (limit)
203
- requests = requests.slice(0, limit);
204
- const { addedRequests } = await requestQueue.addRequestsBatched(requests, {
205
- forefront,
206
- waitForAllRequestsToBeAdded,
207
- });
208
- return { processedRequests: addedRequests, unprocessedRequests: [] };
209
- }
210
- /**
211
- * @internal
212
- * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
213
- * - If a user provides a base url, we always return it
214
- * - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
215
- * request domain, or a redirected one
216
- * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
217
- */
218
- export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
219
- // User provided base url takes priority
220
- if (userProvidedBaseUrl) {
221
- return userProvidedBaseUrl;
222
- }
223
- const originalUrlOrigin = new URL(originalRequestUrl).origin;
224
- const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
225
- // We can assume users want to go off the domain in this case
226
- if (enqueueStrategy === EnqueueStrategy.All) {
227
- return finalUrlOrigin;
228
- }
229
- // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match
230
- // Returning undefined here is intentional! If the domains don't match, having no baseUrl in enqueueLinks will cause it to not enqueue anything
231
- // which is the intended behavior (since we went off domain)
232
- if (enqueueStrategy === EnqueueStrategy.SameDomain) {
233
- const originalHostname = getDomain(originalUrlOrigin, { mixedInputs: false });
234
- const finalHostname = getDomain(finalUrlOrigin, { mixedInputs: false });
235
- if (originalHostname === finalHostname) {
236
- return finalUrlOrigin;
237
- }
238
- return undefined;
239
- }
240
- // Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect
241
- // before actually finding the urls
242
- return originalUrlOrigin;
243
- }
244
- /**
245
- * Internal function that changes the enqueue globs to match both http and https
246
- */
247
- function ignoreHttpSchema(pattern) {
248
- return pattern.replace(/^(https?):\/\//, 'http{s,}://');
249
- }
250
- //# sourceMappingURL=enqueue_links.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAGlC,OAAO,GAAG,MAAM,YAAY,CAAC;AAK7B,OAAO,EACH,6BAA6B,EAC7B,oCAAoC,EACpC,iCAAiC,EACjC,oBAAoB,EACpB,cAAc,EACd,wBAAwB,GAC3B,MAAM,aAAa,CAAC;AA+JrB;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAN,IAAY,eAgCX;AAhCD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;;;OAMG;IACH,iDAA8B,CAAA;IAE9B;;;;;;OAMG;IACH,6CAA0B,CAAA;IAE1B;;;;;;OAMG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAhCW,eAAe,KAAf,eAAe,QAgC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,OAAkE;IAElE,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,UAAU,CAChB;YACI,4JAA4J;YAC5J,kHAAkH;SACrH,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;IACN,CAAC;IAED,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACtD,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QACtC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;QACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAClE,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACnD,CAAC,CACL,CAAC;IAEF,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,OAAO,EACP,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,SAAS,EACT,2BAA2B,EAC3B,aAAa,EACb,gBAAgB,GACnB,GAAG,OAAO,CAAC;IAEZ,MAAM,wBAAwB,GAAuB,EAAE,CAAC;IACxD,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC7C,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACpD,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,GAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,oCAAoC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,CAAC,QAAQ,KAAK,eAAe,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE,CAAC;oBAClB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CACjD,CAAC;gBACN,CAAC;qBAAM,CAAC;oBACJ,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjF,CAAC;gBAED,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4EAA4E;gBAC5E,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxD,MAAM;QACd,CAAC;IACL,CAAC;IAED,IAAI,cAAc,GAAG,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,aAAa,EAAE,CAAC;QAChB,MAAM,eAAe,GAAqB,EAAE,CAAC;QAE7C,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/C,IAAI,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvC,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,IAAI,gBAAgB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,OAAO,CAAC,GAAG,CACb,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,OAAO,gBAAgB,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;YACvE,CAAC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,IAAI,wBAAwB,EAAE,CAAC;QAC3B,cAAc,GAAG,cAAc;aAC1B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC;aACnD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;IAChD,CAAC;IAED,SAAS,sBAAsB;QAC3B,sEAAsE;QACtE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,cAAc,CAAC,cAAc,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/G,CAAC;QAED,qDAAqD;QACrD,MAAM,gCAAgC,GAAG,cAAc,CACnD,cAAc,EACd,iBAAiB,EACjB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,+EAA+E;QAC/E,OAAO,wBAAwB,CAAC,gCAAgC,EAAE,uBAAuB,CAAC,CAAC;IAC/F,CAAC;IAED,IAAI,QAAQ,GAAG,sBAAsB,EAAE,CAAC;IACxC,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QACtE,SAAS;QACT,2BAA2B;KAC9B,CAAC,CAAC;IAEH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE,CAAC;QACtB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,SAAS,CAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO,cAAc,CAAC;QAC1B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAYD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC"}
@@ -1,3 +0,0 @@
1
- export * from './enqueue_links.js';
2
- export * from './shared.js';
3
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,aAAa,CAAC"}
@@ -1,3 +0,0 @@
1
- export * from './enqueue_links.js';
2
- export * from './shared.js';
3
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/enqueue_links/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,aAAa,CAAC"}
@@ -1,67 +0,0 @@
1
- import type { RequestOptions } from '../request.js';
2
- import { Request } from '../request.js';
3
- import type { EnqueueLinksOptions } from './enqueue_links.js';
4
- export { tryAbsoluteURL } from '@crawlee/utils';
5
- export type UrlPatternObject = {
6
- glob?: string;
7
- regexp?: RegExp;
8
- } & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
9
- export type PseudoUrlObject = {
10
- purl: string;
11
- } & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
12
- export type PseudoUrlInput = string | PseudoUrlObject;
13
- export type GlobObject = {
14
- glob: string;
15
- } & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
16
- export type GlobInput = string | GlobObject;
17
- export type RegExpObject = {
18
- regexp: RegExp;
19
- } & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
20
- export type RegExpInput = RegExp | RegExpObject;
21
- /**
22
- * @ignore
23
- */
24
- export declare function updateEnqueueLinksPatternCache(item: GlobInput | RegExpInput | PseudoUrlInput, pattern: RegExpObject | GlobObject): void;
25
- /**
26
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
27
- * to construct RegExps from PseudoUrl strings.
28
- * @ignore
29
- */
30
- export declare function constructRegExpObjectsFromPseudoUrls(pseudoUrls: readonly PseudoUrlInput[]): RegExpObject[];
31
- /**
32
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
33
- * to construct Glob objects from Glob pattern strings.
34
- * @ignore
35
- */
36
- export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[];
37
- /**
38
- * @internal
39
- */
40
- export declare function validateGlobPattern(glob: string): string;
41
- /**
42
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
43
- * to check RegExps input and return valid RegExps.
44
- * @ignore
45
- */
46
- export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
47
- /**
48
- * @ignore
49
- */
50
- export declare function createRequests(requestOptions: (string | RequestOptions)[], urlPatternObjects?: UrlPatternObject[], excludePatternObjects?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy']): Request[];
51
- export declare function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[]): Request[];
52
- /**
53
- * @ignore
54
- */
55
- export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'strategy'>): RequestOptions[];
56
- /**
57
- * Takes an Apify {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
58
- * {@link enqueueLinks} to modify requests before enqueuing them.
59
- */
60
- export interface RequestTransform {
61
- /**
62
- * @param original Request options to be modified.
63
- * @returns The modified request options to enqueue.
64
- */
65
- (original: RequestOptions): RequestOptions | false | undefined | null;
66
- }
67
- //# sourceMappingURL=shared.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAYhD,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CACjD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAC5C,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAChD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAC1C,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAC9C,OAAO,EAAE,YAAY,GAAG,UAAU,GACnC,IAAI,CAMN;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,EAAE,CAiB1G;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,EAAE,CAkCvF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBjG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC1B,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAC3C,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,EACtC,qBAAqB,GAAE,gBAAgB,EAAO,EAC9C,QAAQ,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,GAC3C,OAAO,EAAE,CA8BX;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CAoBtG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,EACtD,OAAO,GAAE,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,gBAAgB,GAAG,UAAU,CAAM,GAC1G,cAAc,EAAE,CA+BlB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;OAGG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC;CACzE"}
@@ -1,190 +0,0 @@
1
- import { URL } from 'node:url';
2
- import { minimatch } from 'minimatch';
3
- import { purlToRegExp } from '@apify/pseudo_url';
4
- import { Request } from '../request.js';
5
- export { tryAbsoluteURL } from '@crawlee/utils';
6
- const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
7
- /**
8
- * To enable direct use of the Actor UI `globs`/`regexps`/`pseudoUrls` output while keeping high performance,
9
- * all the regexps from the output are only constructed once and kept in a cache
10
- * by the `enqueueLinks()` function.
11
- * @ignore
12
- */
13
- const enqueueLinksPatternCache = new Map();
14
- /**
15
- * @ignore
16
- */
17
- export function updateEnqueueLinksPatternCache(item, pattern) {
18
- enqueueLinksPatternCache.set(item, pattern);
19
- if (enqueueLinksPatternCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
20
- const key = enqueueLinksPatternCache.keys().next().value;
21
- enqueueLinksPatternCache.delete(key);
22
- }
23
- }
24
- /**
25
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
26
- * to construct RegExps from PseudoUrl strings.
27
- * @ignore
28
- */
29
- export function constructRegExpObjectsFromPseudoUrls(pseudoUrls) {
30
- return pseudoUrls.map((item) => {
31
- // Get pseudoUrl object from cache.
32
- let regexpObject = enqueueLinksPatternCache.get(item);
33
- if (regexpObject)
34
- return regexpObject;
35
- if (typeof item === 'string') {
36
- regexpObject = { regexp: purlToRegExp(item) };
37
- }
38
- else {
39
- const { purl, ...requestOptions } = item;
40
- regexpObject = { regexp: purlToRegExp(purl), ...requestOptions };
41
- }
42
- updateEnqueueLinksPatternCache(item, regexpObject);
43
- return regexpObject;
44
- });
45
- }
46
- /**
47
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
48
- * to construct Glob objects from Glob pattern strings.
49
- * @ignore
50
- */
51
- export function constructGlobObjectsFromGlobs(globs) {
52
- return globs
53
- .filter((glob) => {
54
- // Skip possibly nullish, empty strings
55
- if (!glob) {
56
- return false;
57
- }
58
- if (typeof glob === 'string') {
59
- return glob.trim().length > 0;
60
- }
61
- if (glob.glob) {
62
- return glob.glob.trim().length > 0;
63
- }
64
- return false;
65
- })
66
- .map((item) => {
67
- // Get glob object from cache.
68
- let globObject = enqueueLinksPatternCache.get(item);
69
- if (globObject)
70
- return globObject;
71
- if (typeof item === 'string') {
72
- globObject = { glob: validateGlobPattern(item) };
73
- }
74
- else {
75
- const { glob, ...requestOptions } = item;
76
- globObject = { glob: validateGlobPattern(glob), ...requestOptions };
77
- }
78
- updateEnqueueLinksPatternCache(item, globObject);
79
- return globObject;
80
- });
81
- }
82
- /**
83
- * @internal
84
- */
85
- export function validateGlobPattern(glob) {
86
- const globTrimmed = glob.trim();
87
- if (globTrimmed.length === 0)
88
- throw new Error(`Cannot parse Glob pattern '${globTrimmed}': it must be an non-empty string`);
89
- return globTrimmed;
90
- }
91
- /**
92
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
93
- * to check RegExps input and return valid RegExps.
94
- * @ignore
95
- */
96
- export function constructRegExpObjectsFromRegExps(regexps) {
97
- return regexps.map((item) => {
98
- // Get regexp object from cache.
99
- let regexpObject = enqueueLinksPatternCache.get(item);
100
- if (regexpObject)
101
- return regexpObject;
102
- if (item instanceof RegExp) {
103
- regexpObject = { regexp: item };
104
- }
105
- else {
106
- regexpObject = item;
107
- }
108
- updateEnqueueLinksPatternCache(item, regexpObject);
109
- return regexpObject;
110
- });
111
- }
112
- /**
113
- * @ignore
114
- */
115
- export function createRequests(requestOptions, urlPatternObjects, excludePatternObjects = [], strategy) {
116
- return requestOptions
117
- .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
118
- .filter(({ url }) => {
119
- return !excludePatternObjects.some((excludePatternObject) => {
120
- const { regexp, glob } = excludePatternObject;
121
- return (regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }));
122
- });
123
- })
124
- .map(({ url, opts }) => {
125
- if (!urlPatternObjects || !urlPatternObjects.length) {
126
- return new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts });
127
- }
128
- for (const urlPatternObject of urlPatternObjects) {
129
- const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;
130
- if ((regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }))) {
131
- const request = typeof opts === 'string'
132
- ? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
133
- : { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
134
- return new Request(request);
135
- }
136
- }
137
- // didn't match any positive pattern
138
- return null;
139
- })
140
- .filter((request) => request);
141
- }
142
- export function filterRequestsByPatterns(requests, patterns) {
143
- if (!patterns?.length) {
144
- return requests;
145
- }
146
- const filtered = [];
147
- for (const request of requests) {
148
- for (const urlPatternObject of patterns) {
149
- const { regexp, glob } = urlPatternObject;
150
- if ((regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true }))) {
151
- filtered.push(request);
152
- // Break the pattern loop, as we already matched this request once
153
- break;
154
- }
155
- }
156
- }
157
- return filtered;
158
- }
159
- /**
160
- * @ignore
161
- */
162
- export function createRequestOptions(sources, options = {}) {
163
- return sources
164
- .map((src) => typeof src === 'string'
165
- ? { url: src, enqueueStrategy: options.strategy }
166
- : { ...src, enqueueStrategy: options.strategy })
167
- .filter(({ url }) => {
168
- try {
169
- return new URL(url, options.baseUrl).href;
170
- }
171
- catch (err) {
172
- return false;
173
- }
174
- })
175
- .map((requestOptions) => {
176
- requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
177
- requestOptions.userData ??= options.userData ?? {};
178
- if (typeof options.label === 'string') {
179
- requestOptions.userData = {
180
- ...requestOptions.userData,
181
- label: options.label,
182
- };
183
- }
184
- if (options.skipNavigation) {
185
- requestOptions.skipNavigation = true;
186
- }
187
- return requestOptions;
188
- });
189
- }
190
- //# sourceMappingURL=shared.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGjD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AA4B3C;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAC1C,IAA8C,EAC9C,OAAkC;IAElC,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oCAAoC,CAAC,UAAqC;IACtF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAAC,KAA2B;IACrE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAClG,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAAC,OAA+B;IAC7E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC1B,cAA2C,EAC3C,iBAAsC,EACtC,wBAA4C,EAAE,EAC9C,QAA0C;IAE1C,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC1E,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,oBAAoB,EAAE,EAAE;YACxD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,CAAC;YAC9C,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7F,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE;QACnB,IAAI,CAAC,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAClD,OAAO,IAAI,OAAO,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,KAAK,MAAM,gBAAgB,IAAI,iBAAiB,EAAE,CAAC;YAC/C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,GAAG,gBAAgB,CAAC;YACnE,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpF,MAAM,OAAO,GACT,OAAO,IAAI,KAAK,QAAQ;oBACpB,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE;oBACnE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;gBAE1E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAc,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,QAAmB,EAAE,QAA6B;IACvF,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,KAAK,MAAM,gBAAgB,IAAI,QAAQ,EAAE,CAAC;YACtC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,kEAAkE;gBAClE,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAChC,OAAsD,EACtD,UAAuG,EAAE;IAEzG,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACT,OAAO,GAAG,KAAK,QAAQ;QACnB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE;QACjD,CAAC,CAAE,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAqB,CAC1E;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC"}
package/errors.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,iBAAiB;gBACnC,OAAO,CAAC,EAAE,MAAM;CAG/B"}
package/errors.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,MAAM,OAAO,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACxC,YAAY,OAAgB;QACxB,KAAK,CAAC,OAAO,IAAI,gDAAgD,CAAC,CAAC;IACvE,CAAC;CACJ;AAED;;;;GAIG;AACH,MAAM,OAAO,YAAa,SAAQ,iBAAiB;IAC/C,YAAY,OAAgB;QACxB,KAAK,CAAC,iDAAiD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;CACJ"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAGpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,wBAA2B;IAC3C,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;gBAE3B,MAAM,gBAAkC;IAI7D;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IAeX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;IAI3D;;OAEG;IACG,6BAA6B;CAGtC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAE1E,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,CAAN,IAAkB,SAMjB;AAND,WAAkB,SAAS;IACvB,2CAA8B,CAAA;IAC9B,uCAA0B,CAAA;IAC1B,oCAAuB,CAAA;IACvB,kCAAqB,CAAA;IACrB,0BAAa,CAAA;AACjB,CAAC,EANiB,SAAS,KAAT,SAAS,QAM1B;AASD,MAAM,OAAgB,YAAY;IAMT;IALX,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;IACjC,WAAW,GAAG,KAAK,CAAC;IACpB,SAAS,GAAc,EAAE,CAAC;IAC1B,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEhD,YAAqB,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QACzD,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,IAAI,CAAC,SAAS,CAAC,YAAY,GAAG,iBAAiB,CAAC,CAAC,gBAA+B,EAAE,EAAE;YAChF,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,gBAAgB,EAAE,CAAC;QACvB,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB,iCAAiC;QACjC,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QAE3D,oCAAoC;QACpC,MAAM,IAAI,CAAC,6BAA6B,EAAE,CAAC;IAC/C,CAAC;IAED,EAAE,CAAC,KAAoB,EAAE,QAAiC;QACtD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CAAC,KAAoB,EAAE,QAAkC;QACxD,IAAI,QAAQ,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC1C,CAAC;IACL,CAAC;IAED,IAAI,CAAC,KAAoB,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,aAAa;QACT,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAoB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAoB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAA+B,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,6BAA6B;QAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,6BAA6B,EAAE,CAAC;IACvD,CAAC;CACJ"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/events/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,0BAA0B,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/events/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,0BAA0B,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAa,MAAM,oBAAoB,CAAC;AAE7D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C;;;OAGG;IACY,IAAI;IAYnB;;OAEG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD;;OAEG;IACG,sBAAsB;IAI5B,OAAO,CAAC,kBAAkB;IAc1B;;OAEG;YACW,gBAAgB;YAQhB,aAAa;YAoBb,gBAAgB;CAiBjC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEvG,OAAO,GAAG,MAAM,YAAY,CAAC;AAC7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1E,OAAO,EAAE,YAAY,EAAa,MAAM,oBAAoB,CAAC;AAE7D,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IACvC,aAAa,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAE9C;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,eAAe,EAAE,CAAC,CAAC;IACrE,CAAC;IAEO,kBAAkB;QACtB,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CACd,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EACD,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CACxB,CAAC;IACN,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;YACrF,OAAO;gBACH,eAAe,EAAE,YAAY,GAAG,GAAG;gBACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;aAC1D,CAAC;QACN,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC;QAC7D,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC;QAChE,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QAChF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEzC,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;gBAC3E,OAAO;oBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;iBAC1E,CAAC;YACN,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;YACtC,OAAO;gBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}