@crawlee/utils 4.0.0-beta.9 → 4.0.0-beta.90

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +17 -13
  2. package/index.d.ts +4 -6
  3. package/index.js +3 -5
  4. package/internals/blocked.d.ts +0 -1
  5. package/internals/blocked.js +0 -1
  6. package/internals/cheerio.d.ts +3 -2
  7. package/internals/cheerio.js +4 -5
  8. package/internals/extract-urls.d.ts +5 -1
  9. package/internals/extract-urls.js +8 -5
  10. package/internals/general.d.ts +16 -19
  11. package/internals/general.js +44 -96
  12. package/internals/iterables.d.ts +126 -0
  13. package/internals/iterables.js +230 -0
  14. package/internals/open_graph_parser.d.ts +2 -3
  15. package/internals/open_graph_parser.js +8 -9
  16. package/internals/robots.d.ts +14 -5
  17. package/internals/robots.js +31 -35
  18. package/internals/sitemap.d.ts +65 -8
  19. package/internals/sitemap.js +174 -34
  20. package/internals/social.d.ts +1 -2
  21. package/internals/social.js +7 -5
  22. package/internals/typedefs.d.ts +0 -1
  23. package/internals/typedefs.js +0 -1
  24. package/internals/url.d.ts +1 -2
  25. package/internals/url.js +1 -2
  26. package/package.json +6 -6
  27. package/index.d.ts.map +0 -1
  28. package/index.js.map +0 -1
  29. package/internals/blocked.d.ts.map +0 -1
  30. package/internals/blocked.js.map +0 -1
  31. package/internals/cheerio.d.ts.map +0 -1
  32. package/internals/cheerio.js.map +0 -1
  33. package/internals/chunk.d.ts +0 -2
  34. package/internals/chunk.d.ts.map +0 -1
  35. package/internals/chunk.js +0 -40
  36. package/internals/chunk.js.map +0 -1
  37. package/internals/debug.d.ts +0 -31
  38. package/internals/debug.d.ts.map +0 -1
  39. package/internals/debug.js +0 -29
  40. package/internals/debug.js.map +0 -1
  41. package/internals/extract-urls.d.ts.map +0 -1
  42. package/internals/extract-urls.js.map +0 -1
  43. package/internals/general.d.ts.map +0 -1
  44. package/internals/general.js.map +0 -1
  45. package/internals/open_graph_parser.d.ts.map +0 -1
  46. package/internals/open_graph_parser.js.map +0 -1
  47. package/internals/robots.d.ts.map +0 -1
  48. package/internals/robots.js.map +0 -1
  49. package/internals/sitemap.d.ts.map +0 -1
  50. package/internals/sitemap.js.map +0 -1
  51. package/internals/social.d.ts.map +0 -1
  52. package/internals/social.js.map +0 -1
  53. package/internals/system-info/cpu-info.d.ts +0 -64
  54. package/internals/system-info/cpu-info.d.ts.map +0 -1
  55. package/internals/system-info/cpu-info.js +0 -211
  56. package/internals/system-info/cpu-info.js.map +0 -1
  57. package/internals/system-info/memory-info.d.ts +0 -28
  58. package/internals/system-info/memory-info.d.ts.map +0 -1
  59. package/internals/system-info/memory-info.js +0 -118
  60. package/internals/system-info/memory-info.js.map +0 -1
  61. package/internals/system-info/ps-tree.d.ts +0 -18
  62. package/internals/system-info/ps-tree.d.ts.map +0 -1
  63. package/internals/system-info/ps-tree.js +0 -145
  64. package/internals/system-info/ps-tree.js.map +0 -1
  65. package/internals/typedefs.d.ts.map +0 -1
  66. package/internals/typedefs.js.map +0 -1
  67. package/internals/url.d.ts.map +0 -1
  68. package/internals/url.js.map +0 -1
  69. package/tsconfig.build.tsbuildinfo +0 -1
@@ -2,9 +2,12 @@ import { createHash } from 'node:crypto';
2
2
  import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
3
3
  import { StringDecoder } from 'node:string_decoder';
4
4
  import { createGunzip } from 'node:zlib';
5
+ import { FetchHttpClient } from '@crawlee/http-client';
6
+ import { fileTypeStream } from 'file-type';
5
7
  import sax from 'sax';
6
8
  import MIMEType from 'whatwg-mimetype';
7
- import log from '@apify/log';
9
+ import { mergeAsyncIterables } from './iterables.js';
10
+ import { RobotsTxtFile } from './robots.js';
8
11
  class SitemapTxtParser extends Transform {
9
12
  decoder = new StringDecoder('utf8');
10
13
  buffer = '';
@@ -91,8 +94,10 @@ class SitemapXmlParser extends Transform {
91
94
  if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
92
95
  this.currentTag = undefined;
93
96
  }
94
- if (name === 'url' && this.url.loc !== undefined) {
95
- this.push({ type: 'url', ...this.url, loc: this.url.loc });
97
+ if (name === 'url') {
98
+ if (this.url.loc !== undefined) {
99
+ this.push({ type: 'url', ...this.url, loc: this.url.loc });
100
+ }
96
101
  this.url = {};
97
102
  }
98
103
  }
@@ -108,7 +113,10 @@ class SitemapXmlParser extends Transform {
108
113
  }
109
114
  text = text.trim();
110
115
  if (this.currentTag === 'lastmod') {
111
- this.url.lastmod = new Date(text);
116
+ const lastmod = new Date(text);
117
+ if (!Number.isNaN(lastmod.getTime())) {
118
+ this.url.lastmod = lastmod;
119
+ }
112
120
  }
113
121
  if (this.currentTag === 'priority') {
114
122
  this.url.priority = Number(text);
@@ -121,9 +129,7 @@ class SitemapXmlParser extends Transform {
121
129
  }
122
130
  }
123
131
  export async function* parseSitemap(initialSources, proxyUrl, options) {
124
- const { gotScraping } = await import('got-scraping');
125
- const { fileTypeStream } = await import('file-type');
126
- const { emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, networkTimeouts } = options ?? {};
132
+ const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, logger, } = options ?? {};
127
133
  const sources = [...initialSources];
128
134
  const visitedSitemapUrls = new Set();
129
135
  const createParser = (contentType = '', url) => {
@@ -145,7 +151,6 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
145
151
  while (sources.length > 0) {
146
152
  const source = sources.shift();
147
153
  if ((source?.depth ?? 0) > maxDepth) {
148
- log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
149
154
  continue;
150
155
  }
151
156
  let items = null;
@@ -155,23 +160,28 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
155
160
  let retriesLeft = sitemapRetries + 1;
156
161
  while (retriesLeft-- > 0) {
157
162
  try {
158
- const sitemapStream = await new Promise((resolve, reject) => {
159
- const request = gotScraping.stream({
160
- url: sitemapUrl,
161
- proxyUrl,
163
+ let sitemapResponse;
164
+ try {
165
+ sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
162
166
  method: 'GET',
163
- timeout: networkTimeouts,
164
167
  headers: {
165
- accept: 'text/plain, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8',
168
+ accept: '*/*',
166
169
  },
170
+ }), {
171
+ proxyUrl,
172
+ timeoutMillis: timeout,
167
173
  });
168
- request.on('response', () => resolve(request));
169
- request.on('error', reject);
170
- });
174
+ }
175
+ catch (error) {
176
+ sitemapResponse = null;
177
+ }
171
178
  let error = null;
172
- if (sitemapStream.response.statusCode >= 200 && sitemapStream.response.statusCode < 300) {
173
- let contentType = sitemapStream.response.headers['content-type'];
174
- const streamWithType = await fileTypeStream(sitemapStream);
179
+ if (sitemapResponse && sitemapResponse.status >= 200 && sitemapResponse.status < 300) {
180
+ let contentType = sitemapResponse.headers.get('content-type');
181
+ if (sitemapResponse.body === null) {
182
+ break;
183
+ }
184
+ const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
175
185
  if (streamWithType.fileType !== undefined) {
176
186
  contentType = streamWithType.fileType.mime;
177
187
  }
@@ -184,29 +194,37 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
184
194
  sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
185
195
  }
186
196
  }
187
- items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType, sitemapUrl), (e) => {
188
- if (e !== undefined) {
189
- error = e;
197
+ items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
198
+ if (e !== undefined && e !== null) {
199
+ error = { type: 'parser', error: e };
190
200
  }
191
201
  });
192
202
  }
193
203
  else {
194
- error = new Error(`Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapStream.response.statusCode}`);
204
+ error = {
205
+ type: 'fetch',
206
+ error: new Error(`Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapResponse?.status}`),
207
+ };
195
208
  }
196
209
  if (error !== null) {
197
- throw error;
210
+ const shouldIgnoreError = error.type === 'fetch' && !reportNetworkErrors;
211
+ if (!shouldIgnoreError) {
212
+ throw error.error;
213
+ }
214
+ }
215
+ else {
216
+ break;
198
217
  }
199
- break;
200
218
  }
201
219
  catch (e) {
202
- log.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
220
+ logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
203
221
  }
204
222
  }
205
223
  }
206
224
  else if (source.type === 'raw') {
207
225
  items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
208
226
  if (error !== undefined) {
209
- log.warning(`Malformed sitemap content: ${error}`);
227
+ logger?.warning(`Malformed sitemap content: ${error}`);
210
228
  }
211
229
  });
212
230
  }
@@ -215,6 +233,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
215
233
  }
216
234
  for await (const item of items) {
217
235
  if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
236
+ if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
237
+ logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
238
+ continue;
239
+ }
218
240
  sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
219
241
  if (emitNestedSitemaps) {
220
242
  yield { loc: item.url, originSitemapUrl: null };
@@ -254,7 +276,7 @@ export class Sitemap {
254
276
  * @param url The domain URL to fetch the sitemap for.
255
277
  * @param proxyUrl A proxy to be used for fetching the sitemap file.
256
278
  */
257
- static async tryCommonNames(url, proxyUrl) {
279
+ static async tryCommonNames(url, proxyUrl, parseSitemapOptions) {
258
280
  const sitemapUrls = [];
259
281
  const sitemapUrl = new URL(url);
260
282
  sitemapUrl.search = '';
@@ -262,7 +284,7 @@ export class Sitemap {
262
284
  sitemapUrls.push(sitemapUrl.toString());
263
285
  sitemapUrl.pathname = '/sitemap.txt';
264
286
  sitemapUrls.push(sitemapUrl.toString());
265
- return Sitemap.load(sitemapUrls, proxyUrl);
287
+ return Sitemap.load(sitemapUrls, proxyUrl, { reportNetworkErrors: false, ...parseSitemapOptions });
266
288
  }
267
289
  /**
268
290
  * Fetch sitemap content from given URL or URLs and return URLs of referenced pages.
@@ -277,8 +299,8 @@ export class Sitemap {
277
299
  * @param content XML sitemap content
278
300
  * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
279
301
  */
280
- static async fromXmlString(content, proxyUrl) {
281
- return await this.parse([{ type: 'raw', content }], proxyUrl);
302
+ static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
303
+ return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
282
304
  }
283
305
  static async parse(sources, proxyUrl, parseSitemapOptions) {
284
306
  const urls = [];
@@ -287,10 +309,128 @@ export class Sitemap {
287
309
  urls.push(item.loc);
288
310
  }
289
311
  }
290
- catch {
312
+ catch (e) {
313
+ parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
291
314
  return new Sitemap([]);
292
315
  }
293
316
  return new Sitemap(urls);
294
317
  }
295
318
  }
296
- //# sourceMappingURL=sitemap.js.map
319
+ /**
320
+ * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
321
+ * the default `sitemap.xml` & `sitemap.txt` files and the URLs themselves.
322
+ * @param `urls` The list of URLs to discover sitemaps for.
323
+ * @param `options` Options for sitemap discovery
324
+ * @returns An async iterable with the discovered sitemap URLs.
325
+ */
326
+ export async function* discoverValidSitemaps(urls, options = {}) {
327
+ const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
328
+ const controller = new AbortController();
329
+ const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
330
+ const onExternalAbort = () => controller.abort();
331
+ if (externalSignal) {
332
+ if (externalSignal.aborted) {
333
+ controller.abort();
334
+ }
335
+ else {
336
+ externalSignal.addEventListener('abort', onExternalAbort, { once: true });
337
+ }
338
+ }
339
+ const signal = controller.signal;
340
+ const sitemapUrls = new Set();
341
+ const addSitemapUrl = (url) => {
342
+ const sizeBefore = sitemapUrls.size;
343
+ sitemapUrls.add(url);
344
+ if (sitemapUrls.size > sizeBefore) {
345
+ return url;
346
+ }
347
+ return undefined;
348
+ };
349
+ const urlExists = async (url) => {
350
+ if (!httpClient) {
351
+ return false;
352
+ }
353
+ try {
354
+ const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
355
+ proxyUrl,
356
+ timeoutMillis: requestTimeoutMillis,
357
+ signal,
358
+ });
359
+ return response.status >= 200 && response.status < 400;
360
+ }
361
+ catch {
362
+ return false;
363
+ }
364
+ };
365
+ const discoverSitemapsForDomainUrls = async function* (hostname, domainUrls) {
366
+ if (!hostname) {
367
+ return;
368
+ }
369
+ try {
370
+ const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
371
+ proxyUrl,
372
+ timeoutMillis: requestTimeoutMillis,
373
+ signal,
374
+ httpClient,
375
+ logger,
376
+ });
377
+ for (const sitemapUrl of robotsFile.getSitemaps()) {
378
+ if (addSitemapUrl(sitemapUrl)) {
379
+ yield sitemapUrl;
380
+ }
381
+ }
382
+ }
383
+ catch (err) {
384
+ logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
385
+ }
386
+ const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
387
+ if (sitemapUrl !== undefined) {
388
+ if (addSitemapUrl(sitemapUrl)) {
389
+ yield sitemapUrl;
390
+ }
391
+ }
392
+ else {
393
+ const firstUrl = new URL(domainUrls[0]);
394
+ const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
395
+ const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
396
+ firstUrl.pathname = pathname;
397
+ return firstUrl.toString();
398
+ });
399
+ const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
400
+ for (const [index, result] of candidateResults.entries()) {
401
+ const candidateSitemapUrl = candidateSitemapUrls[index];
402
+ if (result.status === 'fulfilled') {
403
+ if (result.value && addSitemapUrl(candidateSitemapUrl)) {
404
+ yield candidateSitemapUrl;
405
+ }
406
+ }
407
+ else {
408
+ logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
409
+ error: result.reason,
410
+ });
411
+ }
412
+ }
413
+ }
414
+ };
415
+ const groupedUrls = urls.reduce((acc, url) => {
416
+ const hostname = new URL(url)?.hostname ?? '';
417
+ acc[hostname] ??= [];
418
+ acc[hostname].push(url);
419
+ return acc;
420
+ }, {});
421
+ const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
422
+ const discoveredUrls = new Set();
423
+ try {
424
+ for await (const url of mergeAsyncIterables(...iterables)) {
425
+ if (discoveredUrls.has(url)) {
426
+ continue;
427
+ }
428
+ discoveredUrls.add(url);
429
+ yield url;
430
+ }
431
+ }
432
+ finally {
433
+ clearTimeout(timeoutHandle);
434
+ externalSignal?.removeEventListener('abort', onExternalAbort);
435
+ }
436
+ }
@@ -488,5 +488,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
488
488
  * so that the caller doesn't need to parse the HTML document again, if needed.
489
489
  * @return An object with the social handles.
490
490
  */
491
- export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles;
492
- //# sourceMappingURL=social.d.ts.map
491
+ export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;
@@ -1,7 +1,9 @@
1
- import * as cheerio from 'cheerio';
2
1
  import { htmlToText } from './cheerio.js';
3
2
  // Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
4
- const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
3
+ // The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
4
+ // instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
5
+ // dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
6
+ const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
5
7
  /**
6
8
  * Regular expression to exactly match a single email address.
7
9
  * It has the following form: `/^...$/i`.
@@ -587,7 +589,8 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
587
589
  * so that the caller doesn't need to parse the HTML document again, if needed.
588
590
  * @return An object with the social handles.
589
591
  */
590
- export function parseHandlesFromHtml(html, data = null) {
592
+ export async function parseHandlesFromHtml(html, data = null) {
593
+ const cheerio = await import('cheerio');
591
594
  const result = {
592
595
  emails: [],
593
596
  phones: [],
@@ -606,7 +609,7 @@ export function parseHandlesFromHtml(html, data = null) {
606
609
  const $ = cheerio.load(html, { xml: { decodeEntities: true } });
607
610
  if (data)
608
611
  data.$ = $;
609
- const text = htmlToText($);
612
+ const text = await htmlToText($);
610
613
  if (data)
611
614
  data.text = text;
612
615
  // NOTE: we need to parse each text separately, orherwise we might concatenate unrelated texts
@@ -644,4 +647,3 @@ export function parseHandlesFromHtml(html, data = null) {
644
647
  }
645
648
  return result;
646
649
  }
647
- //# sourceMappingURL=social.js.map
@@ -2,4 +2,3 @@
2
2
  export declare function entries<T extends {}>(obj: T): [keyof T, T[keyof T]][];
3
3
  /** @ignore */
4
4
  export declare function keys<T extends {}>(obj: T): (keyof T)[];
5
- //# sourceMappingURL=typedefs.d.ts.map
@@ -6,4 +6,3 @@ export function entries(obj) {
6
6
  export function keys(obj) {
7
7
  return Object.keys(obj);
8
8
  }
9
- //# sourceMappingURL=typedefs.js.map
@@ -1,4 +1,4 @@
1
- export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
1
+ import type { SearchParams } from '@crawlee/types';
2
2
  /**
3
3
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
4
4
  *
@@ -7,4 +7,3 @@ export type SearchParams = string | URLSearchParams | Record<string, string | nu
7
7
  * @internal
8
8
  */
9
9
  export declare function applySearchParams(url: URL, searchParams: SearchParams | undefined): void;
10
- //# sourceMappingURL=url.d.ts.map
package/internals/url.js CHANGED
@@ -19,7 +19,7 @@ export function applySearchParams(url, searchParams) {
19
19
  }
20
20
  else {
21
21
  newSearchParams = new URLSearchParams();
22
- for (const [key, value] of Object.entries(newSearchParams)) {
22
+ for (const [key, value] of Object.entries(searchParams)) {
23
23
  if (value === undefined) {
24
24
  newSearchParams.delete(key);
25
25
  }
@@ -33,4 +33,3 @@ export function applySearchParams(url, searchParams) {
33
33
  }
34
34
  url.search = newSearchParams.toString();
35
35
  }
36
- //# sourceMappingURL=url.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.9",
3
+ "version": "4.0.0-beta.90",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -35,19 +35,19 @@
35
35
  },
36
36
  "homepage": "https://crawlee.dev",
37
37
  "scripts": {
38
- "build": "yarn clean && yarn compile && yarn copy",
38
+ "build": "pnpm clean && pnpm compile && pnpm copy",
39
39
  "clean": "rimraf ./dist",
40
40
  "compile": "tsc -p tsconfig.build.json",
41
41
  "copy": "tsx ../../scripts/copy.ts"
42
42
  },
43
43
  "dependencies": {
44
- "@apify/log": "^2.5.18",
45
44
  "@apify/ps-tree": "^1.2.0",
46
- "@crawlee/types": "4.0.0-beta.9",
45
+ "@crawlee/http-client": "4.0.0-beta.90",
46
+ "@crawlee/types": "4.0.0-beta.90",
47
47
  "@types/sax": "^1.2.7",
48
48
  "cheerio": "^1.0.0",
49
+ "domhandler": "^5.0.3",
49
50
  "file-type": "^21.0.0",
50
- "got-scraping": "^4.1.1",
51
51
  "ow": "^2.0.0",
52
52
  "robots-parser": "^3.0.1",
53
53
  "sax": "^1.4.1",
@@ -61,5 +61,5 @@
61
61
  }
62
62
  }
63
63
  },
64
- "gitHead": "d2489f2800cd7d9f15815fc6b5e195dc243703d2"
64
+ "gitHead": "7afdd44c5b6850b4775cfc08ffbbdd1791ae6402"
65
65
  }
package/index.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,qCAAqC,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AAEnF,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}
package/index.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAa,MAAM,qCAAqC,CAAC;AACtF,OAAO,EAAE,aAAa,EAAc,MAAM,wCAAwC,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"blocked.d.ts","sourceRoot":"","sources":["../../src/internals/blocked.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,8BAA8B,UAA0E,CAAC;AAEtH;;GAEG;AACH,eAAO,MAAM,mBAAmB,UAI/B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,mBAAmB,UAM/B,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"blocked.js","sourceRoot":"","sources":["../../src/internals/blocked.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,qEAAqE,CAAC,CAAC;AAEtH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAC/B,GAAG,8BAA8B;IACjC,0DAA0D;IAC1D,oCAAoC;CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAC/B,YAAY;IACZ,cAAc;IACd,6BAA6B;IAC7B,8BAA8B;IAC9B,sBAAsB;CACzB,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"cheerio.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAK1C,MAAM,MAAM,WAAW,GAAG,UAAU,CAAC;AAOrC;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,UAAU,CAAC,oBAAoB,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,CAwC7E;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,SAAM,EAAE,OAAO,SAAK,GAAG,MAAM,EAAE,CAwB5F"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAInD,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAClB,sGAAsG,CAAC;AAE3G;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,CAAC,GAAG,OAAO,oBAAoB,KAAK,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjH,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9E,IAAI,IAAI,KAAK,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,qCAAqC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;YACzD,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAAC,CAAa,EAAE,QAAQ,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC9E,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,uCAAuC;gBAC5D,2DAA2D,CAClE,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAa,CAAC;AACrC,CAAC"}
@@ -1,2 +0,0 @@
1
- export declare function chunk<T>(array: readonly T[], chunkSize: number): T[][];
2
- //# sourceMappingURL=chunk.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":"AA2BA,wBAAgB,KAAK,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,EAAE,CAQtE"}
@@ -1,40 +0,0 @@
1
- /*
2
- The MIT License (MIT)
3
-
4
- Copyright © `2020` `The Sapphire Community and its contributors`
5
-
6
- Permission is hereby granted, free of charge, to any person
7
- obtaining a copy of this software and associated documentation
8
- files (the “Software”), to deal in the Software without
9
- restriction, including without limitation the rights to use,
10
- copy, modify, merge, publish, distribute, sublicense, and/or sell
11
- copies of the Software, and to permit persons to whom the
12
- Software is furnished to do so, subject to the following
13
- conditions:
14
-
15
- The above copyright notice and this permission notice shall be
16
- included in all copies or substantial portions of the Software.
17
-
18
- THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
19
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
20
- OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
21
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
22
- HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
23
- WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24
- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
25
- OTHER DEALINGS IN THE SOFTWARE.
26
- */
27
- export function chunk(array, chunkSize) {
28
- if (!Array.isArray(array))
29
- throw new TypeError('entries must be an array.');
30
- if (!Number.isInteger(chunkSize))
31
- throw new TypeError('chunkSize must be an integer.');
32
- if (chunkSize < 1)
33
- throw new RangeError('chunkSize must be 1 or greater.');
34
- const clone = array.slice();
35
- const chunks = [];
36
- while (clone.length)
37
- chunks.push(clone.splice(0, chunkSize));
38
- return chunks;
39
- }
40
- //# sourceMappingURL=chunk.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;EAyBE;AAEF,MAAM,UAAU,KAAK,CAAI,KAAmB,EAAE,SAAiB;IAC3D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,2BAA2B,CAAC,CAAC;IAC5E,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,+BAA+B,CAAC,CAAC;IACvF,IAAI,SAAS,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAQ,KAAK,CAAC,KAAK,EAAE,CAAC;IACjC,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,OAAO,KAAK,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7D,OAAO,MAAM,CAAC;AAClB,CAAC"}
@@ -1,31 +0,0 @@
1
- import type { IncomingMessage } from 'node:http';
2
- import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
3
- interface BrowserResponseLike {
4
- status(): number;
5
- }
6
- interface Request<UserData extends Dictionary = Dictionary> {
7
- id?: string;
8
- url: string;
9
- loadedUrl?: string;
10
- uniqueKey: string;
11
- method: AllowedHttpMethods;
12
- payload?: string;
13
- noRetry: boolean;
14
- retryCount: number;
15
- errorMessages: string[];
16
- headers?: Record<string, string>;
17
- userData: UserData;
18
- handledAt?: string;
19
- }
20
- /**
21
- * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
22
- *
23
- * @param request [Request](https://sdk.apify.com/docs/api/request) object.
24
- * @param [response]
25
- * Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
26
- * or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
27
- * @param [additionalFields] Object containing additional fields to be added.
28
- */
29
- export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
30
- export {};
31
- //# sourceMappingURL=debug.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"debug.d.ts","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGrE,UAAU,mBAAmB;IACzB,MAAM,IAAI,MAAM,CAAC;CACpB;AAED,UAAU,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACtD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAClC,OAAO,EAAE,OAAO,EAChB,QAAQ,GAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAM,EAC7D,gBAAgB,GAAE,UAAe,GAClC,UAAU,CAmBZ"}
@@ -1,29 +0,0 @@
1
- import ow from 'ow';
2
- /**
3
- * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
4
- *
5
- * @param request [Request](https://sdk.apify.com/docs/api/request) object.
6
- * @param [response]
7
- * Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
8
- * or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
9
- * @param [additionalFields] Object containing additional fields to be added.
10
- */
11
- export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
12
- ow(request, ow.object);
13
- ow(response, ow.object);
14
- ow(additionalFields, ow.object);
15
- return {
16
- requestId: request.id,
17
- url: request.url,
18
- loadedUrl: request.loadedUrl,
19
- method: request.method,
20
- retryCount: request.retryCount,
21
- errorMessages: request.errorMessages,
22
- // Puppeteer response has .status() function and NodeJS response, statusCode property.
23
- statusCode: 'status' in response && response.status instanceof Function
24
- ? response.status()
25
- : response.statusCode,
26
- ...additionalFields,
27
- };
28
- }
29
- //# sourceMappingURL=debug.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"debug.js","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,IAAI,CAAC;AAqBpB;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAClC,OAAgB,EAChB,WAA2D,EAAE,EAC7D,mBAA+B,EAAE;IAEjC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACvB,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACxB,EAAE,CAAC,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO;QACH,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,sFAAsF;QACtF,UAAU,EACN,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,YAAY,QAAQ;YACvD,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE;YACnB,CAAC,CAAE,QAA4B,CAAC,UAAU;QAClD,GAAG,gBAAgB;KACtB,CAAC;AACN,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAuB9F;AAED,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,EAAE,CAiBjE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMhF"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAyBnD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACvE,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B,CAAC,CACL,CAAC;IACF,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,EAAE,SAAS,GAAG,mBAAmB,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAEtF,0GAA0G;IAC1G,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACzF,IAAI,QAAQ,GAAG,GAAG,CAAC;IAEnB,IAAI,KAAK,EAAE,CAAC;QACR,QAAQ,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjD,CAAC;IAED,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IAElF,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC;AAeD;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAA2B;IACnD,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CACL,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,OAAe;IACxD,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"general.d.ts","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAC+G,CAAC;AAEhJ;;;GAGG;AACH,eAAO,MAAM,qBAAqB,QAC8G,CAAC;AAoBjJ;;GAEG;AACH,wBAAsB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAKrE;AAID;;GAEG;AACH,wBAAsB,eAAe,qBA0BpC;AAED,wBAAgB,QAAQ,YAEvB;AAGD;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,UAAU,CAAC,EAAE,OAAO,+BAoB3D;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAW7E;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAQjE;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CA8B5D"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAC5B,2IAA2I,CAAC;AAEhJ;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAC9B,4IAA4I,CAAC;AAEjJ,IAAI,oBAAkD,CAAC;AAEvD,KAAK,UAAU,qBAAqB;IAChC,MAAM,QAAQ,GAAG,EAAE;SACd,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,QAAQ,GAAG,EAAE;SACd,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEnE,OAAO,OAAO,IAAI,OAAO,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,UAAoB;IAC/C,wDAAwD;IACxD,IAAI,CAAC,oBAAoB,IAAI,UAAU;QAAE,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;IAExF,OAAO,oBAAoB,CAAC;AAChC,CAAC;AAED,IAAI,qBAA0C,CAAC;AAE/C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe;IACjC,gFAAgF;IAChF,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED,wCAAwC;IACxC,IAAI,QAAQ,EAAE,EAAE,CAAC;QACb,qBAAqB,GAAG,KAAK,CAAC;QAC9B,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED,MAAM,cAAc,GAAG,EAAE;SACpB,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,WAAW,GAAG,EAAE;SACjB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,eAAe,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC;IAEzF,qBAAqB,GAAG,eAAe,IAAI,YAAY,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IACjG,OAAO,qBAAqB,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,QAAQ;IACpB,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;AACzD,CAAC;AAED,IAAI,eAAmC,CAAC;AACxC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,UAAoB;IACxD,wDAAwD;IACxD,IAAI,eAAe,KAAK,SAAS,IAAI,CAAC,UAAU,EAAE,CAAC;QAC/C,OAAO,eAAe,CAAC;IAC3B,CAAC;IACD,IAAI,CAAC;QACD,+DAA+D;QAC/D,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,eAAe,GAAG,IAAI,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,eAAe,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC;QACD,8EAA8E;QAC9E,MAAM,EAAE,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,eAAe,GAAG,IAAI,CAAC;IAC3B,CAAC;IACD,OAAO,eAAe,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,SAAmB,EAAE,UAAoB;IACjE,MAAM,MAAM,GAAG,SAAS;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACd,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1D,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,MAAe;IACvC,OAAO,UAAU,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;AAC3C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,YAAoB;IACrD,OAAO,YAAY;SACd,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACjB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAkB;IAChD,oCAAoC;IACpC,SAAS,gBAAgB,CAAC,UAAe;QACrC,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACrC,UAAU,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC;QACrD,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,oDAAoD;IACpD,SAAS,yBAAyB,CAAC,WAAgB;QAC/C,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;gBAChB,yBAAyB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;gBACzC,IAAI,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,uBAAuB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAErE,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;gBACpD,CAAC;gBACD,EAAE,CAAC,SAAS,IAAI,OAAO,CAAC;YAC5B,CAAC;QACL,CAAC;IACL,CAAC;IAED,yBAAyB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEzC,OAAO,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;AAC9C,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"open_graph_parser.d.ts","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAG1C,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CACjC;AAED,KAAK,eAAe,GAAG,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;AA2X3E;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,oBAAoB,CAAC,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC;AACrH,wBAAgB,cAAc,CAAC,CAAC,EAAE,UAAU,EAAE,oBAAoB,CAAC,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"open_graph_parser.js","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAU/B;;;;;;GAMG;AACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE,CAC9C,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAE/E,MAAM,qBAAqB,GAAwB;IAC/C;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,MAAM;QAClB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,eAAe;QACrB,UAAU,EAAE,YAAY;QACxB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,WAAW;QACjB,UAAU,EAAE,QAAQ;QACpB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD,kDAAkD;IAClD,iHAAiH;IACjH,mDAAmD;IACnD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,mBAAmB;wBACzB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,YAAY;gBAClB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,iBAAiB;wBACvB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,sBAAsB;gBAC5B,UAAU,EAAE,eAAe;gBAC3B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,cAAc;gBAC1B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,uBAAuB;gBAC7B,UAAU,EAAE,gBAAgB;gBAC5B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,UAAU;gBAChB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,kBAAkB;gBACxB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;CACJ,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,kBAAkB,IAAI,IAAI,CAAC;AAE3E,MAAM,sBAAsB,GAAG,CAAC,QAA2B,EAAE,CAAa,EAAuC,EAAE;IAC/G,6GAA6G;IAC7G,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnG,4FAA4F;IAC5F,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAE,MAAM,CAAC,CAAC,CAAY,CAAC,CAAC,CAAE,MAAmB,CAAC;IAElF,sEAAsE;IACtE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IAE9C,wGAAwG;IACxG,OAAO;QACH,qGAAqG;QACrG,+EAA+E;QAC/E,wGAAwG;QACxG,uDAAuD;QACvD,GAAG,cAAc,CAAC,GAAG,QAAQ,CAAC,UAAU,OAAO,EAAE,OAAO,CAAC;QACzD,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CACvB,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YACV,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/C,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,GAAG,CAAC;YAErC,OAAO;gBACH,GAAG,GAAG;gBACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;aACtE,CAAC;QACN,CAAC,EACD,EAAqC,CACxC;KACJ,CAAC;AACN,CAAC,CAAC;AAYF,MAAM,UAAU,cAAc,CAAC,IAAyB,EAAE,oBAA0C;IAChG,MAAM,CAAC,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvD,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,EAAE,GAAG,qBAAqB,CAAC,CAAC,MAAM,CACrE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QACV,OAAO;YACH,GAAG,GAAG;YACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;SACtE,CAAC;IACN,CAAC,EACD,EAAiC,CACpC,CAAC;AACN,CAAC"}