@crawlee/utils 4.0.0-beta.29 → 4.0.0-beta.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +10 -17
- package/index.js +8 -16
- package/internal.d.ts +12 -0
- package/internal.js +11 -0
- package/internals/blocked.d.ts +7 -1
- package/internals/blocked.js +11 -1
- package/internals/cheerio.d.ts +1 -3
- package/internals/cheerio.js +33 -23
- package/internals/extract-microdata.d.ts +24 -0
- package/internals/extract-microdata.js +136 -0
- package/internals/extract-urls.d.ts +1 -2
- package/internals/extract-urls.js +18 -23
- package/internals/general.d.ts +0 -25
- package/internals/general.js +2 -110
- package/internals/http-request.d.ts +5 -0
- package/internals/http-request.js +36 -0
- package/internals/iterables.d.ts +0 -80
- package/internals/iterables.js +0 -126
- package/internals/mutation-journal.d.ts +29 -0
- package/internals/mutation-journal.js +173 -0
- package/internals/{open_graph_parser.d.ts → open-graph-parser.d.ts} +2 -3
- package/internals/{open_graph_parser.js → open-graph-parser.js} +8 -9
- package/internals/robots.d.ts +35 -15
- package/internals/robots.js +66 -30
- package/internals/schemas.d.ts +114 -0
- package/internals/schemas.js +115 -0
- package/internals/sitemap.d.ts +44 -3
- package/internals/sitemap.js +243 -116
- package/internals/social.d.ts +4 -3
- package/internals/social.js +36 -11
- package/internals/url.d.ts +67 -1
- package/internals/url.js +118 -2
- package/internals/validation.d.ts +9 -0
- package/internals/validation.js +16 -0
- package/package.json +16 -15
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/blocked.d.ts.map +0 -1
- package/internals/blocked.js.map +0 -1
- package/internals/cheerio.d.ts.map +0 -1
- package/internals/cheerio.js.map +0 -1
- package/internals/chunk.d.ts +0 -2
- package/internals/chunk.d.ts.map +0 -1
- package/internals/chunk.js +0 -40
- package/internals/chunk.js.map +0 -1
- package/internals/debug.d.ts +0 -32
- package/internals/debug.d.ts.map +0 -1
- package/internals/debug.js +0 -41
- package/internals/debug.js.map +0 -1
- package/internals/extract-urls.d.ts.map +0 -1
- package/internals/extract-urls.js.map +0 -1
- package/internals/general.d.ts.map +0 -1
- package/internals/general.js.map +0 -1
- package/internals/iterables.d.ts.map +0 -1
- package/internals/iterables.js.map +0 -1
- package/internals/open_graph_parser.d.ts.map +0 -1
- package/internals/open_graph_parser.js.map +0 -1
- package/internals/robots.d.ts.map +0 -1
- package/internals/robots.js.map +0 -1
- package/internals/sitemap.d.ts.map +0 -1
- package/internals/sitemap.js.map +0 -1
- package/internals/social.d.ts.map +0 -1
- package/internals/social.js.map +0 -1
- package/internals/system-info/cpu-info.d.ts +0 -64
- package/internals/system-info/cpu-info.d.ts.map +0 -1
- package/internals/system-info/cpu-info.js +0 -211
- package/internals/system-info/cpu-info.js.map +0 -1
- package/internals/system-info/memory-info.d.ts +0 -28
- package/internals/system-info/memory-info.d.ts.map +0 -1
- package/internals/system-info/memory-info.js +0 -118
- package/internals/system-info/memory-info.js.map +0 -1
- package/internals/system-info/ps-tree.d.ts +0 -18
- package/internals/system-info/ps-tree.d.ts.map +0 -1
- package/internals/system-info/ps-tree.js +0 -145
- package/internals/system-info/ps-tree.js.map +0 -1
- package/internals/typedefs.d.ts +0 -5
- package/internals/typedefs.d.ts.map +0 -1
- package/internals/typedefs.js +0 -9
- package/internals/typedefs.js.map +0 -1
- package/internals/url.d.ts.map +0 -1
- package/internals/url.js.map +0 -1
package/internals/social.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import * as cheerio from 'cheerio';
|
|
2
1
|
import { htmlToText } from './cheerio.js';
|
|
3
2
|
// Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
|
|
4
|
-
|
|
3
|
+
// The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
|
|
4
|
+
// instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
|
|
5
|
+
// dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
|
|
6
|
+
const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
|
|
5
7
|
/**
|
|
6
8
|
* Regular expression to exactly match a single email address.
|
|
7
9
|
* It has the following form: `/^...$/i`.
|
|
@@ -28,6 +30,8 @@ export function emailsFromText(text) {
|
|
|
28
30
|
/**
|
|
29
31
|
* The function extracts email addresses from a list of URLs.
|
|
30
32
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
33
|
+
* A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
|
|
34
|
+
* such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
|
|
31
35
|
* Note that the function preserves the order of emails and keep duplicates.
|
|
32
36
|
* @param urls Array of URLs.
|
|
33
37
|
* @return Array of emails addresses found.
|
|
@@ -42,9 +46,20 @@ export function emailsFromUrls(urls) {
|
|
|
42
46
|
continue;
|
|
43
47
|
if (!EMAIL_URL_PREFIX_REGEX.test(url))
|
|
44
48
|
continue;
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
49
|
+
// Characters like `?` and `,` inside an address must be percent-encoded, so it's safe
|
|
50
|
+
// to split on them before decoding.
|
|
51
|
+
const [addresses] = url.replace(EMAIL_URL_PREFIX_REGEX, '').split('?');
|
|
52
|
+
for (const address of addresses.split(',')) {
|
|
53
|
+
let email = address.trim();
|
|
54
|
+
try {
|
|
55
|
+
email = decodeURIComponent(email).trim();
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// Keep the raw value if it isn't valid percent-encoding
|
|
59
|
+
}
|
|
60
|
+
if (EMAIL_REGEX.test(email))
|
|
61
|
+
emails.push(email);
|
|
62
|
+
}
|
|
48
63
|
}
|
|
49
64
|
return emails;
|
|
50
65
|
}
|
|
@@ -136,8 +151,18 @@ export function phonesFromUrls(urls) {
|
|
|
136
151
|
continue;
|
|
137
152
|
if (!PHONE_URL_PREFIX_REGEX.test(url))
|
|
138
153
|
continue;
|
|
139
|
-
|
|
140
|
-
|
|
154
|
+
// Per RFC 3966, parameters such as `;ext=89` can follow the number, and `-`, `.`, `(` and `)`
|
|
155
|
+
// are only visual separators. They are checked without separators too, so a one-digit country
|
|
156
|
+
// code like in `tel:+1-555-123-4567` is accepted.
|
|
157
|
+
const [number] = url.replace(PHONE_URL_PREFIX_REGEX, '').split(';');
|
|
158
|
+
let phone = number.trim();
|
|
159
|
+
try {
|
|
160
|
+
phone = decodeURIComponent(phone).trim();
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
// Keep the raw value if it isn't valid percent-encoding
|
|
164
|
+
}
|
|
165
|
+
if (PHONE_REGEX.test(phone) || PHONE_REGEX.test(phone.replace(/[-.() ]/g, '')))
|
|
141
166
|
phones.push(phone);
|
|
142
167
|
}
|
|
143
168
|
return phones;
|
|
@@ -324,7 +349,7 @@ export const TWITTER_REGEX = new RegExp(`^${TWITTER_REGEX_STRING}$`, 'i');
|
|
|
324
349
|
* ```
|
|
325
350
|
* import { social } from 'crawlee';
|
|
326
351
|
*
|
|
327
|
-
* const matches = text.match(social.
|
|
352
|
+
* const matches = text.match(social.TWITTER_REGEX_GLOBAL);
|
|
328
353
|
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
329
354
|
* ```
|
|
330
355
|
*/
|
|
@@ -587,7 +612,8 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
|
|
|
587
612
|
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
588
613
|
* @return An object with the social handles.
|
|
589
614
|
*/
|
|
590
|
-
export function parseHandlesFromHtml(html, data = null) {
|
|
615
|
+
export async function parseHandlesFromHtml(html, data = null) {
|
|
616
|
+
const cheerio = await import('cheerio/slim');
|
|
591
617
|
const result = {
|
|
592
618
|
emails: [],
|
|
593
619
|
phones: [],
|
|
@@ -606,7 +632,7 @@ export function parseHandlesFromHtml(html, data = null) {
|
|
|
606
632
|
const $ = cheerio.load(html, { xml: { decodeEntities: true } });
|
|
607
633
|
if (data)
|
|
608
634
|
data.$ = $;
|
|
609
|
-
const text = htmlToText($);
|
|
635
|
+
const text = await htmlToText($);
|
|
610
636
|
if (data)
|
|
611
637
|
data.text = text;
|
|
612
638
|
// NOTE: we need to parse each text separately, orherwise we might concatenate unrelated texts
|
|
@@ -644,4 +670,3 @@ export function parseHandlesFromHtml(html, data = null) {
|
|
|
644
670
|
}
|
|
645
671
|
return result;
|
|
646
672
|
}
|
|
647
|
-
//# sourceMappingURL=social.js.map
|
package/internals/url.d.ts
CHANGED
|
@@ -1,4 +1,71 @@
|
|
|
1
1
|
import type { SearchParams } from '@crawlee/types';
|
|
2
|
+
/**
|
|
3
|
+
* The different enqueueing strategies available.
|
|
4
|
+
*
|
|
5
|
+
* Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
|
|
6
|
+
*
|
|
7
|
+
* ```md
|
|
8
|
+
* Protocol Domain
|
|
9
|
+
* ┌────┐ ┌─────────┐
|
|
10
|
+
* https://example.crawlee.dev/...
|
|
11
|
+
* │ └─────────────────┤
|
|
12
|
+
* │ Hostname │
|
|
13
|
+
* │ │
|
|
14
|
+
* └─────────────────────────┘
|
|
15
|
+
* Origin
|
|
16
|
+
*```
|
|
17
|
+
*
|
|
18
|
+
* - The `Protocol` is usually `http` or `https`
|
|
19
|
+
* - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
|
|
20
|
+
* - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
|
|
21
|
+
* - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
|
|
22
|
+
*/
|
|
23
|
+
export declare enum EnqueueStrategy {
|
|
24
|
+
/**
|
|
25
|
+
* Matches any URLs found
|
|
26
|
+
*/
|
|
27
|
+
All = "all",
|
|
28
|
+
/**
|
|
29
|
+
* Matches any URLs that have the same hostname.
|
|
30
|
+
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
|
|
31
|
+
* `https://example.com/hello` will not be matched.
|
|
32
|
+
*
|
|
33
|
+
* > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
|
|
34
|
+
*/
|
|
35
|
+
SameHostname = "same-hostname",
|
|
36
|
+
/**
|
|
37
|
+
* Matches any URLs that have the same domain as the base URL.
|
|
38
|
+
* For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
|
|
39
|
+
* `https://example.com`.
|
|
40
|
+
*
|
|
41
|
+
* > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
|
|
42
|
+
*/
|
|
43
|
+
SameDomain = "same-domain",
|
|
44
|
+
/**
|
|
45
|
+
* Matches any URLs that have the same hostname and protocol.
|
|
46
|
+
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
|
|
47
|
+
* `http://wow.example.com/hello` will not be matched.
|
|
48
|
+
*
|
|
49
|
+
* > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
|
|
50
|
+
*/
|
|
51
|
+
SameOrigin = "same-origin"
|
|
52
|
+
}
|
|
53
|
+
/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
|
|
54
|
+
export declare const UNSUPPORTED_SCHEME_MESSAGE = "unsupported URL scheme (only http and https are allowed)";
|
|
55
|
+
/**
|
|
56
|
+
* Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
|
|
57
|
+
* considered here (use {@link filterUrl} for the combined scheme + strategy check). This is the single
|
|
58
|
+
* definition of the strategies, shared by `enqueueLinks`, sitemaps and robots.txt.
|
|
59
|
+
*/
|
|
60
|
+
export declare function matchesEnqueueStrategy(strategy: EnqueueStrategy | `${EnqueueStrategy}`, target: URL, origin: URL): boolean;
|
|
61
|
+
/**
|
|
62
|
+
* Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
|
|
63
|
+
* scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
|
|
64
|
+
*/
|
|
65
|
+
export declare function filterUrl(target: string | URL, origin: string | URL, strategy: EnqueueStrategy | `${EnqueueStrategy}`): {
|
|
66
|
+
allowed: boolean;
|
|
67
|
+
reason?: string;
|
|
68
|
+
};
|
|
2
69
|
/**
|
|
3
70
|
* Appends search (query string) parameters to a URL, replacing the original value (if any).
|
|
4
71
|
*
|
|
@@ -7,4 +74,3 @@ import type { SearchParams } from '@crawlee/types';
|
|
|
7
74
|
* @internal
|
|
8
75
|
*/
|
|
9
76
|
export declare function applySearchParams(url: URL, searchParams: SearchParams | undefined): void;
|
|
10
|
-
//# sourceMappingURL=url.d.ts.map
|
package/internals/url.js
CHANGED
|
@@ -1,3 +1,120 @@
|
|
|
1
|
+
import { getDomain } from 'tldts';
|
|
2
|
+
/**
|
|
3
|
+
* The different enqueueing strategies available.
|
|
4
|
+
*
|
|
5
|
+
* Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
|
|
6
|
+
*
|
|
7
|
+
* ```md
|
|
8
|
+
* Protocol Domain
|
|
9
|
+
* ┌────┐ ┌─────────┐
|
|
10
|
+
* https://example.crawlee.dev/...
|
|
11
|
+
* │ └─────────────────┤
|
|
12
|
+
* │ Hostname │
|
|
13
|
+
* │ │
|
|
14
|
+
* └─────────────────────────┘
|
|
15
|
+
* Origin
|
|
16
|
+
*```
|
|
17
|
+
*
|
|
18
|
+
* - The `Protocol` is usually `http` or `https`
|
|
19
|
+
* - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
|
|
20
|
+
* - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
|
|
21
|
+
* - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
|
|
22
|
+
*/
|
|
23
|
+
export var EnqueueStrategy;
|
|
24
|
+
(function (EnqueueStrategy) {
|
|
25
|
+
/**
|
|
26
|
+
* Matches any URLs found
|
|
27
|
+
*/
|
|
28
|
+
EnqueueStrategy["All"] = "all";
|
|
29
|
+
/**
|
|
30
|
+
* Matches any URLs that have the same hostname.
|
|
31
|
+
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
|
|
32
|
+
* `https://example.com/hello` will not be matched.
|
|
33
|
+
*
|
|
34
|
+
* > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
|
|
35
|
+
*/
|
|
36
|
+
EnqueueStrategy["SameHostname"] = "same-hostname";
|
|
37
|
+
/**
|
|
38
|
+
* Matches any URLs that have the same domain as the base URL.
|
|
39
|
+
* For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
|
|
40
|
+
* `https://example.com`.
|
|
41
|
+
*
|
|
42
|
+
* > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
|
|
43
|
+
*/
|
|
44
|
+
EnqueueStrategy["SameDomain"] = "same-domain";
|
|
45
|
+
/**
|
|
46
|
+
* Matches any URLs that have the same hostname and protocol.
|
|
47
|
+
* For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
|
|
48
|
+
* `http://wow.example.com/hello` will not be matched.
|
|
49
|
+
*
|
|
50
|
+
* > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
|
|
51
|
+
*/
|
|
52
|
+
EnqueueStrategy["SameOrigin"] = "same-origin";
|
|
53
|
+
})(EnqueueStrategy || (EnqueueStrategy = {}));
|
|
54
|
+
/** Reusable suffix for log messages explaining why a non-`http(s)` URL was rejected. */
|
|
55
|
+
export const UNSUPPORTED_SCHEME_MESSAGE = 'unsupported URL scheme (only http and https are allowed)';
|
|
56
|
+
const ALLOWED_SCHEMES = new Set(['http:', 'https:']);
|
|
57
|
+
function toUrl(value) {
|
|
58
|
+
if (value instanceof URL) {
|
|
59
|
+
return value;
|
|
60
|
+
}
|
|
61
|
+
try {
|
|
62
|
+
return new URL(value);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/** Strip a trailing dot so `example.com.` equals `example.com`. */
|
|
69
|
+
function normalizeHostname(hostname) {
|
|
70
|
+
return hostname.endsWith('.') ? hostname.slice(0, -1) : hostname;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Check whether `target` matches `origin` under the given enqueue `strategy`. The URL scheme is not
|
|
74
|
+
* considered here (use {@link filterUrl} for the combined scheme + strategy check). This is the single
|
|
75
|
+
* definition of the strategies, shared by `enqueueLinks`, sitemaps and robots.txt.
|
|
76
|
+
*/
|
|
77
|
+
export function matchesEnqueueStrategy(strategy, target, origin) {
|
|
78
|
+
switch (strategy) {
|
|
79
|
+
case 'all':
|
|
80
|
+
return true;
|
|
81
|
+
case 'same-hostname':
|
|
82
|
+
return normalizeHostname(target.hostname) === normalizeHostname(origin.hostname);
|
|
83
|
+
case 'same-domain': {
|
|
84
|
+
const originDomain = getDomain(origin.hostname, { mixedInputs: false });
|
|
85
|
+
if (originDomain) {
|
|
86
|
+
return originDomain === getDomain(target.hostname, { mixedInputs: false });
|
|
87
|
+
}
|
|
88
|
+
// No registrable domain (e.g. an IP address or `localhost`), fall back to the same hostname.
|
|
89
|
+
return matchesEnqueueStrategy('same-hostname', target, origin);
|
|
90
|
+
}
|
|
91
|
+
case 'same-origin':
|
|
92
|
+
// Compare scheme/host/port directly so a trailing-dot host is normalized.
|
|
93
|
+
return (target.protocol === origin.protocol &&
|
|
94
|
+
normalizeHostname(target.hostname) === normalizeHostname(origin.hostname) &&
|
|
95
|
+
target.port === origin.port);
|
|
96
|
+
default:
|
|
97
|
+
throw new Error(`Unknown enqueue strategy '${strategy}'.`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Check whether `target` may be enqueued under `strategy` relative to `origin`: it must use an `http(s)`
|
|
102
|
+
* scheme and match the strategy. On rejection, `reason` is a human-readable message for log output.
|
|
103
|
+
*/
|
|
104
|
+
export function filterUrl(target, origin, strategy) {
|
|
105
|
+
const targetUrl = toUrl(target);
|
|
106
|
+
if (targetUrl === null || !ALLOWED_SCHEMES.has(targetUrl.protocol)) {
|
|
107
|
+
return { allowed: false, reason: UNSUPPORTED_SCHEME_MESSAGE };
|
|
108
|
+
}
|
|
109
|
+
const originUrl = toUrl(origin);
|
|
110
|
+
if (originUrl === null) {
|
|
111
|
+
return { allowed: false, reason: 'invalid origin URL' };
|
|
112
|
+
}
|
|
113
|
+
if (!matchesEnqueueStrategy(strategy, targetUrl, originUrl)) {
|
|
114
|
+
return { allowed: false, reason: `does not match enqueue strategy '${strategy}'` };
|
|
115
|
+
}
|
|
116
|
+
return { allowed: true };
|
|
117
|
+
}
|
|
1
118
|
/**
|
|
2
119
|
* Appends search (query string) parameters to a URL, replacing the original value (if any).
|
|
3
120
|
*
|
|
@@ -19,7 +136,7 @@ export function applySearchParams(url, searchParams) {
|
|
|
19
136
|
}
|
|
20
137
|
else {
|
|
21
138
|
newSearchParams = new URLSearchParams();
|
|
22
|
-
for (const [key, value] of Object.entries(
|
|
139
|
+
for (const [key, value] of Object.entries(searchParams)) {
|
|
23
140
|
if (value === undefined) {
|
|
24
141
|
newSearchParams.delete(key);
|
|
25
142
|
}
|
|
@@ -33,4 +150,3 @@ export function applySearchParams(url, searchParams) {
|
|
|
33
150
|
}
|
|
34
151
|
url.search = newSearchParams.toString();
|
|
35
152
|
}
|
|
36
|
-
//# sourceMappingURL=url.js.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Dictionary } from '@crawlee/types';
|
|
2
|
+
export { ArgumentValidationError, parseArgument } from '@apify/validations';
|
|
3
|
+
/**
|
|
4
|
+
* Rejects options that exist only to configure the browser pool the crawler would have built for itself.
|
|
5
|
+
* Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
|
|
6
|
+
* into a second, half-working way of configuring the same pool.
|
|
7
|
+
* @internal
|
|
8
|
+
*/
|
|
9
|
+
export declare function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export { ArgumentValidationError, parseArgument } from '@apify/validations';
|
|
2
|
+
/**
|
|
3
|
+
* Rejects options that exist only to configure the browser pool the crawler would have built for itself.
|
|
4
|
+
* Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
|
|
5
|
+
* into a second, half-working way of configuring the same pool.
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
export function assertBrowserPoolNotConfigured(crawlerName, ignoredOptions) {
|
|
9
|
+
const names = Object.keys(ignoredOptions).filter((name) => ignoredOptions[name] !== undefined);
|
|
10
|
+
if (names.length === 0) {
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
throw new Error(`${crawlerName}: ${names.map((name) => `\`${name}\``).join(', ')} cannot be combined with \`browserPool\`, ` +
|
|
14
|
+
`${names.length > 1 ? 'they configure' : 'it configures'} the browser pool the crawler would build for ` +
|
|
15
|
+
'itself. Configure the pool you pass in instead.');
|
|
16
|
+
}
|
package/package.json
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.290",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=22.
|
|
6
|
+
"node": ">=22.13.0"
|
|
7
7
|
},
|
|
8
8
|
"type": "module",
|
|
9
9
|
"exports": {
|
|
10
10
|
".": "./index.js",
|
|
11
|
+
"./internal": "./internal.js",
|
|
11
12
|
"./package.json": "./package.json"
|
|
12
13
|
},
|
|
13
14
|
"keywords": [
|
|
@@ -35,25 +36,25 @@
|
|
|
35
36
|
},
|
|
36
37
|
"homepage": "https://crawlee.dev",
|
|
37
38
|
"scripts": {
|
|
38
|
-
"build": "
|
|
39
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
39
40
|
"clean": "rimraf ./dist",
|
|
40
41
|
"compile": "tsc -p tsconfig.build.json",
|
|
41
42
|
"copy": "tsx ../../scripts/copy.ts"
|
|
42
43
|
},
|
|
43
44
|
"dependencies": {
|
|
44
|
-
"@apify/
|
|
45
|
-
"@
|
|
46
|
-
"@crawlee/
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.29",
|
|
45
|
+
"@apify/validations": "^1.1.4",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.290",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.290",
|
|
48
48
|
"@types/sax": "^1.2.7",
|
|
49
|
-
"cheerio": "^1.
|
|
50
|
-
"domhandler": "^
|
|
51
|
-
"file-type": "^21.
|
|
52
|
-
"ow": "^2.0.0",
|
|
49
|
+
"cheerio": "^1.2.0",
|
|
50
|
+
"domhandler": "^6.0.1",
|
|
51
|
+
"file-type": "^21.3.4",
|
|
53
52
|
"robots-parser": "^3.0.1",
|
|
54
|
-
"sax": "^1.
|
|
55
|
-
"
|
|
56
|
-
"
|
|
53
|
+
"sax": "^1.6.1",
|
|
54
|
+
"tldts": "^7.5.3",
|
|
55
|
+
"tslib": "^2.9.1",
|
|
56
|
+
"whatwg-mimetype": "^5.0.0",
|
|
57
|
+
"zod": "^4.6.7"
|
|
57
58
|
},
|
|
58
59
|
"lerna": {
|
|
59
60
|
"command": {
|
|
@@ -62,5 +63,5 @@
|
|
|
62
63
|
}
|
|
63
64
|
}
|
|
64
65
|
},
|
|
65
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "c87ef92185664d7c0c0e84ef2a04fe1d13da6ffb"
|
|
66
67
|
}
|
package/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,0BAA0B,CAAC;AACzC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,qCAAqC,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AAEnF,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}
|
package/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,0BAA0B,CAAC;AACzC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAa,MAAM,qCAAqC,CAAC;AACtF,OAAO,EAAE,aAAa,EAAc,MAAM,wCAAwC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"blocked.d.ts","sourceRoot":"","sources":["../../src/internals/blocked.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,8BAA8B,UAA0E,CAAC;AAEtH;;GAEG;AACH,eAAO,MAAM,mBAAmB,UAI/B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,mBAAmB,UAM/B,CAAC"}
|
package/internals/blocked.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"blocked.js","sourceRoot":"","sources":["../../src/internals/blocked.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,qEAAqE,CAAC,CAAC;AAEtH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAC/B,GAAG,8BAA8B;IACjC,0DAA0D;IAC1D,oCAAoC;CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAC/B,YAAY;IACZ,cAAc;IACd,6BAA6B;IAC7B,8BAA8B;IAC9B,sBAAsB;CACzB,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAK1C,MAAM,MAAM,WAAW,GAAG,UAAU,CAAC;AACrC,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACnD,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAO1C;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,UAAU,CAAC,oBAAoB,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,CAwC7E;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,SAAM,EAAE,OAAO,SAAK,GAAG,MAAM,EAAE,CAwB5F"}
|
package/internals/cheerio.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAMnD,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAClB,sGAAsG,CAAC;AAE3G;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,CAAC,GAAG,OAAO,oBAAoB,KAAK,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjH,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9E,IAAI,IAAI,KAAK,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,qCAAqC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;YACzD,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAAC,CAAa,EAAE,QAAQ,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC9E,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,uCAAuC;gBAC5D,2DAA2D,CAClE,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAa,CAAC;AACrC,CAAC"}
|
package/internals/chunk.d.ts
DELETED
package/internals/chunk.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":"AA2BA,wBAAgB,KAAK,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,EAAE,CAQtE"}
|
package/internals/chunk.js
DELETED
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
The MIT License (MIT)
|
|
3
|
-
|
|
4
|
-
Copyright © `2020` `The Sapphire Community and its contributors`
|
|
5
|
-
|
|
6
|
-
Permission is hereby granted, free of charge, to any person
|
|
7
|
-
obtaining a copy of this software and associated documentation
|
|
8
|
-
files (the “Software”), to deal in the Software without
|
|
9
|
-
restriction, including without limitation the rights to use,
|
|
10
|
-
copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
11
|
-
copies of the Software, and to permit persons to whom the
|
|
12
|
-
Software is furnished to do so, subject to the following
|
|
13
|
-
conditions:
|
|
14
|
-
|
|
15
|
-
The above copyright notice and this permission notice shall be
|
|
16
|
-
included in all copies or substantial portions of the Software.
|
|
17
|
-
|
|
18
|
-
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
|
|
19
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
|
20
|
-
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
21
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
22
|
-
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
23
|
-
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
24
|
-
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
25
|
-
OTHER DEALINGS IN THE SOFTWARE.
|
|
26
|
-
*/
|
|
27
|
-
export function chunk(array, chunkSize) {
|
|
28
|
-
if (!Array.isArray(array))
|
|
29
|
-
throw new TypeError('entries must be an array.');
|
|
30
|
-
if (!Number.isInteger(chunkSize))
|
|
31
|
-
throw new TypeError('chunkSize must be an integer.');
|
|
32
|
-
if (chunkSize < 1)
|
|
33
|
-
throw new RangeError('chunkSize must be 1 or greater.');
|
|
34
|
-
const clone = array.slice();
|
|
35
|
-
const chunks = [];
|
|
36
|
-
while (clone.length)
|
|
37
|
-
chunks.push(clone.splice(0, chunkSize));
|
|
38
|
-
return chunks;
|
|
39
|
-
}
|
|
40
|
-
//# sourceMappingURL=chunk.js.map
|
package/internals/chunk.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;EAyBE;AAEF,MAAM,UAAU,KAAK,CAAI,KAAmB,EAAE,SAAiB;IAC3D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,2BAA2B,CAAC,CAAC;IAC5E,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,+BAA+B,CAAC,CAAC;IACvF,IAAI,SAAS,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAQ,KAAK,CAAC,KAAK,EAAE,CAAC;IACjC,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,OAAO,KAAK,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7D,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
package/internals/debug.d.ts
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import type { IncomingMessage } from 'node:http';
|
|
2
|
-
import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
|
|
3
|
-
interface BrowserResponseLike {
|
|
4
|
-
status(): number;
|
|
5
|
-
}
|
|
6
|
-
interface Request<UserData extends Dictionary = Dictionary> {
|
|
7
|
-
id?: string;
|
|
8
|
-
url: string;
|
|
9
|
-
loadedUrl?: string;
|
|
10
|
-
uniqueKey: string;
|
|
11
|
-
method: AllowedHttpMethods;
|
|
12
|
-
payload?: string;
|
|
13
|
-
noRetry: boolean;
|
|
14
|
-
retryCount: number;
|
|
15
|
-
errorMessages: string[];
|
|
16
|
-
headers?: Record<string, string>;
|
|
17
|
-
userData: UserData;
|
|
18
|
-
handledAt?: string;
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
|
|
22
|
-
*
|
|
23
|
-
* @param request [Request](https://sdk.apify.com/docs/api/request) object.
|
|
24
|
-
* @param [response]
|
|
25
|
-
* Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
|
|
26
|
-
* or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
|
|
27
|
-
* @param [additionalFields] Object containing additional fields to be added.
|
|
28
|
-
*/
|
|
29
|
-
export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
|
|
30
|
-
export declare function getObjectType(value: unknown): string;
|
|
31
|
-
export {};
|
|
32
|
-
//# sourceMappingURL=debug.d.ts.map
|
package/internals/debug.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"debug.d.ts","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGrE,UAAU,mBAAmB;IACzB,MAAM,IAAI,MAAM,CAAC;CACpB;AAED,UAAU,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACtD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAClC,OAAO,EAAE,OAAO,EAChB,QAAQ,GAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAM,EAC7D,gBAAgB,GAAE,UAAe,GAClC,UAAU,CAmBZ;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAepD"}
|
package/internals/debug.js
DELETED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
import ow from 'ow';
|
|
2
|
-
/**
|
|
3
|
-
* Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
|
|
4
|
-
*
|
|
5
|
-
* @param request [Request](https://sdk.apify.com/docs/api/request) object.
|
|
6
|
-
* @param [response]
|
|
7
|
-
* Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
|
|
8
|
-
* or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
|
|
9
|
-
* @param [additionalFields] Object containing additional fields to be added.
|
|
10
|
-
*/
|
|
11
|
-
export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
|
|
12
|
-
ow(request, ow.object);
|
|
13
|
-
ow(response, ow.object);
|
|
14
|
-
ow(additionalFields, ow.object);
|
|
15
|
-
return {
|
|
16
|
-
requestId: request.id,
|
|
17
|
-
url: request.url,
|
|
18
|
-
loadedUrl: request.loadedUrl,
|
|
19
|
-
method: request.method,
|
|
20
|
-
retryCount: request.retryCount,
|
|
21
|
-
errorMessages: request.errorMessages,
|
|
22
|
-
// Puppeteer response has .status() function and NodeJS response, statusCode property.
|
|
23
|
-
statusCode: 'status' in response && response.status instanceof Function
|
|
24
|
-
? response.status()
|
|
25
|
-
: response.statusCode,
|
|
26
|
-
...additionalFields,
|
|
27
|
-
};
|
|
28
|
-
}
|
|
29
|
-
export function getObjectType(value) {
|
|
30
|
-
const simple = typeof value;
|
|
31
|
-
if (['string', 'number', 'boolean', 'bigint'].includes(simple)) {
|
|
32
|
-
return simple;
|
|
33
|
-
}
|
|
34
|
-
const objectType = Object.prototype.toString.call(value);
|
|
35
|
-
const type = objectType.match(/\[object (\w+)]/)[1];
|
|
36
|
-
if (type === 'Uint8Array') {
|
|
37
|
-
return 'Buffer';
|
|
38
|
-
}
|
|
39
|
-
return ['Date', 'Buffer', 'RegExp'].includes(type) ? type : type.toLowerCase();
|
|
40
|
-
}
|
|
41
|
-
//# sourceMappingURL=debug.js.map
|
package/internals/debug.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"debug.js","sourceRoot":"","sources":["../../src/internals/debug.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,IAAI,CAAC;AAqBpB;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAClC,OAAgB,EAChB,WAA2D,EAAE,EAC7D,mBAA+B,EAAE;IAEjC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACvB,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACxB,EAAE,CAAC,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO;QACH,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,sFAAsF;QACtF,UAAU,EACN,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,YAAY,QAAQ;YACvD,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE;YACnB,CAAC,CAAE,QAA4B,CAAC,UAAU;QAClD,GAAG,gBAAgB;KACtB,CAAC;AACN,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAc;IACxC,MAAM,MAAM,GAAG,OAAO,KAAK,CAAC;IAE5B,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,iBAAiB,CAAE,CAAC,CAAC,CAAC,CAAC;IAErD,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;AACnF,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAKrD,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAkC9F;AAED,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,EAAE,CAiBjE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAMhF"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AA8BnD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACvE,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KACjC,CAAC,CACL,CAAC;IACF,MAAM,EACF,GAAG,EACH,QAAQ,GAAG,MAAM,EACjB,SAAS,GAAG,mBAAmB,EAC/B,QAAQ,EACR,UAAU,GAAG,IAAI,eAAe,EAAE,GACrC,GAAG,OAAO,CAAC;IAEZ,0GAA0G;IAC1G,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACzF,IAAI,QAAQ,GAAG,GAAG,CAAC;IAEnB,IAAI,KAAK,EAAE,CAAC;QACR,QAAQ,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACjD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;QACpF,QAAQ;KACX,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAE9F,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC;AAeD;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAA2B;IACnD,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CACL,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,CAAC;IAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,OAAe;IACxD,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"general.d.ts","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAC+G,CAAC;AAEhJ;;;GAGG;AACH,eAAO,MAAM,qBAAqB,QAC8G,CAAC;AAoBjJ;;GAEG;AACH,wBAAsB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAKrE;AAID;;GAEG;AACH,wBAAsB,eAAe,qBA0BpC;AAED,wBAAgB,QAAQ,YAEvB;AAGD;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,UAAU,CAAC,EAAE,OAAO,+BAoB3D;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAW7E;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAQjE;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CA8B5D"}
|
package/internals/general.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAC5B,2IAA2I,CAAC;AAEhJ;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAC9B,4IAA4I,CAAC;AAEjJ,IAAI,oBAAkD,CAAC;AAEvD,KAAK,UAAU,qBAAqB;IAChC,MAAM,QAAQ,GAAG,EAAE;SACd,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,QAAQ,GAAG,EAAE;SACd,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEnE,OAAO,OAAO,IAAI,OAAO,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,UAAoB;IAC/C,wDAAwD;IACxD,IAAI,CAAC,oBAAoB,IAAI,UAAU;QAAE,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;IAExF,OAAO,oBAAoB,CAAC;AAChC,CAAC;AAED,IAAI,qBAA0C,CAAC;AAE/C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe;IACjC,gFAAgF;IAChF,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED,wCAAwC;IACxC,IAAI,QAAQ,EAAE,EAAE,CAAC;QACb,qBAAqB,GAAG,KAAK,CAAC;QAC9B,OAAO,qBAAqB,CAAC;IACjC,CAAC;IAED,MAAM,cAAc,GAAG,EAAE;SACpB,IAAI,CAAC,aAAa,CAAC;SACnB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,WAAW,GAAG,EAAE;SACjB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACrC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,eAAe,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC;IAEzF,qBAAqB,GAAG,eAAe,IAAI,YAAY,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IACjG,OAAO,qBAAqB,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,QAAQ;IACpB,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;AACzD,CAAC;AAED,IAAI,eAAmC,CAAC;AACxC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,UAAoB;IACxD,wDAAwD;IACxD,IAAI,eAAe,KAAK,SAAS,IAAI,CAAC,UAAU,EAAE,CAAC;QAC/C,OAAO,eAAe,CAAC;IAC3B,CAAC;IACD,IAAI,CAAC;QACD,+DAA+D;QAC/D,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,eAAe,GAAG,IAAI,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,eAAe,GAAG,IAAI,CAAC;IACvB,IAAI,CAAC;QACD,8EAA8E;QAC9E,MAAM,EAAE,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,eAAe,GAAG,IAAI,CAAC;IAC3B,CAAC;IACD,OAAO,eAAe,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,SAAmB,EAAE,UAAoB;IACjE,MAAM,MAAM,GAAG,SAAS;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACd,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1D,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,MAAe;IACvC,OAAO,UAAU,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;AAC3C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,YAAoB;IACrD,OAAO,YAAY;SACd,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACjB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAkB;IAChD,oCAAoC;IACpC,SAAS,gBAAgB,CAAC,UAAe;QACrC,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,KAAK,MAAM,EAAE,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;YACrC,UAAU,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC;QACrD,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,oDAAoD;IACpD,SAAS,yBAAyB,CAAC,WAAgB;QAC/C,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;gBAChB,yBAAyB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;gBACzC,IAAI,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,uBAAuB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAErE,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,gBAAgB,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;gBACpD,CAAC;gBACD,EAAE,CAAC,SAAS,IAAI,OAAO,CAAC;YAC5B,CAAC;QACL,CAAC;IACL,CAAC;IAED,yBAAyB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEzC,OAAO,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;AAC9C,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"iterables.d.ts","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,CAUlE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC,CAM5E;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAuB,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAErG;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAuB,oBAAoB,CAAC,CAAC,EACzC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACxC,SAAS,EAAE,MAAM,GAClB,aAAa,CAAC,CAAC,EAAE,CAAC,CAmBpB;AAED;;;;GAIG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;IAChF;;;;;OAKG;IACH,IAAI,IAAI,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC;IAC9D,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,CAAC;CACtD;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,qBAAqB,CAAC,CAAC,CAAC,CA+D3G;AAGD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAuB,mBAAmB,CAAC,CAAC,EAAE,GAAG,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,CAgChG"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"iterables.js","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,UAAU,CAAI,KAAc;IACxC,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC;AAChE,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,eAAe,CAAI,KAAc;IAC7C,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC;AACrE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAI,QAAwC;IAC/E,KAAK,CAAC,CAAC,QAAQ,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACvC,QAAwC,EACxC,SAAiB;IAEjB,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,KAAK,GAAQ,EAAE,CAAC;IAEpB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,KAAK,CAAC;YACZ,KAAK,GAAG,EAAE,CAAC;QACf,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACf,MAAM,KAAK,CAAC;IAChB,CAAC;AACL,CAAC;AAyBD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,qBAAqB,CAAI,QAAwC;IAC7E,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IACpE,IAAI,WAAoD,CAAC;IACzD,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,MAAM,gBAAgB,GAA6B;QAC/C,KAAK,CAAC,IAAI;YACN,0DAA0D;YAC1D,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,MAAM,MAAM,GAAG,WAAW,CAAC;gBAC3B,WAAW,GAAG,SAAS,CAAC;gBAExB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBACd,WAAW,GAAG,IAAI,CAAC;oBACnB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;gBAC5C,CAAC;gBAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAChD,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YAC5C,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAErC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,KAAK,CAAC,IAAI;YACN,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;YAC5D,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrC,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAElE,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;gBACnB,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC;QACxB,CAAC;QAED,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;KACJ,CAAC;IAEF,OAAO;QACH,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,gBAAgB,CAAC;QAC5B,CAAC;KACJ,CAAC;AACN,CAAC;AAED,gDAAgD;AAChD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAI,GAAG,SAA6B;IAC1E,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACrF,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,IAAI,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC;IAClC,MAAM,KAAK,GAAmB,IAAI,OAAO,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACpD,KAAK,UAAU,OAAO,CAAC,aAA+B,EAAE,KAAa;QACjE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO;YACH,KAAK;YACL,MAAM;SACT,CAAC;IACN,CAAC;IACD,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACjD,IAAI,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3D,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAC5B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC9B,KAAK,EAAE,CAAC;YACZ,CAAC;iBAAM,CAAC;gBACJ,YAAY,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC5D,MAAM,MAAM,CAAC,KAAK,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC;YAAS,CAAC;QACP,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,kFAAkF;YAClF,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI;gBAAE,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QACzF,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"open_graph_parser.d.ts","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAG1C,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CACjC;AAED,KAAK,eAAe,GAAG,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;AA2X3E;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,oBAAoB,CAAC,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC;AACrH,wBAAgB,cAAc,CAAC,CAAC,EAAE,UAAU,EAAE,oBAAoB,CAAC,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC"}
|