@crawlee/utils 4.0.0-beta.253 → 4.0.0-beta.254
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/social.d.ts +2 -0
- package/internals/social.js +28 -5
- package/package.json +4 -4
package/internals/social.d.ts
CHANGED
|
@@ -19,6 +19,8 @@ export declare function emailsFromText(text: string): string[];
|
|
|
19
19
|
/**
|
|
20
20
|
* The function extracts email addresses from a list of URLs.
|
|
21
21
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
22
|
+
* A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
|
|
23
|
+
* such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
|
|
22
24
|
* Note that the function preserves the order of emails and keep duplicates.
|
|
23
25
|
* @param urls Array of URLs.
|
|
24
26
|
* @return Array of emails addresses found.
|
package/internals/social.js
CHANGED
|
@@ -30,6 +30,8 @@ export function emailsFromText(text) {
|
|
|
30
30
|
/**
|
|
31
31
|
* The function extracts email addresses from a list of URLs.
|
|
32
32
|
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
33
|
+
* A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
|
|
34
|
+
* such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
|
|
33
35
|
* Note that the function preserves the order of emails and keep duplicates.
|
|
34
36
|
* @param urls Array of URLs.
|
|
35
37
|
* @return Array of emails addresses found.
|
|
@@ -44,9 +46,20 @@ export function emailsFromUrls(urls) {
|
|
|
44
46
|
continue;
|
|
45
47
|
if (!EMAIL_URL_PREFIX_REGEX.test(url))
|
|
46
48
|
continue;
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
49
|
+
// Characters like `?` and `,` inside an address must be percent-encoded, so it's safe
|
|
50
|
+
// to split on them before decoding.
|
|
51
|
+
const [addresses] = url.replace(EMAIL_URL_PREFIX_REGEX, '').split('?');
|
|
52
|
+
for (const address of addresses.split(',')) {
|
|
53
|
+
let email = address.trim();
|
|
54
|
+
try {
|
|
55
|
+
email = decodeURIComponent(email).trim();
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// Keep the raw value if it isn't valid percent-encoding
|
|
59
|
+
}
|
|
60
|
+
if (EMAIL_REGEX.test(email))
|
|
61
|
+
emails.push(email);
|
|
62
|
+
}
|
|
50
63
|
}
|
|
51
64
|
return emails;
|
|
52
65
|
}
|
|
@@ -138,8 +151,18 @@ export function phonesFromUrls(urls) {
|
|
|
138
151
|
continue;
|
|
139
152
|
if (!PHONE_URL_PREFIX_REGEX.test(url))
|
|
140
153
|
continue;
|
|
141
|
-
|
|
142
|
-
|
|
154
|
+
// Per RFC 3966, parameters such as `;ext=89` can follow the number, and `-`, `.`, `(` and `)`
|
|
155
|
+
// are only visual separators. They are checked without separators too, so a one-digit country
|
|
156
|
+
// code like in `tel:+1-555-123-4567` is accepted.
|
|
157
|
+
const [number] = url.replace(PHONE_URL_PREFIX_REGEX, '').split(';');
|
|
158
|
+
let phone = number.trim();
|
|
159
|
+
try {
|
|
160
|
+
phone = decodeURIComponent(phone).trim();
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
// Keep the raw value if it isn't valid percent-encoding
|
|
164
|
+
}
|
|
165
|
+
if (PHONE_REGEX.test(phone) || PHONE_REGEX.test(phone.replace(/[-.() ]/g, '')))
|
|
143
166
|
phones.push(phone);
|
|
144
167
|
}
|
|
145
168
|
return phones;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.254",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -43,8 +43,8 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@apify/validations": "^1.1.2",
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.254",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.254",
|
|
48
48
|
"@types/sax": "^1.2.7",
|
|
49
49
|
"cheerio": "^1.2.0",
|
|
50
50
|
"domhandler": "^6.0.1",
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "fd5e81ed20e33a43a281602f76be93169980bcd5"
|
|
67
67
|
}
|