@crawlee/utils 4.0.0-beta.253 → 4.0.0-beta.255

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -19,6 +19,8 @@ export declare function emailsFromText(text: string): string[];
19
19
  /**
20
20
  * The function extracts email addresses from a list of URLs.
21
21
  * Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
22
+ * A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
23
+ * such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
22
24
  * Note that the function preserves the order of emails and keep duplicates.
23
25
  * @param urls Array of URLs.
24
26
  * @return Array of emails addresses found.
@@ -30,6 +30,8 @@ export function emailsFromText(text) {
30
30
  /**
31
31
  * The function extracts email addresses from a list of URLs.
32
32
  * Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
33
+ * A `mailto:` URL can list several comma-separated, percent-encoded addresses and end with a query
34
+ * such as `?subject=Hello`, see [RFC 6068](https://www.rfc-editor.org/rfc/rfc6068).
33
35
  * Note that the function preserves the order of emails and keep duplicates.
34
36
  * @param urls Array of URLs.
35
37
  * @return Array of emails addresses found.
@@ -44,9 +46,20 @@ export function emailsFromUrls(urls) {
44
46
  continue;
45
47
  if (!EMAIL_URL_PREFIX_REGEX.test(url))
46
48
  continue;
47
- const email = url.replace(EMAIL_URL_PREFIX_REGEX, '').trim();
48
- if (EMAIL_REGEX.test(email))
49
- emails.push(email);
49
+ // Characters like `?` and `,` inside an address must be percent-encoded, so it's safe
50
+ // to split on them before decoding.
51
+ const [addresses] = url.replace(EMAIL_URL_PREFIX_REGEX, '').split('?');
52
+ for (const address of addresses.split(',')) {
53
+ let email = address.trim();
54
+ try {
55
+ email = decodeURIComponent(email).trim();
56
+ }
57
+ catch {
58
+ // Keep the raw value if it isn't valid percent-encoding
59
+ }
60
+ if (EMAIL_REGEX.test(email))
61
+ emails.push(email);
62
+ }
50
63
  }
51
64
  return emails;
52
65
  }
@@ -138,8 +151,18 @@ export function phonesFromUrls(urls) {
138
151
  continue;
139
152
  if (!PHONE_URL_PREFIX_REGEX.test(url))
140
153
  continue;
141
- const phone = url.replace(PHONE_URL_PREFIX_REGEX, '').trim();
142
- if (PHONE_REGEX.test(phone))
154
+ // Per RFC 3966, parameters such as `;ext=89` can follow the number, and `-`, `.`, `(` and `)`
155
+ // are only visual separators. They are checked without separators too, so a one-digit country
156
+ // code like in `tel:+1-555-123-4567` is accepted.
157
+ const [number] = url.replace(PHONE_URL_PREFIX_REGEX, '').split(';');
158
+ let phone = number.trim();
159
+ try {
160
+ phone = decodeURIComponent(phone).trim();
161
+ }
162
+ catch {
163
+ // Keep the raw value if it isn't valid percent-encoding
164
+ }
165
+ if (PHONE_REGEX.test(phone) || PHONE_REGEX.test(phone.replace(/[-.() ]/g, '')))
143
166
  phones.push(phone);
144
167
  }
145
168
  return phones;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/utils",
3
- "version": "4.0.0-beta.253",
3
+ "version": "4.0.0-beta.255",
4
4
  "description": "A set of shared utilities that can be used by crawlers",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -43,8 +43,8 @@
43
43
  },
44
44
  "dependencies": {
45
45
  "@apify/validations": "^1.1.2",
46
- "@crawlee/http-client": "4.0.0-beta.253",
47
- "@crawlee/types": "4.0.0-beta.253",
46
+ "@crawlee/http-client": "4.0.0-beta.255",
47
+ "@crawlee/types": "4.0.0-beta.255",
48
48
  "@types/sax": "^1.2.7",
49
49
  "cheerio": "^1.2.0",
50
50
  "domhandler": "^6.0.1",
@@ -63,5 +63,5 @@
63
63
  }
64
64
  }
65
65
  },
66
- "gitHead": "f93aa3970468c422d9626db0d21175e8a39fdfed"
66
+ "gitHead": "e1b2f6c415f90b3eca1686a39a8af5272b54046e"
67
67
  }