@crawlee/utils 4.0.0-beta.281 → 4.0.0-beta.283
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/general.js +2 -2
- package/package.json +4 -4
package/internals/general.js
CHANGED
|
@@ -3,12 +3,12 @@ import { setTimeout } from 'node:timers/promises';
|
|
|
3
3
|
* Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
|
|
4
4
|
* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
|
|
5
5
|
*/
|
|
6
|
-
export const URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()'*]*)?/giu;
|
|
6
|
+
export const URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}\p{M}0-9@:%._+~#=]{0,254}[\p{L}\p{M}0-9])\.(xn--[a-z0-9-]{1,59}|[a-z]{2,63}|[\p{L}\p{M}]{2,63})(:\d{1,5})?(\/[-\p{L}\p{M}0-9@:%_+.~#?&/=()'*]*)?/giu;
|
|
7
7
|
/**
|
|
8
8
|
* Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
|
|
9
9
|
* Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
|
|
10
10
|
*/
|
|
11
|
-
export const URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;
|
|
11
|
+
export const URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}\p{M}0-9@:%._+~#=]{0,254}[\p{L}\p{M}0-9])\.(xn--[a-z0-9-]{1,59}|[a-z]{2,63}|[\p{L}\p{M}]{2,63})(:\d{1,5})?(\/[-\p{L}\p{M}0-9@:%_+,.~#?&/=()'*]*)?/giu;
|
|
12
12
|
/**
|
|
13
13
|
* Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
|
|
14
14
|
* in your code, e.g. to prevent overloading of target website or to avoid bot detection.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.283",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -43,8 +43,8 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@apify/validations": "^1.1.4",
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.283",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.283",
|
|
48
48
|
"@types/sax": "^1.2.7",
|
|
49
49
|
"cheerio": "^1.2.0",
|
|
50
50
|
"domhandler": "^6.0.1",
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "204d05fc6843b3e220598ad342b6547a7f583eb1"
|
|
67
67
|
}
|