crawler-user-agents 1.0.141 → 1.0.142
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__init__.py +18 -15
- package/crawler-user-agents.json +7 -2
- package/package.json +1 -1
package/__init__.py
CHANGED
|
@@ -1,22 +1,25 @@
|
|
|
1
|
-
import crawleruseragents
|
|
2
1
|
import re
|
|
3
2
|
import json
|
|
4
|
-
from
|
|
5
|
-
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
|
|
6
6
|
def load_json():
|
|
7
|
-
|
|
7
|
+
cwd = Path(__file__).parent
|
|
8
|
+
user_agents_file_path = cwd / "crawler-user-agents.json"
|
|
9
|
+
with user_agents_file_path.open() as patterns_file:
|
|
10
|
+
return json.load(patterns_file)
|
|
8
11
|
|
|
9
|
-
DATA = load_json()
|
|
10
12
|
|
|
11
|
-
|
|
12
|
-
# print(s)
|
|
13
|
-
for i in DATA:
|
|
14
|
-
test=re.search(i["pattern"],s,re.IGNORECASE)
|
|
15
|
-
if test:
|
|
16
|
-
return True
|
|
17
|
-
return False
|
|
13
|
+
CRAWLER_USER_AGENTS_DATA = load_json()
|
|
18
14
|
|
|
19
|
-
def is_crawler2(s):
|
|
20
|
-
regexp = re.compile("|".join([i["pattern"] for i in DATA]))
|
|
21
|
-
return regexp.search(s) != None
|
|
22
15
|
|
|
16
|
+
def is_crawler(user_agent: str) -> bool:
|
|
17
|
+
for crawler_user_agent in CRAWLER_USER_AGENTS_DATA:
|
|
18
|
+
if re.search(crawler_user_agent["pattern"], user_agent, re.IGNORECASE):
|
|
19
|
+
return True
|
|
20
|
+
return False
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def is_crawler2(s):
|
|
24
|
+
regexp = re.compile("|".join([i["pattern"] for i in CRAWLER_USER_AGENTS_DATA]))
|
|
25
|
+
return regexp.search(s) is not None
|
package/crawler-user-agents.json
CHANGED
|
@@ -12,8 +12,7 @@
|
|
|
12
12
|
"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
|
|
13
13
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36"
|
|
14
14
|
]
|
|
15
|
-
}
|
|
16
|
-
,
|
|
15
|
+
},
|
|
17
16
|
{
|
|
18
17
|
"pattern": "Googlebot-Mobile",
|
|
19
18
|
"instances": [
|
|
@@ -5314,5 +5313,11 @@
|
|
|
5314
5313
|
"addition_date": "2024/05/14",
|
|
5315
5314
|
"instances": ["Mozilla/5.0 (compatible; Monsidobot/2.2; +http://monsido.com/bot.html; info@monsido.com)"],
|
|
5316
5315
|
"url": "http://monsido.com/bot.html"
|
|
5316
|
+
},
|
|
5317
|
+
{
|
|
5318
|
+
"pattern": "GroupMeBot",
|
|
5319
|
+
"addition_date": "2024/05/19",
|
|
5320
|
+
"instances": ["GroupMeBot/1.0"],
|
|
5321
|
+
"url": "https://groupme.com/"
|
|
5317
5322
|
}
|
|
5318
5323
|
]
|