crawler-user-agents 1.0.140 → 1.0.142

package/README.md

@@ -6,13 +6,15 @@ This repository contains a list of of HTTP user-agents used by robots, crawlers,
 * Go package: <https://pkg.go.dev/github.com/monperrus/crawler-user-agents>
 * PyPi package: <https://pypi.org/project/crawler-user-agents/>
 
+Each `pattern` is a regular expression. It should work out-of-the-box wih your favorite regex library:
+
 ## Install
 
 ### Direct download
 
 Download the [`crawler-user-agents.json` file](https://raw.githubusercontent.com/monperrus/crawler-user-agents/master/crawler-user-agents.json) from this repository directly.
 
-### Npm / Yarn
+### Javascript
 
 crawler-user-agents is deployed on npmjs.com: <https://www.npmjs.com/package/crawler-user-agents>
 
@@ -31,14 +33,21 @@ const crawlers = require('crawler-user-agents');
 console.log(crawlers);
 ```
 
-## Usage
+### Python
 
-Each `pattern` is a regular expression. It should work out-of-the-box wih your favorite regex library:
+Install with `pip install crawler-user-agents`
+
+Then:
+
+```python
+import crawleruseragents
+if crawleruseragents.is_crawler("googlebot/"):
+   # do something
+```
+
+### Go
 
-* JavaScript: `if (RegExp(entry.pattern).test(req.headers['user-agent']) { ... }`
-* PHP: add a slash before and after the pattern: `if (preg_match('/'.$entry['pattern'].'/', $_SERVER['HTTP_USER_AGENT'])): ...`
-* Python: `if re.search(entry['pattern'], ua): ...`
-* Go: use [this package](https://pkg.go.dev/github.com/monperrus/crawler-user-agents),
+Go: use [this package](https://pkg.go.dev/github.com/monperrus/crawler-user-agents),
   it provides global variable `Crawlers` (it is synchronized with `crawler-user-agents.json`),
   functions `IsCrawler` and `MatchingCrawlers`.
 

package/__init__.py

@@ -0,0 +1,25 @@
+import re
+import json
+from pathlib import Path
+
+
+def load_json():
+    cwd = Path(__file__).parent
+    user_agents_file_path = cwd / "crawler-user-agents.json"
+    with user_agents_file_path.open() as patterns_file:
+        return json.load(patterns_file)
+
+
+CRAWLER_USER_AGENTS_DATA = load_json()
+
+
+def is_crawler(user_agent: str) -> bool:
+    for crawler_user_agent in CRAWLER_USER_AGENTS_DATA:
+        if re.search(crawler_user_agent["pattern"], user_agent, re.IGNORECASE):
+            return True
+    return False
+
+
+def is_crawler2(s):
+    regexp = re.compile("|".join([i["pattern"] for i in CRAWLER_USER_AGENTS_DATA]))
+    return regexp.search(s) is not None

package/crawler-user-agents.json

@@ -12,8 +12,7 @@
       "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
       "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36"
     ]
-  }
-  ,
+  },
   {
     "pattern": "Googlebot-Mobile",
     "instances": [
@@ -5314,5 +5313,11 @@
     "addition_date": "2024/05/14",
     "instances": ["Mozilla/5.0 (compatible; Monsidobot/2.2; +http://monsido.com/bot.html; info@monsido.com)"],
     "url": "http://monsido.com/bot.html"
+  },
+  {
+    "pattern": "GroupMeBot",
+    "addition_date": "2024/05/19",
+    "instances": ["GroupMeBot/1.0"],
+    "url": "https://groupme.com/"
   }
 ]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawler-user-agents",
-  "version": "1.0.140",
+  "version": "1.0.142",
   "main": "crawler-user-agents.json",
   "typings": "./index.d.ts",
   "author": "Martin Monperrus <martin.monperrus@gnieh.org>",