crawler-user-agents 1.0.107 → 1.0.109

package/.travis.yml

@@ -1,5 +1,6 @@
+dist: jammy
 language: python
-python: 3.6
+python: 3.11.0
 
 install: 
 - pip3 install jsonschema pytest
@@ -19,7 +20,7 @@ deploy:
   email: martin.monperrus@gnieh.org
   skip_cleanup: true
   api_key:
-    secure: YgdBbKjxy+DRdQcItgiwq3gMjxzFbZsplv13Lb5Ewzb+n6CO7U/RHEdLMHRfE/3xlQIaMMvbjjBSBUR9rDpMqClrfnkwf2RtXKQGImQ9Z8vvRKIWEn1UDoFLPtRtVrPU/7K96LbJsSyncM8HDEsF3s22WdiDcFhuH3/PjWxxr5g=
+    secure: DAS1IgbrU085kpY4acrByrU2oWQq2HbP8aNye/Wb7nJIjT2Hv+DrbGoZ38Qiz8szHGmOy0DwtwJGPL6dhM7cC+2PBn5Pu6GzeORCSh+KlrRFIVI09A9BRc6TGNkJIr7ddP0H69U+OcdL53noBeMK4q2nxM6neCZM4Aa45r1jptI=
   on:
     repo: monperrus/crawler-user-agents
     branch: master

package/crawler-user-agents.json

@@ -101,6 +101,32 @@
     ]
   }
   ,
+  {
+    "pattern": "Google-InspectionTool",
+    "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
+    "instances": [
+      "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)",
+      "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)"
+    ]
+  }
+  ,
+  {
+    "pattern": "Storebot-Google",
+    "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
+    "instances": [
+      "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
+      "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36"
+    ]
+  }
+  ,
+  {
+    "pattern": "GoogleOther",
+    "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
+    "instances": [
+      "GoogleOther"
+    ]
+  }
+  ,
   {
     "pattern": "bingbot",
     "url": "http://www.bing.com/bingbot.htm",
@@ -198,7 +224,7 @@
     "instances": [
       "python-httpx/0.16.1",
       "python-httpx/0.13.0.dev1"
-      
+
     ],
     "url": "https://www.python-httpx.org"
   }
@@ -994,11 +1020,12 @@
   }
   ,
   {
-    "pattern": "TurnitinBot",
+    "pattern": "Turnitin",
     "addition_date": "2010/09/26",
     "url": "http://www.turnitin.com",
     "instances": [
-      "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)"
+      "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)",
+      "Turnitin (https://bit.ly/2UvnfoQ)"
     ]
   }
   ,
@@ -4829,7 +4856,7 @@
     "instances": [
       "Mozilla/5.0 (compatible; Go-http-client/1.1; +centurybot9@gmail.com)"
     ],
-    "url": "unknown" 
+    "url": "unknown"
   },
   {
     "pattern": "Viber",
@@ -4990,5 +5017,120 @@
       "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
       "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
     ]
+  },
+  {
+    "pattern": "Valve\\/Steam",
+    "addition_date": "2023/05/24",
+    "instances": [
+      "Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)"
+    ]
+  },
+  {
+    "pattern": "OdklBot\\/",
+    "addition_date": "2023/05/24",
+    "instances": [
+      "OdklBot/1.0 (share@odnoklassniki.ru)",
+        "Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)"
+    ],
+    "url": "https://odnoklassniki.ru/"
+  },
+  {
+    "pattern": "GPTBot",
+    "addition_date": "2023/08/09",
+    "instances": [
+      "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)"
+    ],
+    "url": "https://platform.openai.com/docs/gptbot"
+  },
+  {
+    "pattern": "YandexRenderResourcesBot\\/",
+    "addition_date": "2023/08/16",
+    "instances": [
+      "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0"
+    ],
+    "url": "http://yandex.com/bots"
+  },
+  {
+    "pattern": "LightspeedSystemsCrawler",
+    "addition_date": "2023/08/16",
+    "instances": [
+      "LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US"
+    ]
+  },
+  {
+    "pattern": "ev-crawler\\/",
+    "addition_date": "2023/08/16",
+    "instances": [
+      "Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)"
+    ],
+    "url": "https://headline.com/legal/crawler"
+  },
+  {
+    "pattern": "BitSightBot\\/",
+    "addition_date": "2023/08/16",
+    "instances": [
+      "Mozilla/5.0 (compatible; BitSightBot/1.0)"
+    ],
+    "url": "https://www.bitsight.com"
+  },
+  {
+    "pattern": "woorankreview\\/",
+    "addition_date": "2023/08/16",
+    "instances": [
+      "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)",
+      "Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)"
+    ],
+    "url": "https://www.woorank.com/"
+  },
+  {
+    "pattern": "Google-Safety",
+    "addition_date": "2023/08/17",
+    "instances": [
+      "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)",
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)",
+      "Google-Safety"
+    ],
+    "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
+  },
+  {
+    "pattern": "AwarioBot",
+    "addition_date": "2023/08/23",
+    "instances": [
+      "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)"
+    ],
+    "url": "https://awario.com/bots.html"
+  },
+  {
+    "pattern": "DataForSeoBot",
+    "addition_date": "2023/08/23",
+    "instances": [
+      "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)"
+    ],
+    "url": "https://dataforseo.com/dataforseo-bot"
+  },
+  {
+    "pattern": "Linespider",
+    "addition_date": "2023/08/24",
+    "instances": [
+      "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)",
+	    "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Linespider/1.1; +https://lin.ee/4dwXkTH) Chrome/W.X.Y.Z Safari/537.36"
+    ],
+    "url": "https://help2.line.me/linesearchbot/web/?contentId=50006055&lang=en"
+  },
+  {
+    "pattern": "WellKnownBot",
+    "addition_date": "2023/08/29",
+    "instances": [
+      "Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)"
+    ],
+    "url": "https://well-known.dev/about/#bot)"
+  },
+  {
+    "pattern": "A Patent Crawler",
+    "addition_date": "2023/08/29",
+    "instances": [
+      "E. Orliac, G. Fourestey/2.3 (A Patent Crawler; http://scitas.epfl.ch/; etienne.orliac@epfl.ch, gilles.fourestey@epfl.ch)"
+    ],
+    "url": "http://scitas.epfl.ch/"
   }
 ]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawler-user-agents",
-  "version": "1.0.107",
+  "version": "1.0.109",
   "main": "crawler-user-agents.json",
   "typings": "./index.d.ts",
   "author": "Martin Monperrus <martin.monperrus@gnieh.org>",

package/.pytest_cache/README.md

@@ -1,8 +0,0 @@
-# pytest cache directory #
-
-This directory contains data from the pytest's cache plugin,
-which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
-
-**Do not** commit this to version control.
-
-See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.