crawler-user-agents 1.0.106 → 1.0.108

package/crawler-user-agents.json

@@ -101,6 +101,32 @@
     ]
   }
   ,
+  {
+    "pattern": "Google-InspectionTool",
+    "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
+    "instances": [
+      "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)",
+      "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)"
+    ]
+  }
+  ,
+  {
+    "pattern": "Storebot-Google",
+    "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
+    "instances": [
+      "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
+      "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36"
+    ]
+  }
+  ,
+  {
+    "pattern": "GoogleOther",
+    "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
+    "instances": [
+      "GoogleOther"
+    ]
+  }
+  ,
   {
     "pattern": "bingbot",
     "url": "http://www.bing.com/bingbot.htm",
@@ -198,7 +224,7 @@
     "instances": [
       "python-httpx/0.16.1",
       "python-httpx/0.13.0.dev1"
-      
+
     ],
     "url": "https://www.python-httpx.org"
   }
@@ -4829,7 +4855,7 @@
     "instances": [
       "Mozilla/5.0 (compatible; Go-http-client/1.1; +centurybot9@gmail.com)"
     ],
-    "url": "unknown" 
+    "url": "unknown"
   },
   {
     "pattern": "Viber",
@@ -4990,5 +5016,87 @@
       "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
       "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
     ]
+  },
+  {
+    "pattern": "Valve\\/Steam",
+    "addition_date": "2023/05/24",
+    "instances": [
+      "Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)"
+    ]
+  },
+  {
+    "pattern": "OdklBot\\/",
+    "addition_date": "2023/05/24",
+    "instances": [
+      "OdklBot/1.0 (share@odnoklassniki.ru)",
+        "Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)"
+    ],
+    "url": "https://odnoklassniki.ru/"
+  },
+  {
+    "pattern": "GPTBot",
+    "addition_date": "2023/08/09",
+    "instances": [
+      "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)"
+    ],
+    "url": "https://platform.openai.com/docs/gptbot"
+  },
+  {
+    "pattern": "YandexRenderResourcesBot\\/",
+    "addition_date": "2023/08/16",
+    "instances": [
+      "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0"
+    ],
+    "url": "http://yandex.com/bots"
+  },
+  {
+    "pattern": "LightspeedSystemsCrawler",
+    "addition_date": "2023/08/16",
+    "instances": [
+      "LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US"
+    ]
+  },
+  {
+    "pattern": "ev-crawler\\/",
+    "addition_date": "2023/08/16",
+    "instances": [
+      "Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)"
+    ],
+    "url": "https://headline.com/legal/crawler"
+  },
+  {
+    "pattern": "BitSightBot\\/",
+    "addition_date": "2023/08/16",
+    "instances": [
+      "Mozilla/5.0 (compatible; BitSightBot/1.0)"
+    ],
+    "url": "https://www.bitsight.com"
+  },
+  {
+    "pattern": "woorankreview\\/",
+    "addition_date": "2023/08/16",
+    "instances": [
+      "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)",
+      "Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)"
+    ],
+    "url": "https://www.woorank.com/"
+  },
+  {
+    "pattern": "Google-Safety",
+    "addition_date": "2023/08/17",
+    "instances": [
+      "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)",
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)",
+      "Google-Safety"
+    ],
+    "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
+  },
+  {
+    "pattern": "AwarioBot",
+    "addition_date": "2023/08/23",
+    "instances": [
+      "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)"
+    ],
+    "url": "https://awario.com/bots.html"
   }
 ]

package/main.php

@@ -1,6 +1,6 @@
 <?php
 if (!function_exists('get_crawler_user_agents_list')) {
     function get_crawler_user_agents_list() {
-        return json_decode(file_get_contents('crawler-user-agents.json'), true);
+        return json_decode(file_get_contents(__DIR__.'/crawler-user-agents.json'), true);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawler-user-agents",
-  "version": "1.0.106",
+  "version": "1.0.108",
   "main": "crawler-user-agents.json",
   "typings": "./index.d.ts",
   "author": "Martin Monperrus <martin.monperrus@gnieh.org>",