es6-crawler-detect 3.2.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/example/node/server.js +38 -32
  2. package/package.json +1 -1
  3. package/src/lib/crawler/crawlers.js +1403 -1402
  4. package/src/lib/crawler/exclusions.js +62 -62
  5. package/src/lib/crawler.js +130 -135
  6. package/test/lib/crawler.test.js +94 -52
  7. package/test/lib/database/crawlers.txt +3651 -0
  8. package/test/lib/database/devices.txt +165636 -0
package/test/lib/database/crawlers.txt ADDED
@@ -0,0 +1,3651 @@
1
+ TinEye-bot/0.51 (see http://www.tineye.com/crawler.html)
2
+ Whoismindbot/1.0 (+http://www.whoismind.com/bot.html)
3
+ Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.3; www.alertra.com)
4
+ R6_FeedFetcher(www.radian6.com/crawler)
5
+ CheckHost (http://check-host.net/)
6
+ YandeG 1.03
7
+ yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html
8
+ SEMrushBot
9
+ Protopage/3.0 (http://www.protopage.com)
10
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html
11
+ StatoolsBot (+http://www.statools.com/bot.html)
12
+ adidxbot/2.0 (+http://search.msn.com/msnbot.htm)
13
+ Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
14
+ Mozilla/5.0 (Linux; Android 5.0; Nexus 5 Build/LRX21O) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.76 Mobile Safari/537.36 PTST/281
15
+ TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)
16
+ Scrapy/1.0.5 (+http://scrapy.org)
17
+ yacybot (/global; amd64 Linux 4.4.0-31-generic; java 1.8.0_91; Europe/en) http://yacy.net/bot.html
18
+ XmlSitemapGenerator - http://xmlsitemapgenerator.org
19
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com
20
+ Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)
21
+ WatchMouse/18990 (http://watchmouse.com/ ; gab)
22
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; hkhkg02.watchmouse.net)
23
+ Mozilla/5.0 (compatible; LinkpadBot/1.06; +http://www.linkpad.ru)
24
+ Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)
25
+ yacybot (webportal/global; amd64 Linux 2.6.32-5-amd64; java 1.6.0_18; Europe/en) http://yacy.net/bot.html
26
+ PercolateCrawler/4 (ops@percolate.com)
27
+ msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)
28
+ Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daum 4.1
29
+ Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )
30
+ Mozilla/5.0 (compatible; LoadTimeBot/0.9; +http://www.loadtime.net/bot.html)
31
+ UnwindFetchor/1.0 (+http://www.gnip.com/)
32
+ nrsbot/5.0(loopip.com/robot.html)
33
+ SemrushBot/0.9
34
+ Mozilla/5.0 (compatible; UASlinkChecker/2.1; +https://udger.com/support/UASlinkChecker)
35
+ yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html
36
+ Netvibes (http://www.netvibes.com)
37
+ Acoon v4.1.0 (www.acoon.de)
38
+ msnbot/2.0b (+http://search.msn.com/msnbot.htm).
39
+ Mozilla/5.0 (compatible; ltbot/3.2.0.10 +http://www.kdsl.tu-darmstadt.de/de/kdsl/research-program/crawling-and-semantic-structuring/)
40
+ smart.apnoti.com Robot/v1.34 (http://smart.apnoti.com/en/aboutApnotiWebCrawler.html)
41
+ HubSpot Links Crawler 2.0 http://www.hubspot.com/
42
+ yacybot (/global; x86 Windows XP 5.1; java 1.7.0_55; Europe/de) http://yacy.net/bot.html
43
+ Mozilla/5.0 (compatible; DCPbot/1.0; +http://domains.checkparams.com/)
44
+ Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)
45
+ WebDoc (abuse-webdoc at service.moquadv.com)
46
+ coccoc
47
+ Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )
48
+ Mozilla/5.0 (compatible; ExchangleBot/3.0; +https://www.exchangle.com/exchangling)
49
+ Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)
50
+ OOZBOT/0.20 ( Setooz výrazný ako say-th-uuz, znamená mosty. ; http://www.setooz.com/oozbot.html ; agentname at setooz dot_com )
51
+ SpeedySpider - http://www.entireweb.com
52
+ Mozilla/5.0 (compatible; Heurekabot/3.1; +http://sluzby.heureka.cz/)
53
+ crawler for netopian (http://www.netopian.co.uk/)
54
+ L.webis/0.51 (http://webalgo.iit.cnr.it/index.php?pg=lwebis)
55
+ Influencebot/0.9; (Automatic classification of websites; http://www.influencebox.com/; info@influencebox.com)
56
+ Baiduspider+(+http://www.baidu.com/search/spider.htm)
57
+ Mozilla/5.0 (X11; compatible; semantic-visions.com crawler; HTTPClient 3.1)
58
+ Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.2.28) Gecko/20120306 Firefox/99.0 YottaaMonitor
59
+ Scrapy/1.1.1 (+http://scrapy.org)
60
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Marketing Grader
61
+ RyzeCrawler/1.1.1 ( http://www.domain2day.nl/crawler/)
62
+ eBot / v.1.0a (http://alfa.elchron.cz)
63
+ Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
64
+ Crowsnest/0.5 (+http://www.crowsnest.tv/)
65
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.4 (KHTML, like Gecko; Google Page Speed Insights) Chrome/22.0.1229 Safari/537.4
66
+ DoCoMo/2.0 N902iS(c100;TB;W24H12)(compatible; moba-crawler; http://crawler.dena.jp/)
67
+ Yeti/1.1 (NHN Corp.; http://help.naver.com/robots/)
68
+ Experibot_v1
69
+ Nekstbot - http://www.ipipan.waw.pl/nekst/nekstbot/
70
+ Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)
71
+ Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36; SSL-Crawler: http://crawler.dcsec.uni-hannover.de
72
+ CorpusCrawler 2.0.19 (http://corpora.fi.muni.cz/crawler/);Project:CzCorpus
73
+ Mozilla/5.0 (compatible; Semager/1.4; http://www.semager.de/blog/semager-bots/)
74
+ Mozilla/5.0 (compatible; dlcbot/0.1; +http://www.drlinkcheck.com/)
75
+ yacybot (freeworld/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; Europe/de) http://yacy.net/bot.html
76
+ A6-Indexer/1.0 (http://www.a6corp.com/a6-web-scraping-policy/)
77
+ CopperEgg/RevealUptime/DallasTX(linode)
78
+ Mozilla/5.0 (compatible; Qwantify/2.1dw; +https://www.qwant.com/)/*
79
+ Curious George - www.analyticsseo.com
80
+ GozaikBot (www.gozaik.com;webmaster@gozaik.com;www.gozaik.com/gozaikbot.html)
81
+ Pu_iN Crawler (+http://semanticjuice.com/)
82
+ Mozilla/5.0 (compatible; OpenindexDeepSpider/Nutch-1.5-dev; +http://www.openindex.io/en/webmasters/spider.html; systemsATopenindexDOTio)
83
+ yacybot (freeworld/global; i386 Linux 3.0.0-17-generic-pae; java 1.6.0_23; Europe/en) http://yacy.net/bot.html
84
+ yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html
85
+ FAST Enteprise Crawler/6 (www dot fastsearch dot com)
86
+ Mozilla/5.0 (compatible; parsijoo-update-crawler; +http://www.parsijoo.ir/; ehsan.mousakazemi@gmail.com)
87
+ yacybot (freeworld/global; amd64 Windows Server 2008 6.0; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
88
+ Inspingbot/1.0 (+https://www.insping.com/)
89
+ OrgProbe/0.9.4 (+http://www.blocked.org.uk)
90
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider
91
+ Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko PTST/276
92
+ Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36 PTST/276
93
+ Mozilla/5.0 (compatible; SemrushBot/0.97~bl; +http://www.semrush.com/bot.html)
94
+ Feedspot http://www.feedspot.com
95
+ Zookabot/2.0;++http://zookabot.com
96
+ drupact/0.7; http://www.arocom.de/drupact
97
+ RobotsChecker/0.6 (+http://www.blocked.org.uk)
98
+ yacybot (/global; i386 Linux 3.13.0-37-generic; java 1.7.0_65; Europe/en) http://yacy.net/bot.html
99
+ Scrapy/1.0.3 (+http://scrapy.org)
100
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/4.0.0~beta12
101
+ Slack-ImgProxy 0.66 (+https://api.slack.com/robots)
102
+ Mozilla/5.0 (compatible; WebCookies/1.0; +https://webcookies.org/faq/#agent)
103
+ NalezenCzBot/1.0 (http://www.nalezen.cz)
104
+ yacybot (freeworld/global; amd64 Windows Server 2008 6.0; java 1.7.0_03; Europe/en) http://yacy.net/bot.html
105
+ Mozilla/5.0 (compatible; AhrefsBot/5.1; +http://ahrefs.com/robot/)
106
+ yacybot (freeworld/global; amd64 Windows 8 6.2; java 1.7.0_51; Europe/de) http://yacy.net/bot.html
107
+ yacybot (/global; amd64 Windows 7 6.1; java 1.8.0_65; Europe/de) http://yacy.net/bot.html
108
+ Mozilla/4.0 compatible ZyBorg/1.0 (wn-16.zyborg@looksmart.net; http://www.WISEnutbot.com)
109
+ Mozilla/5.0 (en-us) AppleWebKit/537.36(KHTML, like Gecko; Google-Adwords-DisplayAds-WebRender;) Chrome/27.0.1453Safari/537.36
110
+ Crawler powered by contentDetection (www.mindup.de)
111
+ PagePeeker.com (info: http://pagepeeker.com/robots)
112
+ Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +http://go.mail.ru/help/robots)
113
+ coccoc/1.0 (http://help.coccoc.com/)
114
+ Favicon downloader (+https://favico.be/bot.html)
115
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) (larbin2.6.3@unspecified.mail)
116
+ Mozilla/5.0 (compatible; DIY-SEOBot/0.1a; +http://www.upcity.com/bot.html)
117
+ Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36 PTST/276
118
+ Mozilla/5.0 (compatible; theoldreader.com)
119
+ Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)
120
+ hawkReader/1.8 (Link Parser; http://www.hawkreader.com/; Allow like Gecko) Build/f2b2566
121
+ Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)
122
+ Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)
123
+ yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html
124
+ Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
125
+ yacybot (/global; amd64 Windows 7 6.1; java 1.8.0_45; Europe/de) http://yacy.net/bot.html
126
+ larbin_2.6.3 gqnmgsp@ruc.edu.cn
127
+ yacybot (freeworld/global; amd64 Linux 3.10.17-gentoo; java 1.7.0_45; UTC/en) http://yacy.net/bot.html
128
+ MeMoNewsBot/2.0 (http://www.memonews.com/en/crawler)
129
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0) PTST/281
130
+ Mozilla/5.0 (compatible; DCPbot/1.5; +http://domains.checkparams.com/)
131
+ Mozilla/5.0 (compatible; SemrushBot/0.97; +http://www.semrush.com/bot.html)
132
+ BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent
133
+ Mozilla/5.0 (compatible; Baiduspider-cpro; +http://www.baidu.com/search/spider.html)
134
+ yacybot (/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html
135
+ Mozilla/5.0 (compatible; electricmonk/3.1.1 +https://www.duedil.com/our-crawler/)
136
+ Hatena Antenna/0.5 (http://a.hatena.ne.jp/help)
137
+ Backlink-Ceck.de (+http://www.backlink-check.de/bot.html)
138
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0) commoncrawl.org/research//Nutch-1.7-SNAPSHOT
139
+ Mozilla/5.0 (compatible; Steeler/3.5; http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)
140
+ LapozzBot/1.4 (+http://robot.lapozz.com)
141
+ Mozilla/5.0 (WhatsMyIP.org HTTP_Compression_Test) http://whatsmyip.org/ua
142
+ Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)
143
+ Mozilla/5.0 (compatible; ExaleadCloudview/6;)
144
+ ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com
145
+ msnbot/1.0 (+http://search.msn.com/msnbot.htm)
146
+ Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4 LinkTiger 2.0
147
+ Pixray-Seeker/1.1 (Pixray-Seeker; crawler@pixray.com)
148
+ mbot v.1.16
149
+ MXT/Nutch-1.12-SNAPSHOT (http://t.co/GSRLLKex24; informatique at mixdata dot com)
150
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/3.0.0
151
+ polybot 1.0 (http://cis.poly.edu/polybot/)
152
+ Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots mtweb01t.yandex.ru)
153
+ Mozilla/5.0 (compatible; memoryBot/1.21.46 +http://internetmemory.org/en/)
154
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)
155
+ yacybot (amd64 Linux 2.6.26-2-amd64; java 1.6.0_0; Europe/en) http://yacy.net/bot.html
156
+ yacybot (/global; x86 Windows 8.1 6.3; java 1.8.0_45; America/en) http://yacy.net/bot.html
157
+ Mozilla/5.0 (compatible; startmebot/1.0; +http://www.start.me/bot)
158
+ yacybot (amd64 Windows 7 6.1; java 1.6.0_21; Europe/fr) http://yacy.net/bot.html
159
+ yacybot (/global; amd64 Windows 7 6.1; java 1.8.0_101; Asia/ru) http://yacy.net/bot.html
160
+ yacybot (freeworld/global; amd64 Linux 3.0.0-17-generic; java 1.6.0_23; Europe/de) http://yacy.net/bot.html
161
+ alexa v0.1.4 (http://www.openwebspider.org/)
162
+ http://arachnode.net 1.4
163
+ Photon/1.0
164
+ NetpeakCheckerBot
165
+ GIDBot/3.0 (+http://www.gidnetwork.com/tools/gzip-test.php)
166
+ Yandex/1.01.001 (compatible; Win16; P)
167
+ w3dt.net httphr/2.0
168
+ yacybot (/global; amd64 Windows 7 6.1; java 1.8.0_51; Europe/de) http://yacy.net/bot.html
169
+ iqdb/0.1 (+http://iqdb.org/)
170
+ Mozilla/5.0 (compatible; GimmeUSAbot/1.0; +https://gimmeusa.com/pages/crawler)
171
+ Motoricerca-Robots.txt-Checker/1.0 (http://tool.motoricerca.info/robots-checker.phtml)
172
+ Mozilla/5.0 (compatible; Tagoobot/3.0; +http://www.tagoo.ru)
173
+ Mozilla/5.0 (Windows NT 6.1; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0 Evidon (lab@evidon.com)
174
+ Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)
175
+ yacybot (i386 Linux 2.6.32-22-generic; java 1.6.0_20; Europe/de) http://yacy.net/bot.html
176
+ Mozilla/5.0 (Linux; Android 4.4.3; HTC One Build/KTU84L) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2125.111 Mobile Safari/537.36 DareBoost
177
+ mindUpBot (datenbutler.de)
178
+ Mozilla/5.0 (compatible; monitis - premium monitoring service; http://www.monitis.com)
179
+ Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)
180
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2125.111 Safari/537.36 DareBoost
181
+ Zookabot/2.1;++http://zookabot.com
182
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.6.0_24; Europe/de) http://yacy.net/bot.html
183
+ Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)
184
+ Mozilla/5.0 (compatible; Plukkie/1.2; http://www.botje.com/plukkie.htm)
185
+ focusbot/1.1
186
+ Mozilla/5.0 (compatible; idmarch Automatic.beta/1.2; +http://www.idmarch.org/bot.html)
187
+ CorpusCrawler 2.0.14 (http://corpora.fi.muni.cz/crawler/)
188
+ Mozilla/5.0 (compatible; XML Sitemaps Generator; https://www.xml-sitemaps.com) Gecko XML-Sitemaps/1.0
189
+ ImplisenseBot 1.1
190
+ Promotion_Tools_www.searchenginepromotionhelp.com
191
+ Aboundex/0.2 (http://www.aboundex.com/crawler/)
192
+ CorpusCrawler 2.0.22 (http://corpora.fi.muni.cz/crawler/);Project:CzCorpus
193
+ yacybot (/global; amd64 no-os-name no-os-version; java no-java-version; Europe/en) http://yacy.net/bot.html
194
+ LumpImageSearch/0.1 (+http://lump.co/about/bot)
195
+ Mozilla/4.0 (compatible; NaverBot/1.0; http://help.naver.com/customer_webtxt_02.jsp)
196
+ Mozilla/5.0 (compatible; UASlinkChecker/2.0; +http://udger.com/support/UASlinkChecker)
197
+ GAChecker (+http://www.gachecker.com)
198
+ yacybot (freeworld/global; x86 Windows 7 6.1; java 1.7.0_25; Europe/de) http://yacy.net/bot.html
199
+ Influencebot/0.9; (Automatic classification of websites; http://www.influencebox.com/; info@influencebox.com)User-Agent: Mozilla/5.0 (X11; Linux i686; rv:9.0) Gecko/20100101 Firefox/9.0
200
+ Experibot_v1 (https://dl.dropboxusercontent.com/u/8024465/site/Info.html)
201
+ DNSPod-reporting(http://www.dnspod.cn/reporting)
202
+ yacybot (freeworld/global; i386 Linux 3.2.0-23-generic; java 1.6.0_27; Europe/en) http://yacy.net/bot.html
203
+ Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)
204
+ PostPost/1.0 (+http://postpo.st/crawlers)
205
+ envolk/1.7 (+http://www.envolk.com/envolkspiderinfo.html)
206
+ Snapbot/1.0 (Snap Shots, +http://www.snap.com)
207
+ Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; Google-Publisher-Plugin) Chrome/27.0.1453 Mobile Safari/537.36
208
+ Mozilla/5.0 (compatible; Uptimebot/0.2.18; +http://www.uptime.com/uptimebot)
209
+ MIA DEV/search:robot/0.0.1 (This is the MIA Bot - crawling for mia research project. If you feel unhappy and do not want to be visited by our crawler send an email to spider@neofonie.de; http://spider.neofonie.de; spider@neofonie.de)
210
+ Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0; PTST 2.386) like Gecko
211
+ Jyxobot/1
212
+ WebAlta Crawler/2.0 (http://www.webalta.net/ru/about_webmaster.html) (Windows; U; Windows NT 5.1; ru-RU)
213
+ GC3pro+dir SEO Tools - Vers. 3.00b - For more informations: http://chkme.com/
214
+ Mozilla/5.0 (X11; Linux x86_64; rv:45.0; GTmetrix https://gtmetrix.com/) Gecko/20100101 Firefox/45.0
215
+ Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0 AppEngine-Google; (+http://code.google.com/appengine; appid: s~aeshortener)
216
+ Priceonomics Analysis Engine - Fetch/1.0
217
+ Ruky-Roboter (Version: 1.06, powered by www.ruky.de +http://www.ruky.de/bot.html)
218
+ Baiduspider+(+http://help.baidu.jp/system/05.html)
219
+ Openstat/0.1
220
+ Yandex/1.01.001 (compatible; Win16; m)
221
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)
222
+ Mozilla/5.0 (compatible; OpenindexSpider/Nutch-1.5-dev; +http://www.openindex.io/en/webmasters/spider.html)
223
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot
224
+ CERT.at-Statistics-Survey/1.0 (http://www.cert.at/about/consec/content.html)
225
+ Mozilla/5.0 (compatible; pmoz.info ODP link checker; +http://pmoz.info/doc/botinfo.htm)
226
+ yacybot (freeworld/global; x86_64 Mac OS X 10.6.8; java 1.6.0_29; Asia/ru) http://yacy.net/bot.html
227
+ gonzo/1[P] (+http://www.suchen.de/faq.html)
228
+ MixBot (+http://t.co/GSRLLKex24)
229
+ yacybot (/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_60; Europe/en) http://yacy.net/bot.html
230
+ Mozilla/5.0 (compatible; imbot/0.1 +http://internetmemory.org/en/
231
+ Mozilla/5.0 (en-us) AppleWebKit/525.13 (KHTML, like Gecko; Google Web Preview) Version/3.1 Safari/525.13
232
+ Mozilla/5.0 (compatible; Pi-Monster; https://pricepi.com/)
233
+ ThumbSniper (http://thumbsniper.com)
234
+ Shelob (shelob@gmx.net)
235
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/537.36 (KHTML, like Gecko, Google-Publisher-Plugin) Chrome/27.0.1453 Safari/537.36
236
+ Mozilla/5.0 (compatible; KaloogaBot; http://www.kalooga.com/info.html?page=crawler)
237
+ yacybot (/global; arm Linux 4.1.13+; java 1.8.0_40-internal; Etc/de) http://yacy.net/bot.html
238
+ CorporateNewsSearchEngine/Nutch-1.7 (http://pibs.co/news-search-engine)
239
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0) PTST/276
240
+ Yandex/1.01.001 (compatible; Win16; I)
241
+ FlightDeckReportsBot/2.0 (http://www.flightdeckreports.com/pages/bot)
242
+ Scrapy/0.24.4 (+http://scrapy.org)
243
+ ADmantX Platform Semantic Analyzer US - Turn - ADmantX Inc. - www.admantx.com - support@admantx.com
244
+ Kyoto-Tohoku-Crawler/v1 (Mozilla-compatible; kyoto-crawler-contact@nlp.ist.i.kyoto-u.ac.jp; http://nlp.ist.i.kyoto-u.ac.jp/?crawling-kt)
245
+ Mozilla/5.0 (compatible; Scarlett/ 1.0; +http://www.ellerdale.com/crawler.html)
246
+ Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)
247
+ www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36
248
+ http://arachnode.net 1.2
249
+ Mozilla/5.0 (compatible; Plukkie/1.5; http://www.botje.com/plukkie.htm)
250
+ yacybot (freeworld/global; arm Linux 4.4.11-v7+; java 1.7.0_101; Etc/en) http://yacy.net/bot.html
251
+ Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.16) Gecko/2009121601 Ubuntu/9.04 (jaunty) Firefox/3.0.16 Specificfeeds- http://www.specificfeeds.com
252
+ Mozilla/5.0 (compatible; evc-batch/2.0)
253
+ Orbiter/1.2 (http://dailyorbit.com/)
254
+ crawler4j (https://github.com/yasserg/crawler4j/)
255
+ Mozilla/5.0 (compatible; SEOdiver/1.0; +http://www.seodiver.com/bot)
256
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 (https://shrinktheweb.com)
257
+ Scopia crawler 1.0 (+http://www.scopia.co)
258
+ yacybot (i386 Linux 2.6.23; java 1.6.0_06; Europe/en) http://yacy.net/bot.html
259
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)
260
+ Mozilla/5.0 (compatible; LXRbot/1.0;http://www.lxrmarketplace.com/,support@lxrmarketplace.com)
261
+ yacybot (freeworld/global; amd64 Linux 3.8.13-gentoo; java 1.7.0_21; Europe/de) http://yacy.net/bot.html
262
+ Mozilla/5.0 (compatible; Sitemap Generator/1.3; +http://www.check-domains.com/sitemap/index.php)
263
+ DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)
264
+ Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)
265
+ Slack-ImgProxy 0.59 (+https://api.slack.com/robots)
266
+ Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)
267
+ Mozilla/5.0 (compatible; YandexPagechecker/2.0; +http://yandex.com/bots)
268
+ CoinCornerBot/1.1 ( https://www.coincorner.com/BitcoinBot)
269
+ yacybot (freeworld/global; amd64 Linux 3.8.0-21-generic; java 1.6.0_27; Pacific/en) http://yacy.net/bot.html
270
+ ScreenerBot Crawler Beta 2.0 (+http://www.ScreenerBot.com)
271
+ gonzo1[P] +http://www.suchen.de/faq.html
272
+ Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1
273
+ Mozilla/5.0 (compatible; 008/0.83; http://www.80legs.com/spider.html;) Gecko/2008032620
274
+ Sogou web spider/4.02525A
275
+ Visbot/2.0 (+http://www.visvo.com/en/webmasters.jsp;bot@visvo.com)
276
+ Mozilla/5.0 (compatible; AcoonBot/4.10.8; +http://www.acoon.de/robot.asp)
277
+ WatchMouse/18990 (http://watchmouse.com/ ; d3.watchmouse.com)
278
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) Qt/4.8.1 Safari/534.34 ShoppimonAgent/1.0 (feedback+agent@shoppimon.com)
279
+ webinatorbot 1.1; +http://www.webinator.de
280
+ findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)
281
+ WikiDo/1.1 (http://wikido.com; crawler@wikido.com)
282
+ yacybot (freeworld/global; amd64 Linux 3.5.0-27-generic; java 1.7.0_03; Europe/de) http://yacy.net/bot.html
283
+ Mozilla/5.0 (compatible; Uptimebot/0.1.73; +http://www.uptime.com/uptimebot)
284
+ Semantifire1/0.20 ( -- ; http://www.setooz.com/oozbot.html ; agentname at setooz dot_com )
285
+ Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)
286
+ Mozilla/5.0 compatible; yelpspider/yelpspider-1.0 (Crawlerbot run by Yelp Inc; yelpbot at yelp dot com)
287
+ Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)
288
+ Mozilla/5.0 (compatible; emefgebot/beta; +http://emefge.de/bot.html)
289
+ Mozilla/5.0 (compatible; OpenindexShallowSpider/Nutch-1.5-dev; +http://www.openindex.io/en/webmasters/spider.html; systemsATopenindexDOTio)
290
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/)
291
+ Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)
292
+ CCBot/1.0 (+http://www.commoncrawl.org/bot.html)
293
+ Lijit Crawler (+http://www.lijit.com/robot/crawler)
294
+ baypup/1.1 (Baypup; http://www.baypup.com/; jason@baypup.com)
295
+ Mozilla/5.0 (FauBot/0.1; +http://buzzvolume.com/fau/)
296
+ Mozilla/5.0 (compatible; NLNZ_IAHarvester2016/3.3.0 +https://natlib.govt.nz/publishers-and-authors/web-harvesting/domain-harvest)
297
+ yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html
298
+ yacybot (amd64 Windows 7 6.1; java 1.6.0_18; Europe/de) http://yacy.net/bot.html
299
+ SEOENGWorldBot/1.0 (+http://www.seoengine.com/seoengbot.htm)
300
+ Mozilla/5.0 (compatible; Finderbots finder bot; +http://wiki.github.com/bixo/bixo/bixocrawler; bixo-dev@yahoogroups.com)
301
+ Pompos/1.3 http://dir.com/pompos.html
302
+ Mozilla/5.0 (compatible; Gimme60bot/1.0; +http://gimme60.com) Firefox/16.0
303
+ Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)
304
+ Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)
305
+ urlfan-bot/1.0; +http://www.urlfan.com/site/bot/350.html
306
+ al_viewer (larbin2.6.3@unspecified.mail)
307
+ LoadImpactRload/3.1.1 (Load Impact; http://loadimpact.com);
308
+ findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)
309
+ Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )
310
+ Mozilla/5.0 (compatible; AcoonBot/4.11.0; +http://www.acoon.de/robot.asp)
311
+ WillyBot/1.1 (http://www.willyfogg.com/info/willybot)
312
+ Norton-Safeweb
313
+ WMCAI-robot (http://www.topicmaster.jp/wmcai/crawler.html)
314
+ rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)
315
+ Szukacz/1.5 (robot; www.szukacz.pl/jakdzialarobot.html; info@szukacz.pl)
316
+ rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)
317
+ ROR Sitemap Generator (http://www.rorweb.com)
318
+ http://domino.research.ibm.com/comm/research_projects.nsf/pages/sai-crawler.callingcard.html
319
+ MozillaTest/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )
320
+ Mozilla/5.0 (compatible; Peew/1.0; http://www.peew.de/crawler/)
321
+ Mozilla/5.0 (compatible; Website Analyzer/1.1; +http://www.check-domains.com/website-analysis/website-analyzer.php)
322
+ Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)
323
+ PagePeeker.com
324
+ CorpusCrawler 2.0.10 (http://corpora.fi.muni.cz/crawler/)
325
+ yacybot (/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_101; Europe/cs) http://yacy.net/bot.html
326
+ yacybot (/global; amd64 Linux 4.0.7-1-ck; java 1.8.0_45; Europe/de) http://yacy.net/bot.html
327
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 PTST/284
328
+ yacybot (freeworld/global; i386 Linux 3.12-1-686-pae; java 1.7.0_21; Europe/fr) http://yacy.net/bot.html
329
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; PTST 2.295)
330
+ CovarioIDS/1.1 (http://www.covario.com/ids; support at covario dot com)
331
+ Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)
332
+ www.adressendeutschland.de
333
+ DialogSearch.com Bot 1.0;http://dialogsearch.com/webmasters
334
+ librabot/2.0 (+http://search.msn.com/msnbot.htm)
335
+ Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)
336
+ wsAnalyzer/1.0; ++http://www.wsanalyzer.com/bot.html
337
+ Mozilla/5.0 (compatible; websays; +http://wiki.github.com/bixo/bixo/bixocrawler; bixo-dev@yahoogroups.com)
338
+ facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)
339
+ yacybot (/global; amd64 Linux 4.4.5-1-ARCH; java 1.8.0_77; America/en) http://yacy.net/bot.html
340
+ websitepulse checker/1.1 (compatible; MSIE 5.5; Netscape 4.75; Linux)
341
+ Mozilla/5.0 (compatible; memoryBot/1.21.14 +http://mignify.com/bot.html)
342
+ SalesIntelligent/v1.0
343
+ larbin_2.6.2 pierre@micro-fun.ch
344
+ Mozilla/5.0 (compatible; SiteCondor; http://www.sitecondor.com)
345
+ yacybot (freeworld/global; amd64 Windows Server 2012 6.2; java 1.7.0_51; Europe/de) http://yacy.net/bot.html
346
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 43063)
347
+ yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html
348
+ Mozilla/5.0 (compatible; SemrushBot/0.96.2; +http://www.semrush.com/bot.html)
349
+ Linguee Bot (http://www.linguee.com/bot)
350
+ yacybot (/global; x86 Windows XP 5.1; java 1.7.0_51; Europe/de) http://yacy.net/bot.html
351
+ ICC-Crawler(Mozilla-compatible; ; http://kc.nict.go.jp/project1/crawl.html)
352
+ Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.8.1.11) Gecko/20080109 (Charlotte/0.9t; http://www.searchme.com/support/) (Charlotte/0.9t; http://www.searchme.com/support/)
353
+ Mozilla/5.0 (compatible; Goodzer/2.0; crawler@goodzer.com)
354
+ Acoon v4.10.5 (www.acoon.de)
355
+ CorpusCrawler 2.0.20 (http://corpora.fi.muni.cz/crawler/);Project:CzCorpus
356
+ Mozilla/5.0 (compatible; AMZNKAssocBot/4.0 +http://affiliate-program.amazon.com)
357
+ Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)
358
+ yacybot (/global; amd64 Linux 4.2.0-22-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html
359
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) Qt/4.8.3 Safari/534.34 https://linkpeek.com
360
+ MetaTagRobot/0.2 (http://www.seocentro.com/tools/search-engines/metatag-analyzer.html)
361
+ Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)
362
+ Mozilla/5.0 (compatible; Uptimebot/0.2.29; +http://www.uptime.com/uptimebot)
363
+ Mozilla/5.0 (compatible; Pandeo Bot; +http://pandeo.de/bot.php)
364
+ Pixray-Seeker/1.1 (Pixray-Seeker; http://www.pixray.com/pixraybot; crawler@pixray.com)
365
+ EasyBib AutoCite (http://autocite-info.citation-api.com/)
366
+ Mozilla/5.0 (compatible; OptimizationCrawler/0.2; +http://www.domainoptima.com/robot)
367
+ AboutUsBot/Harpy (Website Analysis; http://www.aboutus.org/Aboutus:Bot; help@aboutus.org)
368
+ Gigabot/2.0
369
+ DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
370
+ Morning Paper 1.0 (robots.txt compliant!)
371
+ Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html; Linux; Android 4; iPhone; CPU iPhone OS 6_0_1 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A523 Safari/8536.25
372
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; GWX:RED; PTST 2.386)
373
+ SuperPagesUrlVerifyBot/1.0
374
+ CopperEgg/RevealUptime/LondonUK(linode)
375
+ Mozilla/5.0 (compatible; www.monitor.us - free monitoring service; http://www.monitor.us)
376
+ Mozilla/5.0 (compatible; LinkMarketbot/1.2; +http://www.linkmarket.com/)
377
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider
378
+ wangling
379
+ Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)
380
+ RSSMicro.com RSS/Atom Feed Robot
381
+ GarlikCrawler/1.1 (http://garlik.com/, crawler@garik.com)
382
+ al_org_viewer (larbin2.6.3@unspecified.mail)
383
+ Mozilla/5.0 (compatible; JadynAveBot; +http://www.jadynave.com/robot)
384
+ dj-research/Nutch-1.11 (analytics@@demandjump.com)
385
+ Karneval-Bot (Version: 1.06, powered by www.karnevalsuchmaschine.de +http://www.karnevalsuchmaschine.de/bot.html)
386
+ Baiduspider-image+(+http://www.baidu.com/search/spider.htm)\nReferer: http://image.baidu.com/i?ct=503316480&z=0&tn=baiduimagedetail
387
+ SauceNAO/1.0 (+http://saucenao.com/)
388
+ Mozilla/5.0 (compatible; JobdiggerSpider +http://www.jobdigger.nl/spider)
389
+ Yepi/1.0 (NHN Corp.; http://help.naver.com/robots/)
390
+ Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)
391
+ SSL Labs (https://www.ssllabs.com/about/assessment.html)
392
+ Mozilla/5.0 (compatible; http://alyze.info)
393
+ GigablastOpenSource/1
394
+ Mozilla/5.0 (Windows; U; Windows NT 5.1;fr;rv:1.8.1) VoilaBotCollector BETA 0.1 (http://www.voila.com/)
395
+ Vorboss Web Crawler [crawl@vorboss.net]/Nutch-2.3
396
+ Mozilla/5.0 (compatible; SecretSerachEngineLabs.com-SBSearch/0.9; http://www.secretsearchenginelabs.com/secret-web-crawler.php)
397
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Search Console) Chrome/27.0.1453 Safari/537.36
398
+ YahooCacheSystem
399
+ search.KumKie.com
400
+ yacybot (webportal-global; x86 Windows Vista 6.0; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
401
+ Embedly +support@embed.ly
402
+ stq_bot (+http://www.searchteq.de)
403
+ MSRBOT
404
+ Mozilla/5.0 (compatible; WBSearchBot/1.1; +http://www.warebay.com/bot.html)
405
+ CorpusCrawler 2.0.17 (http://corpora.fi.muni.cz/crawler/);Project:CzCorpus
406
+ Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)
407
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) PTST/276
408
+ Mozilla/5.0 (compatible; WebCookies/1.0; +http://webcookies.org/faq/#agent)
409
+ L.webis/0.44 (http://webalgo.iit.cnr.it/index.php?pg=lwebis)
410
+ yacybot (freeworld/global; amd64 Linux 3.0.0-17-generic; java 1.6.0_23; America/en) http://yacy.net/bot.html
411
+ Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0 PTST/279
412
+ Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0
413
+ Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots mtmon01i.yandex.ru)
414
+ Y!J-BSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/blog-search/)
415
+ Mozilla/5.0 (compatible; UptimeRobot/1.0; http://www.uptimerobot.com/)
416
+ NG-Search/0.86 (+http://www.ng-search.com)
417
+ ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)
418
+ TwengaBot/1.1 (+http://www.twenga.com/bot.html)
419
+ WebImages 0.3 ( http://herbert.groot.jebbink.nl/?app=WebImages )
420
+ Mozilla/5.0 (compatible; aiHitBot/2.8; +http://endb-consolidated.aihit.com/)
421
+ RyzeCrawler/1.1.1 (+http://www.domain2day.nl/crawler/)
422
+ Mozilla/5.0 (Nekstbot; http://www.ipipan.waw.pl/nekst/nekstbot/)
423
+ Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)
424
+ Mozilla/5.0 (compatible; SWEBot/1.0; +http://swebot-crawler.net)
425
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; gblon01.watchmouse.net)
426
+ yacybot (/global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html
427
+ Page Analyzer v4.0 ( http://www.ranks.nl/ )
428
+ web_bh (larbin2.6.3@unspecified.mail)
429
+ findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)
430
+ Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)
431
+ findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)
432
+ Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)
433
+ PagesInventory (robot http://www.pagesinventory.com)
434
+ Mozilla/5.0 (compatible; aiHitBot/1.0-DS; +http://www.aihit.com/)
435
+ tagSeoBot/1.0 (http://www.tagseoblog.de/tools)
436
+ Mozilla/5.0 (en-us) AppleWebKit/537.36 (KHTML, like Gecko; Google PP Default) Chrome/27.0.1453 Safari/537.36
437
+ 404 Checker [http://www.404checker.com/user-agent]
438
+ CopperEgg/RevealUptime/
439
+ Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)
440
+ Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)
441
+ CopperEgg/RevealUptime/TokyoJapan
442
+ Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)
443
+ GoSquared-Status-Checker/0.2
444
+ Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0 Safari/537.36 YottaaMonitor
445
+ WatchMouse/18990 (http://watchmouse.com/ ; bc.watchmouse.com)
446
+ yacybot (freeworld/global; amd64 Linux 3.2.1-gentoo-r2; java 1.6.0_24; Europe/de) http://yacy.net/bot.html
447
+ Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)
448
+ linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)
449
+ Heurekabot-Feed/1.0 (+http://sluzby.heureka.cz/napoveda/heurekabot/)
450
+ Mozilla/5.0 (compatible; Charlotte/1.1; http://www.searchme.com/support/)
451
+ yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html
452
+ LSSRocketCrawler/1.0 LightspeedSystems
453
+ Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; Jobboerse.com; http://www.xn--jobbrse-d1a.com) Gecko/20100401 Firefox/24.0
454
+ findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)
455
+ Mozilla/5.0 (compatible; kulturarw3 +http://www.kb.se/om/projekt/Svenska-webbsidor---Kulturarw3/)
456
+ Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34
457
+ Mozilla/5.0 (compatible; SemrushBot/0.96.4; +http://www.semrush.com/bot.html)
458
+ woobot/2.0
459
+ GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)
460
+ yacybot (webportal-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html
461
+ Mozilla/5.0 (compatible; AboutUsBot Johnny5/2.0; +http://www.AboutUs.org/)
462
+ yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html
463
+ Mozilla/5.0 (Linux; Android 4.1.2; Galaxy Nexus Build/JZO54K; GTmetrix http://gtmetrix.com/) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/26.0.1410.58 Mobile Safari/537.22
464
+ Nuhk/2.4 ( http://www.neti.ee/cgi-bin/abi/Otsing/Nuhk/)
465
+ Grahambot/0.1 (+http://www.sunaga-lab.com/graham-bot)
466
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html
467
+ SEO Consulting; Redirect Checker Tool V.02; IP:
468
+ Mozilla/5.0 (compatible; Pro Sitemaps Generator; https://pro-sitemaps.com) Gecko Pro-Sitemaps/1.0
469
+ yacybot (/global; amd64 Linux 4.4.10-antix.1-amd64-smp; java 1.8.0_101; Europe/en) http://yacy.net/bot.html
470
+ Mozilla/5.0 (Windows NT 6.2; WOW64) Runet-Research-Crawler (itrack.ru/research/cmsrate; rating@itrack.ru)
471
+ Mozilla/5.0 (compatible; Crawler/0.9; http://linkfluence.net/)
472
+ ADmantX Platform Semantic Analyzer US Async - ADmantX Inc. - www.admantx.com - support@admantx.com
473
+ GetProxi.es-bot/1.1 (http://getproxi.es/spiderinfo/)
474
+ Mozilla/5.0 (Windows NT 6.3; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0 PTST/277
475
+ Pinterest/0.2 (+http://www.pinterest.com/)
476
+ CopperEgg/RevealUptime/AtlantaGA(linode)
477
+ OdklBot/1.0 (klass@odnoklassniki.ru)
478
+ Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)
479
+ yacybot (freeworld/global; amd64 Linux 3.2.1-gentoo-r2; java 1.6.0_22; Europe/de) http://yacy.net/bot.html
480
+ MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)
481
+ Sogou web spider/4.05252A
482
+ Mozilla/5.0 (Windows NT 6.3; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0 PTST/276
483
+ Mozilla/5.0 (compatible; MotoMinerBot/1.0; +https://motominer.com/Bot)
484
+ NG/2.0
485
+ Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)
486
+ StackRambler/2.0 (MSIE incompatible)
487
+ Baiduspider+(+http://www.baidu.jp/spider/)
488
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.6.0_25; Europe/de) http://yacy.net/bot.html
489
+ Mozilla/5.0 (compatible; houzzbot; +http://www.houzz.com/)
490
+ Woko robot 3.0
491
+ Mozilla/5.0 (compatible; Qwantify/2.0; +https://www.qwant.com/)
492
+ yacybot (/global; amd64 Linux 4.2.0-27-generic; java 1.8.0_66-internal; America/en) http://yacy.net/bot.html
493
+ ADmantX Platform Semantic Analyzer - APAC - ADmantX Inc. - www.admantx.com - support@admantx.com
494
+ Mozilla/5.0 (compatible; Uptimebot/0.2.40; +http://www.uptime.com/uptimebot)
495
+ Mozilla/5.0 (compatible; ExpertSearchSpider +http://www.expertsearch.nl/spider)
496
+ Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)
497
+ Acoon v4.10.4 (www.acoon.de)
498
+ Mozilla/5.0 (compatible; memoryBot/1.20.210 +http://internetmemory.org/en/)
499
+ Readability/740ec9 - http://readability.com/about/
500
+ Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)
501
+ yacybot (i386 Linux 2.6.28-gentoo-r5; java 1.5.0_18; Europe/en) http://yacy.net/bot.html
502
+ Mozilla/5.0 (compatible; suggybot v0.01a, http://blog.suggy.com/was-ist-suggy/suggy-webcrawler/)
503
+ yacybot (amd64 Windows 7 6.1; java 1.6.0_14; Europe/de) http://yacy.net/bot.html
504
+ yacybot (freeworld/global; amd64 Linux 3.3.4-1-ARCH; java 1.6.0_24; Europe/en) http://yacy.net/bot.html
505
+ Mozilla/5.0 (compatible; AportWorm/3.2; +http://www.aport.ru/help)
506
+ Mozilla/5.0 (compatible; memoryBot/1.20.235 +http://internetmemory.org/en/)
507
+ findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)
508
+ Mozilla/5.0 (compatible; Hailoobot/1.2; +http://www.hailoo.com/spider.html)
509
+ eCommerceBot (http://www.ehandel.se/botinfo.html)
510
+ Mozilla/5.0(compatible;Sosospider/2.0;+http://help.soso.com/webspider.htm)
511
+ yacybot (/global; amd64 Linux 4.3.0-gentoo-ARCH; java 1.7.0_85; Europe/en) http://yacy.net/bot.html
512
+ Nymesis/2.0 (http://nymesis.com)
513
+ CopperEgg/RevealUptime/OregonUSA
514
+ uclassify.com/1.0
515
+ Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly.html) Gecko/2009032608 Firefox/3.0.8
516
+ Mozilla/5.0 (compatible; Prlog/1.0; +http://prlog.ru/)
517
+ Slack-ImgProxy 1.106 (+https://api.slack.com/robots)
518
+ AdnormCrawler www.adnorm.com/crawler
519
+ Mozilla/5.0 (compatible; YandexZakladki/3.0; +http://yandex.com/bots)
520
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon
521
+ Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)
522
+ Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
523
+ ICC-Crawler/2.0 (Mozilla-compatible; ; http://kc.nict.go.jp/project1/crawl.html)
524
+ Mozilla/4.0 (xcm@huaweisymantec.com)
525
+ bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)
526
+ Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/)
527
+ Mozilla/5.0 (compatible; Webmaster tools +http://sitexy.com/)
528
+ yacybot (/global; amd64 Windows 8.1 6.3; java 1.8.0_40; Europe/de) http://yacy.net/bot.html
529
+ Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)
530
+ Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )
531
+ Mozilla/5.0 (compatible; EuripBot/2.0; +http://www.eurip.com)
532
+ findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)
533
+ Sogou web spider/4.025251
534
+ SETOOZBOT/5.0 ( http://www.setooz.com/bot.html )
535
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) (larbin@unspecified.mail)
536
+ Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )
537
+ Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0
538
+ mozilla/5.0 (larbin2.6.3@unspecified.mail)
539
+ Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )
540
+ Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/98 Safari/537.4 (StatusCake SSL Monitor)
541
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; PTST 2.385)
542
+ Mozilla/5.0 (compatible; evc-batch/2.0.20160608212921)
543
+ Mozilla/5.0 (compatible; Mail.RU_Bot/2.0)
544
+ seebot/2.0 (+http://www.seegnify.com/bot)
545
+ bl.uk_lddc_bot/3.3.0-LBS-2016-02 (+http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)
546
+ CommaFeed/2.3.0-SNAPSHOT (https://www.commafeed.com)
547
+ OmniExplorer_Bot/5.91c (+http://www.omni-explorer.com) WorldIndexer
548
+ hledejLevne.cz/2.0
549
+ page_verifier (http://www.securecomputing.com/goto/pv)
550
+ url_test (larbin2.6.3@unspecified.mail)
551
+ Mozilla/5.0 (X11; Linux x86_64; rv:10.0.12) Gecko/20100101 Firefox/21.0 WordPress.com mShots
552
+ Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
553
+ Is is up? (+http://isitup.org)
554
+ Metaspinner/0.01 (Metaspinner; http://www.meta-spinner.de/; support@meta-spinner.de/)
555
+ TwengaBot-2.0 Champigny (+http://www.twenga.com/bot.html)
556
+ LivelapBot/0.2 (http://site.livelap.com/crawler)
557
+ HubSpot Crawler 1.0 http://www.hubspot.com/
558
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 TinEye/1.0 (via http://www.tineye.com/)
559
+ yacybot (/global; x86 Windows 10 10.0; java 1.8.0_73; Europe/de) http://yacy.net/bot.html
560
+ Scrapy/0.24.6 (+http://scrapy.org)
561
+ FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)
562
+ Baiduspider+(+http://www.baidu.com/search/spider_jp.html)
563
+ Mozilla/5.0 (compatible; seplinkbot/1.0 )
564
+ Mozilla/5.0 (compatible; Falconsbot; +http://ws.nju.edu.cn/falcons/)
565
+ Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0 PTST/277
566
+ Mozilla/4.0 (Toread-Crawler/1.1; +http://news.toread.cc/crawler.php)
567
+ TinEye-bot/0.02 (see http://www.tineye.com/crawler.html)
568
+ yacybot (freeworld/global; i386 Linux 2.6.32-39-generic-pae; java 1.6.0_20; Europe/en) http://yacy.net/bot.html
569
+ Mozilla/5.0 (compatible; parsijoo; +http://www.parsijoo.ir/; ehsan.mousakazemi@gmail.com)
570
+ Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )
571
+ MaxPoint Bot (+http://www.maxpoint.com)
572
+ Mozilla/5.0 (compatible; Infohelfer/1.2.0; +http://www.infohelfer.de/crawler.php)
573
+ ExB Language Crawler 2.1.1 (+http://www.exb.de/crawler)
574
+ cg-eye interactive
575
+ ZumBot/1.0 (ZUM Search; http://help.zum.com/inquiry)
576
+ rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)
577
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 47129)
578
+ Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:14.0) Gecko/20100101 Firefox/14.0.1 AppEngine-Google; (+http://code.google.com/appengine; appid: s~metacdn-hr)
579
+ Mozilla/5.0 (compatible; WoTBoT; +https://www.wslta.com/WoTBoT.html)
580
+ HolmesBot (http://holmes.ge)
581
+ Baiduspider-image+(+http://www.baidu.com/search/spider.htm)
582
+ Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; no-reply@alexa.com)
583
+ Mediapartners-Google
584
+ Mozilla/5.0 (compatible; MFGPagesBot/2.1; http://www.mfgpages.com)
585
+ larbin_2.6.2 kalou@kalou.net
586
+ Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11
587
+ BlogPulseLive (support@blogpulse.com)
588
+ WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)
589
+ yacybot (freeworld/global; i386 Linux 3.0.0-17-generic; java 1.6.0_23; America/en) http://yacy.net/bot.html
590
+ ImplisenseBot 1.0
591
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; PTST 2.386)
592
+ gonzo2[P] +http://www.suchen.de/faq.html
593
+ Mozilla/5.0 (compatible; LXRbot/1.0; http://lxrseo.com/, support@lxrseo.com)
594
+ Mozilla/5.0 (compatible; Arachnophilia/1.0; +http://arachnys.com/)
595
+ Mozilla/5.0 (compatible; CloudServerMarketSpider/1.0; +http://cloudservermarket.com/spider.html)
596
+ kalooga/KaloogaBot (Kalooga; http://www.kalooga.com/info.html?page=crawler)
597
+ yacybot (webportal/global; x86_64 Mac OS X 10.9.2; java 1.6.0_65; Europe/de) http://yacy.net/bot.html
598
+ Mozilla/5.0 (compatible; DotBot/1.1; http://www.dotnetdotcom.org/, crawler@dotnetdotcom.org)
599
+ yacybot (/global; amd64 Linux 4.1.19-gentoo; java 1.7.0_95; Europe/pl) http://yacy.net/bot.html
600
+ Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko PTST/276
601
+ Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)
602
+ audisto.com full crawler 3.26.431 (refer to in robots.txt as audisto, see https://audisto.com/bot)
603
+ Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )
604
+ Mozilla/5.0 (compatible; OpenindexShallowSpider/Nutch-1.5-dev; +http://www.openindex.io/en/webmasters/spider.html)
605
+ mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)
606
+ gonzo2[p] (+http://www.suchen.de/faq.html)
607
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)
608
+ Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )
609
+ WatchMouse/18990 (http://watchmouse.com/ ; uk)
610
+ yacybot (freeworld/global; amd64 Linux 3.1.10-hardened; java 1.7.0_03-icedtea; Europe/en) http://yacy.net/bot.html
611
+ VeBot (+http://www.veinteractive.com/vebot)
612
+ Mozilla/5.0 (compatible; NLNZ_IAHarvester2013 +http://natlib.govt.nz/about-us/current-initiatives/web-harvest-2012)
613
+ findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)
614
+ Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36
615
+ DialogSearch.com Bot 1.4;http://dialogsearch.com/webmasters
616
+ Mozilla/5.0 (compatible; GurujiBot/1.0; +http://www.guruji.com/en/WebmasterFAQ.html)
617
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9) AppleWebKit/537.71 (KHTML, like Gecko) Version/7.0 Safari/537.71 (Rival IQ, rivaliq.com)
618
+ wscheck.com/1.0.0 (+http://wscheck.com/)
619
+ Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.69 Safari/537.36 Webthumb/2.0
620
+ Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)
621
+ WeSEE:Ads/PictureBot (http://www.wesee.com/bot/)
622
+ Mozilla/5.0 (compatible; Dataprovider/6.101; +https://www.dataprovider.com/)
623
+ Kyoto-Crawler/n1.0 (Mozilla-compatible; kyoto-crawler-contact@nlp.ist.i.kyoto-u.ac.jp; http://nlp.ist.i.kyoto-u.ac.jp/?crawling)
624
+ Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)
625
+ Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )
626
+ Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )
627
+ seo-nastroj.cz
628
+ LoadImpactPageAnalyzer/1.3.0 (Load Impact; http://loadimpact.com/)
629
+ CSS Certificate Spider (http://www.css-security.com/certificatespider/)
630
+ MetaGeneratorCrawler/1.3.2 (www.metagenerator.info)
631
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/4.0.0~beta8
632
+ BLEXBot
633
+ Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
634
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/4.0.0~beta7
635
+ Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)
636
+ Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )
637
+ ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)
638
+ HeartRails Robot/0.1 (http://www.heartrails.com)
639
+ Mozilla/5.0 (compatible; Faveeo/1.0; +http://www.faveeo.com)
640
+ yacybot (/global; amd64 Linux 3.16.0-49-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html
641
+ Castabot/0.1 (+http://topixtream.com/)
642
+ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.5) Gecko/2010033101 Gentoo Firefox/3.0.5 (Dot TK - spider 3.0)
643
+ Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server; +http://ws.daum.net/aboutWebSearch.html) Daumoa/2.0
644
+ istellabot-nutch/Nutch-1.10
645
+ Mail.RU/2.0
646
+ ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)
647
+ LexxeBot/1.0 (lexxebot@lexxe.com)
648
+ Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)
649
+ Mozilla/5.0 (compatible; FlipboardRSS/1.1; +http://flipboard.com/browserproxy)
650
+ Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)
651
+ yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html
652
+ it2media-domain-crawler/1.0 on crawler-prod.it2media.de
653
+ yacybot (/global; amd64 Windows 8.1 6.3; java 1.8.0_25; Europe/de) http://yacy.net/bot.html
654
+ yacybot (freeworld-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html
655
+ Mozilla/5.0 (WhatsMyIP.org HTTP_Headers) http://whatsmyip.org/ua
656
+ checkgzipcompression.com robot
657
+ Mozilla/5.0 eCairn-Grabber/1.0 (+http://ecairn.com/grabber)
658
+ RankurBot/3.3 (+http://rankur.com)
659
+ L.webis/0.50 (http://webalgo.iit.cnr.it/index.php?pg=lwebis)
660
+ Speedy Spider (Submit your site at http://www.entireweb.com/free_submission/)
661
+ oBot
662
+ Snappy/2.0 ( http://www.urltrends.com/ )
663
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0) CrawlerProcess (http://www.PowerMapper.com) /5.23.770.0
664
+ Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )
665
+ flatlandbot/baypup (Flatland Industries Web Spider; http://www.flatlandindustries.com/flatlandbot; jason@flatlandindustries.com)
666
+ istellabot/Nutch-1.11
667
+ GetintentCrawler getintent.com
668
+ Covario-IDS/1.0 (Covario; http://www.covario.com/ids; support at covario dot com)
669
+ Mozilla/5.0 (compatible; FatBot 2.0; http://www.thefind.com/crawler)
670
+ Mozilla/5.0 (compatible; MegaIndex.com/2.0; +http://megaindex.com/crawler)
671
+ Microsearch.ru/1.0; http://microsearch.ru/webmasters
672
+ TurnitinBot/3.0 (http://www.turnitin.com/robot/crawlerinfo.html)
673
+ BacklinkCrawler V (http://www.backlinktest.com/crawler.html)
674
+ FeedlyBot/1.0 (http://feedly.com)
675
+ Clickagy Intelligence Bot v2
676
+ GetURLInfo/1.0
677
+ DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)
678
+ Scrapy/1.1.0 (+http://scrapy.org)
679
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html
680
+ Mozilla/5.0 (compatible; VSAgent/1.2)
681
+ Experibot_v1 [goo.gl/n6zrAf]
682
+ Mozilla/5.0 (compatible; DNS-Digger-Explorer/1.0; +http://www.dnsdigger.com)
683
+ boitho.com-robot/1.0
684
+ yacybot (/global; amd64 Windows Server 2012 6.2; java 1.7.0_51; Europe/de) http://yacy.net/bot.html
685
+ RelateIQ Crawler www.relateiq.com
686
+ Mozilla/4.0 (compatible; Netcraft Web Server Survey)
687
+ Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; https://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0
688
+ yacybot (i386 Linux 2.6.9-023stab046.2-smp; java 1.6.0_05; Europe/en) http://yacy.net/bot.html
689
+ Scrubby/2.2 (http://www.scrubtheweb.com/)
690
+ Mozilla/5.0 (Yahoo-MMCrawler/4.0; mailto:vertical-crawl-support@yahoo-inc.com)
691
+ Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)
692
+ bitlybot/3.0 (+http://bit.ly/)
693
+ Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)
694
+ yacybot (freeworld/global; amd64 Windows Server 2012 6.2; java 1.7.0_25; Europe/de) http://yacy.net/bot.html
695
+ agentslug.com - website monitoring tool
696
+ SafeDNSBot (https://www.safedns.com/searchbot)
697
+ Feedbin
698
+ Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)
699
+ Mozilla/5.0 (compatible; AcoonBot/4.11.1; +http://www.acoon.de/robot.asp)
700
+ Mozilla/5.0 (compatible; MagiBot/3.4.3; +http://magi.peak-labs.com/robots.txt)
701
+ Mozilla/5.0 (compatible; XML Sitemaps Generator; http://www.xml-sitemaps.com) Gecko XML-Sitemaps/1.0
702
+ baypup/colbert (Baypup; http://sf.baypup.com/webmasters; jason@baypup.com)
703
+ MergeFlow-PageReader/0.91;+(+http://mergeflow.net/info/pagereader) Mozilla/5.0 (Windows) compatible
704
+ Mozilla/5.0 (Mobile; rv:18.0) Gecko/18.0 Firefox/18.0 commoncrawl.org/research//Nutch-1.7-SNAPSHOT
705
+ Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )
706
+ Mozilla/5.0 (compatible; OsO; http://oso.octopodus.com/abot.html)
707
+ Mozilla/5.0 (compatible; gofind; +http://govid.mobi/bot.php)
708
+ CatchBot/2.0; +http://www.catchbot.com
709
+ Mozilla/5.0 (compatible; BusinessSeek.biz_Spider; http://www.businessseek.biz/)
710
+ Quora Link Preview/1.0 (http://www.quora.com)
711
+ radian6_default_(www.radian6.com/crawler)
712
+ DWDS-Crawler +http://odo.dwds.de/dwds-crawler.html
713
+ Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)
714
+ sogou spider
715
+ Mozilla/5.0 (compatible; GigablastOpenSource/1.0)
716
+ Mozilla/5.0 (compatible; HomeTags/1.0; +http://www.hometags.nl/bot)
717
+ CorpusCrawler 2.0.24 (http://corpora.fi.muni.cz/crawler/);Project:CzCorpus
718
+ Scooter/3.3
719
+ Zookabot/2.5;++http://zookabot.com
720
+ Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)
721
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.34 (KHTML, like Gecko) Safari/534.34; +http://sniptracker.com
722
+ Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)
723
+ es_com_viewer (larbin2.6.3@unspecified.mail)
724
+ Mozilla/5.0 (compatible; Uptimebot/0.2.14; +http://www.uptime.com/uptimebot)
725
+ yacybot (freeworld/global; amd64 Linux 3.0.0-14-generic; java 1.6.0_23; Europe/en) http://yacy.net/bot.html
726
+ Mozilla/5.0 (X11; Linux x86_64; rv:35.0) Gecko/20100101 Firefox/35.0 DareBoost
727
+ yacybot (freeworld/global; amd64 Linux 2.6.32-41-server; java 1.6.0_26; Europe/de) http://yacy.net/bot.html
728
+ yacybot (amd64 Linux 2.6.28-18-generic; java 1.6.0_16; GMT/en) http://yacy.net/bot.html
729
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; PTST 2.386)
730
+ KD Bot
731
+ Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.9.2.19) Gecko WebThumb/1.0
732
+ Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)
733
+ rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)
734
+ MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)
735
+ Superarama.com-Tarama-Botu-v.01
736
+ Mozilla/5.0 (compatible; Urlfilebot/2.2; +http://urlfile.com/bot.html)
737
+ Vegi bot (we follow your robots.txt settings before crawling, you can slow down the bot by change the Crawl-Delay parameter in the settings.if you have an enquiry, please email to: abuse-report@terrykyleseoagency.com)
738
+ MnoGoSearch/3.3.9
739
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b
740
+ JyxobotRSS/0.06
741
+ Mozilla/5.0 (compatible; BigBozz/2.2.1; +http://www.bigbozz.com/)
742
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2 PTST/277
743
+ KiNShooboT (compatible; KiNShooboT/1.0.C; +http://www.kinshoo.com/bot.html)
744
+ SentiBot www.sentibot.eu (compatible with Googlebot)
745
+ Topicbot/1.0 (Mozilla;I;+http://92.42.190.57/)
746
+ IDG/IT (http://spaziodati.eu/)
747
+ LinkAider (http://linkaider.com/crawler/)
748
+ Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)
749
+ yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html
750
+ bitlybot
751
+ Mozilla/5.0 (compatible; Mp3Bot/0.7; +http://mp3realm.org/mp3bot/)
752
+ WWW::LayeredExtractor::Handler::Feed/0.01
753
+ Mozilla/5.0 (compatible; SWEBot/1.0; +http://swebot.net)
754
+ Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)
755
+ MetaTagRobot/2.1 (http://www.seocentro.com/tools/search-engines/metatag-analyzer.html)
756
+ Mozilla/5.0 (compatible; Goodzer/1.0)
757
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_40; Europe/en) http://yacy.net/bot.html
758
+ Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (NetShelter ContentScan)
759
+ Facebot/1.0
760
+ SafeAds.xyz bot
761
+ Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)
762
+ yacybot (/global; amd64 Windows 7 6.1; java 1.7.0_55; Europe/en) http://yacy.net/bot.html
763
+ scrapyproject (+http://www.profound.net)
764
+ Mozilla/5.0 (compatible; Online Domain Tools - Online Website Link Checker/1.2; +http://website-link-checker.online-domain-tools.com)
765
+ Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)
766
+ Mozilla/5.0 (compatible; Scrubby/3.1; +http://www.scrubtheweb.com/help/technology.html)
767
+ Semantifire1/0.20 ( http://www.setooz.com/oozbot.html ; agentname at setooz dot_com )
768
+ Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1
769
+ UnisterBot (Mozilla/5.0 compatible; crawler@unister-gmbh.de)
770
+ Mozilla/5.0 (compatible; Ezooms/1.0; help@moz.com)
771
+ Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)
772
+ Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )
773
+ findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)
774
+ Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)
775
+ Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)
776
+ Mozilla/5.0 (compatible; kazbtbot/0.1; +http://kazbt.com/)
777
+ Orgbybot/OrgbyBot v1.3 (Spider; http://orgby.com/bot/ ; Orgby.com Search Engine)
778
+ Mozilla/5.0 (compatible; aiHitBot-DM/2.0.2 +http://www.aihit.com)
779
+ YahooSeeker-Testing/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/)
780
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 40409)
781
+ Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)
782
+ yacybot (/global; amd64 Linux 3.19.2-1-ARCH; java 1.8.0_40; Europe/de) http://yacy.net/bot.html
783
+ Mozilla/5.0 (compatible; evc-batch/2.0.20161017175820)
784
+ datagnionbot (+http://www.datagnion.com/bot.html)
785
+ CopperEgg/RevealUptime/TokyoJP(linode)
786
+ MB-SiteCrawler
787
+ WatchMouse/18990 (http://watchmouse.com/ ; liz)
788
+ HybridBot (hybrid.ru/about. If our bot caused problems please contact us. Contact email: m.lyashkov@targetix.net)
789
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 24522)
790
+ yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html
791
+ findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)
792
+ yacybot (freeworld/global; amd64 Linux 3.13.0-24-generic; java 1.7.0_55; Europe/de) http://yacy.net/bot.html
793
+ Mozilla/5.0 (compatible; OpenfosBot/2.4; +http://www.openfos.com)
794
+ Nusearch Spider (www.nusearch.com)
795
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; sesto02.watchmouse.net)
796
+ ICC-Crawler/2.0 (Mozilla-compatible; ; http://www.nict.go.jp/en/univ-com/plan/crawl.html)
797
+ Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )
798
+ Orbiter/1.3 (http://dailyorbit.com/)
799
+ CCBot/2.0
800
+ Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)
801
+ Mozilla/5.0 (compatible; DNS-Digger/1.0; +http://www.dnsdigger.com)
802
+ AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)
803
+ yacybot (/global; x86 Windows 7 6.1; java 1.8.0_71; Europe/de) http://yacy.net/bot.html
804
+ Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)
805
+ Dex Social Bot
806
+ yacybot (/global; amd64 Linux 4.7.6-200.fc24.x86_64; java 1.8.0_102; Etc/en) http://yacy.net/bot.html
807
+ Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)
808
+ CRAZYWEBCRAWLER 0.9.10, http://www.crazywebcrawler.com
809
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36 PTST/277
810
+ Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3
811
+ SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)
812
+ Zemanta Aggregator/0.9 +http://www.zemanta.com
813
+ Mozilla/5.0 (compatible; special_archiver/3.2.0 +http://www.loc.gov/webarchiving/notice_to_webmasters.html)
814
+ Mozilla/5.0 (compatible; GimmeUSAbot/1.0; +https://gimmeusa.com/crawler.html)
815
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.7.0_45; Europe/de) http://yacy.net/bot.html
816
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider
817
+ FyberSpider/1.3 (http://www.fybersearch.com/fyberspider.php)
818
+ yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html
819
+ Searchie/1.0 (a Storm-based crawler; https://www.searchie.org; admin@searchie.org)
820
+ yacybot (freeworld/global; amd64 Linux 3.1.10-1-desktop; java 1.6.0_22; Europe/de) http://yacy.net/bot.html
821
+ holmes/3.12.4 (http://morfeo.centrum.cz/bot)
822
+ WatchMouse/18990 (http://watchmouse.com/ ; se.watchmouse.com)
823
+ MXT/Nutch-1.10 (http://t.co/GSRLLKex24; informatique at mixdata dot com)
824
+ Mozilla/5.0 (compatible; Shareaholicbot/1.0; +http://www.shareaholic.com/bot)
825
+ ZoomInformation Bot
826
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; www.alertra.com)
827
+ Mozilla/5.0 (compatible; HomeTags/1.0; http://www.hometags.nl/bot)
828
+ Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.17) Gecko/20110515 HeartRails_Capture/1.0.4 (+http://capture.heartrails.com/) Namoroka/3.6.17
829
+ OmniExplorer_Bot/5.20 (+http://www.omni-explorer.com) WorldIndexer
830
+ WatchMouse/18990 (http://watchmouse.com/ ; it)
831
+ Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)
832
+ Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )
833
+ Readability/1900e6 - http://readability.com/about/
834
+ Mozilla/5.0 (compatible; Dataprovider/6.92; +https://www.dataprovider.com/)
835
+ Mozilla/5.0 (iPhone; CPU iPhone OS 6_0_1 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko; Google Page Speed Insights) Version/6.0 Mobile/10A525 Safari/8536.25
836
+ VegeBot (we follow your robots.txt settings before crawling, you can slow down the bot by change the Crawl-Delay parameter in the settings.if you have an enquiry, please email to: abuse-report@terrykyleseoagency.com)
837
+ yacybot (webportal-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html
838
+ FeedBucket/1.0 (+http://www.feedbucket.com)
839
+ topster.de HTTP-Header 1.0
840
+ MaxPointCrawler/Nutch-1.10 (maxpoint.crawler at maxpointinteractive dot com)
841
+ Mozilla/5.0 (Windows NT 6.0; rv:45.0) Gecko/20100101 Firefox/45.0 PTST/276
842
+ yacybot (/global; amd64 Windows 7 6.1; java 1.8.0_05; Europe/es) http://yacy.net/bot.html
843
+ NETCRAFT
844
+ Mozilla/5.0 (compatible; WbSrch/1.0; +https://wbsrch.com)
845
+ WebCookies/1.0 (+http://webcookies.info/faq/#agent)
846
+ netEstate Impressumscrawler (+http://www.netestate.de/De/Loesungen/Impressumscrawler)
847
+ msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)
848
+ CopperEgg/RevealUptime/DublinIEUSA
849
+ StormCrawler/1.0 (a Storm-based crawler; https://github.com/DigitalPebble/storm-crawler; stormcrawler@digitalpebble.com)
850
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; uschi02.watchmouse.net)
851
+ Mozilla/5.0 (Linux; Android 5.0.2; SM-G920T Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2125.111 Mobile Safari/537.36 DareBoost
852
+ Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)
853
+ Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)
854
+ Mozilla/5.0 (compatible; SemrushBot/0.99~bl; +http://www.semrush.com/bot.html)
855
+ WebWatch/Robot_txtChecker
856
+ Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)
857
+ Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)
858
+ Toweyabot: toweya.com
859
+ Mozilla/5.0 (compatible; Infohelfer/1.4.3; +http://www.infohelfer.de/crawler.php)
860
+ Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2.1; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/20021130
861
+ FeedCatBot/3.0 (+http://www.feedcat.net/)
862
+ LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com), libot/Nutch-1.9 (http://www.linkedin.com; libot@linkedin.com)
863
+ R6_CommentReader(www.radian6.com/crawler)
864
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 70350)
865
+ Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)
866
+ Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com
867
+ Riddler (http://riddler.io/about.html)
868
+ Mozilla/5.0 (compatible; Esribot/1.0; http://www.esrihu.hu/)
869
+ Favicon downloader (http://favicon.netk6.com/)
870
+ findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)
871
+ ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl)
872
+ Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0 Google favicon
873
+ GetFoundBot (+http://www.getfound.cz/getfoundbot/)
874
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; usdal02.watchmouse.net)
875
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36 Yandex.Translate
876
+ Mozilla/5.0 (compatible; NLNZ_IAHarvester2013 +http://natlib.govt.nz/about-us/current-initiatives/web-harvest-2013)
877
+ COMODOSpider/Nutch-1.2
878
+ Mozilla/5.0 (compatible; DomainMacroCrawler/0.1; +http://domainmacro.com)
879
+ Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)
880
+ Mozilla/5.0 (compatible; MagiBot/3.6.2; +http://magi.peak-labs.com/robots.txt)
881
+ Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)
882
+ psbot-image (+http://www.picsearch.com/bot.html)
883
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.6; +http://flipboard.com/browserproxy)
884
+ larbin_2.6.4 (atyzos@yahoo.com)
885
+ yacybot (/global; amd64 Windows 7 6.1; java 1.7.0_55; Europe/ru) http://yacy.net/bot.html
886
+ BacklinkCrawler (http://www.backlinktest.com/crawler.html)
887
+ Mozilla/5.0 (Windows NT 6.1; WOW64) adbeat.com/policy AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36
888
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2 PTST/276
889
+ Tools4noobs.com/1.0 Spider
890
+ yacybot (freeworld/global; amd64 Linux 2.6.32-40-generic; java 1.6.0_20; Europe/de) http://yacy.net/bot.html
891
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)
892
+ istellabot/t.1
893
+ Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)
894
+ Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)
895
+ SemrushBot/Nutch-1.5-SNAPSHOT
896
+ Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
897
+ Mozilla/5.0 (X11; Linux x86_64; rv:47.0; GTmetrix https://gtmetrix.com/) Gecko/20100101 Firefox/47.0
898
+ CopperEgg/RevealUptime/Oregon(aws)
899
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_102; Europe/en) http://yacy.net/bot.html
900
+ rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)
901
+ yacybot (amd64 Linux 2.6.26-2-amd64; java 1.6.0_20; Europe/en) http://yacy.net/bot.html
902
+ Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko
903
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MDDS; PTST 2.386)
904
+ Mozilla/5.0 (compatible; RavenCrawler/2.0; +https://raventools.com/seo-website-auditor/)
905
+ Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)
906
+ Mozilla/5.0 (compatible; Scrubby/3.2; +http://seotools.scrubtheweb.com/webpage-analyzer.html)
907
+ dubaiindex (adressendeutschland.de)
908
+ Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )
909
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Netcraft SSL Server Survey - contact info@netcraft.com)
910
+ Mozilla/5.0 (compatible; CompSpyBot/1.0; +http://www.compspy.com/spider.html)
911
+ Mozilla/5.0 (compatible; LA2; +http://www.zeerch.com/zeerch2/bot.php)
912
+ yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html
913
+ updated/0.1-alpha (updated crawler; http://www.updated.com; crawler@updated.com)
914
+ CopperEgg/RevealUptime/FrankfurtGermany
915
+ Scrapy/1.1.2 (+http://scrapy.org)
916
+ yrspider (Mozilla/5.0 (compatible; YRSpider; +http://www.yunrang.com/yrspider.html))
917
+ Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)
918
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.6.0_31; Europe/de) http://yacy.net/bot.html
919
+ Surphace Scout&v4.0 - scout at surphace dot com
920
+ yacybot (freeworld/global; amd64 Linux 2.6.32-5-xen-amd64; java 1.6.0_18; Europe/fr) http://yacy.net/bot.html
921
+ MetaGeneratorCrawler/1.1 (www.metagenerator.info)
922
+ Porkbun/Mustache (Website Analysis; http://porkbun.com; tech@porkbun.com)
923
+ Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)
924
+ Ruby, link_thumbnailer
925
+ yacybot (freeworld/global; amd64 Windows Server 2008 R2 6.1; java 1.7.0_25; Europe/de) http://yacy.net/bot.html
926
+ yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html
927
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11 GotSiteMonitor.com
928
+ Mozilla/5.0 (Linux; Android 4.4; Nexus 5 Build/KRT16M) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/40.0.2125.111 Mobile Safari/537.36 DareBoost
929
+ Mozilla/5.0 (KeepRight OpenStreetMap Checker; http://keepright.at) Gecko/20100101 Firefox/22.0
930
+ Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0; topster.de Linkchecker 5.0) like Gecko (194.228.205.74)
931
+ Mozilla/5.0 AppleWebkit/537.36 (KHTML, like Gecko) Trident/7.0; rv:11.0, Chrome/42.0.2311.135, Edge/12.10136, http://www.shrinktheweb.com/, Webshot/0.9
932
+ topicbot/1.0 (Mozilla;I;+http://www.topic.bot/contact_page.html)
933
+ DataparkSearch/4.53 (+http://dataparksearch.org/bot)
934
+ Mozilla/5.0 (compatible; Linux; InfegyAtlas/1.0; en-US; collection@infegy.com)
935
+ Mozilla/5.0 (X11; Linux x86_64; rv:41.0; GTmetrix https://gtmetrix.com/) Gecko/20100101 Firefox/41.0
936
+ Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)
937
+ Mozilla/5.0 (compatible; parsijoo-update-crawler; +http://www.parsijoo.ir/; ehsanmousa@parsijoo.ir)
938
+ Abrave Spider v4 Robot 1 (http://robot.abrave.co.uk)
939
+ Mozilla/5.0 (compatible; memoryBot/1.21.24 +http://internetmemory.org/en/)
940
+ Mozilla/5.0 (compatible; STINGbot/1.0; +http://136.186.231.16)
941
+ domainsbot (+http://www.domainsbot.com)
942
+ WEPA/3.1 (http://www.wepa.com/; webmaster@wepa.com)
943
+ Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)
944
+ Mozilla/5.0 (compatible; forensiq; +http://www.forensiq.com)
945
+ 192.comAgent
946
+ JUST-CRAWLER(+http://www.justsystems.com/jp/tech/crawler/)
947
+ Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe
948
+ HuaweiSymantecSpider/1.0+DSE-support@huaweisymantec.com+(compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR ; http://www.huaweisymantec.com/en/IRL/spider)
949
+ Mozilla/5.0 ( Macintosh; Intel Mac OS X 10_10_1 ) AppleWebKit/600.2.5 ( KHTML, like Gecko ) Version/8.0.2 Safari/600.2.5 ( compatible; CloudServerMarketSpider/1.0; +http://cloudservermarket.com/spider.html )
950
+ Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)
951
+ Mozilla/5.0 (compatible; AcoonBot/4.10.6; +http://www.acoon.de/robot.asp)
952
+ Scrapy/1.0.1 (+http://scrapy.org)
953
+ seegnifybot/1.0.0 (http://www.seegnify.com/bot)
954
+ omgili/0.5 +http://omgili.com
955
+ Slack-ImgProxy 149 (+https://api.slack.com/robots)
956
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html
957
+ www.integromedb.org/Crawler
958
+ Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm
959
+ Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com
960
+ Mozilla/5.0 (compatible; IstellaBot/1.18.81 +http://www.tiscali.it/)
961
+ Gigabot/1.0
962
+ page_test (larbin2.6.3@unspecified.mail)
963
+ istellabot/Nutch-1.10
964
+ Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)
965
+ ldspider (http://code.google.com/p/ldspider/wiki/Robots)
966
+ yacybot (/global; amd64 Windows 8.1 6.3; java 1.8.0_45; Europe/ru) http://yacy.net/bot.html
967
+ facebookplatform/1.0 (+http://developers.facebook.com)
968
+ Mozilla/5.0 (compatible; RTGI; http://linkfluence.net/)
969
+ yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html
970
+ Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)
971
+ Mozilla/5.0 (compatible; DCPbot/1.2; +http://domains.checkparams.com/)
972
+ WatchMouse/18990 (http://watchmouse.com/ ; d2.watchmouse.com)
973
+ MnoGoSearch/3.3.15
974
+ Mozilla/5.0 (compatible; AboutUsBot/0.9; +http://www.aboutus.org/AboutUsBot)
975
+ Web-sniffer/1.1.0 (+http://web-sniffer.net/)
976
+ Mozilla/4.0 (compatible; Fooooo_Web_Video_Crawl http://fooooo.com/bot.html)
977
+ 50.nu/0.01 ( +http://50.nu/bot.html )
978
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows; Windows NT 5.1) BrokenLinkCheck.com/1.1
979
+ Comodo-Certificates-Spider
980
+ Iframely/0.9.8 (+https://iframely.com/;)
981
+ MnoGoSearch/3.3.12
982
+ CopperEgg/RevealUptime/FrankfurtEU
983
+ Python-urllib/2.7 (+http://ella.juls.savba.sk/aranea_about)
984
+ Mozilla/5.0 (compatible; JobKereso; +http://www.kozvetlen-allasok.hu/robot.jsp info@kozvetlen-allasok.hu)
985
+ Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36 PTST/201
986
+ Mozilla/5.0 (compatible; Google Keyword Tool; +http://adwords.google.com/select/KeywordToolExternal)
987
+ WinWebBot/1.0; (Balaena Ltd, UK); http://www.balaena.com/winwebbot.html; winwebbot@balaena.com;)
988
+ AppleNewsBot
989
+ classbot (+http://allclasses.com)
990
+ yacybot (freeworld/global; amd64 Windows Server 2008 R2 6.1; java 1.6.0_31; America/pt) http://yacy.net/bot.html
991
+ Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
992
+ yacybot (/global; arm Linux 3.12.28+; java 1.7.0_71; Europe/en) http://yacy.net/bot.html
993
+ Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots mtmon01e.yandex.ru)
994
+ CopperEgg/RevealUptime/TokyoJP
995
+ Mozilla/5.0 (compatible; AhrefsBot/3.1; +http://ahrefs.com/robot/)
996
+ CJNetworkQuality; http://www.cj.com/networkquality
997
+ psbot-page (+http://www.picsearch.com/bot.html)
998
+ DealGates Bot/1.1 by Luc Michalski (http://spider.dealgates.com/bot.html)
999
+ Pingdom GIGRIB (http://www.pingdom.com)
1000
+ Mozilla/5.0 (compatible; SEOlyticsCrawler/3.0; +http://crawler.seolytics.net/)
1001
+ Mozilla/5.0 (compatible; EventGuruBot/1.0; +http://www.eventguru.com/spider.html)
1002
+ Python-urllib/2.7 (+http://aranea.juls.savba.sk/aranea_about.html)
1003
+ yacybot (/global; amd64 Windows Vista 6.0; java 1.7.0_55; America/en) http://yacy.net/bot.html
1004
+ The Lemur Web Crawler/Nutch-1.3 (Lemur Web Crawler; http://boston.lti.cs.cmu.edu/crawler_12/; admin@lemurproject.org)
1005
+ SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
1006
+ Feedfetcher-Nuesbyte; (+http://www.nuesbyte.com)
1007
+ copyright sheriff (+http://www.copyrightsheriff.com/)
1008
+ yacybot (/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html
1009
+ Touche (+http://www.touche.com.ve)
1010
+ Panopta v1.1
1011
+ Mozilla/5.0 (compatible; parsijoo-bot; +http://www.parsijoo.ir/; ehsan.mousakazemi@gmail.com)
1012
+ Zookabot/2.2;++http://zookabot.com
1013
+ yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html
1014
+ it2media-domain-crawler/2.0
1015
+ yacybot (/global; amd64 Linux 4.2.0-27-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html
1016
+ Mozilla/5.0 (Android; Mobile; rv:21.0) Gecko/21.0 Firefox/21.0 commoncrawl.org/research//Nutch-1.7-SNAPSHOT
1017
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Data Center; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; PTST 2.386)
1018
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0 Safari/537.36 YottaaMonitor
1019
+ CopperEgg/RevealUptime/AtlantaGAUSA
1020
+ Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
1021
+ yacybot (/global; amd64 Windows 7 6.1; java 1.7.0_55; Europe/fr) http://yacy.net/bot.html
1022
+ Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)
1023
+ Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)
1024
+ ICC-Crawler/2.0 (Mozilla-compatible; ; http://www.nict.go.jp/en/ucri/plan/crawl.html)
1025
+ Mozilla/5.0 (compatible; YYSpider; +http://www.yunyun.com/spider.html)
1026
+ Mozilla/5.0 (iPhone; CPU iPhone OS 5_0_1 like Mac OS X) (compatible; Yeti-Mobile/0.1; +http://help.naver.com/robots/)
1027
+ Mozilla/5.0 (WhatsMyIP.org Text_to_Code_Ratio_Tool) http://whatsmyip.org/ua
1028
+ Thumbnail.CZ robot 1.1 (http://thumbnail.cz/why-no-robots-txt.html)
1029
+ Google-Calendar-Importer
1030
+ Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/534.34 (KHTML, like Gecko) Websnapr/3.0 Safari/534.34
1031
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36 PTST/276
1032
+ WeSEE
1033
+ librabot/2.0 (+http://academic.research.microsoft.com/)
1034
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/3.0.1
1035
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) PTST/276
1036
+ Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)
1037
+ Scrubby/3.0 (+http://www.scrubtheweb.com/help/technology.html)
1038
+ Pinterest/0.1 +http://pinterest.com/
1039
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 52327)
1040
+ Comodo-Webinspector-Crawler 2.2.0, http://www.comodorobot.com
1041
+ Iframely/0.6.0 (+http://iframely.com/;)
1042
+ Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)
1043
+ Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)
1044
+ Y!J-BSC/1.0 (http://help.yahoo.co.jp/help/jp/blog-search/)
1045
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36 PTST/276
1046
+ Mozilla/5.0 (compatible; U; Hootsuite-WebFeed/1.0; mailto:abuse@hootsuite.com)
1047
+ RankFlex.com Webspider
1048
+ TipTop http://feeltiptop.com
1049
+ vebidoobot
1050
+ Mozilla/5.0 (compatible; Dow Jones Searchbot)
1051
+ Mozilla/5.0 (compatible; OrangeBot-Collector/2.0; support.orangebot@orange.com)
1052
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) MsnBot-Media /1.0b
1053
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 Ara.com.tr AraBot 1.0
1054
+ Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)
1055
+ Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots mtmon01g.yandex.ru)
1056
+ BUbiNG (+http://law.di.unimi.it/BUbiNG.html)
1057
+ downnotifier.com monitoring
1058
+ A6-Indexer
1059
+ Mozilla/5.0 (compatible; htrix/1.14.4 ++https://webarchiv.onb.ac.at/robot.html)
1060
+ MetaURI API/2.0 +metauri.com
1061
+ Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)
1062
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36 TinEye/1.0 (via http://www.tineye.com/)
1063
+ Mozilla/5.0 Project 25499 (project25499.com)
1064
+ COMODOspider/Nutch-1.0
1065
+ Mozilla/5.0 (compatible; CareerBot/1.1; +http://www.career-x.de/bot.html)
1066
+ Uptime.com_(http://uptime.com/)
1067
+ AcoonBot/4.10.5 (+http://www.acoon.de)
1068
+ BDFetch
1069
+ 20090717_aa_page (larbin2.6.3@unspecified.mail)
1070
+ Metaspinner/1.0 (Metaspinner Search Engine; http://www.meta-spinner.de/; support@meta-spinner.de)
1071
+ Mozilla/5.0 (compatible; DBLBot/1.0; +http://www.dontbuylists.com/)
1072
+ Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )
1073
+ LYT.SR v1.5 +http://lyt.sr/ (running on http://lyt.sr)
1074
+ Putin (+http://semanticjuice.com/)
1075
+ Mozilla/5.0 (compatible; LA1; +http://www.zeerch.com/bot.php)
1076
+ WatchMouse/18990 (http://watchmouse.com/ ; ny)
1077
+ bitlybot/2.0
1078
+ yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html
1079
+ Webshot/0.9 ( http://www.shrinktheweb.com/ )
1080
+ MetaURI API +metauri.com
1081
+ Mozilla/5.0 (compatible; ProductoDownloadUrlBot/1.0; +http://www.producto.de/)
1082
+ Google-Adwords-Instant (+http://www.google.com/adsbot.html)
1083
+ yacybot (freeworld/global; amd64 Linux 3.8.0-19-generic; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
1084
+ yacybot (freeworld/global; i386 Linux 2.6.32-5-686; java 1.6.0_18; Europe/en) http://yacy.net/bot.html
1085
+ Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36 PTST/277
1086
+ Mozilla/5.0 (compatible; DomainSigmaCrawler/0.1; +http://domainsigma.com/robot)
1087
+ HyperZbozi.cz Feeder/3.1
1088
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)
1089
+ yacybot (/global; amd64 Windows Server 2012 R2 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html
1090
+ Google-AdWords-Express
1091
+ Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)
1092
+ SMTBot (similartech.com/smtbot)
1093
+ Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)
1094
+ MetaInspector/4.7.2 (+https://github.com/jaimeiniesta/metainspector)
1095
+ Mozilla/5.0 (Compatible; Vedma/0.91Beta; +http://www.vedma.ru/bot.htm)
1096
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.21 (KHTML, like Gecko) web-capture.net/0.9 Safari/537.21
1097
+ Nokia6680/1.0 (4.04.07) SymbianOS/8.0 Series60/2.6 Profile/MIDP-2.0 Configuration/CLDC-1.1 (botmobi find.mobi/bot.html find@mtld.mobi)
1098
+ Mozilla/5.0 (compatible; Exploratodo/1.0; +http://www.exploratodo.com
1099
+ iZSearch.com
1100
+ Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)
1101
+ bl.uk_lddc_bot/3.3.0-SNAPSHOT-2014-10-07T09:33:31Z (+http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)
1102
+ yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html
1103
+ HuaweiSymantecSpider/1.0+DSE-support@huaweisymantec.com+(compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR ; http://www.huaweisymantec.com/cn/IRL/spider)
1104
+ FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)
1105
+ Mozilla/5.0 (compatible; DripfeedBot/2.0; +http://dripfeedbookmark.com/bot.html
1106
+ Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)
1107
+ yacybot (freeworld/global; i386 Linux 3.2.0-23-generic-pae; java 1.7.0_03; Europe/en) http://yacy.net/bot.html
1108
+ hawkReader/0.9b (Feed Parser; http://www.hawkreader.com; Allow like Gecko)
1109
+ ruijie (larbin@unspecified.mail)
1110
+ CorpusCrawler 2.0.0 (http://corpora.fi.muni.cz/crawler/)
1111
+ ExactSeek Crawler (http://www.exactseek.com/)/Nutch-1.4
1112
+ WordPress.com; https://botd.wordpress.com
1113
+ Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*
1114
+ Mozilla/5.0 (compatible; memoryBot/1.20.268 +http://internetmemory.org/en/)
1115
+ ShowyouBot (http://showyou.com/crawler)
1116
+ Yoleo Consumer v0.2
1117
+ Mozilla/5.0 (compatible; IstellaBot/1.10.2 +http://www.tiscali.it/)
1118
+ Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server; +http://ws.daum.net/aboutWebSearch.html) Daumoa/3.0
1119
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 YaBrowser/15.12.0.6151 Safari/537.36 Yandex.Translate
1120
+ Mozilla/5.0 (compatible; TeeRaidBot; +https://teeraid.com/bot/)
1121
+ Mozilla/5.0 (compatible; Scopia Crawler 1.1; +http://www.scopia.co)
1122
+ CopperEgg/RevealUptime/DublinIE(aws)
1123
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.7.0_04; Asia/ja) http://yacy.net/bot.html
1124
+ RankurBot/Rankur2.1 (http://rankur.com; info at rankur dot com)
1125
+ Gaisbot/3.0+(robot06@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php)
1126
+ Mozilla/5.0 (compatible; zitebot support [at] zite [dot] com +http://zite.com)
1127
+ MetaTagRobot/2.0 (http://www.check-domains.com/website-analysis/website-analyzer.php)
1128
+ Virusdie crawler/2.1
1129
+ findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)
1130
+ Mozilla/5.0 (compatible; AcoonBot/4.10.7; +http://www.acoon.de/robot.asp)
1131
+ Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)
1132
+ yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html
1133
+ Google_Analytics_Snippet_Validator
1134
+ SurcentroBot
1135
+ Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)
1136
+ findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)
1137
+ Mozilla/5.0 (compatible; memoryBot/1.20.199 +http://internetmemory.org/en/)
1138
+ Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)
1139
+ TwengaBot-Discover (http://www.twenga.fr/bot-discover.html)
1140
+ Mozilla/5.0 (bl.uk_lddc_bot; Linux x86_64) PhantomJS/1.9.7 (+http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)
1141
+ panscient.com
1142
+ Mozilla/5.0 (compatible; TestCrawler; +http://wiki.github.com/bixo/bixo/bixocrawler; bixo-dev@yahoogroups.com)
1143
+ DataparkSearch/4.37-23012006 ( http://www.dataparksearch.org/)
1144
+ GOFORITBOT ( http://www.goforit.com/about/ )
1145
+ CopperEgg/RevealUptime/SydneyAustralia
1146
+ msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)
1147
+ Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)
1148
+ MetaCommentBot; http://metacomment.io/about
1149
+ Mozilla/5.0 (compatible; YesupBot/1.0; +http://www.yesup.net/bot.html)
1150
+ Twikle/1.0 , http://twikle.com , contact@twikle.com
1151
+ yacybot (/global; amd64 Windows Server 2008 R2 6.1; java 1.8.0_60; Europe/de) http://yacy.net/bot.html
1152
+ Mozilla/4.0 (compatible; Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)
1153
+ Comodo-Webinspector-Crawler 2.1
1154
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11 KimonoLabs/0.2
1155
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1 + FairShare-http://fairshare.cc)
1156
+ Scrapy/0.24.5 (+http://scrapy.org)
1157
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) Site-Shot/2.1 (http://www.site-shot.com/) Safari/534.34
1158
+ Port Monitor check service 1.0 | http://www.port-monitor.com
1159
+ Mozilla/5.0 (compatible; Uptimebot/0.1.43; +http://www.uptime.com/uptimebot)
1160
+ yacybot (freeworld/global; amd64 Linux 2.6.32-5-amd64; java 1.6.0_18; Europe/en) http://yacy.net/bot.html
1161
+ SolomonoBot/1.04 (http://www.solomono.ru)
1162
+ Easy-Thumb (https://www.easy-thumb.net/)
1163
+ Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36 PTST/276
1164
+ nic.it crawler v.1.221115 (7.35.0)
1165
+ Mozilla/5.0 (compatible; memorybot/1.24.12 +https://internetmemory.net/en/bots)
1166
+ Mozilla/5.0 (compatible; inoreader.com-like FeedFetcher-Google)
1167
+ yacybot (/global; amd64 Windows 7 6.1; java 1.7.0_72; Europe/de) http://yacy.net/bot.html
1168
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; demuc02.watchmouse.net)
1169
+ Mozilla/5.0 (X11; Linux x86_64; rv:43.0; GTmetrix https://gtmetrix.com/) Gecko/20100101 Firefox/43.0
1170
+ ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)
1171
+ Mozilla/5.0 (compatible; UnisterBot; http://www.bluekiwi.de/misc/imprint)
1172
+ Web-sniffer/1.0.31 (+http://web-sniffer.net/)
1173
+ woobot
1174
+ Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; https://www.jobboerse.com/bot.htm) Gecko/20100401 Firefox/24.0
1175
+ AntBot/Ant-Nutch-1.1 (Ant Nutch Crawler; http://www.ant.com; crawler@ant.com)
1176
+ Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)
1177
+ Mozilla/5.0 (compatible; Evrinid Iudex 1.0.0; +http://www.evri.com/evrinid)
1178
+ adidxbot/1.1 (+http://search.msn.com/msnbot.htm)
1179
+ Mozilla/5.0 (compatible; WebThumbnail/2.2; Website Thumbnail Generator; +http://webthumbnail.org)
1180
+ Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)
1181
+ EasyBib AutoCite (http://content.easybib.com/autocite/)
1182
+ yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html
1183
+ Mozilla/5.0 (compatible; woriobot +http://worio.com)
1184
+ Mozilla/4.0 (compatible; Vagabondo/4.0; http://webagent.wise-guys.nl/)
1185
+ SniffRSS/0.5beta (+http://www.blogator.com/)
1186
+ woobot/1.1
1187
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) WordPress.com mShots Safari/537.36
1188
+ ADmantX Platform Semantic Analyzer - Turn - ADmantX Inc. - www.admantx.com - support@admantx.com
1189
+ Mozilla/5.0 (compatible; UnisterBot; crawler@unister.de)
1190
+ Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)
1191
+ UptimeDog Robot (www.uptimedog.com)
1192
+ Googlebot/2.1 (+http://www.google.com/bot.html)
1193
+ Mozilla/5.0 (Windows NT 6.3; WOW64; rv:36.0) Gecko/20100101 Firefox/36.0 (NetShelter ContentScan, contact abuse@inpwrd.com for information)
1194
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.5.30729; .NET4.0C; .NET CLR 3.0.30729; .NET4.0E) PTST/276
1195
+ ServiceUptime.robot
1196
+ Mozilla/4.0 (compatible;HostTracker/2.0;+http://www.host-tracker.com/)
1197
+ SEO Browser
1198
+ Leikibot/1.0 (+http://www.leiki.com)
1199
+ yacybot (webportal-global; amd64 Windows 7 6.1; java 1.7.0_04; America/en) http://yacy.net/bot.html
1200
+ AppEngine-Google; (+http://code.google.com/appengine; appid: downforeveryoneorjustme)
1201
+ Mozilla/5.0 (Windows NT 5.1) BrokenLinkCheck.com/1.1
1202
+ Seobility (SEO-Check; http://bit.ly/1dJuuzs)
1203
+ Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
1204
+ Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.
1205
+ Site24x7
1206
+ websitepulse checker/3.0 (compatible; MSIE 5.5; Netscape 4.75; Linux)
1207
+ Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)
1208
+ Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)
1209
+ Abrave Spider v4 Robot 2 (http://robot.abrave.co.uk)
1210
+ HubPages V0.2.2 (http://hubpages.com/help/crawlingpolicy)
1211
+ lmspider (lmspider@scansoft.com)
1212
+ magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)
1213
+ Mozilla/5.0 (compatible; WbSrch/1.1 +http://wbsrch.com
1214
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.0.7; Google-SearchByImage) Gecko/2009021910 Firefox/3.0.7
1215
+ Mozilla/5.0 (compatible; Najdi.si/3.1)
1216
+ larbin_2.6.3 (larbin2.6.3@unspecified.mail)
1217
+ Mozilla/5.0 (compatible; aiHitBot/1.1; +http://www.aihit.com/)
1218
+ Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36 PTST/277
1219
+ Mozilla/5.0 (YahooYSMcm/3.0.0; http://help.yahoo.com)
1220
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html
1221
+ OOZBOT/0.20 ( http://www.setooz.com/oozbot.html ; agentname at setooz dot_com )
1222
+ findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)
1223
+ Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)
1224
+ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)
1225
+ Mozilla/5.0 (Twiceler-0.9 http://www.cuil.com/twiceler/robot.html)
1226
+ Mozilla/5.0 (compatible; memoryBot/1.24.11 +http://internetmemory.org/en/)
1227
+ RED/1 (https://redbot.org/)
1228
+ Twitterbot/1.0
1229
+ Mozilla/5.0 (compatible; ptd-crawler; +http://bixolabs.com/crawler/ptd/; crawler@bixolabs.com)
1230
+ Open Web Analytics Bot 1.5.4
1231
+ JikeSpider Mozilla/5.0 (compatible; JikeSpider; +http://shoulu.jike.com/spider.html)
1232
+ Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)
1233
+ Mozilla/5.0 (compatible; oBot/2.3.1; +http://www-935.ibm.com/services/us/index.wss/detail/iss/a1029077?cntxt=a1027244)
1234
+ elefent/Elefent 1.2 (A friendly web elefent.; http://elefent.eu/; webmaster@elefent.eu)
1235
+ Genderanalyzer/1.0
1236
+ elefent/1.2 (Web Crawler; https://crawler.elefent.net; webmaster at elefent dot net)
1237
+ Yandex/1.01.001 (compatible; Win16; H)
1238
+ crawler4j for XQuery
1239
+ Mozilla/5.0 (compatible; Plukkie/1.4; http://www.botje.com/plukkie.htm)
1240
+ LinguaBot/v0.001-dev (MultiLinual Sarch Engine v0.001; LinguaSeek; admin at linguaseek dot com)
1241
+ MrakyPraceCzBot/1.0 (http://www.mrakyprace.cz/about-crawler)
1242
+ Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)
1243
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; plgdn01.watchmouse.net)
1244
+ Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
1245
+ AskQuickly v2 (http://askquickly.org/)
1246
+ Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)
1247
+ Miniflux (http://miniflux.net)
1248
+ Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )
1249
+ Mozilla/5.0 (compatible; wmtips.com/1.0; +http://www.wmtips.com/tools/)
1250
+ Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)
1251
+ Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)
1252
+ Google favicon
1253
+ Mozilla/5.0(compatible; Sosospider/2.0; +http://help.soso.com/webspider.htm)
1254
+ MiaDev/0.0.1 (MIA Bot for research project MIA (www.MIA-marktplatz.de); http://www.mia-marktplatz.de/spider; spider@mia-marktplatz.de)
1255
+ Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)
1256
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.146 Safari/537.36 Specificfeeds- http://www.specificfeeds.com
1257
+ Mozilla/5.0 (compatible; socialbm_bot/1.0; +http://spider.socialbm.net)
1258
+ wp.com feedbot/1.0 (+https://wp.com)
1259
+ Mozilla/5.0 (compatible; CligooRobot/2.0; +http://www.cligoo.de/wk/technik.php)
1260
+ Slack-ImgProxy (+https://api.slack.com/robots)
1261
+ Mozilla/5.0 (compatible; Online Domain Tools - HTTP Headers Online/1.0; +http://http-headers.online-domain-tools.com)
1262
+ Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html)
1263
+ WGETbot/1.0 (+http://wget.alanreed.org)
1264
+ yacybot (/global; amd64 Linux 4.0.9-2-ck; java 1.8.0_51; Europe/de) http://yacy.net/bot.html
1265
+ Y!J-BRI/0.0.1 crawler ( http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html )
1266
+ dlvr.it/1.0 (+http://dlvr.it/) Mozilla/5.0
1267
+ yacybot (/global; arm Linux 4.1.13-v7+; java 1.8.0; Etc/de) http://yacy.net/bot.html
1268
+ Mozilla/5.0 (compatible; GimmeUSAbot/1.0; +http://gimmeusa.com/crawler.html)
1269
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.6.0_29; Europe/fr) http://yacy.net/bot.html
1270
+ Link Valet Online 1.2
1271
+ Mozilla/5.0 (compatible; METASpider; +http://meta.ua/spider)
1272
+ NeutrinoAPI/2.0.1
1273
+ thumbshots-de-bot (+http://www.thumbshots.de/)
1274
+ Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)
1275
+ argenfybot
1276
+ MergadoBot
1277
+ Ant/Ant-Nutch-1.1 (Ant Nutch Crawler; http://www.ant.com; crawler@ant.com)
1278
+ CopperEgg/RevealUptime/SaoPauloBrazil
1279
+ Mozilla/4.0 (compatible; Vagabondo/4.0/EU; http://webagent.wise-guys.nl/)
1280
+ Mozilla/4.0 (compatible; DAWINCI ANTIPLAG SPIDER)
1281
+ Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0 PTST/276
1282
+ HetrixTools.COM Uptime Monitoring Bot. https://hetrixtools.com/uptime-monitoring-bot.html
1283
+ yacybot (/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html
1284
+ Mozilla/5.0 (Windows NT 6.1) (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)
1285
+ Mozilla/5.0 (compatible; InbyBot/2.1; +http://inbyapp.com)
1286
+ all_web1 (larbin2.6.3@unspecified.mail)
1287
+ Mozilla/5.0 (compatible; BusinessSeek.biz_Directory; http://www.businessseek.biz/)
1288
+ Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)
1289
+ Mediatoolkitbot (info@mediatoolkit.com)
1290
+ OpenWebSpider v0.1.2.B (http://www.openwebspider.org/)
1291
+ Mozilla/5.0 (compatible; WebCorp/5.0; +http://www.webcorp.org.uk)
1292
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/4.0.0
1293
+ fastbot crawler beta 2.0 (+http://www.fastbot.de)
1294
+ Sogou Web Spider
1295
+ DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)
1296
+ Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)
1297
+ XoviBot/1.0
1298
+ Mozilla/5.0 (compatible; Online Domain Tools - Online Website Link Checker/1.1; +http://website-link-checker.online-domain-tools.com)
1299
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; decgn02.watchmouse.net)
1300
+ yacybot (i386 Linux 2.6.31-21-generic; java 1.6.0_0; Europe/en) http://yacy.net/bot.html
1301
+ ZnajdzFoto/ImageBot 2.0b
1302
+ Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)
1303
+ Mozilla/5.0 (compatible; ProCogSEOBot/1.0; +http://www.procog.com/ )
1304
+ Mozilla/5.0 (compatible; Plukkie/1.6; http://www.botje.com/plukkie.htm)
1305
+ yacybot (freeworld/global; amd64 Linux 3.2.0-21-generic; java 1.7.0_03-icedtea; America/en) http://yacy.net/bot.html
1306
+ Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
1307
+ Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server; + http://tab.search.daum.net/aboutWebSearch.html) Daumoa/3.0
1308
+ Mozilla/5.0 (compatible; NLNZ_IAHarvester2014 +http://natlib.govt.nz/about-us/current-initiatives/web-harvest-2014)
1309
+ Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)
1310
+ Mozilla/5.0 (compatible; special_archiver/3.3.0 +http://www.loc.gov/webarchiving/notice_to_webmasters.html)
1311
+ yacybot (freeworld/global; i386 Linux 3.4.2-linode44; java 1.6.0_27; America/en) http://yacy.net/bot.html
1312
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36 Yandex.Translate
1313
+ yacybot (webportal-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
1314
+ FreeWebMonitoring SiteChecker/0.1 (+http://www.freewebmonitoring.com)
1315
+ Mozilla/5.0 (compatible; Infohelfer/1.3.0; +http://www.infohelfer.de/crawler.php)
1316
+ Kemvibot/1.0 (http://kemvi.com)
1317
+ Mozilla/4.0 (compatible; HostTracker.com/1.0;+http://host-tracker.com/)
1318
+ AdsBot-Google-Mobile (+http://www.google.com/mobile/adsbot.html) Mozilla (iPhone; U; CPU iPhone OS 3 0 like Mac OS X) AppleWebKit (KHTML, like Gecko) Mobile Safari
1319
+ Mozilla/5.0 (compatible; GeliyooBot/1.0; +http://www.geliyoo.com/)
1320
+ yacybot (x86 Windows XP 5.1; java 1.6.0_21; Europe/de) http://yacy.net/bot.html
1321
+ Mozilla/5.0 (compatible; archive.org_bot/3.1.2 +http://natlib.govt.nz/about-us/current-initiatives/web-harvest-2012)
1322
+ Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)
1323
+ infohelfer/1.1.0 (http://www.infohelfer.de/)
1324
+ yacybot (/global; amd64 Linux 3.16.0-41-generic; java 1.7.0_80; Europe/de) http://yacy.net/bot.html
1325
+ SeoCheckBot (FischerNetzDesign Seo Checker, info@fischernetzdesign.de)
1326
+ netEstate NE Crawler (+http://www.website-datenbank.de/)
1327
+ Mail.Ru/1.0
1328
+ binlar_2.6.3 (larbin2.6.3@unspecified.mail)
1329
+ LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)
1330
+ bot.wsowner.com/1.0.0 (+http://wsowner.com/)
1331
+ Ronzoobot/1.5 (http://www.ronzoo.com/about/)
1332
+ Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)
1333
+ Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)
1334
+ findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)
1335
+ Mozilla/5.0 (compatible; LoadTimeBot/0.7; +http://www.load-time.com/bot.html)
1336
+ Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)
1337
+ Mozilla/4.0 (compatible; MSIE 5.01; GomezAgent 2.0; Windows NT)
1338
+ Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)
1339
+ SEOENGBot/1.2 (+http://learn.seoeng.com/seoengbot.htm)
1340
+ yacybot (i386 Linux 2.6.31-18-generic; java 1.6.0_0; Europe/en) http://yacy.net/bot.html
1341
+ Online Virus Scanner: http://tools.geek-tools.org
1342
+ DwnldBot (+http://dwnld.me)
1343
+ Mozilla/5.0 (compatible; OpenfosBot/2.1; http://www.openfos.com)
1344
+ taptubot *** please read http://www.taptu.com/corp/taptubot ***
1345
+ yacybot (freeworld/global; amd64 Linux 3.9.3-1-ARCH; java 1.7.0_21; America/en) http://yacy.net/bot.html
1346
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/4.0.0~beta10
1347
+ rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)
1348
+ Mozilla/5.0 (Windows NT 6.1; Win64; x64) KomodiaBot/1.0
1349
+ Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )
1350
+ Mozilla/5.0 (compatible; DomainTunoCrawler/0.1; +https://www.domaintuno.com/robot)
1351
+ Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36
1352
+ yacybot (amd64 Linux 2.6.32-gentoo; java 1.6.0_17; Europe/en) http://yacy.net/bot.html
1353
+ Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko; Google Page Speed Insights) Chrome/27.0.1453 Mobile Safari/537.36
1354
+ Mozilla/5.0 (compatible; GimmeUSAbot/1.0; +http://gimmeusa-update.com/crawler)
1355
+ Cloud mapping experiment. Contact research@pdrlabs.net
1356
+ emefgebot/beta (+http://emefge.de/bot.html)
1357
+ DoCoMo/2.0 D903i(c100;TB;W28H20) (compatible; Hatena-Mobile-Gateway/1.2; +http://mgw.hatena.ne.jp/help)
1358
+ ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)
1359
+ Mozilla/5.0 (compatible; SecurityResearch.bot; +http://besome1.info/securityresearchbot.html)
1360
+ WPScan v2.9 (http://wpscan.org)
1361
+ Ronzoobot/1.6 (http://www.ronzoo.com/about/)
1362
+ livedoor ScreenShot/0.10
1363
+ Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) KomodiaBot/1.0
1364
+ Mozilla/5.0 (compatible; memoryBot/1.20.289 +http://internetmemory.org/en/)
1365
+ Mozilla/5.0 (compatible; DomainTunoCrawler/0.1; +http://www.domaintuno.com/robot)
1366
+ yacybot (freeworld/global; amd64 Linux 2.6.26-2-amd64; java 1.6.0_18; Europe/de) http://yacy.net/bot.html
1367
+ Mozilla/5.0 (compatible; GimmeUSAbot/1.0; +http://gimmeusa.com/crawler)
1368
+ Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )
1369
+ Mozilla/5.0 (compatible; Webauskunft.org; http://www.webauskunft.org)
1370
+ CCResearchBot/1.0 commoncrawl.org/research//Nutch-1.7-SNAPSHOT
1371
+ Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot)
1372
+ linkdex.com/v2.0
1373
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 73393)
1374
+ Mozilla/5.0 (compatible; SEODat/0.1 http://crawler.seodat.com)
1375
+ yacybot (freeworld/global; amd64 Linux 3.2.0-4-amd64; java 1.6.0_27; Europe/en) http://yacy.net/bot.html
1376
+ CorpusCrawler 2.0.12 (http://corpora.fi.muni.cz/crawler/)
1377
+ Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )
1378
+ Mozilla/5.0 (compatible; aiHitBot/2.7; +http://www.aihit.com/)
1379
+ Mozilla/5.0 (compatible; SiteAnalyzerBot/4.0; +https://www.site-analyzer.com/)
1380
+ DataparkSearch/4.40.1 (+http://www.dataparksearch.org/)
1381
+ yacybot (freeworld-global; amd64 Linux 3.0.0-21-generic; java 1.6.0_26; America/en) http://yacy.net/bot.html
1382
+ findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)
1383
+ CopperEgg/RevealUptime/SydneyAU(aws)
1384
+ OpenAcoon v4.1.0 (www.openacoon.de)
1385
+ semanticbot (info@semanticaudience.com)
1386
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 75425)
1387
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 11912)
1388
+ Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
1389
+ Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)
1390
+ yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html
1391
+ CorpusCrawler 2.0.25 (http://corpora.fi.muni.cz/crawler/);Project:CzCorpus
1392
+ ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)
1393
+ R6_FeedFetcher_(www.radian6.com/crawler)
1394
+ sosoimagespider+(+http://help.soso.com/soso-image-spider.htm)
1395
+ Mozilla/5.0 (X11; Linux x86_64; rv:39.0; GTmetrix http://gtmetrix.com/) Gecko/20100101 Firefox/39.0
1396
+ yacybot (freeworld/global; amd64 Linux 3.2.0-24-generic; java 1.6.0_24; Europe/en) http://yacy.net/bot.html
1397
+ Mozilla/4.0 (compatible; Vagabondo/4.0; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)
1398
+ CommaFeed/2.4.0-SNAPSHOT (https://github.com/Athou/commafeed)
1399
+ Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
1400
+ yacybot (/global; x86 Windows 7 6.1; java 1.8.0_31; Europe/de) http://yacy.net/bot.html
1401
+ Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)
1402
+ Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.at/crawler)
1403
+ bl.uk_lddc_bot/3.3.0-SNAPSHOT-20140519-0844 (+http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)
1404
+ FFbot/1.0; +http://feedfury.com
1405
+ Mozzila/5.0 (compatible; YamanaLab-Robot/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)
1406
+ LinqiaRSSBot/1.0 (eng@linqia.com)
1407
+ FreeWebMonitoring SiteChecker/0.2 (+http://www.freewebmonitoring.com/bot.html)
1408
+ Mozilla/5.0 (compatible; Web-Monitoring/1.0; +http://monoid.nic.ru/)
1409
+ Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)
1410
+ Mozilla/5.0 (compatible; SpiderLing (a SPIDER for LINGustic research); http://nlp.fi.muni.cz/projects/biwec/)
1411
+ UrlTrends Ranking Updater/2.0 ( http://www.urltrends.com/ )
1412
+ yacybot (/global; amd64 Linux 3.16.1-gentoo; java 1.7.0_55; Europe/en) http://yacy.net/bot.html
1413
+ Mozilla/5.0 (compatible; gimme60bot/1.0 ; +http://gimme60.com)
1414
+ CatchBot/5.0; +http://www.catchbot.com
1415
+ elefent/Elefent 1.1 (Friendly web walker.; http://elefent.eu/; jason@elefent.eu)
1416
+ findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)
1417
+ Googlebot-Video/1.0
1418
+ Mozilla/5.0 (compatible; XmarksFetch/1.0; +http://www.xmarks.com/about/crawler; info@xmarks.com)
1419
+ XRL/3.00 (Linux; i686; en-us) (+http://metamark.net/about)
1420
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36 PTST/276
1421
+ CorpusCrawler 2.0.8 (http://corpora.fi.muni.cz/crawler/)
1422
+ Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)
1423
+ Mozilla/5.0 (compatible; OpenindexDeepSpider/Nutch-1.5-dev; +http://openindex.io/spider.html; systemsATopenindexDOTio)
1424
+ Ronzoobot/1.3 (http://www.ronzoo.com/about.php)
1425
+ MetaInspector/2.2.1 (+https://github.com/jaimeiniesta/metainspector)
1426
+ Mozilla/5.0 (compatible; WASALive-Bot ; http://blog.wasalive.com/wasalive-bots/)
1427
+ Mozilla/5.0 (compatible; YioopBot; +http://www.yioop.com/bot.php)
1428
+ Mozilla/5.0 (compatible; Mozilla/5.0; +http://wiki.github.com/bixo/bixo/bixocrawler; bixo-dev@yahoogroups.com)
1429
+ Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; bot@linkfluence.net)
1430
+ Mozilla/5.0 (compatible; Qseero; +http://www.q0.com)
1431
+ yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html
1432
+ Mozilla/5.0 (compatible; ProCogBot/1.0; +http://www.procog.com/spider.html)
1433
+ Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)
1434
+ Mozilla/5.0 (Anturis Agent)
1435
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) Chromium/31.0.1650.63 Site-Shot/2.1 (http://www.site-shot.com/) Safari/534.34
1436
+ webnumbrFetcher/1.0 (http://webnumbr.com/)
1437
+ Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)
1438
+ NIF/1.1 (http://www.newsisfree.com/robot.php users:)
1439
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 PTST/284
1440
+ DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)
1441
+ Iframely/0.8.8 (+https://iframely.com/;)
1442
+ yacybot (ppc Mac OS X 10.5.2; java 1.5.0_13; Europe/de) http://yacy.net/bot.html
1443
+ Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)
1444
+ yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html
1445
+ Mozilla/5.0 (compatible; FeedBooster; +http://feeds.qsensei.com)
1446
+ yacybot (freeworld/global; amd64 Linux 2.6.32-5-amd64; java 1.6.0_18; Europe/de) http://yacy.net/bot.html
1447
+ Mozilla/5.0 (compatible; SputnikFaviconBot/1.2; +http://corp.sputnik.ru/webmaster)
1448
+ Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)
1449
+ Mozilla/5.0 (compatible; OrangeBot/2.0; support.voilabot@orange.com)
1450
+ Yeti-FeedItemCrawler/1.0 (NHN Corp.; http://help.naver.com/robots/)
1451
+ Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)
1452
+ Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)
1453
+ yacybot (/global; amd64 Linux 4.0.5-gentoo; java 1.8.0_51; Europe/en) http://yacy.net/bot.html
1454
+ pr-cy.ru Screenshot Bot
1455
+ GentleSource Short URL Checker ( http://www.gentlesource.com/ )
1456
+ crazywebcrawler 0.9.0, http://www.crazywebcrawler.com
1457
+ a.pr-cy.ru
1458
+ Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)
1459
+ findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)
1460
+ ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)
1461
+ yacybot (/global; amd64 Windows 7 6.1; java 1.8.0_25; Europe/en) http://yacy.net/bot.html
1462
+ Yeti/1.1 (Naver Corp.; http://help.naver.com/robots/)
1463
+ Technoratibot/8.0
1464
+ Setooz/Nutch-1.0 (http://www.setooz.com)
1465
+ Mozilla/5.0 (Windows NT 5.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36 PTST/201
1466
+ semanticbot
1467
+ Mozilla/5.0 (compatible; JikeSpider; +http://shoulu.jike.com/spider.html)
1468
+ Mozilla/5.0 (compatible; MetamojiCrawler/1.0; +http://www.metamoji.com/jp/crawler.html
1469
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html
1470
+ Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://ecommerce.bigdatacorp.com.br/faq.aspx) ppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36
1471
+ Mozilla/5.0 (compatible; CloudServerMarketSpider/1.0; +http://www.cloudservermarket.com/spider.html)
1472
+ Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0; Neustar WPM) Gecko/20100101 Firefox/38.0
1473
+ Mozilla/5.0 (compatible;+ParchBot/1.0;++http://www.parchmenthill.com/search.htm)
1474
+ Sogou-Test-Spider/4.0 (compatible; MSIE 5.5; Windows 98)
1475
+ MetaHeadersBot (+http://www.metaheaders.com/bot.html)
1476
+ Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.4.2; trendiction media ssppiiddeerr; http://www.trendiction.com/bot/; please let us know of any problems; ssppiiddeerr at trendiction.com) Gecko/20071127 Firefox/2.0.0.11
1477
+ voyager/2.0 (http://www.kosmix.com/crawler.html)
1478
+ Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )
1479
+ hivaBot/hivaBot-1.0 (Iranian Search Engine bot; http://yooz.ir; info@yooz.ir)
1480
+ yacybot (freeworld/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html
1481
+ AppEngine-Google; (+http://code.google.com/appengine; appid: s~closure-compiler-hrd)
1482
+ larbin_2.5.0 (larbin2.5.0@unspecified.mail)
1483
+ Scrapy/0.22.2 (+http://scrapy.org)
1484
+ Ronzoobot/1.2 (http://www.ronzoo.com/about.php)
1485
+ gooblog/2.0 (http://help.goo.ne.jp/contact/)
1486
+ Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)
1487
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21 PagePeeker/2.1; +http://pagepeeker.com/robots/
1488
+ WeSEE:Ads/PageBot (http://www.wesee.com/bot/)
1489
+ yacybot (amd64 Linux 2.6.18-164.el5; java 1.6.0; Europe/de) http://yacy.net/bot.html
1490
+ SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)
1491
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535. Safari/535.22+; (+http://immediatenet.com/thumbnail_api.html; free thumbnails)
1492
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 85747)
1493
+ SBIder/Nutch-1.0-dev (http://www.sitesell.com/sbider.html)
1494
+ Scrapy/1.0.4 (+http://scrapy.org)
1495
+ SanszBot/1.7(http://www.sansz.org/sanszbot, spider@sansz.org) (spider@sansz.org)
1496
+ Mozilla/5.0 (compatible; KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36 collection@infegy.com
1497
+ Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3 like Mac OS X; en-GB) adbeat.com/policy AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F190 Safari/6533.18.5
1498
+ Curious George - www.analyticsseo.com/crawler
1499
+ Grammarly/1.0 (http://www.grammarly.com)
1500
+ OpenCalaisSemanticProxy
1501
+ yacybot (freeworld/global; amd64 Linux 2.6.32-49-server; java 1.6.0_27; Europe/en) http://yacy.net/bot.html
1502
+ netEstate RSS crawler (+http://www.rss-directory.info/)
1503
+ ipv6-test.com validator
1504
+ Mozilla/5.0 (compatible; Qualidator.com Bot 1.0;)
1505
+ ichiro/2.0 (ichiro@nttr.co.jp)
1506
+ yacybot (freeworld/global; amd64 Linux 2.6.38-14-generic; java 1.6.0_22; Europe/en) http://yacy.net/bot.html
1507
+ cg-eye live
1508
+ Mozilla/5.0 (compatible; OpenindexSpider; +http://www.openindex.io/en/webmasters/spider.html)
1509
+ Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http://code.google.com/appengine; appid: s~virustotalcloud)
1510
+ Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)
1511
+ MetaGeneratorCrawler/1.3.4 (www.metagenerator.info)
1512
+ Mozilla/5.0 (Linux; Android 5.0; LG-D855 Build/LRX21R.A1422018487) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.109 Mobile Safari/537.36 DareBoost
1513
+ Mozilla/5.0 (compatible; Nigma.ru/3.0; crawler@nigma.ru)
1514
+ Mozilla/5.0 (compatible; Wappalyzer; +https://github.com/AliasIO/Wappalyzer)
1515
+ yacybot (/global; x86 Windows XP 5.1; java 1.7.0_71; Europe/de) http://yacy.net/bot.html
1516
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36 PTST/277
1517
+ OpenWebSpider v0.1.4 (http://www.openwebspider.org/)
1518
+ HubSpot Connect 1.0 (http://dev.hubspot.com/)
1519
+ Page Valet/4.1pre5
1520
+ msnbot/2.0b (+http://search.msn.com/msnbot.htm)
1521
+ amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831
1522
+ SETOOZBOT/5.0 ( compatible; SETOOZBOT/0.30 ; http://www.setooz.com/bot.html )
1523
+ Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )
1524
+ Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0
1525
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com
1526
+ EdisterBot (http://www.edister.com/bot.html)
1527
+ Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)
1528
+ Tarantula/BETA-0.5 (Tarantula de la Enlle, el mejor buscador en la red; http://www.enlle.com; tarantula@enlle.com)
1529
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)
1530
+ yacybot (freeworld/global; amd64 Linux 2.6.38-8-generic; java 1.6.0_22; Europe/en) http://yacy.net/bot.html
1531
+ Mozilla/5.0 (compatible; SemrushBot/0.95; +http://www.semrush.com/bot.html)
1532
+ CatchBot/3.0; +http://www.catchbot.com
1533
+ Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: crawling@ubermetrics-technologies.com)
1534
+ Mozilla/5.0 (compatible; RukiCrawler/1.0; +http://ruki.rezko.net)
1535
+ adbeat_bot
1536
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler
1537
+ Mozilla/5.0 (compatible; YandexCatalog/3.0; +http://yandex.com/bots)
1538
+ NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)
1539
+ Statastico/4.0
1540
+ facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)
1541
+ Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko, Google Keyword Suggestion) Chrome/10.0.648.127 Safari/534.16
1542
+ Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b
1543
+ EuripBot/2.0 (+http://www.eurip.com) GetRobots
1544
+ Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)
1545
+ Braintree-Webhooks
1546
+ Nutch Checksem/Nutch-1.10
1547
+ Mozilla/5.0 (compatible; HyperCrawl/0.2; +http://www.seograph.net/bot.html)
1548
+ quickobot/quickobot-1 (Quicko Labs; http://quicko.co; robot at quicko dot co)
1549
+ Whibse
1550
+ Yahoo! Site Explorer Feed Validator http://help.yahoo.com/l/us/yahoo/search/siteexplorer/manage/
1551
+ Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)
1552
+ Siteshooter B0t 2.1
1553
+ GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)
1554
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Page Speed Insights) Chrome/27.0.1453 Safari/537.36
1555
+ eBot / v.0.1e (http://alfa.elchron.cz)
1556
+ Mozilla/5.0 (compatible; memoryBot/1.20.247 +http://internetmemory.org/en/)
1557
+ yacybot (x86 Windows 2003 5.2; java 1.6.0_20; America/de) http://yacy.net/bot.html
1558
+ YandexSomething/1.0
1559
+ Scoop
1560
+ Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)
1561
+ Mozilla/5.0 (compatible; SputnikBot/2.3; +http://corp.sputnik.ru/webmaster)
1562
+ TurnitinBot/2.1 (http://www.turnitin.com/robot/crawlerinfo.html)
1563
+ Mozilla/5.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)
1564
+ Mozilla/5.0 (compatible; CrawlForMe/3.0.2; +info@crawlforme.com)
1565
+ Seologies/0.1 Bot
1566
+ www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36
1567
+ dubaiindex (addressendeutschland.de)
1568
+ yacybot (/global; x86 Windows 7 6.1; java 1.8.0_40; America/es) http://yacy.net/bot.html
1569
+ Mozilla/5.0+(compatible;+PiplBot;++http://www.pipl.com/bot/)
1570
+ LinkStats Bot
1571
+ Mozilla/5.0 (compatible; adbeat-publisher-description-fetcher; +crawler@scaleunlimited.com; crawler@scaleunlimited.com)
1572
+ CISPA Vulnerability Notification (https://notify.mmci.uni-saarland.de)
1573
+ librabot/1.0 (+http://search.msn.com/msnbot.htm)
1574
+ SeoCheckBot (Seomira Seo Check, http://www.kfsw.de/bot.html)
1575
+ yacybot (x86 Windows 2003 5.2; java 1.6.0_16; Europe/de) http://yacy.net/bot.html
1576
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; itpda01.watchmouse.net)
1577
+ yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html
1578
+ BrainbruBot/1.0 (+http://www.brainbru.com/)
1579
+ Docoloc
1580
+ Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 Prerender (+https://github.com/prerender/prerender)
1581
+ dlvr.it/1.0 (+http://dlvr.it/)
1582
+ Mozilla/5.0 (compatible; parsijoo-bot; +http://www.parsijoo.ir/; ehsanmousa@parsijoo.ir)
1583
+ DataparkSearch/4.35-02122005 ( http://www.dataparksearch.org/)
1584
+ Pixray-Seeker/2.0 (Pixray-Seeker; +http://www.pixray.com/pixraybot; +crawler@pixray.com)
1585
+ Abrave v5.5 (http://robot.abrave.co.uk)
1586
+ Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.8.1.20) Gecko/20090429 HeartRails_Capture/0.6 (+http://capture.heartrails.com/) BonEcho/2.0.0.20
1587
+ yacybot (/global; amd64 Linux 3.19.0-26-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html
1588
+ MLBot (www.metadatalabs.com/mlbot)
1589
+ yacybot (freeworld/global; amd64 Linux 2.6.32-34-server; java 1.6.0_26; Europe/de) http://yacy.net/bot.html
1590
+ Mozilla/5.0 (compatible; LinkpadBot/1.08; +http://www.linkpad.ru)
1591
+ Mozilla/5.0 (compatible; SearchmetricsBot; http://www.searchmetrics.com/en/searchmetrics-bot/)
1592
+ msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)
1593
+ ZnajdzFoto/Image 2.0
1594
+ Mozilla/5.0 (compatible; Magibot/4.0.0; +http://magi.peak-labs.com/robots.txt)
1595
+ Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )
1596
+ Eurobot/1.2 (http://eurobot.ayell.eu)
1597
+ Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)
1598
+ Yaanb/1.5.001 (compatible; Win64;+http://www.yaanb.com/company/bot.hmtl)
1599
+ yacybot (i386 Linux 2.6.31-16-generic; java 1.6.0_15; Europe/en) http://yacy.net/bot.html
1600
+ Mozilla/5.0 (compatible; YioopBot; +http://173.13.143.74/bot.php)
1601
+ CopperEgg/RevealUptime/Singapore(aws)
1602
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.24 (KHTML, like Gecko; Google Web Preview) Chrome/11.0.696 Safari/534.24
1603
+ yacybot (freeworld/global; amd64 Windows 8 6.2; java 1.7.0_25; Europe/de) http://yacy.net/bot.html
1604
+ Server Density Service Monitoring v2
1605
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/3.0~beta
1606
+ TinEye/1.0; +http://www.tineye.com/
1607
+ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.19; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/2010040121 Firefox/3.0.19
1608
+ GigablastOpenSource/1.0
1609
+ Googlebot (gocrawl v0.4)
1610
+ CSmartCrawler (email: abuse[at]adsadv[dot]com)
1611
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/4.0.0~beta9
1612
+ KOCMOHABT (+http://kozmonavt.tk/) AppEngine-Google; (+http://code.google.com/appengine; appid: s~web-explorer)
1613
+ Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.18) Gecko/20110622 HeartRails_Capture/1.0.5 (+http://capture.heartrails.com/) Namoroka/3.6.18
1614
+ PingSpot/1.0 (http://www.pingspot.org/)
1615
+ AppEngine-Google; (+http://code.google.com/appengine; appid: s~buzzstream-tools-hrd)
1616
+ AddThis.com (http://support.addthis.com/)
1617
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36
1618
+ ExactSeekCrawler/1.0
1619
+ Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)
1620
+ Mozilla/5.0 (compatible; jpg-newsbot/2.0; +http://vipnytt.no/bot.html)
1621
+ g2reader-bot/1.0 (+http://www.g2reader.com/)
1622
+ MetaGeneratorCrawler/1.3.12 (www.metagenerator.info)
1623
+ QuerySeekerSpider ( http://queryseeker.com/bot.html )
1624
+ KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)
1625
+ Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)
1626
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; WOW64; Trident/6.0; BingPreview/1.0b)
1627
+ Mozilla/5.0 (compatible; firmilybot/0.3; +http://www.firmily.com/bot.php
1628
+ Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; http://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0
1629
+ Mozilla/5.0 (compatible; metager2-verification-bot; +http://metager2.de/technology.php)
1630
+ crawler4j (http://code.google.com/p/crawler4j/)
1631
+ CirrusExplorer/1.1 (http://www.cireu.com/explorer.php)
1632
+ Mozilla/5.0 (compatible; RankvalBot/1.0; +https://rankval.com)
1633
+ Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)
1634
+ Mozilla/5.0 (compatible; BLEXBotTest/1.0; +http://webmeup.com/crawler.html)
1635
+ nWormFeedFinder (http://www.nworm.com)
1636
+ netEstate NE Crawler (+http://www.sengine.info/)
1637
+ Mozila/5.0 (compatible; StudioFACA Search; +http://www.studiofaca.com/)
1638
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 92751)
1639
+ wikiwix-bot-3.0
1640
+ SEOCentro Page Keyword Analyzer v1.2
1641
+ yacybot (amd64 Linux 2.6.18-164.15.1.el5xen; java 1.6.0_0; Europe/en) http://yacy.net/bot.html
1642
+ Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)
1643
+ Automattic Analytics Crawler/0.1; http://wordpress.com/crawler/
1644
+ SemrushBot/0.92
1645
+ http://www.uni-koblenz.de/~flocke/robot-info.txt
1646
+ Mozilla/5.0 (compatible; idmarch Automatic.beta/1.3; +http://www.idmarch.org/bot.html)
1647
+ Mozilla/5.0 (X11; Linux i686 on x86_64; rv:25.0; GTmetrix http://gtmetrix.com/) Gecko/20100101 Firefox/25.0
1648
+ msnbot/2.0b (+http://search.msn.com/msnbot.htm)._
1649
+ Stroke.cz (http://stroke.cz)
1650
+ Mozilla (+http://web.nli.org.il/sites/NLI/English/collections/israel-collection/internet_archive/Pages/default.aspx)
1651
+ audisto.com full crawler 4.33.470 (refer to in robots.txt as audisto, see https://audisto.com/bot)
1652
+ http://arachnode.net 2.5
1653
+ SurveyBot/2.3 (Whois Source)
1654
+ ant.com/Nutch-1.7 (http://ant.com)
1655
+ CorpusCrawler 2.0.13 (http://corpora.fi.muni.cz/crawler/)
1656
+ Seo Servis - Analyza zdrojoveho kodu
1657
+ SEOENGBot/1.2 (+http://www.seoengine.com/seoengbot.htm)
1658
+ Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)
1659
+ Mozilla/5.0 (compatible; evc-batch/2.0.20160708184126)
1660
+ HubSpot Marketing Grader
1661
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
1662
+ nrsbot/6.0(loopip.com/robot.html)
1663
+ Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup.com/crawler.html)
1664
+ yacybot (freeworld-global; amd64 Windows 7 6.1; java 1.7.0_02-ea; America/en) http://yacy.net/bot.html
1665
+ Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)
1666
+ Mozilla/5.0 (compatible; Topicbot/12.0rc-2; +http://topicbot.awardspace.us/)
1667
+ Mozilla/5.0 (compatible; Gimme60bot/1.0; +http://gimme60.com)
1668
+ yacybot (i386 Linux 2.6.28-11-generic; java 1.6.0_13; Europe/de) http://yacy.net/bot.html
1669
+ Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)
1670
+ Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)
1671
+ CorpusCrawler 2.0.15 (http://corpora.fi.muni.cz/crawler/)
1672
+ dotbot
1673
+ Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)
1674
+ page_viewer (larbin2.6.3@unspecified.mail)
1675
+ HootSuite Image proxy (hosting@hootsuite.com)
1676
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)
1677
+ PINGOMETER_BOT_(HTTPS://PINGOMETER.COM)
1678
+ Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)
1679
+ VSE/1.0 (rabraham@multiview.com)
1680
+ Scrapy/0.24.0 (+http://scrapy.org)
1681
+ Speedy Spider (http://www.entireweb.com)
1682
+ Mozilla/5.0 (X11; Linux x86_64; GTmetrix https://gtmetrix.com/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36
1683
+ WatchMouse/18990 (http://watchmouse.com/ ; hk)
1684
+ Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)
1685
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 87214)
1686
+ Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)
1687
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.8.0_11; Europe/de) http://yacy.net/bot.html
1688
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/4.0.0~beta6
1689
+ yacybot (freeworld/global; amd64 Linux 3.0.0-15-server; java 1.6.0_23; Europe/en) http://yacy.net/bot.html
1690
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)
1691
+ YisouSpider
1692
+ Slack-ImgProxy 1.138 (+https://api.slack.com/robots)
1693
+ mozilla/5.0 (compatible; webmastercoffee/0.7; +http://webmastercoffee.com/about)
1694
+ Mozilla/5.0 (compatible; Uptimebot/0.2.22; +http://www.uptime.com/uptimebot)
1695
+ Mozilla/5.0 (compatible; AhrefsBot/1.0; +http://ahrefs.com/robot/)
1696
+ Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)
1697
+ sg-Orbiter/1.0 (+http://searchgears.de/uber-uns/crawling-faq.html)
1698
+ Site24x7 Tools
1699
+ NextGenSearchBot 1 (for information visit http://www.zoominfo.com/About/misc/NextGenSearchBot.aspx)
1700
+ Aboundex/0.3 (http://www.aboundex.com/crawler/)
1701
+ yacybot (freeworld/global; i386 Linux 2.6.37.6-0.5-desktop; java 1.6.0_20; Europe/en) http://yacy.net/bot.html
1702
+ Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/98 Safari/537.4 (StatusCake)
1703
+ Mozilla/5.0 (compatible; DomainSONOCrawler/0.1; +http://domainsono.com)
1704
+ Mozilla/5.0 (compatible; Online Domain Tools - Server Monitor/1.0; +http://server-monitoring.online-domain-tools.com)
1705
+ Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)
1706
+ Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )
1707
+ Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
1708
+ Factbot 1.09 (see http://www.factbites.com/webmasters.php)
1709
+ CopperEgg/RevealUptime/SaoPauloBR(aws)
1710
+ CRIM Crawler/Nutch-2.3 (Crawler du Centre de Recherche Informatique de Montr\xc3\xa9al (CRIM))
1711
+ Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)
1712
+ Mozilla/5.0 (compatible; WormlyBot; +http://wormly.com)
1713
+ yacybot (i386 Linux 2.6.23; java 1.6.0_17; Europe/en) http://yacy.net/bot.html
1714
+ Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)
1715
+ URLitor.com
1716
+ RavenCrawler
1717
+ Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.18) Gecko/20130331 HeartRails_Capture/1.0.5 (+http://capture.heartrails.com/) Namoroka/3.6.18
1718
+ Mozilla/5.0 (compatible; WebThumbnail/3.x; Website Thumbnail Generator; +http://webthumbnail.org)
1719
+ Kyoto-Crawler/2.0 (Mozilla-compatible; kyoto-crawler-contact(at)nlp(dot)kuee(dot)kyoto-u(dot)ac(dot)jp; http://nlp.ist.i.kyoto-u.ac.jp/)
1720
+ Mozilla/5.0 (compatible; SputnikImageBot/2.3; +http://corp.sputnik.ru/webmaster)
1721
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; lvps91-250-96-109.dedicated.hosteurope.de)
1722
+ MetaGeneratorCrawler/1.3.3 (www.metagenerator.info)
1723
+ LongURL API
1724
+ Mozilla/5.0 ( compatible; SETOOZBOT/0.30 ; http://www.setooz.com/bot.html ; agentname at setooz dot_com )
1725
+ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/2.0; +http://flipboard.com/browserproxy)
1726
+ http://tools.geek-tools.org/link-counter/
1727
+ SCFCrawler/Nutch-1.8 (Image Crawler for StolenCameraFinder.com; http://www.stolencamerafinder.com/; crawler@stolencamerafinder.com)
1728
+ Mozilla/5.0 (compatible; Uptimebot/0.2.41; +http://www.uptime.com/uptimebot)
1729
+ Mozilla/5.0 (compatible; DomainAppender /1.0; +http://www.profound.net/domainappender)
1730
+ Mozilla/5.0 (compatible; hypestat/1.0; +http://www.hypestat.com/bot)
1731
+ yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html
1732
+ PagesInventory (robot +http://www.pagesinventory.com)
1733
+ Sosospider+(+http://help.soso.com/webspider.htm)
1734
+ Mozilla/5.0 (compatible; AntBot/1.0; +http://www.ant.com/)
1735
+ PagesInventory (robot http://www.pagesinvenotry.com)
1736
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )
1737
+ Wotbox/2.01 (+http://www.wotbox.com/bot/)
1738
+ Mozilla/5.0 (compatible; bixolabs/1.0; +http://bixolabs.com/crawler/general; crawler@bixolabs.com)
1739
+ Mozilla/5.0 (compatible; SpiderLing (a SPIDER for LINGustic research); +http://nlp.fi.muni.cz/projects/biwec/)
1740
+ CopperEgg/RevealUptime/DallasTXUSA
1741
+ yacybot (freeworld/global; amd64 Linux 2.6.32-custom; java 1.6.0_26; Europe/en) http://yacy.net/bot.html
1742
+ Bad-Neighborhood Link Analyzer (http://www.bad-neighborhood.com/)
1743
+ Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)
1744
+ Pixray-Seeker/2.0 (Pixray-Seeker; http://www.pixray.com/pixraybot; crawler@pixray.com)
1745
+ Cityreview Robot (+http://www.cityreview.org/crawler/)
1746
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html
1747
+ Dlvr.it/1.0 (http://dlvr.it/)
1748
+ Zookabot/2.4;++http://zookabot.com
1749
+ AddThis.com robot tech.support@clearspring.com
1750
+ Mozilla/5.0 (compatible; discoverybot/2.0; +http://discoveryengine.com/discoverybot.html)
1751
+ TwengaBot
1752
+ Mozilla/5.0 (compatible; 4SeoHuntBot; +http://4seohunt.biz/about.html)
1753
+ Web-sniffer.me/1.0.0 (+http://web-sniffer.me/)
1754
+ KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)
1755
+ gosquared-thumbnailer/1.0
1756
+ yacybot (/global; amd64 Windows 7 6.1; java 1.7.0_55; Asia/zh) http://yacy.net/bot.html
1757
+ BPImageWalker/2.0 (www.bdbrandprotect.com)
1758
+ Acoon v4.10.3 (www.acoon.de)
1759
+ Mozilla/4.0 (CMS Crawler: http://www.cmscrawler.com)
1760
+ WP Engine Install Performance API
1761
+ Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24
1762
+ Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko PTST/277
1763
+ CamontSpider/1.0 +http://epweb2.ph.bham.ac.uk/user/slater/camont/info.html
1764
+ SEOENGBot/1.2 (+http://learn.seoengine.com/seoengbot.htm)
1765
+ Y!J-BRJ/YATS crawler (http://listing.yahoo.co.jp/support/faq/int/other/other_001.html)
1766
+ CopperEgg/RevealUptime/N.Virginia(aws)
1767
+ Mozilla/5.0, Google-AdWords-Express
1768
+ Mozilla/5.0 (compatible;YodaoBot-Image/1.0;http://www.youdao.com/help/webmaster/spider/;)
1769
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_91; Europe/en) http://yacy.net/bot.html
1770
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.7.0_09; Europe/de) http://yacy.net/bot.html
1771
+ Acoon v4.10.1 (www.acoon.de)
1772
+ Mediatoolkitbot (complaints@mediatoolkit.com)
1773
+ Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)
1774
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.3) SPEng
1775
+ 2Bone_LinkChkr/1.0 libwww-perl/6.03
1776
+ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardBrowserProxy/0.0.5; +http://flipboard.com/browserproxy)
1777
+ Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)
1778
+ Comodo SSL Checker
1779
+ Comodo Spider 1.2
1780
+ Who.is Bot
1781
+ Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)
1782
+ yacybot (freeworld/global; amd64 Windows Server 2008 R2 6.1; java 1.6.0_29; Europe/en) http://yacy.net/bot.html
1783
+ psbot/0.1 (+http://www.picsearch.com/bot.html)
1784
+ IlTrovatore-Setaccio/1.2 (http://www.iltrovatore.it/aiuto/faq.html)
1785
+ Mozilla/5.0 (compatible; Twingly Recon; twingly.com)
1786
+ Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com; webmaster@moreover.com)
1787
+ Orgbybot/OrgbyBot v1.2 (Spidering the net for Orgby; http://www.orgby.com/ ; Orgby.com Search Engine)
1788
+ Browsershots
1789
+ AdsBot-Google (+http://www.google.com/adsbot.html)
1790
+ Qseero v1.0.0
1791
+ Mozilla/5.0 (compatible; ltbot/0.3.6 +http://www.kdsl.tu-darmstadt.de/de/kdsl/research-program/crawling-and-semantic-structuring/)
1792
+ yacybot (x86 Windows XP 5.1; java 1.6.0_18; Europe/de) http://yacy.net/bot.html
1793
+ visionutils/0.2
1794
+ Mozilla/5.0 (compatible; Googlebot/2.1; https://www.deepcrawl.com/bot)
1795
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.4 (KHTML, like Gecko; Google Web Preview) Chrome/22.0.1229 Safari/537.4
1796
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 8942)
1797
+ Mozilla/4.0 (compatible; LinkMarket-Bot v1.2)
1798
+ Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)
1799
+ WordPress.com; https://public-api.wordpress.com
1800
+ Mozilla/5.0 ( compatible; SETOOZBOT/0.30 ; http://www.setooz.com/bot.html )
1801
+ CopperEgg/RevealUptime/FremontCAUSA
1802
+ Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.4.5; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11
1803
+ yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html
1804
+ Mozilla/5.0 (compatible; Windows NT 6.1?; ZumBot/1.0; http://help.zum.com/inquiry)
1805
+ Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0; topster.de Linkchecker 5.0) like Gecko (90.177.52.133)
1806
+ Mozilla/5.0+(compatible; Monitority/1.0; http://www.monitority.com/)
1807
+ yacybot (freeworld/global; x86 Windows 7 6.1; java 1.8.0_25; Europe/de) http://yacy.net/bot.html
1808
+ Mozilla/5.0 (compatible; abby/1.0; +http://www.ellerdale.com/crawler.html)
1809
+ Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)
1810
+ YowedoBot/Yowedo 1.0 (Search Engine crawler for yowedo.com; http://yowedo.com/en/partners.html; crawler@yowedo.com)
1811
+ Mozilla/5.0 (compatible; oBot/2.3.1; http://filterdb.iss.net/crawler/)
1812
+ Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
1813
+ Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)
1814
+ GetIntent Crawler (http://getintent.com/bot.html)
1815
+ Mozilla/5.0 (compatible; Uptimebot/0.1.89; +http://www.uptime.com/uptimebot)
1816
+ Cliqzbot
1817
+ AboutUsBot
1818
+ yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html
1819
+ Sufog/Nutch-2.2.1 (www.sufog.com; www.sufog.com)
1820
+ Mozilla/5.0 (compatible; imbot/0.1 +http://internetmemory.org/en/)
1821
+ KOCMOHABT (+http://kozmonavt.tk/) Mozilla/5.0 (compatible; Web Explorer 1) AppEngine-Google; (+http://code.google.com/appengine; appid: s~web-explorer)
1822
+ yacybot (/global; amd64 Linux 3.16.0-53-generic; java 1.7.0_85; Europe/en) http://yacy.net/bot.html
1823
+ L.webis/0.53 (http://webalgo.iit.cnr.it/index.php?pg=lwebis)
1824
+ Mozilla/5.0 (compatible; memoryBot/1.21.23 +http://internetmemory.org/en/)
1825
+ ExB Language Crawler 2.1.5 (+http://www.exb.de/crawler)
1826
+ Peeplo Screenshot Bot/0.20 ( abuse at peeplo dot_com )
1827
+ Mozilla/5.0 (compatible; Qwantify/2.1w; +https://www.qwant.com/)/*
1828
+ Experibot_v1 [bit.ly/2bLMJdw]
1829
+ WikioFeedBot 1.0 (http://www.wikio.com)
1830
+ SalesIntelligent/v1.1 (https://www.salesintelligent.com)
1831
+ Mozilla/5.0 (compatible; ExaleadCloudView/5;)
1832
+ Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)
1833
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36 TinEye/1.0 (via http://www.tineye.com/)
1834
+ Mozilla/5.0 (compatible; Search17Bot/1.1; http://www.search17.com/bot.php)
1835
+ FollowSite Bot ( http://www.followsite.com/bot.html )
1836
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler - web-crawlers@hubspot.com
1837
+ Mozilla/5.0 (compatible; BlinkaCrawler/1.0; +http://www.blinka.jp/crawler/)
1838
+ Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*
1839
+ findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)
1840
+ yacybot (/global; amd64 Linux 4.4.0-28-generic; java 1.8.0_91; Europe/pl) http://yacy.net/bot.html
1841
+ yacybot (/global; amd64 Windows Server 2008 R2 6.1; java 1.8.0_40; Europe/de) http://yacy.net/bot.html
1842
+ Mozilla/5.0 (compatible; 200PleaseBot/1.0; +http://www.200please.com/bot)
1843
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; chzrh01.watchmouse.net)
1844
+ Iframely/0.8.8 (+http://iframely.com/;)
1845
+ Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)
1846
+ LoadImpactRload/3.2.0 (Load Impact; http://loadimpact.com);
1847
+ RSSingBot (http://www.rssing.com)
1848
+ RADaR-Bot/Nutch-1.3 (http://radar-bot.com/)
1849
+ Ocelli/1.4 (http://www.globalspec.com/Ocelli)
1850
+ Mozilla/5.0 (compatible; BuzzSumo; +http://www.buzzsumo.com/bot.html)
1851
+ Mozilla/4.0 (compatible; IDwhois/2.0alpha-10102619; Linux; en-US; +http://idwhois.info/about)
1852
+ omgilibot/0.4 +http://omgili.com
1853
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.6.0_23; Europe/de) http://yacy.net/bot.html
1854
+ Y!J-BRW/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)
1855
+ Mozilla/5.0 (compatible; AhrefsBot/2.0; +http://ahrefs.com/robot/)
1856
+ Mozilla/5.0 (compatible; VideoSurf_bot +http://www.videosurf.com/bot.html)
1857
+ Symfony Spider (http://symfony.com/spider)
1858
+ Mozilla/5.0 (compatible; Sosospider/2.0; +http://help.soso.com/webspider.htm)
1859
+ Mozilla/5.0 (compatible; OpenindexDeepSpider/Nutch-1.5-dev; +http://www.openindex.io/en/webmasters/spider.html)
1860
+ ADmantX Platform Semantic Analyzer - ADform - ADmantX Inc. - www.admantx.com - support@admantx.com
1861
+ Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)
1862
+ cuwhois/1.0 (+http://www.cuwhois.com/)
1863
+ Mozilla/5.0 (compatible; memoryBot/1.20.311 +http://internetmemory.org/en/)
1864
+ Mozilla/4.0 (compatible; Vagabondo/4.0; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)
1865
+ yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html
1866
+ Mozilla/5.0 (compatible; Dazoobot/1.0; http://dazoo.fr)
1867
+ Mozilla/5.0 (compatible; Semager/1.4c; +http://www.semager.de/blog/semager-bots/)
1868
+ EuripBot/1.1 (+http://www.eurip.com) GetRobots
1869
+ yacybot (freeworld/global; amd64 Linux 3.8.0-23-generic; java 1.6.0_27; Pacific/en) http://yacy.net/bot.html
1870
+ yacybot (amd64 Linux 2.6.24-23-generic; java 1.6.0_07; Europe/de) http://yacy.net/bot.html
1871
+ yacybot (freeworld/global; x86 Windows 2003 5.2; java 1.6.0_32; Europe/de) http://yacy.net/bot.html
1872
+ yacybot (freeworld/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_25; Europe/de) http://yacy.net/bot.html
1873
+ socialbm_bot http://spider.socialbm.net
1874
+ Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)
1875
+ webinatorbot 1.0; +http://www.webinator.de
1876
+ WorldBrewBot/2.1 (+http://www.marketbrew.com/)
1877
+ BotOnParade, http://www.bots-on-para.de/bot.html
1878
+ ltx71 - (http://ltx71.com/)
1879
+ VSB-TUO/1.0 (+http://www.vsb.cz)
1880
+ yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html
1881
+ CopperEgg/RevealUptime/TokyoJP(aws)
1882
+ Mozilla/5.0 (compatible; etoolsbot/1.0; +https://etools.io/bots/)
1883
+ Comodo-Webinspector-Crawler 2.2.2, http://www.comodorobot.com
1884
+ Mozilla/5.0 (compatible; TWMBot/0.1; +http://thewebminer.com)
1885
+ Mozilla/5.0 (compatible; Plukkie/1.3; http://www.botje.com/plukkie.htm)
1886
+ Autonomy
1887
+ yacybot (i386 Linux 2.6.30-2-686; java 1.6.0_0; SystemV/en) http://yacy.net/bot.html
1888
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/4.1.0
1889
+ CatchBot/1.0; +http://www.catchbot.com
1890
+ Sitedomain-Bot(Sitedomain-Bot 1.0, http://www.sitedomain.de/sitedomain-bot/)
1891
+ Mozilla/5.0 (compatible; Uptimebot/0.2.42; +http://www.uptime.com/uptimebot)
1892
+ yacybot (/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_03; Europe/en) http://yacy.net/bot.html
1893
+ SiteTruth.com site rating system
1894
+ SiteUptime.com
1895
+ yacybot (i386 Linux 2.6.24-23-generic; java 1.6.0_16; Europe/en) http://yacy.net/bot.html
1896
+ UrlTrends Ranking Updater/2.0
1897
+ yacybot (/global; amd64 Windows 7 6.1; java 1.7.0_55; Europe/de) http://yacy.net/bot.html
1898
+ yacybot (freeworld/global; amd64 Linux 3.5.0-27-generic; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
1899
+ GarlikCrawler/1.1 (http://garlik.com/, crawler@garlik.com)
1900
+ Mozilla/5.0 (compatible; ParsijooBot; +http://www.parsijoo.ir/; crawler@parsijoo.ir)
1901
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)
1902
+ Willow Internet Crawler by Twotrees V2.1
1903
+ KrOWLer/0.1.0, matentzn at cs dot man dot ac dot uk
1904
+ SimplyFast.info Headers
1905
+ yacybot (freeworld/global; amd64 Linux 2.6.32-431.29.2.el6.x86_64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html
1906
+ Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)
1907
+ msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)
1908
+ Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0 PTST/278
1909
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2 PTST/278
1910
+ gipo-crawler/Nutch-1.10 (Global Internet Policy Observatory crawler)
1911
+ Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)
1912
+ PayPal IPN ( https://www.paypal.com/ipn )
1913
+ Mozilla/5.0 (compatible; KaloogaBot; http://kalooga.com/crawler)
1914
+ eCairn-Grabber/1.0 (+http://ecairn.com/grabber) curl/7.15
1915
+ Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML like Gecko; +http://www.diffbot.com) Chrome/24.0.1295.0 Safari/537.15
1916
+ ImageEngine/1.0
1917
+ DER-bot (+https://goo.gl/9ZibHG)
1918
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.7.0_60; Europe/de) http://yacy.net/bot.html
1919
+ Mozilla/5.0 (compatible; SemrushBot/0.96.3; +http://www.semrush.com/bot.html)
1920
+ Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)
1921
+ Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)
1922
+ Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)
1923
+ Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)
1924
+ MetaGeneratorCrawler/1.3.9 (www.metagenerator.info)
1925
+ TelegramBot (like TwitterBot)
1926
+ Mozilla/5.0 (compatible; Siteliner/1.0; +http://www.siteliner.com/bot)
1927
+ calculon spider/0.1.0 mailto:abuse@yieldbot.com
1928
+ yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html
1929
+ PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)
1930
+ montastic-monitor http://www.montastic.com
1931
+ Mozilla/5.0 (compatible; trovitBot 1.0; +http://www.trovit.com/bot.html)
1932
+ Mozilla/5.0 (Windows NT 6.1)+(compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)
1933
+ Yaanb/1.5.001 (compatible; Win64;)
1934
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 70054)
1935
+ Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )
1936
+ yacybot (freeworld-global; amd64 Linux 3.2.0-4-amd64; java 1.6.0_24; Europe/en) http://yacy.net/bot.html
1937
+ Mozilla/5.0 (compatible; ImageFetcher/5.6; +http://images.weserv.nl/)
1938
+ CopperEgg/RevealUptime/N.VirginiaUSA
1939
+ Huaweisymantecspider (compatible; MSIE 8.0; DSE-support@huaweisymantec.com)
1940
+ OrangeCrawler/Nutch-1.0 (ldorange.crawler@orange-ftgroup.com)
1941
+ CopperEgg/RevealUptime/AmsterdamHolland
1942
+ Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)
1943
+ Mozilla/5.0 (X11; Linux x86_64; GTmetrix https://gtmetrix.com/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36
1944
+ Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html) AppEngine-Google; (+http://code.google.com/appengine; appid: s~natmacdevice)
1945
+ yacybot (/global; amd64 Linux 3.2.41-042stab094.7; java 1.7.0_75; Europe/en) http://yacy.net/bot.html
1946
+ yacybot (x86 Windows Vista 6.1; java 1.6.0_13; Europe/de) http://yacy.net/bot.html
1947
+ SmabblerBot/1.0 (http://www.smabbler.com/)
1948
+ Mozilla/5.0 (compatible; grapeFX/0.9; crawler@grapeshot.co.uk
1949
+ Mozilla/5.0 (compatible; SemrushBot/1~bl; +http://www.semrush.com/bot.html)
1950
+ MetaGeneratorCrawler/1.3.14 (www.metagenerator.info)
1951
+ oku-taka-lab-bot (Mozilla-compatible; htakimoto@lr.pi.titech.ac.jp; http://lr-www.pi.titech.ac.jp/wp/)
1952
+ Mozilla/5.0 (Linux; Android 4.1.2; Galaxy Nexus Build/JZO54K; GTmetrix https://gtmetrix.com/) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/26.0.1410.58 Mobile Safari/537.22
1953
+ Mozilla/5.0 (compatible; Feedage/2.0; +http://www.feedage.com/bot.php)
1954
+ yacybot (amd64 Linux 2.6.26-2-openvz-amd64; java 1.6.0_12; UTC/en) http://yacy.net/bot.html
1955
+ yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html
1956
+ Mozilla/5.0 (compatible; DKIMRepBot/1.0; +http://www.dkim-reputation.org)
1957
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11; 360Spider
1958
+ Mozilla/5.0 (compatible; pub-crawler; +http://wiki.github.com/bixo/bixo/bixocrawler; bixo-dev@yahoogroups.com)
1959
+ mfibot/1.1 (http://www.mfisoft.ru/analyst/; <admin@mfisoft.ru>; en-RU)
1960
+ Visited by http://tools.geek-tools.org
1961
+ peerindex
1962
+ Mozilla/5.0 (compatible; janforman.com/2.4; +http://janforman.com/)
1963
+ http://www.amagit.com/
1964
+ yacybot (freeworld/global; amd64 Linux 2.6.32-40-server; java 1.6.0_20; Europe/en) http://yacy.net/bot.html
1965
+ Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
1966
+ Mozilla/5.0 (compatible; NLNZ_IAHarvester2014 +https://natlib.govt.nz/publishers-and-authors/web-harvesting/domain-harvest)
1967
+ Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)
1968
+ mahonie, neofonie search:robot/search:robot/0.0.1 (This is the MIA Bot - crawling for mia research project. If you feel unhappy and do not want to be visited by our crawler send an email to spider@neofonie.de; http://spider.neofonie.de; spider@neofonie.de)
1969
+ Mozilla/5.0 (Linux; Android 5.0; Nexus 5 Build/LRX21O) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.76 Mobile Safari/537.36 PTST/276
1970
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)
1971
+ Mozilla/5.0 (compatible; OptimizationCrawler/0.2; +http://www.domainoptima.com/bot.html)
1972
+ Mozilla/5.0 (compatible; Cloudinary/1.0)
1973
+ rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)
1974
+ Iframely/0.9.8 (+http://iframely.com/;)
1975
+ yacybot (webportal-global; amd64 Linux 2.6.32-5-amd64; java 1.6.0_18; Europe/en) http://yacy.net/bot.html
1976
+ Interdose AntiSpamBot/2.0 (+http://www.piep.net)
1977
+ CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)
1978
+ Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Google Search Console)
1979
+ rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)
1980
+ findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)
1981
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0) PTST/277
1982
+ yacybot (/global; x86 Windows 2003 5.2; java 1.7.0_67; Europe/de) http://yacy.net/bot.html
1983
+ NeutrinoAPI/2.0.3
1984
+ Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )
1985
+ Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)
1986
+ Site24x7 Agent
1987
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.2; WOW64; Trident/6.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR 2.0.50727; .NET CLR 3.0.30729) CrawlerProcess (http://www.PowerMapper.com) /5.7.704.0
1988
+ Mozilla/5.0 (compatible; Falconsbot; +http://iws.seu.edu.cn/services/falcons/contact_us.jsp)
1989
+ Super Monitoring
1990
+ Seznam-Zbozi-robot/3.0
1991
+ LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)
1992
+ Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)
1993
+ Mozilla/5.0 (compatible; IstellaBot/1.01.18 +http://www.tiscali.it/)
1994
+ Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)
1995
+ Mozilla/5.0 (compatible; BegunAdvertising/3.0; +http://begun.ru/begun/technology/indexer/)
1996
+ Netcraft SSL Server Survey - contact info@netcraft.com
1997
+ Mozilla/5.0 (compatible; Qualidator.com SiteAnalyzer 1.0;)
1998
+ Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com)
1999
+ Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online)
2000
+ seebot/1.0.0 (http://www.seegnify.com/bot)
2001
+ Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)
2002
+ Eurobot/1.1 (http://eurobot.ayell.eu)
2003
+ Mozilla/5.0 (compatible; IXEbot; +http://medialab.di.unipi.it/IXEbot.html)
2004
+ Mozilla/5.0 (GetLinkInfo.com - http://www.getlinkinfo.com)
2005
+ Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )
2006
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.6.0_31; America/en) http://yacy.net/bot.html
2007
+ Mozilla/5.0 (Windows NT 6.1; WOW64; rv:48.0) Gecko/20100101 Firefox/48.0 PTST/314
2008
+ Mozilla/5.0 (compatible; AhrefsBot/4.0; +http://ahrefs.com/robot/)
2009
+ UXCrawlerBot
2010
+ Pixray-Seeker/2.0 (http://www.pixray.com/pixraybot; crawler@pixray.com)
2011
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36 PTST/276
2012
+ FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)
2013
+ yacybot (freeworld/global; amd64 Windows 10 10.0; java 1.8.0_60; Europe/de) http://yacy.net/bot.html
2014
+ Technoratibot/7.0
2015
+ Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; DomainDB-1.1; http://domaindb.com/crawler/)
2016
+ Abrave v6.0 (http://robot.abrave.com)
2017
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/4.0.0~beta11
2018
+ DowntimeDetector/1.0 (+http://downforeveryoneorjustme.com)
2019
+ Comodo Spider 1.1
2020
+ Link Valet Online 1.1
2021
+ yacybot (freeworld/global; amd64 Windows XP 5.2; java 1.7.0_04; America/en) http://yacy.net/bot.html
2022
+ li_viewer (larbin2.6.3@unspecified.mail)
2023
+ Mozilla/5.0 (compatible; Mail.RU/2.0c)
2024
+ Mozilla/5.0 (compatible; aiHitBot/1.0; +http://www.aihit.com/)
2025
+ Mozilla/5.0 (compatible; Uptimebot/0.2.19; +http://www.uptime.com/uptimebot)
2026
+ Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
2027
+ Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 ( compatible; CloudServerMarketSpider/1.0; +http://cloudservermarket.com/spider.html )
2028
+ yacybot (i386 Linux 2.6.28-13-generic; java 1.6.0_13; Europe/en) http://yacy.net/bot.html
2029
+ linguatools-bot/Nutch-1.6 (searching for translated pages; http://www.linguatools.de/linguatoolsbot.html; peter dot kolb at linguatools dot org)
2030
+ Mozilla/5.0 (compatible; Lippershey/5.2; +https://www.lipperhey.com/en/about/)
2031
+ NextGenSearchBot 69 (for information visit http://www.zoominfo.com/business/nextgensearchbot.html)
2032
+ Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)
2033
+ Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
2034
+ yacybot (freeworld-global; amd64 Linux 3.2.0-35-generic; java 1.7.0_09; Europe/en) http://yacy.net/bot.html
2035
+ peerindex/0.1 (http://www.peerindex.com/; crawler AT peerindex DOT com)
2036
+ iqdb/0.2 (+http://iqdb.harry.lu/)
2037
+ Mozilla/5.0 (compatible; LuminateBot/1.0; +http://www.luminate.com/bot/)
2038
+ Mozilla/5.0 (X11; Linux x86_64) adbeat.com/policy AppleWebKit/537.21 (KHTML, like Gecko) Firefox/34.0 Safari/537.21
2039
+ AntBot/1.0 (http://www.ant.com)
2040
+ Mozilla/5.0 (compatible; Thumbshots.ru; +http://thumbshots.ru/bot) Firefox/3
2041
+ Mozilla/5.0 (compatible; value/1.1; +http://www.aihit.com/)
2042
+ Digg Deeper/v1 (http://digg.com/about)
2043
+ Elmer, the Thinglink ImageBot (http://www.thinglink.com/help/ThinglinkImageBot)
2044
+ rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)
2045
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; http://spyonweb.com spyonweb@gmail.com
2046
+ BabalooSpider/1.3 (BabalooSpider; http://www.babaloo.si; spider@babaloo.si)
2047
+ Mozilla/5.0 (compatible; dotSemantic/1.0; +http://www.dotsemantic.org)
2048
+ TinEye/1.1 (http://tineye.com/crawler.html)
2049
+ Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)
2050
+ CorpusCrawler 2.0.9 (http://corpora.fi.muni.cz/crawler/)
2051
+ aboutthedomain
2052
+ Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)
2053
+ Mozilla/5.0 (compatible; Uptimebot/0.2.43; +http://www.uptime.com/uptimebot)
2054
+ LinqiaMetadataDownloaderBot/1.0 (eng@linqia.com)
2055
+ yacybot (freeworld/global; amd64 Linux 3.2.0-32-generic; java 1.6.0_24; Europe/en) http://yacy.net/bot.html
2056
+ NeumobBot/0.2.16 (+http://www.neumob.com/bot)
2057
+ Mozilla/5.0 (compatible; Mp3Bot/0.4; +http://mp3realm.org/mp3bot/)
2058
+ Miniflux (https://miniflux.net)
2059
+ Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko; Google Page Speed Insights) Version/8.0 Mobile/12F70 Safari/600.1.4
2060
+ Digincore crawler bot. See https://www.digincore.com/crawler.html for rules and instructions.
2061
+ Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)
2062
+ NerdyBot
2063
+ Nutch/2.2.1 (page scorer; http://integralads.com/site-indexing-policy/)
2064
+ Mozilla/5.0 (compatible; 008/0.83; http://www.80legs.com/webcrawler.html;) Gecko/2008032620
2065
+ Mozilla/4.0 (compatible; LinkMarket-Bot)
2066
+ SeoCheckBot (Seo-Check, http://www.kfsw.de/bot.html)
2067
+ Mozilla/5.0 (compatible; Whynder Magnet; http://whynder.com/magnet)
2068
+ IdeelaborPlagiaat/1
2069
+ x28-job-bot; +http://x28.ch/bot.html
2070
+ Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )
2071
+ Mozilla/5.0 (compatible; RelateIQ Crawler/0.4; +www.relateiq.com; info@relateiq.com)
2072
+ WebRankSpider/1.37 (+http://ulm191.server4you.de/crawler/)
2073
+ Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)
2074
+ LinkWalker/3.0 (http://www.brandprotect.com)
2075
+ Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)
2076
+ ia_archiver-web.archive.org
2077
+ Mozilla/5.0 (compatible; Online Domain Tools - SEO Checker/1.0; +http://online-domain-tools.com)
2078
+ Mozilla/5.0 (compatible; AhrefsBot/3.0; +http://ahrefs.com/robot/)
2079
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2; Shoppimon Analyzer (http://www.shoppimon.com/)
2080
+ Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)
2081
+ Mozilla/5.0 (compatible; Mozilla; +http://wiki.github.com/bixo/bixo/bixocrawler; bixo-dev@yahoogroups.com)
2082
+ Mozilla/5.0 (compatible; CukBot; Not a spammer; http://www.companiesintheuk.co.uk/bot.html)
2083
+ yacybot (freeworld/global; amd64 Linux 3.1.10-1.9-default; java 1.6.0_24; Europe/de) http://yacy.net/bot.html
2084
+ Nuhk/2.4 (+http://www.neti.ee/cgi-bin/abi/otsing.html)
2085
+ elefent/Elefent 1.1 (Friendly web walker.; http://www.elefent.eu/; jason@elefent.eu)
2086
+ HubSpot Links Crawler 1.0 http://www.hubspot.com/
2087
+ Mozilla/5.0 (compatible; Siteluxbot/1.0; +https://sitelux.co.uk/)
2088
+ CopperEgg/RevealUptime/AmsterdamNL(softlayer)
2089
+ Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )
2090
+ Mediumbot-MetaTagFetcher/0.1 (+https://medium.com/)
2091
+ Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)
2092
+ Mozilla/5.0 (compatible; 008/0.85; http://www.80legs.com/webcrawler.html) Gecko/2008032620
2093
+ yacybot (/global; amd64 Linux 4.4.0ct; java 1.7.0_91; Europe/de) http://yacy.net/bot.html
2094
+ Mozilla/5.0 (compatible; Scrubby/2.2; http://www.scrubtheweb.com/)
2095
+ yacybot (/global; amd64 Linux 3.2.0-4-amd64; java 1.8.0_05; Europe/en) http://yacy.net/bot.html
2096
+ Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.18) Gecko/20110622 HeartRails_Capture/1.0.4 (+http://capture.heartrails.com/) Namoroka/3.6.18
2097
+ Mozilla/5.0 (X11; U; Linux i686; de; rv:1.9.0.1; compatible; iCjobs Stellenangebote Jobs; http://www.icjobs.de) Gecko/20100401 iCjobs/3.2.3
2098
+ KOCMOHABT (+http://www.KOCMOHABT.tk/) AppEngine-Google; (+http://code.google.com/appengine; appid: s~web-explorer)
2099
+ yacybot (freeworld/global; x86 Windows 7 6.1; java 1.6.0_31; Europe/de) http://yacy.net/bot.html
2100
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0) CrawlerProcess (http://www.PowerMapper.com) /5.21.754.0
2101
+ Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots; +http://go.mail.ru/help/robots)
2102
+ Mozilla/5.0 (compatible; Googlebot/2.1; +http://import.io)
2103
+ ownCloud Server Crawler
2104
+ Mozilla/5.0 (compatible; Qwantify/2.1n; +https://www.qwant.com/)/*
2105
+ LoadImpactRload/3.1.5 (Load Impact; http://loadimpact.com);
2106
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0; Girafabot; girafabot at girafa dot com; http://www.girafa.com)
2107
+ OmniExplorer_Bot/6.65a (+http://www.omni-explorer.com) WorldIndexer
2108
+ Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)
2109
+ anonymous coward/1.0 (a Storm-based crawler; https://github.com/DigitalPebble/storm-crawler; someone@company.com)
2110
+ Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)
2111
+ Mozilla/5.0 (compatible; FoundSeoTool/1.0; +http://www.found.co.uk/seo-tool/)
2112
+ findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)
2113
+ yacybot (freeworld/global; amd64 Windows NT (unknown) 6.2; java 1.7.0_05; Africa/en) http://yacy.net/bot.html
2114
+ Mozilla/5.0 (compatible; memorybot/1.20.71 +http://archivethe.net/en/index.php/about/internet_memory1 on behalf of DNB)
2115
+ ownCloud News/5.3.6 (+https://owncloud.org/; 1 subscriber;)
2116
+ YottaBot Spider/Nutch-2.3
2117
+ Mozilla/5.0 (compatible; WeViKaBot/1.0; +http://www.wevika.de/)
2118
+ ThumbShots-Bot (+http://thumbshots.in/bot.html)
2119
+ Mnogosearch-3.1.21
2120
+ Favicon downloader (+https://favico.be/)
2121
+ Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)
2122
+ ExB Language Crawler 2.1.2 (+http://www.exb.de/crawler)
2123
+ PrivateSearch/0.1.0 (Polite Nutch Crawler; grierforensics.com)
2124
+ nodestackbot/0.1 (bot@nodestack.com http://nodestack.com/bot.html)
2125
+ iskanie (+http://www.iskanie.com)
2126
+ yacybot (/global; amd64 Windows 7 6.1; java 1.7.0_67; Europe/de) http://yacy.net/bot.html
2127
+ Mozilla/5.0 (compatible; Dataprovider Site Explorer; http://www.dataprovider.com/)
2128
+ Mozilla/5.0 (compatible; akula/12.0rc-2; +http://k311.fd.cvut.cz/)
2129
+ Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)
2130
+ Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
2131
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0) PTST/284
2132
+ Mozilla/5.0 (compatible; Infohelfer/1.3.3; +http://www.infohelfer.de/crawler.php)
2133
+ www.deadlinkchecker.com XMLHTTP/1.0
2134
+ Mozilla/5.0 (compatible; TweetmemeBot/3.0; +http://tweetmeme.com/)
2135
+ Chrome/49.0.2623.87 (compatible; Wappalyzer; +https://github.com/AliasIO/Wappalyzer)
2136
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0) CrawlerProcess (http://www.PowerMapper.com) /5.11.740.0
2137
+ XING-contenttabreceiver/2.0
2138
+ Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
2139
+ Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)
2140
+ Pingoscope
2141
+ Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)
2142
+ Mozilla/5.0 (compatible; Pagespeed/1.1 Fetcher; +http://www.pagespeed.de)
2143
+ HTTP-Header-Abfrage/1.0 (http://www.internalscripts.de/werkzeuge/http-header-abfrage.php)
2144
+ Mozilla/5.0 (compatible; 80bot/0.71; http://www.80legs.com/spider.html;) Gecko/2008032620
2145
+ LoadImpactRload/3.0.3 (Load Impact; http://loadimpact.com);
2146
+ Mozilla/5.0 (compatible; Peepowbot/1.0; +http://www.peepow.com/bot.php)
2147
+ yacybot (freeworld/global; amd64 Linux 2.6.32-5-amd64; java 1.6.0_26; Atlantic/en) http://yacy.net/bot.html
2148
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; grath01.watchmouse.net)
2149
+ Mozilla/5.0 (compatible; +http://servernfo.com/)
2150
+ Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)
2151
+ CopperEgg/RevealUptime/NewarkNJUSA
2152
+ Mozilla/5.0 (compatible; archive.org_bot/3.3.0 +http://www.archive.org/details/archive.org_bot)
2153
+ Mozilla/5.0 (compatible; Plukkie/1.1; http://www.botje.com/plukkie.htm)
2154
+ Mozilla/5.0 (compatible; musobot/1.0; info@muso.com; +http://www.muso.com)
2155
+ yacybot (freeworld/global; amd64 Linux 3.10.17-gentooxen; java 1.6.0_31; Europe/nb) http://yacy.net/bot.html
2156
+ ResponseCodeTest/1.1
2157
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com
2158
+ Acoon v4.9.5 (www.acoon.de)
2159
+ Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5
2160
+ SiteGuardian/2.0 (Internet Monitoring)
2161
+ Mozilla/5.0 (compatible; DCPbot/1.4; +http://domains.checkparams.com/)
2162
+ BoardReader Favicon Fetcher /1.0 info@boardreader.com
2163
+ baypup/colbert (Baypup; http://www.baypup.com/webmasters; jason@baypup.com)
2164
+ Mozilla/5.0 (compatible; MonTools.com)
2165
+ yacybot (freeworld/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_03; Etc/de) http://yacy.net/bot.html
2166
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.51 (KHTML, like Gecko; Google Web Preview) Chrome/12.0.742 Safari/534.51
2167
+ linkapediabot (+http://www.linkapedia.com)
2168
+ Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7
2169
+ BCKLINKS 1.0
2170
+ CorpusCrawler 2.0.21 (http://corpora.fi.muni.cz/crawler/);Project:CzCorpus
2171
+ Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/support/robots.html)
2172
+ Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)
2173
+ Mozilla/5.0 (compatible; memoryBot/1.20.275 +http://internetmemory.org/en/)
2174
+ Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )
2175
+ yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html
2176
+ BlogSearch/2 +http://www.icerocket.com/
2177
+ Mozilla/5.0 (compatible; LoadTimeBot/0.81; +http://www.load-time.com/bot.html)
2178
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9) Gecko/2008052906 Firefox/3.0 (BrowserSpyBot/1.0) (http://browserspy.dk/)
2179
+ Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)
2180
+ CopperEgg/RevealUptime/NewarkNJ(linode)
2181
+ Automattic Analytics Crawler/0.2; http://wordpress.com/crawler/
2182
+ larbin_2.6.3 capveg@cs.umd.edu
2183
+ HatenaScreenshot
2184
+ CopperEgg/RevealUptime/LondonUK
2185
+ Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)
2186
+ Mozilla/5.0 (compatible; evc-batch/2.0.20160902205649)
2187
+ yacybot (freeworld/global; amd64 Linux 3.2.0-23-generic; java 1.6.0_24; Europe/en) http://yacy.net/bot.html
2188
+ WordPress.com;
2189
+ Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0 Google (+https://developers.google.com/+/web/snippet/)
2190
+ Mozilla/5.0 (compatible; ScoutJet; http://www.scoutjet.com/)
2191
+ Mozilla/5.0 (compatible; Uptimebot/0.1.97; +http://www.uptime.com/uptimebot)
2192
+ WebTarantula.com Crawler
2193
+ WatchMouse/8.4.0.3 (http://watchmouse.com/ ; itmil01.watchmouse.net)
2194
+ LinqiaScrapeBot/1.0 (eng@linqia.com)
2195
+ Orbiter (+http://www.dailyorbit.com/bot.htm)
2196
+ Mozilla/5.0 (compatible; janforman.com/2.5; +http://janforman.com/)
2197
+ InternetSeer.com
2198
+ Mozilla/5.0 (compatible; aiHitBot/2.9; +http://www.aihitdata.com/about)
2199
+ Mozilla/5.0 (compatible; WebNL; +http://www.web.nl/webmasters/spider.html; helpdesk@web.nl)
2200
+ coccoc/1.0 (http://help.coccoc.vn/)
2201
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36 PTST/314
2202
+ Mozilla/5.0 (compatible; Googlebot/2.1; startmebot/1.0; +https://start.me/bot)
2203
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C; .NET4.0E; PTST 2.386)
2204
+ Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)
2205
+ Mozilla/5.0 (compatible; idmarch Automatic.beta/1.4; +http://www.idmarch.org/bot.html)
2206
+ Mozilla/5.0 (compatible; tldstat.com/Bot; +http://tldstat.com/bot.html)
2207
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html
2208
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 84846)
2209
+ Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )
2210
+ Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://www.komodia.com/newwiki/index.php/URL_server_crawler) KomodiaBot/1.0
2211
+ WebarooBot (Webaroo Bot; http://64.124.122.252/feedback.html)
2212
+ OOZBOT/0.20 ( -- ; http://www.setooz.com/oozbot.html ; agentname at setooz dot_com )
2213
+ Mozilla/5.0 (compatible; memoryBot/1.21.16 +http://internetmemory.org/en/)
2214
+ FAST-WebCrawler/3.8
2215
+ Googlebot-Image/1.0
2216
+ enlle punto com/Nutch-1.9
2217
+ OpenAcoon v4.10.5 (www.openacoon.de)
2218
+ yacybot (freeworld/global; amd64 Linux 3.0.0-21-generic; java 1.6.0_26; America/en) http://yacy.net/bot.html
2219
+ Microsearch.ru Bot 1.3;http://microsearch.ru/webmasters
2220
+ Mozilla/5.0 (compatible; Google-Site-Verification/1.0)
2221
+ Mozilla/5.0 (compatible; Qwantify/2.3w; +https://www.qwant.com/)/2.3w
2222
+ rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)
2223
+ HubSpot Connect 2.0 (http://dev.hubspot.com/)
2224
+ Robozilla/1.0
2225
+ Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )
2226
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36 PTST/314
2227
+ facebookexternalhit/1.1
2228
+ PercolateCrawler/3.1.30 (ops@percolate.com)
2229
+ Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)
2230
+ Mozilla/5.0 (compatible; pdffillerbot/1.0; +http://www.pdffiller.com/en/support.htm)
2231
+ fastbot crawler beta 4.0 (+http://www.fastbot.de)
2232
+ SafeDNS search bot/Nutch-1.9 (https://www.safedns.com/searchbot; support [at] safedns [dot] com)
2233
+ Mozilla/5.0 (Nekstbot; http://nekst.ipipan.waw.pl/nekstbot/)
2234
+ yacybot (freeworld/global; amd64 Linux 2.6.32-5-amd64; java 1.6.0_18; US/en) http://yacy.net/bot.html
2235
+ yacybot (freeworld/global; amd64 Linux 3.0.0-12-generic; java 1.6.0_26; Europe/en) http://yacy.net/bot.html
2236
+ coccoc/1.0 ()
2237
+ larbin_2.6.3 (atyzos@yahoo.com)
2238
+ yacybot (i386 Linux 2.6.26-2-686; java 1.6.0_0; Europe/de) http://yacy.net/bot.html
2239
+ Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://ecommerce.bigdatacorp.com.br/faq.aspx) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.107 Safari/535.1
2240
+ yacybot (amd64 Linux 2.6.31-22-server; java 1.6.0_18; Asia/en) http://yacy.net/bot.html
2241
+ yacybot (/global; amd64 Windows Server 2012 6.2; java 1.8.0_91; Europe/de) http://yacy.net/bot.html
2242
+ Mozilla/5.0 (compatible; Netseer crawler/2.0; +http://www.netseer.com/crawler.html; crawler@netseer.com)
2243
+ Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)
2244
+ Scrapy/0.25.1 (+http://scrapy.org)
2245
+ Mozilla/5.0 (compatible; Scopia Crawler 1.2; +http://www.scopia.co)
2246
+ cn_ab_viewer (larbin2.6.3@unspecified.mail)
2247
+ Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv 11.0) like Gecko (compatible; Zombiebot/2.1; +http://www.zombiedomain.net/robot/)
2248
+ Mozilla/5.0 (compatible; Uptimebot/0.1.38; +http://www.uptime.com/uptimebot)
2249
+ yacybot (freeworld/global; amd64 Windows 7 6.1; java 1.7.0_25; Europe/de) http://yacy.net/bot.html
2250
+ Python-urllib/2.7 (+http://sketch.juls.savba.sk/aranea_about)
2251
+ Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )
2252
+ yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html
2253
+ yacybot (/global; amd64 Linux 3.2.41-042stab094.7; java 1.7.0_79; Europe/en) http://yacy.net/bot.html
2254
+ Mozilla/5.0 (compatible; UASlinkChecker/1.0; +http://user-agent-string.info/UASlinkChecker)
2255
+ Y!J-BRO/YFSJ crawler (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html; YahooFeedSeekerJp/2.0)
2256
+ yacybot (freeworld/global; amd64 Linux 3.10.15-1-MANJARO; java 1.7.0_40; Asia/en) http://yacy.net/bot.html
2257
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.2; WOW64; Trident/6.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR 2.0.50727; .NET CLR 3.0.30729) CrawlerProcess (http://www.PowerMapper.com) /5.9.726.0
2258
+ Scrubby/2.1 (http://www.scrubtheweb.com/)
2259
+ Mozilla/5.0 (compatible; JobKeresoBot; +http://www.kozvetlen-allasok.hu/help.jsp; info@kozvetlen-allasok.hu; 5.0
2260
+ yacybot (/global; amd64 Windows 7 6.1; java 1.8.0_66; Europe/de) http://yacy.net/bot.html
2261
+ Mozilla/5.0 (X11; Linux x86_64 UXCrawlerBot http://zium.co)
2262
+ Slackbot-LinkExpanding (+https://api.slack.com/robots)
2263
+ yacybot (freeworld/global; amd64 Linux 3.12.15-gentoo; java 1.7.0_55; Europe/en) http://yacy.net/bot.html
2264
+ Mozilla/5.0 (compatible; LinkpadBot/1.07; +http://www.linkpad.ru)
2265
+ Mozilla/5.0 (compatible; ZB-1; +http://www.zeerch.com/bot.php)
2266
+ Mozilla/5.0 (compatible; WbSrch/1.1 +http://wbsrch.com)
2267
+ WeSEE:Search
2268
+ Mozilla/5.0 (compatible; DCPbot/1.1; +http://domains.checkparams.com/)
2269
+ yacybot (webportal-global; amd64 Linux 2.6.32-5-amd64; java 1.6.0_45; Europe/en) http://yacy.net/bot.html
2270
+ WordPress.com mShots; http://support.wordpress.com/contact/
2271
+ Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; FeedFinder-2.0; http://bloggz.se/crawler)
2272
+ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/0.0.5; +http://flipboard.com/browserproxy)
2273
+ Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)
2274
+ Mozilla/5.0 (X11; Linux x86_64; GTmetrix http://gtmetrix.com/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36
2275
+ https_page_test (larbin2.6.3@unspecified.mail)
2276
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;
2277
+ yacybot (freeworld/global; amd64 Linux 3.2.0-2-amd64; java 1.6.0_24; Europe/en) http://yacy.net/bot.html
2278
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/600.1.4 (KHTML, like Gecko) Safari/600.1.4 (compatible; Princetonbot/1.0; +http://http://tigress-web.princeton.edu/~fy/bot.html)
2279
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider
2280
+ KeywordDensityRobot/0.8 (http://www.seocentro.com/tools/search-engines/keyword-density.html)
2281
+ Mozilla/5.0 (compatible; aiHitBot-BP/1.1; +http://www.aihit.com/)
2282
+ findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)
2283
+ Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
2284
+ CopperEgg/RevealUptime/FremontCA(linode)
2285
+ Mozilla/5.0 (compatible; NTENTbot; +http://www.ntent.com/ntentbot)
2286
+ link_thumbnailer
2287
+ KeywordDensityRobot/0.9 (http://www.seocentro.com/tools/search-engines/keyword-density.html)
2288
+ Mozilla/5.0 (compatible;WI Job Roboter Spider Version 3;+http://www.webintegration.at)
2289
+ Mozilla/5.0 (compatible; Mail.RU/2.0)
2290
+ CopperEgg/RevealUptime/N.California(aws)
2291
+ Searchie Bot/1.0 (a Storm-based crawler; https://www.searchie.org; admin@searchie.org)
2292
+ ^Nail (http://CaretNail.com)
2293
+ MarketBrewBot/2.1 (+http://www.marketbrew.com/)
2294
+ Updownerbot (+http://www.updowner.com/bot)
2295
+ yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html
2296
+ Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)
2297
+ Mozilla/5.0 (compatible; bixolabs/1.0; +http://bixolabs.com/crawler/general; crawler@mail.bixolabs.com)
2298
+ Mozilla/5.0 (compatible; Scopia Crawler 2.0; +http://www.scopia.co)
2299
+ asafaweb.com
2300
+ KrOWLer/0.0.1, matentzn at cs dot man dot ac dot uk
2301
+ Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; siteaudit@alexa.com)
2302
+ findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)
2303
+ SygolBot http://www.sygol.com
2304
+ voltron
2305
+ Mozilla/5.0 (compatible; uMBot-FC/1.0; mailto: crawling@ubermetrics-technologies.com)
2306
+ Comodo-Webinspector-Crawler 2.2.1, http://www.comodorobot.com
2307
+ Superarama.com-Tarama-Botu-v.0.1
2308
+ yacybot (freeworld/global; amd64 Linux 3.2.0-3-amd64; java 1.6.0_24; Europe/en) http://yacy.net/bot.html
2309
+ Mozilla/5.0 (compatible; SSLBot/1.0; http://www.sslstats.com/sslbot)
2310
+ PagesInventory.com (robot http://www.pagesinvenotry.com)
2311
+ Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3 like Mac OS X; en-US) adbeat.com/policy AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F190 Safari/6533.18.5
2312
+ WordPress.com Reader
2313
+ Mozilla/5.0 (compatible; SEOkicks-Robot +http://www.seokicks.de/robot.html)
2314
+ Woko 3.0
2315
+ Gigabot/3.0 (http://www.gigablast.com/spider.html)
2316
+ ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)
2317
+ bl.uk_lddc_bot/3.1.1 (+http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)
2318
+ Urlstat (http://bit.ly/1dJuuzs)
2319
+ yacybot (/global; amd64 Windows 8.1 6.3; java 1.7.0_55; America/ru) http://yacy.net/bot.html
2320
+ Bad Neighborhood Header Detector (http://www.bad-neighborhood.com/header_detector.php)
2321
+ Mozilla/5.0 (compatible; GeliyooBot/1.0beta; +http://www.geliyoo.com/)
2322
+ NalezenCzBot/1.0 (http://www.nalezen.cz/about-crawler)
2323
+ WeSEE_Bot:we_help_monitize_your_site (http://www.wesee.com/bot/)
2324
+ yacybot (freeworld/global; x86 Windows XP 5.1; java 1.7.0_55; Asia/en) http://yacy.net/bot.html
2325
+ studylib download bot, see http://studylib.es/public/bot for additional info
2326
+ SemrushBot/0.91
2327
+ Jabse.com Crawler v.1.0 www.jabse.com/crawler.php//imagecrawler
2328
+ Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com)
2329
+ BizwikiBot/0.3 (http://www.bizwiki.com/bizwikibot.htm)
2330
+ Mozilla/5.0 (compatible; BIXOCRAWLER; +http://wiki.github.com/bixo/bixo/bixocrawler; bixo-dev@yahoogroups.com)
2331
+ PostPost/1.0 (+http://postpost.com/crawlers)
2332
+ Mozilla/5.0 (nomore404.com robot/1.1; +https://nomore404.com/)
2333
+ Riddler (http://riddler.io/about)
2334
+ Mozilla/5.0 (compatible; socketcrawler; http://nlp.fi.muni.cz/projects/biwec/)
2335
+ internetVista monitor (Mozilla compatible)
2336
+ 2Bone_LinkChecker/1.0 libwww-perl/6.03
2337
+ MaxPointCrawler/Nutch-1.1 (maxpoint.crawler at maxpointinteractive dot com)
2338
+ DomainStatsBot/1.0 (http://domainstats.io/our-bot)
2339
+ KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)
2340
+ Mergadobot/3.0.2 (+http://mergado.cz)
2341
+ DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)
2342
+ SalesIntelligent/v1.1
2343
+ SiteAnalyzerBot
2344
+ DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)
2345
+ yacybot (/global; i386 Linux 3.16.3-gentoo; java 1.7.0_55; Australia/en) http://yacy.net/bot.html
2346
+ Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Img/2.0; +http://go.mail.ru/help/robots)
2347
+ Cliqz Bot (+http://www.cliqz.com)
2348
+ SalesIntelligent (+https://www.salesintelligent.com)
2349
+ Mozilla/5.0 (compatible; memoryBot/1.20.243 +http://internetmemory.org/en/)
2350
+ gimme60 (Gimme60 Store ID Bot; gimme60.com)
2351
+ Orbiter/1.0 (http://dailyorbit.com/)
2352
+ msnbot/1.1 (+http://search.msn.com/msnbot.htm)
2353
+ Mozilla/5.0 (compatible; memoryBot/1.20.241 +http://internetmemory.org/en/)
2354
+ Mozilla/5.0 (compatible; akula/k311; +http://k311.fd.cvut.cz/)
2355
+ Irokez.cz monitoring v1.2 - (http://www.irokez.cz, Irokez.cz, crawl)
2356
+ CCBot/2.0 (http://commoncrawl.org/faq/)
2357
+ Mozilla/5.0 (compatible; openstat.ru/Bot)
2358
+ Mozilla/5.0 (compatible; GigaBot/1.0; +http://www.gigablast.com/ )
2359
+ Readability/6a54d4 - http://readability.com/about/
2360
+ percbotspider <ops@percolate.com>
2361
+ Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )
2362
+ HatenaScreenshot/1.0 (checker)
2363
+ Mozilla/5.0 (compatible; AcoonBot/4.12.1; +http://www.acoon.de/robot.asp)
2364
+ CRAZYWEBCRAWLER 0.9.2, http://www.crazywebcrawler.com
2365
+ Mozilla/5.0 (compatible; Online Domain Tools - Online Sitemap Generator/1.1; +http://sitemap-generator.online-domain-tools.com)
2366
+ Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)
2367
+ OnPageBot (compatible; Googlebot 2.1; +https://bot.onpage.org/)
2368
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.103 Safari/537.36 Yandex.Translate
2369
+ Sogou web spider/4.0
2370
+ Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots DEV)
2371
+ LinkWalker/2.0
2372
+ SeoCheck (FischerNetzDesign Seo Checker, info@fischernetzdesign.de)
2373
+ yacybot (freeworld/global; amd64 Linux 3.2.13-1-ARCH; java 1.7.0_03-icedtea; Europe/fr) http://yacy.net/bot.html
2374
+ SiteCheck - http://downforeveryoneorjustme.com
2375
+ Robots_Tester_http_www.searchenginepromotionhelp.com
2376
+ CRAZYWEBCRAWLER 0.9.4, http://www.crazywebcrawler.com
2377
+ Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)
2378
+ Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.10) Gecko/20050716 Thunderbird/1.0.6 - WebCrawler http://cognitiveseo.com/bot.html
2379
+ Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)
2380
+ yacybot (/global; amd64 Linux 4.2.0-gentoo-r1; java 1.8.0_45; Australia/en) http://yacy.net/bot.html
2381
+ DomainsDB.net MetaCrawler v.0.9.7c (http://domainsdb.net/)
2382
+ Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; mowser; http://www.mowser.com)
2383
+ www.probethenet.com scanner
2384
+ Mozilla/5.0 (compatible; Crawlera/1.10.2; UID 6573)
2385
+ LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)
2386
+ Imagga Downloader/v0.14.3
2387
+ SeopultContentAnalyzer/1.0
2388
+ Mozilla/5.0 (compatible; +http://www.hosterstats.com/newgtld-websurvey.php)
2389
+ Mozilla/5.0 (compatible; GoogleDocs; apps-spreadsheets; +http://docs.google.com)
2390
+ Mozilla/5.0 (compatible;acapbot/0.1)
2391
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;Alibaba.Security.Heimdall)
2392
+ HTTPClient/1.0 (2.8.0, ruby 2.0.0 (2014-05-08))
2393
+ Jetty/9.3.z-SNAPSHOT
2394
+ Jetty/9.3.z-SNAPSHOT, SMUrlExpander
2395
+ Jetty/9.3.z-SNAPSHOT, SMViewer
2396
+ Liferea/0.x.x (Linux; en_US.UTF-8; http://liferea.sf.net/)
2397
+ SynHttpClient-Built/5825
2398
+ asynchttp
2399
+ SMUrlExpander
2400
+ KickFire
2401
+ HAAARTLAND http client
2402
+ PEAR HTTPRequest class ( http://pear.php.net/ )
2403
+ Mozilla/4.0 (vBSEO; http://www.vbseo.com)
2404
+ HAARTLAND http client
2405
+ JS-Kit URL Resolver http://js-kit.com/
2406
+ http://www.checkprivacy.or.kr:6600/RS/PRIVACY_ENFAQ.jsp
2407
+ Kml-Google; (+http://code.google.com/apis/kml) gzip
2408
+ gvfs/1.20.3
2409
+ request.js
2410
+ Microsoft URL Control - 6.01.9782
2411
+ git/2.1.4
2412
+ IPS/1.0
2413
+ OxfordCloudService/1.0
2414
+ Mozilla/5.0 (compatible; informatique at mixdata dot com/0.6; +http://t.co/GSRLLKex24; informatique at mixdata dot com)
2415
+ Mozilla/5.0 (compatible; Feedwind/2.0; +http://feed.mikle.com/support/description/)
2416
+ Mozilla/5.0 (compatible; Feedwind/3.0; +http://feed.mikle.com/support/description/)
2417
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 1068) AppleWebKit/534.50 (KHTML like Gecko) Version/5.1 Instapaper/4.0 (+http://www.instapaper.com/)
2418
+ Mozilla/5.0 (compatible; Keywords Research; http://www.check-domains.com/keyword-research/keywords-adwords.php)
2419
+ CAAM/0.1 (CAAM; caam dot crwlr at gmail dot com)
2420
+ i2kconnect/"1.7.0"
2421
+ resolver/0.1.12 (+http://github.com/andris9/resolver)
2422
+ DangDang
2423
+ Mozilla/5.0 (compatible; Tetrahedron/1.0; +https://compute.info/tet)
2424
+ PritTorrent/0.1
2425
+ FeedBunch/1.0 (+http://www.feedbunch.com)
2426
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.21 (KHTML, like Gecko) www.HomepageScreenshot.de - Free ScreenShotService/1.0 Safari/537.21
2427
+ Mozilla/5.0 (compatible; Licorne Image Snapshot 1.0; +http://www.jednorozec.cz)
2428
+ Pinterest/0.2 (+https://www.pinterest.com/bot.html)
2429
+ Pinterest/0.1 (+https://www.pinterest.com/bot.html)
2430
+ Mozilla/5.0 (iplabel; Windows NT 6.3; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0
2431
+ Mozilla/5.0 zgrab/0.x
2432
+ Zend_Http_Client
2433
+ Zend\Http\Client
2434
+ Zao/0.1 (http://www.kototoi.org/zao/)
2435
+ YOURLS v1.7.1 +http://yourls.org/ (running on http://flnx.co)
2436
+ yoogliFetchAgent/0.1
2437
+ Yo-yo
2438
+ Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)
2439
+ Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)
2440
+ Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) AppleWebKit (KHTML, like Gecko) Mobile (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)
2441
+ xpymep.exe
2442
+ xpymep2.exe
2443
+ xpymep3.exe
2444
+ Xenu Link Sleuth 1.2b
2445
+ Xenu Link Sleuth 1.2c
2446
+ Xenu Link Sleuth 1.2d
2447
+ Xenu Link Sleuth 1.2e
2448
+ Xenu Link Sleuth 1.2f
2449
+ Xenu Link Sleuth 1.2g
2450
+ Xenu Link Sleuth 1.2h
2451
+ Xenu Link Sleuth 1.2i
2452
+ Xenu Link Sleuth/1.3.7
2453
+ Xenu Link Sleuth/1.3.8
2454
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36 XaxisSemanticsClassifier/1.0 http://crystalsemantics.com
2455
+ WWW-Mechanize/1.73
2456
+ WordPress/4.0.1; http://bestcomp.hol.es
2457
+ WordPress/4.3.3; http://homebusinessreviews.online
2458
+ WordPress/4.3.3; http://legithomejobs.club
2459
+ WordPress/4.4.2; http://cheapautoinsuranceinva.com
2460
+ WordPress/4.4.2; http://citycollegeinmiami.com
2461
+ WordPress/4.4.2; http://cookingideabook.com
2462
+ WordPress/4.4.2; http://oldmanhealth.com
2463
+ WordPress/4.4.2; http://sapcrmarchitecture.com
2464
+ WordPress/4.4.2; http://www.spar3d.com
2465
+ WordPress/4.5.2; http://1.we-careu.xyz
2466
+ WordPress/x.x.x.x PHP/4.x.xxMozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) PhantomJS/1.6.1 Safari/534.34
2467
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Word/14.0.0
2468
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Word/14.25.0
2469
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Word/14.47.0
2470
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Word/14.48.0
2471
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Word/14.50.0
2472
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Word/14.56.0
2473
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Word/14.61.0
2474
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Word/14.62.0
2475
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Word/14.63.0
2476
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Word/14.64.0
2477
+ WomlpeFactory/0.1 (+http://www.Womple.com/bot.html)
2478
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) wkhtmltoimage Safari/534.34
2479
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) wkhtmltopdf Safari/534.34
2480
+ Mozilla/4.0 (compatible; Win32; WinHttp.WinHttpRequest.5)
2481
+ Windows-RSS-Platform/2.0 (IE 11.0; Windows NT 10.0)
2482
+ WhatsApp
2483
+ WhatsApp/2.12.15/i
2484
+ WhatsApp/2.12.16/i
2485
+ WhatsApp/2.12.17/i
2486
+ WhatsApp/2.12.449 A
2487
+ WhatsApp/2.12.453 A
2488
+ WhatsApp/2.12.510 A
2489
+ WhatsApp/2.12.540 A
2490
+ WhatsApp/2.12.548 A
2491
+ WhatsApp/2.12.555 A
2492
+ WhatsApp/2.12.556 A
2493
+ WhatsApp/2.16.1/i
2494
+ WhatsApp/2.16.13 A
2495
+ WhatsApp/2.16.2/i
2496
+ WhatsApp/2.16.42 A
2497
+ WhatsApp/2.16.57 A
2498
+ WhatWeb/0.4.8-dev
2499
+ http://www.almaden.ibm.com/cs/crawler [wf84]
2500
+ WeLikeLinks - WeAreNotEvil
2501
+ Websquash.com (Add url robot)
2502
+ WeCrawlForThePeace - We are not Evil
2503
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.21 (KHTML, like Gecko) webscreenie Safari/537.21
2504
+ Mozilla/3.0 (compatible; WebMon 1.0.11; Windows XP)
2505
+ Mozilla/5.0 (X11; BSD Four) AppleWebKit/534.34 (KHTML, like Gecko) webkit2png Safari/534.34
2506
+ WebFetch
2507
+ WebIndex
2508
+ webcollage/1.114
2509
+ webcollage/1.117
2510
+ webcollage/1.125
2511
+ webcollage/1.129
2512
+ webcollage/1.93
2513
+ Mozilla/3.0 (compatible; WebCapture 2.0; Auto; Windows)
2514
+ W3C_Unicorn/1.0
2515
+ W3C_I18n-Checker/1.0
2516
+ W3C-mobileOK/DDC-1.0
2517
+ W3C-checklink
2518
+ W3C-checklink/2.90 libwww-perl/5.64
2519
+ W3C-checklink/3.6.2.3 libwww-perl/5.64
2520
+ W3C-checklink/4.2 [4.20] libwww-perl/5.803
2521
+ W3C-checklink/4.2.1 [4.21] libwww-perl/5.803
2522
+ W3C-checklink/4.3 [4.42] libwww-perl/5.805
2523
+ W3C-checklink/4.3 [4.42] libwww-perl/5.808
2524
+ W3C-checklink/4.3 [4.42] libwww-perl/5.820
2525
+ W3C-checklink/4.5 [4.154] libwww-perl/5.823
2526
+ W3C-checklink/4.5 [4.160] libwww-perl/5.823
2527
+ w3af.org
2528
+ VYU2 (GNU; OpenRISC)
2529
+ VORTEX/1.2 ( http://marty.anstey.ca/robots/vortex/)
2530
+ Vortex/2.2 ( http://marty.anstey.ca/robots/vortex/)
2531
+ Vortex/2.2 (+http://marty.anstey.ca/robots/vortex/)
2532
+ Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)
2533
+ URLChecker
2534
+ urlresolver
2535
+ Upflow/1.0
2536
+ Typhoeus - https://github.com/typhoeus/typhoeus
2537
+ UdmSearch/3.1.x
2538
+ Mozilla/5.0 Tweetminster
2539
+ Mozilla/5.0 (compatible; +http://tweetedtimes.com)
2540
+ truwoGPS/1.0 (GNU/Linux; U; i686; en-US; +http://www.lan4lano.net/browser.html )
2541
+ Traackr.com
2542
+ More-Testing (+http://www.theinternetrules.com)
2543
+ The Expert HTML Source Viewer (http://www.expert-html.com)
2544
+ htmlayout 3.3; above-Windows-7; www.terrainformatica.com )
2545
+ WebFuck V2.1 T0PHackTeam www.t0p.xyz
2546
+ Symfony2 BrowserKit
2547
+ Mozilla/5.0 (compatible; SwiteScraper)
2548
+ Summify (Summify/1.0.1; +http://summify.com)
2549
+ Stratagems Kumo
2550
+ Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu)
2551
+ Sprinklr 2.0
2552
+ spray-can/1.3.3
2553
+ SortSite/4.4.500.0
2554
+ Snoopy v2.0.0
2555
+ SMRF URL Expander
2556
+ silk/1.0 (+http://www.slider.com/silk.htm)/3.7
2557
+ SiteBar/3.3.5 (Bookmark Server; http://sitebar.org/)
2558
+ SiteBar/3.3.8 (Bookmark Server; http://sitebar.org/)
2559
+ SimplePie/1.3.1 (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20160226060242
2560
+ ShortLinkTranslate
2561
+ Mozilla/5.0 (compatible; Seznam screenshot-generator 2.1; +http://fulltext.sblog.cz/screenshot/)
2562
+ SEOstats 2.0.9 https://github.com/eyecatchup/SEOstats
2563
+ semanticdiscovery/0.1
2564
+ SearchSight/2.0 (http://SearchSight.com/)
2565
+ ScoutURLMonitor/5.9.8
2566
+ Robosourcer/1.0
2567
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)
2568
+ RestSharp/105.2.3.0
2569
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.78.2 (KHTML, like Gecko) Version/7.0.1(2.5.3) Safari(ReederForMac)
2570
+ redback/v0-570-g26f8c96
2571
+ RebelMouse/0.1 Mozilla/5.0 (compatible; http://rebelmouse.com) Gecko/20100101 Firefox/7.0.1
2572
+ RealPlayer%20Downloader/1569 CFNetwork/720.3.13 Darwin/14.3.0 (x86_64)
2573
+ RealPlayer%20Downloader/1701 CFNetwork/760.5.1 Darwin/15.5.0 (x86_64)
2574
+ RealPlayer%20Downloader/1725 CFNetwork/720.3.13 Darwin/14.3.0 (x86_64)
2575
+ RealPlayer%20Downloader/1750 CFNetwork/720.5.7 Darwin/14.5.0 (x86_64)
2576
+ RealPlayer%20Downloader/1750 CFNetwork/760.0.5 Darwin/15.0.0 (x86_64)
2577
+ RealPlayer%20Downloader/1750 CFNetwork/760.4.2 Darwin/15.4.0 (x86_64)
2578
+ Qirina Hurdler v. 1.05 10.11.01 (+http://www.qirina.com/hurdler.html)
2579
+ python-requests/1.1.0 CPython/2.7.6 Linux/3.13.0-58-generic
2580
+ python-requests/1.2.0 CPython/2.7.3 Linux/3.2.0-29-virtual
2581
+ python-requests/2.1.0 CPython/2.7.3 Linux/3.2.0-4-amd64
2582
+ python-requests/2.10.0
2583
+ python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-48-generic
2584
+ python-requests/2.2.1 CPython/2.7.6 Linux/3.13.0-83-generic
2585
+ python-requests/2.3.0 CPython/2.7.6 Linux/3.13.0-57-generic
2586
+ python-requests/2.5.0 CPython/2.7.9 Linux/3.14.20-20.44.amzn1.x86_64
2587
+ python-requests/2.6.0 CPython/2.7.3 Linux/3.2.0-4-amd64
2588
+ python-requests/2.6.0 CPython/2.7.5 Linux/3.10.0-327.3.1.el7.x86_64
2589
+ python-requests/2.7.0 CPython/2.7.6 Linux/3.13.0-85-generic
2590
+ python-requests/2.8.0
2591
+ python-requests/2.8.1
2592
+ python-requests/2.9.1
2593
+ Python-httplib2/0.7.4 (gzip)
2594
+ Pulsepoint XT3 web scraper
2595
+ Mozilla/5.0 (compatible; ProspectB2B/2.0; http://prospectb2b.com)
2596
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) PowerPoint/14.39.0
2597
+ PostRank/2.0 (postrank.com; 1 subscribers)
2598
+ PostRank/2.0 (postrank.com)
2599
+ postano
2600
+ Ploetz + Zeller (http://www.ploetz-zeller.de) Link Validator v1.0 (support@p-und-z.de) for ARIS Business Architect
2601
+ Pizilla++ ver 2.45
2602
+ phpcrawl
2603
+ Pearltrees
2604
+ Pcore-HTTP/v0.23.20
2605
+ Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.4) Gecko/2008102920 http://ow.ly web crawler (.NET CLR 3.5.30729)
2606
+ Mozilla/5.0 (compatible; Optimizer)
2607
+ Mozilla/5.0 [en] (X11, U; OpenVAS 7.0.10)
2608
+ JetBrains Omea Reader 1.0.x (http://www.jetbrains.com/omea_reader/)
2609
+ JetBrains Omea Reader 2.0 Release Candidate 1 (http://www.jetbrains.com/omea_reader/)
2610
+ okhttp/2.5.0
2611
+ okhttp/4.9.0
2612
+ oegp v. 1.3.0
2613
+ Nuzzel
2614
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36 http://notifyninja.com/monitoring
2615
+ Notifixious/LinkChecker (http://notifixio.us)
2616
+ http://www.nominet.org.uk/privacypolicy
2617
+ node-superagent/1.8.3
2618
+ node.io
2619
+ Kimengi/nineconnections.com
2620
+ newspaper/0.0.9.8
2621
+ newspaper/0.1.7
2622
+ newsme/1.0; feedback@news.me
2623
+ NewsGator/2.0 Bot (http://www.newsgator.com)
2624
+ NewsGator/2.5 (http://www.newsgator.com; Microsoft Windows NT 5.1.2600.0; .NET CLR 1.1.4322.2032)
2625
+ NetTrack Anonymous Web Statistics https://nettrack.info/support.php
2626
+ NetTrack.info Crawler
2627
+ NetResearchServer(http://www.look.com)
2628
+ NetResearchServer/2.5(loopimprovements.com/robot.html)
2629
+ NetResearchServer/2.7(loopimprovements.com/robot.html)
2630
+ NetResearchServer/2.8(loopimprovements.com/robot.html)
2631
+ NetResearchServer/3.5(loopimprovements.com/robot.html)
2632
+ NetResearchServer/4.0(loopimprovements.com/robot.html)
2633
+ NetLyzer FastProbe (See http://netlyzer.com/report/www.domain.com for info)
2634
+ check_http/v2.0.3 (nagios-plugins 2.0.3)
2635
+ MVAClient
2636
+ Link Validity Check From: http://www.w3dir.com/cgi-bin (Using: Hot Links SQL by Mrcgiguy.com)
2637
+ Mojolicious (Perl)
2638
+ mogimogi/1.0
2639
+ Mindjet
2640
+ Mindjet MindManager
2641
+ Microsoft Windows Network Diagnostics
2642
+ Microsoft Office Excel 2013
2643
+ Microsoft Office Excel 2014
2644
+ Microsoft Office Excel 2014 (16.0.6528) Windows NT 10.0
2645
+ Microsoft Office Existence Discovery
2646
+ Microsoft Office Mobile/15.0
2647
+ Microsoft Office OneNote 2013
2648
+ Microsoft Office PowerPoint 2013
2649
+ Microsoft Office PowerPoint 2013 (15.0.4420) Windows NT 6.2
2650
+ Microsoft Office PowerPoint 2013 (15.0.4641) Windows NT 6.2
2651
+ Microsoft Office PowerPoint 2013 (15.0.4797) Windows NT 10.0
2652
+ Microsoft Office PowerPoint 2013 (15.0.4797) Windows NT 6.2
2653
+ Microsoft Office PowerPoint 2013 (15.0.4805) Windows NT 6.1
2654
+ Microsoft Office PowerPoint 2013 (15.0.4805) Windows NT 6.2
2655
+ Microsoft Office PowerPoint 2013 (15.0.4815) Windows NT 10.0
2656
+ Microsoft Office PowerPoint 2014
2657
+ Microsoft Office Word 2013
2658
+ Microsoft Office Word 2013 (15.0.4623) Windows NT 6.2
2659
+ Microsoft Office Word 2013 (15.0.4701) Windows NT 6.1
2660
+ Microsoft Office Word 2013 (15.0.4797) Windows NT 10.0
2661
+ Microsoft Office Word 2013 (15.0.4805) Windows NT 10.0
2662
+ Microsoft Office Word 2013 (15.0.4805) Windows NT 6.1
2663
+ Microsoft Office Word 2013 (15.0.4823) Windows NT 10.0
2664
+ Microsoft Office Word 2014
2665
+ Microsoft Office/16.0 (Windows NT 10.0; Microsoft Excel 16.0.13231; Pro)
2666
+ Melvil Rawi/1.0
2667
+ marketinggrader.com
2668
+ Mandrill-Webhook/1.0
2669
+ MailChimp.com
2670
+ MagpieRSS/0.72 (+http://magpierss.sf.net)
2671
+ http://www.mabontland.com
2672
+ lwp-trivial/1.33
2673
+ lwp-trivial/1.35
2674
+ lwp-trivial/1.36
2675
+ lwp-trivial/1.38
2676
+ lwp-trivial/1.41
2677
+ looksystems.net
2678
+ LinkExaminer/1.01 (Windows)
2679
+ VLC/2.2.4 LibVLC/2.2.4
2680
+ hello@knows.is
2681
+ Mozilla/5.0 (compatible; Jobrapido/1.1; +http://www.jobrapido.com)
2682
+ JoBo/1.x (http://www.matuschek.net/jobo.html)
2683
+ Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0
2684
+ Mozilla/5.0 (compatible; IODC-Internet Engineering Odysseus 001; +http://iodc.co.uk)
2685
+ Mozilla/5.0 (compatible; IODC-Internet Engineering Odysseus 005; +http://iodc.co.uk)
2686
+ Mozilla/5.0 (compatible; IODC-Internet Engineering Odysseus 006; +http://iodc.co.uk)
2687
+ Mozilla/5.0 (compatible; IODC-Internet Engineering Odysseus 007; +http://iodc.co.uk)
2688
+ Mozilla/5.0 (compatible; IODC-Internet Engineering Odysseus 011; +http://iodc.co.uk)
2689
+ Mozilla/5.0 (compatible; IODC-Internet Engineering Odysseus 013; +http://iodc.co.uk)
2690
+ Mozilla/5.0 (compatible; IODC-Odysseus Survey 29691-100-070315021100-72; +http://iodc.co.uk)
2691
+ Mozilla/5.0 (compatible; IODC-Odysseus Survey 29692-100-060415034231-192; +http://iodc.co.uk)
2692
+ Mozilla/5.0 (compatible; IODC-Odysseus Survey 31193-100-080615161452-372; +http://iodc.co.uk)
2693
+ Mozilla/5.0 (compatible; IODC-Odysseus Survey 37044-100-110516191742-025; +https://iodc.co.uk)
2694
+ Mozilla/5.0 (compatible; IODC-Odysseus Survey 41437-100-090715214314-122; +http://iodc.co.uk)
2695
+ Integrity/6
2696
+ InfoWizards Reciprocal Link System PRO - (http://www.infowizards.com)
2697
+ InDesign%20CC/11300 CFNetwork/720.4.4 Darwin/14.4.0 (x86_64)
2698
+ inbound.li parser
2699
+ InAGist URL Resolver (http://inagist.com)
2700
+ igdeSpyder (compatible; igde.ru; +http://igde.ru/doc/tech.html)
2701
+ IDG Twitter Links Resolver
2702
+ Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)
2703
+ Jersey/2.19 (HttpUrlConnection 1.7.0_85)
2704
+ httpunit/1.x
2705
+ httpssites_power/1 CFNetwork/758.3.15 Darwin/15.4.0
2706
+ httpscheck (unknown version) CFNetwork/760.5.1 Darwin/15.5.0 (x86_64)
2707
+ HttpComponents/1.1
2708
+ http_requester/0.1
2709
+ HTTP_Request2/@package_version@ (http://pear.php.net/package/http_request2) PHP/5.5.9-1ubuntu4.14
2710
+ HTTP_Request2/2.2.0 (http://pear.php.net/package/http_request2) PHP/5.3.29
2711
+ HTTP_Request2/2.2.1 (http://pear.php.net/package/http_request2) PHP/5.3.29
2712
+ HTTP_Request2/2.2.1 (http://pear.php.net/package/http_request2) PHP/5.5.33
2713
+ HTTP_Request2/2.2.1 (http://pear.php.net/package/http_request2) PHP/5.5.35
2714
+ HTTP_Request2/2.3.0 (http://pear.php.net/package/http_request2) PHP/5.3.29
2715
+ HTTP-Tiny/0.054
2716
+ http-kit/2.0
2717
+ HTMLParser/2.0
2718
+ htdig/3.1.5
2719
+ htdig/3.1.5 (infosys@storm.rmi.org)
2720
+ htdig/3.1.5 (root@localhost)
2721
+ htdig/3.1.5 (webmaster@online-medien.de)
2722
+ htdig/3.1.6 (mathieu.peltier@inrialpes.fr)
2723
+ htdig/3.1.6 (unconfigured@htdig.searchengine.maintainer)
2724
+ ht://check/2.0.0-rc1 (Linux 2.6.32-573.18.1.el6.x86_64 x86_64)
2725
+ HEADMasterSEO
2726
+ Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)
2727
+ Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)
2728
+ Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)
2729
+ Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)
2730
+ Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)
2731
+ Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)
2732
+ Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)
2733
+ Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)
2734
+ Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)
2735
+ Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)
2736
+ Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)
2737
+ Mozilla/4.0 (compatible; grub-client-2.3)
2738
+ Mozilla/4.0 (compatible; grub-client-2.6.0)
2739
+ Mozilla/4.0 (compatible; grub-client-2.6.1)
2740
+ GoSpotCheck 1.0.6 (iPhone; iPhone OS 5.1.1; en_US)
2741
+ GoScraper
2742
+ GoogleProducer; (+http://goo.gl/7y4SX)
2743
+ GoogleHC/
2744
+ Google-HTTP-Java-Client/1.17.0-rc (gzip)
2745
+ Google-HTTP-Java-Client/1.9.0-alpha
2746
+ Mozilla/5.0 (compatible; Google-Apps-Script)
2747
+ Go-http-client/1.1
2748
+ Go 1.1 package http
2749
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_6) AppleWebKit/534.24 (KHTML, like Gecko) (Contact: backend@getprismatic.com)
2750
+ Mozilla/5.0 (compatible; Funnelback) RPT-HTTPClient/0.3-3E
2751
+ Mozilla/5.0 (compatible; Snacktory; +https://flynxapp.com)
2752
+ Mozilla/3.0 (compatible; Fluffy the spider; http://www.searchhippo.com/; info@searchhippo.com)
2753
+ Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)
2754
+ Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Fetch API Request
2755
+ FeedChecker-Zocle/1.0 (+https://zocle.com/zoclechecker)
2756
+ FeedBurner/1.0 (http://www.FeedBurner.com)
2757
+ FavOrg
2758
+ Faraday v0.9.1
2759
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Excel/14.10.0
2760
+ Mozilla/5.0 (Macintosh; Intel Mac OS X) Excel/14.62.0
2761
+ ExactSearch
2762
+ EventMachine HttpClient
2763
+ Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)
2764
+ Embed PHP Library
2765
+ ECCP/1.2.1+(productlists@eniro.com)
2766
+ ec2linkfinder
2767
+ Mozilla/5.0 (compatible; EARTHCOM/2.2; +http://enter4u.eu)
2768
+ Drupal (+http://drupal.org/)
2769
+ Dragonfly File Reader
2770
+ DNS-Tools Header-Analyzer
2771
+ Dispatch/0.11.1-SNAPSHOT
2772
+ luis@cybo.com
2773
+ CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)
2774
+ curb
2775
+ Mozilla/4.0 (compatible; cron-job.org; http://cron-job.org/abuse/)
2776
+ cosmos/0.9_(robot@xyleme.com)
2777
+ Mozilla/5.0/Firefox/42.0 (contactbigdatafr at gmail.com)
2778
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727; MAXTHON 2.0); Connect Us: webauth@cmcm.com
2779
+ Pattern/2.6 +http://www.clips.ua.ac.be/pattern
2780
+ Mozilla/4.0 (compatible; Cerberian Drtrs Version-3.2-Build-0)
2781
+ Mozilla/4.0 (compatible; Cerberian Drtrs Version-3.2-Build-1)
2782
+ CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper
2783
+ cb crawl (+http://www.companybooknetworking.com)
2784
+ CapsuleChecker (http://www.capsulink.com/)
2785
+ CakePHP
2786
+ Branch-Passthrough
2787
+ Branch Metrics API
2788
+ boitho.com-dc/0.79 ( http://www.boitho.com/dcbot.html )
2789
+ boitho.com-dc/0.81 ( http://www.boitho.com/dcbot.html )
2790
+ boitho.com-dc/0.82 ( http://www.boitho.com/dcbot.html )
2791
+ boitho.com-dc/0.83 ( http://www.boitho.com/dcbot.html )
2792
+ boitho.com-dc/0.85 ( http://www.boitho.com/dcbot.html )
2793
+ boitho.com-robot/1.1
2794
+ Blackboard Safeassign
2795
+ Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.152 Safari/537.36 (BingLocalSearch)
2796
+ BIGLOTRON (Beta 2;GNU/Linux)
2797
+ Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)
2798
+ Mozilla/4.0 (compatible; B-l-i-t-z-B-O-T)
2799
+ Astute SRM
2800
+ aria2/1.18.8
2801
+ Mozilla/5.0 (compatible; archive-com.com/1.1; +http://archive-com.com/bot)
2802
+ Mozilla/4.0 (compatible; Arachmo)
2803
+ Apache-HttpClient/4.2.3 (java 1.5)
2804
+ Apache-HttpClient/4.2.5 (java 1.5)
2805
+ Apache-HttpClient/4.3.1 (java 1.5)
2806
+ Apache-HttpClient/4.3.3 (java 1.5)
2807
+ Apache-HttpClient/4.3.5 (java 1.5)
2808
+ Apache-HttpClient/4.5 (Java/1.8.0_51)
2809
+ Apache-HttpClient/4.5 (Java/1.8.0_72)
2810
+ Apache-HttpClient/4.5 (Java/1.8.0_74)
2811
+ Apache-HttpClient/4.5.1 (Java/1.7.0_79)
2812
+ Apache-HttpClient/4.5.1 (Java/1.8.0_40)
2813
+ Apache-HttpClient/4.5.2 (Java/1.8.0_72)
2814
+ Apache-HttpClient/UNAVAILABLE (java 1.4)
2815
+ Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.22; +http://software.schmorp.de/pkg/AnyEvent)
2816
+ Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.23; +http://software.schmorp.de/pkg/AnyEvent)
2817
+ Anemone/0.7.2
2818
+ AndroidDownloadManager/6.0.1 (Linux; U; Android 6.0.1; SM-G900F Build/MMB29M)
2819
+ AndroidDownloadManager/4.4.2 (Linux; U; Android 4.4.2; SM-T310 Build/KOT49H)
2820
+ AndroidDownloadManager/4.4.2 (Linux; U; Android 4.4.2; SM-T210 Build/KOT49H)
2821
+ Airmail 1.3.3 rv:237 (Macintosh; Mac OS X 10.9.3; sk_SK)
2822
+ Python/3.5 aiohttp/0.21.6
2823
+ AHC/1.0
2824
+ Mozilla/4.0 (compatible; Advanced Email Extractor v2.61)
2825
+ Pulp/1.5.2 (iPad; http://www.acrylicapps.com/pulp/)
2826
+ Accoona-AI-Agent/1.1.1 (crawler at accoona dot com)
2827
+ Accoona-AI-Agent/1.1.2
2828
+ Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com)
2829
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) shipra.goel@a3logics.in
2830
+ scrutiny/6
2831
+ RMA/1.0 (compatible; RealMedia)
2832
+ PHP/5.5
2833
+ NING/1.0
2834
+ Mget/0.1.9
2835
+ Java/1.4.1_04
2836
+ Java/1.6.0
2837
+ Java/1.6.0_04
2838
+ Java/1.6.0_22
2839
+ Java/1.6.0_24
2840
+ Java/1.6.0_26
2841
+ Java/1.6.0_31
2842
+ Java/1.6.0_32
2843
+ Java/1.6.0_34
2844
+ Java/1.6.0_37
2845
+ Java/1.7.0_101
2846
+ Java/1.7.0_45
2847
+ Java/1.7.0_51
2848
+ Java/1.7.0_65
2849
+ Java/1.7.0_80
2850
+ Java/1.8.0_05
2851
+ Java/1.8.0_20
2852
+ Java/1.8.0_31
2853
+ Java/1.8.0_40
2854
+ Java/1.8.0_45
2855
+ Java/1.8.0_51
2856
+ Java/1.8.0_60
2857
+ Java/1.8.0_65
2858
+ Java/1.8.0_66
2859
+ Java/1.8.0_73
2860
+ Java/1.8.0_77
2861
+ Java/1.8.0_91
2862
+ Java/1.8.0_92
2863
+ Goose/1.0.25
2864
+ FDM 3.x
2865
+ bluefish 0.6 HTML editor
2866
+ 2ip.ru CMS Detector (http://2ip.ru/cms/)
2867
+ masscan/1.0 (https://github.com/robertdavidgraham/masscan)
2868
+ VidibleScraper/1.0
2869
+ WhereGoes? Redirect Tracer/1.0
2870
+ unfurlist (https://github.com/Doist/unfurlist)
2871
+ pshtt, https scanning
2872
+ gofetch
2873
+ COMODO DCV
2874
+ CATExplorador/1.0beta (sistemes at domini dot cat; http://domini.cat/catexplorador.html)
2875
+ ZmEu
2876
+ RSSOwl/2.2.1.201312301314 (Windows; U; en)
2877
+ WebClient/1.0
2878
+ Microsoft-WebDAV-MiniRedir/10.0.10586
2879
+ Microsoft-WebDAV-MiniRedir/10.0.14393
2880
+ Microsoft-WebDAV-MiniRedir/5.1.2600
2881
+ Microsoft-WebDAV-MiniRedir/6.0.6001
2882
+ Microsoft-WebDAV-MiniRedir/6.1.7601
2883
+ Microsoft-WebDAV-MiniRedir/6.3.9600
2884
+ Microsoft Data Access Internet Publishing Provider Protocol Discovery
2885
+ Test Certificate Info
2886
+ Needle/1.4.2 (Node.js v6.9.2; linux x64)
2887
+ DavClnt
2888
+ Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 mixed-content-scan
2889
+ Mozilla/5.0 (compatible; Semiocast HTTP client; http://semiocast.com/)
2890
+ Mozilla/5.0 (compatible; The Drop Reaper; +http://tag1consulting.com/dropreaper.html)
2891
+ /n software IPWorks HTTP/S Component - www.nsoftware.com
2892
+ safe-agent-scanner
2893
+ Mozilla/5.0 (compatible; ACHE/Unknown Version; +https://github.com/ViDA-NYU/ache; )
2894
+ ANAM (github.com/dutchcoders/anam)
2895
+ Dolphin http client/11.2.8(385) (Android)
2896
+ Dolphin http client/11.3.4(405) (Android)
2897
+ Calypso v/0.01
2898
+ -Mozilla/4.0 (compatible; http://search.thunderstone.com/texis/websearch/about.html)
2899
+ Mozilla/5.0 (compatible; Tweezler/2.0; http://tweezler.com)
2900
+ Jeode/1.x.x
2901
+ KeyCDN Perf Test
2902
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) YLT Chrome/27.0.1453.110 Safari/537.36
2903
+ Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36 DMBrowser-UV
2904
+ Taringa UGC www.taringa.net/shared-link.php
2905
+ Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) PingdomTMS/0.8.5 Safari/534.34
2906
+ ganarvisitas/1.0 (+http://www.ganarvisitas.com/)
2907
+ FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)
2908
+ Mozilla/5.0 (compatible) Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)
2909
+ Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)
2910
+ Mozilla/5.0 PhantomJS (compatible; Seznam screenshot-generator 2.1; +http://fulltext.sblog.cz/screenshot/)
2911
+ Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)
2912
+ Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)
2913
+ Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://fulltext.sblog.cz/)
2914
+ Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://fulltext.sblog.cz/)
2915
+ Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)
2916
+ Seznam-Zbozi-robot/3.2.1
2917
+ SeznamBot/2.0 (+http://fulltext.sblog.cz/robot/)
2918
+ Seznam-Zbozi-robot/3.2.2
2919
+ Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://fulltext.sblog.cz/)
2920
+ Mozilla/5.0 (compatible; SeznamBot/3.2-test3; +http://fulltext.sblog.cz/)
2921
+ Seznam-Zbozi-robot/3.3
2922
+ SklikBot/2.0 (sklik@firma.seznam.cz;+http://napoveda.sklik.cz/)
2923
+ SeznamBot/3.0 (+http://fulltext.sblog.cz/)
2924
+ Mozilla/5.0 (compatible; SeznamBot/3.1-test1; +http://fulltext.sblog.cz/)
2925
+ Mozilla/5.0 (Linux; U; Android 4.1.2; cs-cz; Seznam screenshot-generator Build/Q3) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30
2926
+ SeznamBot/3.0-test (+http://fulltext.sblog.cz/), I
2927
+ SeznamBot/3.0 (HaF+http://fulltext.sblog.cz/)
2928
+ SeznamBot/3.0-test (+http://fulltext.sblog.cz/)
2929
+ SeznamBot/3.0-beta (+http://fulltext.sblog.cz/), I
2930
+ SeznamBot/3.0-beta (+http://fulltext.sblog.cz/)
2931
+ SeznamBot/3.0-alpha (+http://fulltext.sblog.cz/)
2932
+ SeznamBot/2.0 (+http://fulltext.seznam.cz/)
2933
+ SeznamBot/2.0-Test (+http://fulltext.sblog.cz/robot/)
2934
+ Mozilla/5.0 (compatible; Seznam screenshot-generator 2.0; +http://fulltext.sblog.cz/screenshot/)
2935
+ Mozilla/5.0 (compatible; FlipboardRSS/1.2; +http://flipboard.com/browserproxy)
2936
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)
2937
+ Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)
2938
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)
2939
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/23.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-nikon3)
2940
+ Mozilla/5.0 (compatible; SemrushBot/1.2~bl; +http://www.semrush.com/bot.html)
2941
+ Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots mtmon01it.mtrs.yandex.ru)
2942
+ Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexBot/3.0; +http://yandex.com/bots)
2943
+ Mozilla/5.0 (compatible; memoryBot/1.24.61 +http://internetmemory.org/en/)
2944
+ Mozilla/5.0 (compatible; memoryBot/1.24.54 +http://internetmemory.org/en/)
2945
+ BUbiNG - Research at Brno University of Technology - KNOT group - http://knot.fit.vutbr.cz/crawling/ - Stop? http://law.di.unimi.it/BUbiNG.html
2946
+ Mozilla/5.0 (compatible; MSIE 9.0; Firefox/50.1) Daum/4.1
2947
+ Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/534.34 (KHTML, like Gecko) PhantomJS/1.9.7 Daum Raam Edition b20140820 Safari/534.34
2948
+ Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
2949
+ yacybot (/global; amd64 Linux 4.4.0-59-generic; java 1.8.0_121; Europe/de) http://yacy.net/bot.html
2950
+ yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html
2951
+ yacybot (/global; amd64 Windows 8.1 6.3; java 1.8.0_111; Europe/de) http://yacy.net/bot.html
2952
+ yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_111; Europe/en) http://yacy.net/bot.html
2953
+ yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html
2954
+ yacybot (webportal-global; amd64 Linux 4.4.0-46-generic; java 1.8.0_91; Europe/en) http://yacy.net/bot.html
2955
+ yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html
2956
+ Mozilla/5.0 (compatible; special_archiver/3.3.0 bot@archive.org +https://archive.org/details/archive.org_bot)
2957
+ Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)
2958
+ Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)
2959
+ SeobilityBot (SEO-Check; http://bit.ly/1dJuuzs)
2960
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/5.0.4
2961
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/5.0.3
2962
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/5.0.2
2963
+ Testomatobot/1.0 (Linux x86_64; +http://www.testomato.com/testomatobot) minicrawler/5.0.1
2964
+ Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)
2965
+ Heurekabot-Feed/1.0 (+https://sluzby.heureka.cz/napoveda/heurekabot/)
2966
+ Heurekabot-ImageFullText/1.0 (+https://sluzby.heureka.cz/napoveda/heurekabot/)
2967
+ Heurekabot-ImageFullText/1.0 (+http://sluzby.heureka.cz/napoveda/heurekabot/)
2968
+ Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider
2969
+ ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)
2970
+ Mozilla/5.0 (compatible; SOLOFIELD/1.0 +http://solofield.net/bot.html)
2971
+ Qwantify/1.0
2972
+ Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)
2973
+ Scrapy/1.3.0 (+http://scrapy.org)
2974
+ Scrapy/1.0.5.post4+g4b324a8 (+http://scrapy.org)
2975
+ Scrapy/1.2.1 (+http://scrapy.org)
2976
+ Scrapy/0.16.5 (+http://scrapy.org)
2977
+ Mozilla/5.0 (compatible; Dataprovider.com;)
2978
+ Mozilla/5.0 (compatible; Dataprovider; https://www.dataprovider.com/spider/)
2979
+ Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)
2980
+ ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)
2981
+ DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)
2982
+ Mozilla/5.0 (compatible; SMTBot/1.0; http://www.similartech.com/smtbot)
2983
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) Chromium/31.0.1650.63 Site-Shot/2.1 (http://www.site-shot.com/) Safari/538.1
2984
+ Mozilla/5.0 (Windows NT 6.1; WOW64; Neustar WPM) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.82 Safari/537.36
2985
+ Mozilla/5.0 (iPad; CPU OS 6_1_2 like Mac OS X; Neustar WPM) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25
2986
+ Mozilla/5.0 (Windows NT 6.0; Neustar WPM) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.82 Safari/537.36
2987
+ Mozilla/5.0 (Windows NT 6.1; Neustar WPM) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.82 Safari/537.36
2988
+ Mozilla/5.0 (Windows NT 6.0; rv:38.0; Neustar WPM) Gecko/20100101 Firefox/38.0
2989
+ Mozilla/5.0 (Windows NT 6.1; rv:38.0; Neustar WPM) Gecko/20100101 Firefox/38.0
2990
+ Mozilla/5.0 (Windows NT 6.1; Neustar WPM) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36
2991
+ Mozilla/5.0 (Linux; Android 4.1.1; HTC6435LVW Build/JRO03C; Neustar WPM) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.169 Mobile Safari/537.22
2992
+ Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)
2993
+ Mozilla/5.0 (Linux; Android 5.0.2; SM-G920T Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840 Mobile Safari/537.36 DareBoost
2994
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840 Safari/537.36 DareBoost
2995
+ Mozilla/5.0 (Linux; Android 5.0; LG-D855 Build/LRX21R.A1422018487) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840 Mobile Safari/537.36 DareBoost
2996
+ Mozilla/5.0 (iPhone; CPU iPhone OS 10_0_1 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A403 Safari/602.1 DareBoost
2997
+ Mozilla/5.0 (Linux; Android 4.4.3; HTC One Build/KTU84L) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840 Mobile Safari/537.36 DareBoost
2998
+ Mozilla/5.0 (Linux; Android 4.4; Nexus 5 Build/KRT16M) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/54.0.2840 Mobile Safari/537.36 DareBoost
2999
+ HubSpot Website Grader (web-crawlers@hubspot.com)
3000
+ HubSpot Marketing Grader, HubSpot Marketing Grader
3001
+ com.hubspot.spooks.core.UserAgent@753535c0, HubSpot Marketing Grader
3002
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 (http://www.shrinktheweb.com)
3003
+ scrapy-redis (+https://github.com/rolando/scrapy-redis)
3004
+ Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 PTST/348
3005
+ Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36 PTST/345
3006
+ TeamSpeak3-ImageFetcher-1.0
3007
+ wf_crawler (http://www.websitefigures.com)
3008
+ wf_crawler (http://websitefigures.com)
3009
+ Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 TinEye/1.0 (via http://www.tineye.com/)
3010
+ TinEye-bot/0.61 (see http://www.tineye.com/crawler.html)
3011
+ Mozilla/5.0 (compatible; Googlebot/2.1; https://deepcrawl.com/bot)
3012
+ Googlebot deepcrawl
3013
+ Mozilla/5.0 (compatible; evc-batch/2.0.20170110125844)
3014
+ Mozilla/5.0 (compatible; evc-batch/2.0.20161220184139)
3015
+ Mozilla/5.0 (compatible; evc-batch/2.0.20161122223033)
3016
+ Mozilla/5.0 (compatible; evc-batch/2.0.20161026010401)
3017
+ Mozilla/5.0 (X11; Linux x86_64; rv:49.0; GTmetrix https://gtmetrix.com/) Gecko/20100101 Firefox/49.0
3018
+ Mozilla/5.0 (Linux; Android 4.3; Galaxy Nexus Build/JWR66Y; GTmetrix https://gtmetrix.com/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.68 Mobile Safari/537.36
3019
+ Mozilla/5.0 (X11; Linux x86_64; GTmetrix https://gtmetrix.com/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36
3020
+ Mediumbot-MetaTagFetcher/0.3 (+https://medium.com/)
3021
+ Mediumbot-ProxyFetcher/0.1 (+https://medium.com/)
3022
+ YOOBI.de - WebBot 1.0
3023
+ YOOBI - WebBot 1.0
3024
+ Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)
3025
+ IAS crawler (page scorer; http://integralads.com/site-indexing-policy/)
3026
+ Mozilla/5.0 (compatible; JobKeresoBot; +https://www.kozvetlen-allasok.hu/help.jsp; info@kozvetlen-allasok.hu; 5.0
3027
+ Mozilla/5.0 (compatible; JobKereso; +https://www.kozvetlen-allasok.hu/robot.jsp info@kozvetlen-allasok.hu)
3028
+ pyspider/0.3.9 (+http://pyspider.org/)
3029
+ Mozilla/5.0 (X11; U; Linux x86_64; de-DE; rv:1.9.0.19) Gecko/2010120923 ThumbShotsBot (KFSW 3.0.6-3)
3030
+ Mozilla/5.0 (compatible; Discordbot/1.0; +https://discordapp.com)
3031
+ Mozilla/5.0 (compatible; Dead Link Checker; http://www.dead-link-checker.com/)
3032
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0) CrawlerProcess (http://www.PowerMapper.com) /5.24.776.0
3033
+ Mozilla/4.0 (compatible; BOTW Spider; +http://botw.org)
3034
+ TwengaBot-2.0
3035
+ Iframely/1.0.0 (+https://iframely.com/;)
3036
+ Iframely/0.8.5 (+http://iframely.com/;)
3037
+ rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)
3038
+ rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com
3039
+ Mozilla/5.0 (compatible; Sitemap Generator/1.3; http://www.check-domains.com/sitemap/index.php) Gecko Check-domains)
3040
+ mindUpBot
3041
+ Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36
3042
+ Contacts Crawler (+http://www.scrapinghub.com)
3043
+ TA SEO Crawler v0.1 URLData.pm (smcquaker@tripadvisor.com)
3044
+ Mozilla/5.0 (compatible; scanbot/1.0; +http://dazzlepod.com/ip/)
3045
+ ArielisBot/1.1 (+http://arielis.com/
3046
+ RedesScrapy/0.24.1 (+http://g2pi.tsc.uc3m.es/es)
3047
+ Blackboard Safeassign/0.1 (a Storm-based Blackboard Safeassign web-crawler; https://github.com/DigitalPebble/storm-crawler; stormcrawler@digitalpebble.com)
3048
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:48.0) Gecko/20100101 Firefox/48.0 glindahl-cocrawler/0.01 (+http://www.pbm.com/~lindahl/glindahl-cocrawler.html)
3049
+ botrobin/Nutch-1.13-SNAPSHOT (http://smarter.codes/bot-robin/; botrobin@smarter.codes)
3050
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36 (compatible; aylienbot/0.2; +http://www.aylien.com/bot.html)
3051
+ GloomarBot/1.1 (https://www.gloomar.com/bot)
3052
+ Hatena::Bookmark/2.00 (Hatena::Bookmark; master;)
3053
+ Opendi Screenshot Bot bot@opendi.com
3054
+ Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)
3055
+ Mozilla/5.0 (compatible; KAZ.KZ_Bot/3.0)
3056
+ BuckyOHare/1.3 (Googlebot/2.1; +https://hypefactors.com/webcrawler)
3057
+ uipbot/1.0 (uipbot@semasio.net)
3058
+ gMozilla/5.0 (compatible; Infohelfer/1.4.3; +http://www.infohelfer.de/crawler.php)
3059
+ Mozilla/5.0 (compatible; lincobot/0.3; http://www.linksco.com/about/robot.html) AppEngine-Google; (+http://code.google.com/appengine)
3060
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0) MapperCmd/5.24.780.0
3061
+ Seeker v.1 http://LookSeek.com/seeker/
3062
+ AppManager RPT-HTTPClient/0.3-3E
3063
+ LinkAlarm/3.1
3064
+ Mozilla/5.0 (X11; U; Linux i686) Firefox/2.0.0.11 wprecon.com survey
3065
+ Jeffrey's Exif Viewer (http://regex.info/exif)
3066
+ Untiny
3067
+ Mozilla/5.0 (Grabify.link - https://grabify.link/expander)
3068
+ biNu image cacher (info@binu-inc.com)
3069
+ Mozilla/5.0 (compatible; Google-Youtube-Links)
3070
+ ping.blo.gs/2.0
3071
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; QQDownload 1.7; .NET CLR 2.0.50727)
3072
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/4.0; QQDownload 663; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET CLR 1.1.4322)
3073
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; QQDownload 760; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; InfoPath.2; .NET4.0E; .NET4.0C; .NET CLR 1.1.4322)
3074
+ Mozilla/5.0 (X11; Linux x86_64; rv:52.0.2) Gecko/20100101 Firefox/52.0.2 Ghost Inspector
3075
+ Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0 Ghost Inspector
3076
+ Mozilla/5.0 (X11; Linux x86_64; rv:50.1.0) Gecko/20100101 Firefox/50.1.0 Ghost Inspector
3077
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) QtWebEngine/5.3.0 Safari/538.1 Ghost Inspector
3078
+ Mozilla/5.0 (X11; Linux x86_64; rv:45.0.1) Gecko/20100101 Firefox/45.0.1 Ghost Inspector
3079
+ Mozilla/5.0 (X11; Linux x86_64; rv:46.0.1) Gecko/20100101 Firefox/46.0.1 Ghost Inspector
3080
+ Mozilla/5.0 (X11; Linux x86_64; rv:47.0.2) Gecko/20100101 Firefox/47.0.2 Ghost Inspector
3081
+ Mozilla/5.0 (X11; Linux x86_64; rv:48.0.2) Gecko/20100101 Firefox/48.0.2 Ghost Inspector
3082
+ Mozilla/5.0 (X11; Linux x86_64; rv:49.0.2) Gecko/20100101 Firefox/49.0.2 Ghost Inspector
3083
+ Netpursual/1.0
3084
+ Kaspersky Lab CFR link resolver cfradmins@kaspersky.com
3085
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Safari/537.36 (compatible; Google-HotelAdsVerifier/2.0)
3086
+ b0t
3087
+ "echocrawl 2.0"
3088
+ masscan/1.0
3089
+ Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexBot/3.0; +http://yandex.com/bots)
3090
+ Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots)
3091
+ Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)
3092
+ Mozilla/5.0 (compatible; YandexDirectDyn/1.0; +http://yandex.com/bots
3093
+ Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)
3094
+ Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)
3095
+ Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)
3096
+ Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)
3097
+ Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)
3098
+ Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)
3099
+ Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)
3100
+ Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)
3101
+ Mozilla/5.0 (compatible; YaDirectFetcher/1.0; Dyatel; +http://yandex.com/bots)
3102
+ Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)
3103
+ Mozilla/5.0 (compatible; YandexSitelinks; Dyatel; +http://yandex.com/bots)
3104
+ Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)
3105
+ Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)
3106
+ Mozilla/5.0 (compatible; YandexVertis/3.0; +http://yandex.com/bots)
3107
+ Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)
3108
+ jetmon/1.0 (Jetpack Site Uptime Monitor by WordPress.com)
3109
+ Mozilla/5.0 (compatible; Abonti/0.8 - http://www.abonti.com)
3110
+ CloudEndure Scanner (ops@cloudendure.com)
3111
+ AFB/3.0 (+http://allloadin.com)
3112
+ TLSProbe/1.0 (+https://scan.trustnet.venafi.com/)
3113
+ Sens.ai http://sovereign.ai/contact
3114
+ eContext/1.0 (eContext Classification Engine)
3115
+ Owler
3116
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Slack/1.2.6 Chrome/45.0.2454.85 AtomShell/0.34.3 Safari/537.36 Slack_SSB/1.2.6
3117
+ Mozilla/5.0 (compatible; ChangesMeter/1.9.1; http://intuiware.com/apps/changes-meter)
3118
+ Mozilla/5.0 (compatible; Gookey.co/1.0; +http://gookey.co/)
3119
+ TrapitAgent/0.1 (feed processor; +http://trapit.com/about)
3120
+ sqlmap/1.0-dev-nongit-201612050a8c (http://sqlmap.org)
3121
+ UniversalFeedParser/3.3 +http://feedparser.org/
3122
+ TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8
3123
+ scalaj-http/1.0
3124
+ HggH PhantomJS Screenshoter
3125
+ fasthttp
3126
+ fasthttp, Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36
3127
+ lua-resty-http/0.10 (Lua) ngx_lua/10000
3128
+ GIS-LABS:CertMon
3129
+ GuzzleHttp/6.2.1 PHP/7.0.15-0ubuntu0.16.04.4
3130
+ http-request/v0.7.0 (http://git.io/tl_S2w) node.js/v0.10.29
3131
+ intraVnews/1.x
3132
+ MailChimp
3133
+ G-i-g-a-b-o-t
3134
+ Symfony BrowserKit
3135
+ VCPP71 http-get 1.0a
3136
+ Jaunt/1.2
3137
+ Mozilla/5.0 (compatible; Owler/0.4; +; )
3138
+ MovableType/x.x
3139
+ OMSC
3140
+ websitepulse agent/0.1 (compatible; MSIE 5.5; Netscape 4.75; Linux)
3141
+ Mozilla/5.0 (compatible; Uptimia; www.uptimia.com)
3142
+ ssl-tools.net
3143
+ Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0; .NET CLR 1.1.4322; .NET CLR 1.0.3705; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729) DMBrowser/2.1
3144
+ keycdn-tools/perf
3145
+ keycdn-tools/br
3146
+ keycdn-tools/h2
3147
+ keycdn-tools/perf
3148
+ keycdn-tools/br
3149
+ keycdn-tools/h2
3150
+ KOCMOHABT (+https://kozmonavt.tk/) Mozilla/5.0 (compatible; Web Explorer 2)
3151
+ Donuts Content Explorer (www.donuts.domains)
3152
+ Netsparker
3153
+ Mozilla/5.0+(compatible; MxToolbox/Beta7; http://www.mxtoolbox.com/)
3154
+ Apache-HttpAsyncClient/4.1.3 (Java/1.8.0_121)
3155
+ Tenon.io Automated Web Accessibility Tester
3156
+ ApacheBench/2.3
3157
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Clarsentia)
3158
+ Mozilla/5.0 (compatible; historious/2.0; +http://historio.us/)
3159
+ Mozilla/5.0 (compatible; uCrawlr/1.0 ; +https://blog.ucoz.ru/upolicy)
3160
+ Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)
3161
+ Brodie/1.0 (+http://www.15miles.com/)
3162
+ Jobg8 URL Monitor
3163
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.21 (KHTML, like Gecko) wpif Safari/537.21
3164
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.21 (KHTML, like Gecko) imgsizer Safari/537.21
3165
+ Mozilla/5.0 (compatible; AppBeat/1.0; +http://www.appbeat.io/)
3166
+ Mozilla/5.0 (compatible; Chirp/1.0; +http://www.binarycanary.com)
3167
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SiteMonitor)
3168
+ Mixnode/3.0.1 (+https://www.mixnode.com/; Speed analysis; no data is stored)
3169
+ Pingability.com Monitoring Service
3170
+ SiteSucker for macOS/2.10.6
3171
+ Mozilla/5.0 (compatible; PRTG Network Monitor (www.paessler.com); Windows)
3172
+ Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)
3173
+ Sideqik +http://www.sideqik.com
3174
+ SiteSucker/2.4.6
3175
+ Vulnbusters Meter (see http://vulnbusters.com for details)
3176
+ Express WebPictures (www.express-soft.com)
3177
+ gobyus
3178
+ Search37/1.0 (http://www.search37.com; info@search37.com)
3179
+ Search37/1.2 (http://www.search37.com; info@search37.com)
3180
+ Mozilla/5.0 (compatible; SiteIndexed/1.2; +http://www.siteindexed.com)
3181
+ Jersey/2.25.1 (Apache HttpClient 4.5)
3182
+ URL Verifier by Daanav.com
3183
+ trendspottr.com
3184
+ Kerrigan/2.0 (kerrigan@mashable.com)
3185
+ Keyword Extractor 5000; lucas@hyscore.io
3186
+ Citoid (Wikimedia tool; learn more at https://www.mediawiki.org/wiki/Citoid)
3187
+ grokkit (pdsupport@purediscovery.com)
3188
+ akka-http/10.0.10
3189
+ POE-Component-Client-HTTP/0.943 (perl; N; POE; en; rv:0.943000)
3190
+ RecurPost/1.0 https://recurpost.com (KHTML, like Gecko)
3191
+ Mozilla/5.0 Firefox Windows WhoRunsCoinHive.com
3192
+ Mozilla/5.0 (compatible; Kelny/2.0; +http://www.kelny.com/)
3193
+ colly - https://github.com/gocolly/colly
3194
+ WebCopier vx.x
3195
+ gSOAP/2.8
3196
+ http.rb/2.2.2
3197
+ Grabber - https://sites.google.com/3ziele.de/grabber/
3198
+ JobFeed discovery script (http://www.jobfeed.com)
3199
+ JoinVision Generic Site Loader 1.0
3200
+ SSI-LinkPreviewGenerator/1.0
3201
+ SnowHaze Search/1.0 support@snowhaze.com
3202
+ Saleslift
3203
+ Perlu - HTML Analyzer and Relationship Builder 1.1
3204
+ Offline Explorer 1.*
3205
+ Mozilla/5.0 (compatible; Wappalyzer; https://www.wappalyzer.com)
3206
+ AngleSharp/0.9.9
3207
+ Email%20Extractor%20Lite/26 CFNetwork/720.3.13 Darwin/14.3.0 (x86_64)
3208
+ HubSpot Page Screenshot Service
3209
+ NewRelicPinger/1.0 (543245)
3210
+ APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)
3211
+ updown.io daemon 2.2
3212
+ updown.io daemon 2.3
3213
+ updown.io daemon 2.2
3214
+ updown.io daemon 2.3
3215
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36 PrintFriendly.com
3216
+ Unshorten.It!/1.0 (https://unshorten.it/)
3217
+ github-camo (65c9ebe0)
3218
+ github-camo (abb93c82)
3219
+ github-camo (bacd3d5b)
3220
+ github-camo (df02412a)
3221
+ Jobsearch1.5
3222
+ HubSpot Content Strategy link check
3223
+ Twisted PageGetter
3224
+ node-urllib/2.25.3 Node.js/6.12.2 (Linux 4.4; x64)
3225
+ sentry/8.18.0 (https://sentry.io)
3226
+ Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 (compatible; Sitebulb/1.1; +https://sitebulb.com)
3227
+ Thinklab (thinklab.com)
3228
+ HubSpot inbound link reporting check
3229
+ SetCronJob/1.0 (+http://www.setcronjob.com/)
3230
+ Burf Search Engine
3231
+ Apache Droid
3232
+ Mozilla/5.0 (compatible; adscanner/)
3233
+ hackney/1.11.0
3234
+ Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)
3235
+ Mozilla/5.0 (compatible; dotMailer content retrieval service)
3236
+ Nexgate Ruby Client
3237
+ Miniature.io/2.0 (Linux; en-us)
3238
+ help@dataminr.com
3239
+ Mozilla/5.0 (compatible; Your-Website-Sucks/2.0; +http://www.Your-Website-Sucks.net/WWW.SOMEWEBSITE.COM)
3240
+ axios/0.17.1
3241
+ Wtrace
3242
+ PostmanRuntime/7.1.1
3243
+ hackney/1.11.0
3244
+ Lavf/57.25.100
3245
+ makecontact/1.0 (+https://makecontact.io)
3246
+ QrafterPro/4600 CFNetwork/889.9 Darwin/17.2.0
3247
+ Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)
3248
+ Go http package
3249
+ HonesoSearchEngine/1.0
3250
+ orion-semantics.com 0.1
3251
+ ExperianCrawlUK (andrew dot swanton at phgroup dot com)
3252
+ Trendsmap Resolver/0.0.1
3253
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FunWebProducts; .NET CLR 1.1.4322; PeoplePal 6.2)
3254
+ Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent
3255
+ Mozilla/3.0 (compatible; Indy Library)
3256
+ VB Project
3257
+ webster-internet.de pad browser
3258
+ LWP::Simple/5.805
3259
+ Microsoft Data Access Internet Publishing Provider DAV
3260
+ MSFrontPage/4.0
3261
+ Microsoft Data Access Internet Publishing Provider Cache Manager
3262
+ VB Project
3263
+ eCatch/3.0
3264
+ MSFrontPage/12.0
3265
+ Mozilla/4.7 (compatible; OffByOne; Windows 2000) Webster Pro V3.4
3266
+ Mozilla/4.7 (compatible; OffByOne; Windows 98) Webster Pro V3.2
3267
+ webster-internet.de pad browser
3268
+ Trendsmap Resolver/0.0.1
3269
+ Mozilla/4.0 (compatible; MSIE 6.0; MS Web Services Client Protocol 4.0.30319.17929)
3270
+ Arukereso/1.0
3271
+ Trendsmap Resolver/0.0.1
3272
+ Flock/14.15 (Android 2.9; fr_BE;)
3273
+ Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.0.16) Gecko/2010021003 Firefox/3.0.16 Flock/2.5.6
3274
+ Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.8.1.14) Gecko/20080414 Firefox/2.0.0.14 Flock/1.1.2
3275
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070801 Firefox/2.0.0.6 Flock/0.9.0.2
3276
+ AppStoreScraperZ/1 CFNetwork/889.9 Darwin/17.2.0 (x86_64)
3277
+ Everwall Link Expander ( http://everwall.com )
3278
+ Wfuzz/2.2.2
3279
+ The Knowledge AI the.knowledge.ai@gmail.com
3280
+ The Knowledge AI
3281
+ Mozilla/5.0 (compatible; DataXu/1.0; +http://dataxu.com)
3282
+ Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 (+https://whatis.contentkingapp.com)
3283
+ AdAuth/1.0; +https://www.adauth.com
3284
+ twibble.io
3285
+ JolokiaPwn, https://matmannion.com/jolokiapwn/, mat.mannion@gmail.com
3286
+ CookieReports.com/0.1
3287
+ GoogleCloudMonitoring
3288
+ kube-probe/1.9+
3289
+ Mozilla/5.0 Chrome/10.0.613.0 Safari/534.15 Zombie.js/5.0.7
3290
+ Microsoft Office/16.0 (Microsoft Outlook Mail 16.0.7167; Pro)Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Win64; x64; Trident/7.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; Tablet PC 2.0; InfoPath.3; Microsoft Outlook 14.0.6025; ms-office; MSOf
3291
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Win64; x64; Trident/7.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3; .NET4.0E; Microsoft Outlook 14.0.7167; ms-office; MSOffice 14)
3292
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; Microsoft Outlook 15.0.4805; ms-office; MSOffice 15)
3293
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; Tablet PC 2.0; MALNJS; Microsoft Outlook 16.0.6001; ms-office; MSOffice 16)
3294
+ Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; Microsoft Outlook 14.0.6025)
3295
+ Outlook-Express/7.0 (MSIE 7.0; Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; McAfee; .NET4.0C; .NET4.0E; GWX:QUALIFIED; TmstmpExt)
3296
+ Mail/3445.4.7 CFNetwork/889.9 Darwin/17.2.0 (x86_64)
3297
+ SPDYCheck SPDY Protocol Tester, see http://spdycheck.org/about.html
3298
+ SpamExperts/128741 (Virus Scan)
3299
+ Unshorten.It! /1.0 (unshorten.it)
3300
+ Monit/5.25.2
3301
+ Mozilla/5.0 (compatible; PingAdmin.Ru/1.1; +http://pingadmin.ru/)
3302
+ Mozilla/5.0 (compatible; localsearch-web/2.0; +http://www.localsearch.ch/en/legal-notice)
3303
+ Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36
3304
+ annotate_google; http://ponderer.org/download/annotate_google.user.js
3305
+ Apache OpenOffice/4.1.2
3306
+ Bidtellect/0.0.346.0
3307
+ BlockNote.Net
3308
+ Bloglines/3.1 (http://www.bloglines.com)
3309
+ DemandbasePublisherAnalyzer/0.1 (http://www.demandbase.com; info@demandbase.com)
3310
+ endo/1.0 (Mac OS X; ppc i386; http://kula.jp/endo)
3311
+ Feedable/0.1 (compatible; MSIE 6.0; Windows NT 5.1)
3312
+ Feedreader 3.xx (Powered by Newsbrain)
3313
+ FeedshowOnline (http://www.feedshow.com)
3314
+ FeedZcollector v1.x (Platinum) http://www.feeds4all.com/feedzcollector
3315
+ ImageVisu/v4.x.x
3316
+ InetURL:/1.0
3317
+ Mixmax-LinkPreview 1.0 (+https://sdk.mixmax.com)
3318
+ Mozilla/5.0 (compatible; DNS SSL/TLS HTTP and HTML scanner/0.1; +https://www.htmlyse.com/)
3319
+ Mozilla/5.0 (compatible; DNS SSL/TLS HTTP HTML Website Security Scanner/0.2 beta; +https://www.htmlyse.com/)
3320
+ Mozilla/5.0 (compatible; Scrutiny/8; +https://peacockmedia.software/mac/scrutiny/
3321
+ NetNewsWire/2.x (Mac OS X; http://ranchero.com/netnewswire/)
3322
+ og-scraper/1.0.0
3323
+ Plagger/0.x.xx (http://plagger.org/)
3324
+ raynette_httprequest/1.0
3325
+ RssReader/1.0.xx.x (http://www.rssreader.com) Microsoft Windows NT 5.1.2600.0
3326
+ ScrapeBox Page Scanner
3327
+ SearchExpress
3328
+ Snarfer/0.x.x (http://www.snarfware.com/)
3329
+ unirest-java/1.3.11
3330
+ WinPodder (http://winpodder.com)
3331
+ WordupinfoSearch/1.0
3332
+ 13TABS/13TABS-1.14 (https://www.13tabs.com/aboutUs.jsp)
3333
+ 777Siters/1.05 (+http://777s.ru/siters/)
3334
+ 7Siters/1.07 (+http://7ooo.ru/siters/)
3335
+ Aranea Web Corpora Project (Sk Crawl) (+http://unesco.uniba.sk/guest)
3336
+ censys.io internet census
3337
+ Chromaxa/1.0 Sonar/1.0
3338
+ compatible; Licorne Uri Checker 1.0; +http://www.jednorozec.cz)
3339
+ CronDroid v0.80a http://www.crondroid.com
3340
+ Cronjob.de
3341
+ Cronless/2.0 (+http://cronless.com)
3342
+ EasyCron/1.0 (https://www.easycron.com/)
3343
+ EDS EXIF/1.0 (https://www.exif-search.com)
3344
+ EDS EXIF/1.0 (https:/exif-search.com)
3345
+ faviconkit/1.0 (https://faviconkit.com)
3346
+ HomePage Rss Reader 1.0
3347
+ Hong Kong Education City/1.0 (built with Hong Kong Education City 1.10; http://www.hkedcity.net/; search@hkedcity.net)
3348
+ https://getroot.sh survey
3349
+ isUp.li Website Monitoring 2
3350
+ JAHHO check
3351
+ MB-SitemapGenerator
3352
+ Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; http://basicstate.com/)
3353
+ Mozilla/5.0 (compatible; Aprc/2.9.15-24; +https://aprc.it/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3528.4 Safari/537.36
3354
+ Mozilla/5.0 (compatible; Google AppsViewer; http://drive.google.com)
3355
+ Mozilla/5.0 (compatible; GoogleSites)
3356
+ Mozilla/5.0 (compatible; iThemes Sync/1.0; +http://ithemes.com/sync)
3357
+ Mozilla/5.0 (compatible; iThemes Sync/1.0; ITX WP-Admin API/1.0; +http://ithemes.com/sync)
3358
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; CHKD 1.2; Akamai_Site_Analyzer)
3359
+ Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WAC-OFU)
3360
+ Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)
3361
+ Mozilla/5.0 (Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.143 Safari/537.36 Nodemeter 0.7
3362
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)
3363
+ Mozilla/5.0 (Windows NT 6.1; rv:12.0; StumbleUpon; noc@stumbleupon.com) Gecko/20100101 Firefox/12.0
3364
+ Mozilla/5.0 (Windows NT 6.1; rv:40.0; StumbleUpon; noc@stumbleupon.com) Gecko/20100101 Firefox/40.1
3365
+ Mozilla/5.0 (Windows NT 6.1; stumbleupon.com) Chrome/58.0.3029.110 Safari/537.36
3366
+ Mozilla/5.0 AppleWebKit (compatible; s4a-probe/1.0; +https://www.seo4ajax.com/webscraper)
3367
+ Mozilla/5.0 AppleWebKit (compatible; s4a/1.0; +https://www.seo4ajax.com/webscraper)
3368
+ Mozilla/5.0 Gecko/20100101 Firefox/51.0.1 SnapSearch
3369
+ NodePing
3370
+ PocketImageCache/1.0
3371
+ queuedriver/1.0 (eng@linqia.com)
3372
+ Re-re Studio (+http://re-re.ru/)
3373
+ Tarmot Gezgin/1.0 (compatible; TarmotGezgin/1.1; +http://www.tarmot.com/gezgin)
3374
+ WDT.io
3375
+ webeval.researchready.com - support@imagineeasy.com for info.
3376
+ WWW::Document/0.02
3377
+ Y!J-BRW/1.0 (https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716)
3378
+ YoYs.net
3379
+ YP.PL
3380
+ Mozilla/5.0 (Windows NT 6.1; rv:40.0; StumbleUpon; noc@stumbleupon.com) Gecko/20100101 Firefox/40.1
3381
+ Mozilla/5.0 (Windows NT 6.1; StumbleUpon; noc@stumbleupon.com) Chrome/58.0.3029.110 Safari/537.36
3382
+ Mozilla/5.0 (Windows NT 6.1; stumbleupon.com) Chrome/58.0.3029.110 Safari/537.36
3383
+ Mozilla/5.0 (compatible; tracemyfile/1.0)
3384
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/66.0.3336.0 Safari/537.36
3385
+ Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 Pix/1.0 (Criteo; x64-criteo)
3386
+ CDR/1.7.1 Simulator/0.7(+http://timewe.net) Profile/MIDP-1.0 Configuration/CLDC-1.0
3387
+ Email%20Extractor/1 CFNetwork/760.6.3 Darwin/15.6.0 (x86_64)
3388
+ Email%20Extractor/29 CFNetwork/897.15 Darwin/17.5.0 (x86_64)
3389
+ HTTP Banner Detection (https://security.ipip.net)
3390
+ HTTP Banner Detection(security.ipip.net)
3391
+ Mozilla/5.0 (compatible; SimpleScraper)
3392
+ http_get
3393
+ MeltwaterNews www.meltwater.com
3394
+ Disqus/1.0
3395
+ Datafeedwatch/2.1.x
3396
+ Zabbix
3397
+ Mozilla/5.0 (compatible; Google Desktop/5.9.1005.12335; http://desktop.google.com/)
3398
+ httpRequest
3399
+ Google-Transparency-Report
3400
+ Mozilla/5.0 (compatible; TrendsmapResolver/0.1)
3401
+ Embarcadero URI Client/1.0
3402
+ GetCode
3403
+ Gregarius/0.5.2 ( http://devlog.gregarius.net/docs/ua)
3404
+ GRequests/0.10
3405
+ DAP/NetHTTP
3406
+ Hadi Agent
3407
+ Canary%20Mail/397 CFNetwork/893.13.1 Darwin/17.4.0 (x86_64)
3408
+ Sendsay.Ru/1.0; https://Sendsay.Ru/; ask@sendsay.ru
3409
+ Mozilla/5.0 (Zoom.Mac 10.8.5 x86)
3410
+ Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36 TryJsoup/1.0 (+http://try.jsoup.org/)
3411
+ Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0 ; BacklinkHttpStatus)
3412
+ SilverReader/1.0; http://silverreader.com
3413
+ ExtractorPro
3414
+ WebsiteExtractor
3415
+ Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1 google_partner_monitoring FWSzVTDDBz14547302713138T
3416
+ Outlook-iOS/696.1188109.prod.iphone (2.102.0)
3417
+ Polymail/1.47 (ctrlla.Polymail; build:64; OS X 10.12.6) Alamofire/4.5.1
3418
+ nyawc/1.8.1 CPython/3.6.5 Linux/3.10.0-862.9.1.el7.x86_64
3419
+ commonscan.org cralwer v1.01
3420
+ AdminLabs
3421
+ WebGazer/1.0 (+https://www.webgazer.io)
3422
+ HappyApps-WebCheck/1.0
3423
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6; +feeder.co) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36
3424
+ Mozilla/5.0 (compatible; TestURI; +http://testuri.org/)
3425
+ WebSniffer/1.0 (+http://websniffer.cc/)
3426
+ Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 SeoSiteCheckup (https://seositecheckup.co
3427
+ Adstxtaggregator.com/1.0
3428
+ Discourse Forum Onebox v2.0.0.beta10
3429
+ faviconarchive.org
3430
+ GreatNews/1.0
3431
+ MemGator:1.0-rc2 <@WebSciDL>
3432
+ URLTester/1 CFNetwork/974.1 Darwin/18.0.0 (x86_64)
3433
+ URLTester/1 CFNetwork/975.0.3 Darwin/18.2.0 (x86_64)
3434
+ Urlcheckr/2.0
3435
+ SimpleChecker
3436
+ reqwest/0.9.9
3437
+ ReactorNetty/0.7.10.RELEASE
3438
+ Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)
3439
+ https://www.nominet.uk/privacy-notice
3440
+ <a href='http://www.unchaos.com/'> UnChaos </a> From Chaos To Order Hybrid Web Search Engine.(vadim_gonchar@unchaos.com)
3441
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse
3442
+ Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/61.0.3116.0 Mobile Safari/537.36 Chrome-Lighthouse
3443
+ Trackuity (+http://trackuity.com)
3444
+ deeris/1.0 (+http://www.deeris.com)
3445
+ Mediametric
3446
+ TextRazor Downloader (https://www.textrazor.com)
3447
+ ddline.cn rank history
3448
+ Web spyder
3449
+ AppleSyndication/52.8
3450
+ AutomaticWPTester/1 CFNetwork/975.0.3 Darwin/17.7.0
3451
+ CSSCheck/1.2.2
3452
+ dBpoweramp
3453
+ Thumbor/6.4.1
3454
+ Reeder/1000.02.01 CFNetwork/454.11.5 Darwin/10.5.0 (i386) (Macmini2)
3455
+ worldping-api
3456
+ Geziyor 1.0
3457
+ adstxt-worker (www.column6.com)
3458
+ adstxt.com/1.2
3459
+ Mozilla/5.0 (compatible; ZnHTTP/7.80)
3460
+ khttp/1.0.0-SNAPSHOT
3461
+ IZaBEE/IZaBEE-1.01 (Buzzing Abound The Web; https://izabee.com; info at izabee dot com)
3462
+ HTTPie/1.0.2
3463
+ Craw/1.0
3464
+ internetwache.org v3.4
3465
+ looid.com Search Engine/0.1
3466
+ OgScrper/1.0.0
3467
+ NotionEmbedder
3468
+ facebookexternalua
3469
+ Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; DuplexWeb-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Mobile Safari/537.36
3470
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943)
3471
+ FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)
3472
+ Amazon Simple Notification Service Agent
3473
+ cortex/1.0
3474
+ AWS Security Scanner
3475
+ HaxerMen
3476
+ kubectl/v1.12.0 (linux/amd64) kubernetes/0ed3388
3477
+ NetSystemsResearch studies the availability of various services across the internet. Our website is netsystemsresearch.com
3478
+ Mozilla/5.0(Linux;Android 5.1.1;OPPO A33 Build/LMY47V;wv) AppleWebKit/537.36(KHTML,link Gecko) Version/4.0 Chrome/43.0.2357.121 Mobile Safari/537.36 LieBaoFast/4.51.3
3479
+ sc-downloader/2.0 (http://www.searchconcepts.io)
3480
+ [object Promise] delve.ai website analyse
3481
+ attohttpc/0.9.0
3482
+ Bewica-security-scan
3483
+ BlogBridge 2.13 (http://www.blogbridge.com/)
3484
+ Boost.Beast/248
3485
+ DomainLabz
3486
+ ExoRank
3487
+ Fuzz Faster U Fool v1.0-rc1
3488
+ Hexometer
3489
+ longurl-r-package
3490
+ looid.com Search Engine/0.1
3491
+ Mozilla/5.0 (compatible; RyowlEngine/1.0; +https://ryowl.org)
3492
+ Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0; https://app.limber.io) Gecko/20100101 Firefox/25.0
3493
+ OSPScraper (+https://www.opensyllabusproject.org)
3494
+ python-httpx/0.11.1
3495
+ shortURL lengthener/0.1 +http://kevindriscoll.info/
3496
+ Scamadviser-Frontend
3497
+ Wolfram HTTPClient 12.
3498
+ SEOlizer/1.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13
3499
+ Mozilla/5.0 (pc-x86_64-linux-gnu) Siege/4.0.4
3500
+ Filestack
3501
+ Google-Ads-Overview Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36
3502
+ Google-Ads-Overview Mozilla/5.0 (Linux; U; Android 6.0.1; generic) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Version/4.0 Mobile Safari/537.36
3503
+ Google-Ads-Overview Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Version/4.0 Mobile Safari/537.36
3504
+ Google-Ads-Overview Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Mobile Safari/537.36
3505
+ adreview/1.0
3506
+ Google-speakr
3507
+ Google-speakr,gzip(gfe)
3508
+ PageFreezer
3509
+ Prebid.js Scraper
3510
+ Mr.4x3 Powered
3511
+ Goldfire Server
3512
+ OhDear.app (+https://ohdear.app/docs/checks/uptime)
3513
+ Mozilla/5.0 (compatible;Cloudflare-Healthchecks/1.0;+https://www.cloudflare.com/; healthcheck-id: 755842180c7ae904)
3514
+ WinHttp-Autoproxy-Service/5.1
3515
+ DynatraceSynthetic/
3516
+ acebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)
3517
+ Mozilla/5.0 (compatible; Google-Podcast)
3518
+ Project25499 Scanner (opt-out@project25499.com)
3519
+ GitHub-Hookshot/a6f2714
3520
+ Fiery%20Feeds/272 CFNetwork/1125.2 Darwin/19.4.0 (x86_64)
3521
+ FreshRSS/1.13.0 (Linux; https://freshrss.org)
3522
+ Gofeed/1.0
3523
+ Mozilla/5.0 (Windows NT 6.1) AppleWebKit/538.1 (KHTML, like Gecko) QuiteRSS/0.18.12 Safari/538.1
3524
+ php-requests/1.7
3525
+ nghttp2/1.40.0
3526
+ WinHTTP/1.1
3527
+ Mozilla/5.0 (compatible; Javelin; +https://about.javelin.io/)
3528
+ swcd (unknown version) CFNetwork/1125.2 Darwin/19.4.0
3529
+ YahooMailProxy; https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html
3530
+ Mozilla/5.0 (Java) outbrain
3531
+ DHSH
3532
+ DHSH Virus Check
3533
+ DHSH Ads Check
3534
+ DHSH Statistik Chec
3535
+ DHSH Law Check
3536
+ DHSH User Check
3537
+ Mozilla/5.0+(compatible; Cula/2.0; https://cula.io/)
3538
+ Winds: Open Source RSS & Podcast app: https://getstream.io/winds/
3539
+ XenForo/2.x (https://www.recreationalflying.com)
3540
+ TECHIAITH_CY (+https://techiaith.cymru)
3541
+ ScopeContentAG-HTTP-Client www.thescope.com/0.1
3542
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Lush Http Client
3543
+ Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36 The National Archives UK Government Web Archive: http://www.nationalarchives.gov.uk/webarchive/; webarchive@nationalarchives.gov.uk
3544
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36 (compatible; NameFo; https://namefo.com/)
3545
+ Mozilla/5.0 (Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36 (compatible; Statically-Screenshot; +https://statically.io/screenshot)
3546
+ Mozilla/5.0 (compatible; Domains Project/1.0.8; +https://domainsproject.org)
3547
+ Mozilla/5.0 (compatible; AffiliateLabz/1.0; +https://affiliatelabz.com/)
3548
+ Mozilla/5.0 (compatible; 2GDPR/1.2; https://2gdpr.com)
3549
+ Mozilla/5.0 (+http://www.sli-systems.com/) Mammoth/0.1
3550
+ Microsoft.Data.Mashup (https://go.microsoft.com/fwlink/?LinkID=304225)
3551
+ LinkPreview/1.5 (http://linkpreview.net)
3552
+ https://gdnplus.com:Gather Analyze Provide.
3553
+ Feedshow/x.0 (http://www.feedshow.com; 1 subscriber)
3554
+ apimon.de/http-apis#redirect
3555
+ ZoteroTranslationServer/WMF (mailto:services@lists.wikimedia.org)
3556
+ yomins.com/0.3
3557
+ WakeletLinkExpander/0.1
3558
+ ultimate_sitemap_parser/0.5
3559
+ Swurl Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36
3560
+ postplanner.com/site-scraping
3561
+ Nuclei (@pdiscoveryio)
3562
+ myseosnapshot/1.0
3563
+ Contextual Code Sites Explorer
3564
+ Corax - support@coraxcyber.com
3565
+ Florienzh/1.0
3566
+ BKCTwitterUnshortener/0.1 (Contact: systems@cyber.harvard.edu)
3567
+ Syndirella/0.91pre
3568
+ PodcastAddict/v2 - Dalvik/2.1.0 (Linux; U; Android 6.0.1; SM-G920F Build/MMB29K)
3569
+ PodcastAddict/v5 (+https://podcastaddict.com/; Android podcast app)
3570
+ PDF24 URL To PDF
3571
+ MacOutlook/16.41.20091302 (Intelx64 Mac OS X 10.14.4 (Build 18E226))
3572
+ HubSpot-Link-Resolver
3573
+ Railgun/5.3.3
3574
+ RSSMix/3.0
3575
+ SearchWP
3576
+ Stripe/1.0 (+https://stripe.com/docs/webhooks)
3577
+ ZoominfoBot (zoominfobot at zoominfo dot com)
3578
+ Asana/1.4.0 WebsiteMetadataRetriever
3579
+ mattermost-5.19.0
3580
+ WP Rocket/Homepage Preload
3581
+ Anthill
3582
+ cf-facebook/6.8.48 Node.js/v6.11.1
3583
+ LinkAnalyser/2.0
3584
+ PHP/{5|6|7}.{3|2}.{1|2|3|4|5|6|7|8|9|0}{1|2|3|4|5|6|7|8|9|0}
3585
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:82.0) Gecko/20100101 Firefox/82.0 Observatory/82.0
3586
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) Project-Resonance (http://project-resonance.com/) (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36
3587
+ Mozilla/5.0 (compatible; DNSResearchBot/2.1; +http://195.37.190.77)
3588
+ Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)
3589
+ Mozilla/5.0 (compatible; tchelebi/1.0; +http://tchelebi.io)
3590
+ OnalyticaBot
3591
+ Pleroma 2.1.50-495-gf9ece1a7-develop+dev; https://search.fedi.app <pleroma@search.fedi.app>; Bot
3592
+ SmerBot/0.1 (smerity@smerity.com) Language Model Dataset Fetcher
3593
+ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/71.0.3578.98 Safari/537.36 Prerender (+https://github.com/prerender/prerender)
3594
+ Bitrix link preview
3595
+ Mikrotik/6.x Fetch
3596
+ Mikrotik/7.x Fetch
3597
+ HTTPing v2.5
3598
+ Mozilla/5.0 (compatible; JungleKeyThumbnail/1.1; +http://www.junglekey.fr/)
3599
+ postfix-mta-sts-resolver
3600
+ go-mtasts/1.0
3601
+ Mail::STS
3602
+ Google-SMTP-STS
3603
+ Netlab360
3604
+ TLS tester from https://testssl.sh/dev/
3605
+ TBI-HttpOpenPlugin/0.1.0 (+https://leakix.net/)
3606
+ TBI-WebScanner/0.0.1 (+https://leakix.net/)
3607
+ quic-go-HTTP/3
3608
+ CISPA Web Analyser (https://notify.cispa.saarland)
3609
+ webtech/1.2.11
3610
+ Hardenize (https://www.hardenize.com)
3611
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:46.0) Gecko/20100101 Firefox/46.0 | Hardenize (https://www.hardenize.com)
3612
+ Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:46.0) Gecko/20100101 Firefox/46.0 | Hardenize/v1.1196.1 (https://www.hardenize.com)
3613
+ Expanse indexes the network perimeters of our customers. If you have any questions or concerns, please reach out to: scaninfo@expanseinc.com
3614
+ Expanse, a Palo Alto Networks company, searches across the global IPv4 space multiple times per day to identify customers&#39; presences on the Internet. If you would like to be excluded from our scans, please send IP addresses/domains to: scaninfo@paloaltonetworks.com
3615
+ PageThing http://pagething.com
3616
+ PageThing.com
3617
+ Mozilla/5.0 (Windows; U; Windows NT 5.1; pl; PageThing http://pagething.com); rv:1.9; Gecko/2008052906 Firefox/3.0
3618
+ ureq
3619
+ ureq/2.0.0-rc1
3620
+ ureq/2.0.1
3621
+ URL/Emacs Emacs/26.1 (X11; x86_64-pc-linux-gnu)
3622
+ unshortenit 0.4.0
3623
+ Statically-Images/2.0
3624
+ Statically-Icon/1.4.1
3625
+ RadioPublicImageResizer/2.0
3626
+ nettle (+https://www.nettle.sk)
3627
+ NihilScio - Educational search engine - +https://www.nihilscio.it/NihilScio.htm
3628
+ Miro-HttpClient/1.0
3629
+ mio_httpc 0.9.4
3630
+ Amazon-Route53-Health-Check-Service (ref c96ad03c-acdf-48c6-8ec7-45806bf08123; report http://amzn.to/1vsZADi)
3631
+ pnpm/6.18.0 npm/? node/v16.11.1 linux x64
3632
+ Gradle/7.1 (Linux;4.15.0-162-generic;amd64) (Oracle Corporation;11.0.10;11.0.10+9)
3633
+ npm/6.14.12 node/v10.24.1 linux x64 ci/drone
3634
+ pip/21.3.1 {"ci":null,"cpu":"x86_64","distro":{"name":"macOS","version":"11.6"},"implementation":{"name":"CPython","version":"3.9.7"},"installer":{"name":"pip","version":"21.3.1"},"openssl_version":"OpenSSL 1.1.1l 24 Aug 2021","python":"3.9.7","setuptools_version":"59.2.0","system":{"name":"Darwin","release":"20.6.0"}}
3635
+ pip/21.0.1 {"ci":null,"cpu":"x86_64","distro":{"id":"stretch","libc":{"lib":"glibc","version":"2.24"},"name":"Debian GNU/Linux","version":"9"},"implementation":{"name":"CPython","version":"3.7.9"},"installer":{"name":"pip","version":"21.0.1"},"openssl_version":"OpenSSL 1.1.0l 10 Sep 2019","python":"3.7.9","setuptools_version":"53.0.0","system":{"name":"Linux","release":"4.19.123-coreos"}}
3636
+ twine/3.4.2 importlib_metadata/4.8.1 pkginfo/1.7.1 requests/2.26.0 requests-toolbelt/0.9.1 tqdm/4.62.3 CPython/3.8.12
3637
+ Aether
3638
+ PHP-AYMAPI/1.0
3639
+ Adyen HttpClient 1.0
3640
+ affilimate-puppeteer
3641
+ docker/19.03.8 go/go1.12.17 git-commit/afacb8b7f0 kernel/4.19.123-coreos os/linux arch/amd64 UpstreamClient(Docker-Client/19.03.8 \(linux\))
3642
+ Apache Ant/1.10.9
3643
+ Synapse/1.36.0
3644
+ GoogleProber
3645
+ Lucee (CFML Engine)
3646
+ RapidLoad/1.0
3647
+ SAP NetWeaver Application Server (1.0;750)
3648
+ Y!J-DLC/1.0 (https://support.yahoo-net.jp/PccSearch/s/article/H000007955)
3649
+ ColdFusion
3650
+ node-fetch
3651
+ awin.com - site screen shotter