device_detector 0.8.2 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +11 -3
  3. data/CHANGELOG.md +10 -5
  4. data/README.md +4 -4
  5. data/Rakefile +24 -19
  6. data/device_detector.gemspec +1 -1
  7. data/lib/device_detector.rb +25 -3
  8. data/lib/device_detector/device.rb +1 -1
  9. data/lib/device_detector/model_extractor.rb +4 -0
  10. data/lib/device_detector/os.rb +8 -3
  11. data/lib/device_detector/parser.rb +1 -1
  12. data/lib/device_detector/version.rb +1 -1
  13. data/regexes/bots.yml +211 -27
  14. data/regexes/client/browsers.yml +192 -58
  15. data/regexes/client/feed_readers.yml +6 -0
  16. data/regexes/client/libraries.yml +4 -0
  17. data/regexes/client/mediaplayers.yml +8 -4
  18. data/regexes/client/mobile_apps.yml +39 -4
  19. data/regexes/client/pim.yml +4 -0
  20. data/regexes/device/mobiles.yml +1803 -242
  21. data/regexes/device/televisions.yml +53 -11
  22. data/regexes/oss.yml +180 -12
  23. data/regexes/vendorfragments.yml +1 -0
  24. data/spec/device_detector/concrete_user_agent_spec.rb +36 -0
  25. data/spec/device_detector/detector_fixtures_spec.rb +1 -1
  26. data/spec/fixtures/client/browser.yml +313 -2
  27. data/spec/fixtures/client/feed_reader.yml +8 -1
  28. data/spec/fixtures/client/library.yml +7 -1
  29. data/spec/fixtures/client/mediaplayer.yml +6 -1
  30. data/spec/fixtures/client/mobile_app.yml +6 -0
  31. data/spec/fixtures/client/pim.yml +6 -0
  32. data/spec/fixtures/detector/bots.yml +2307 -612
  33. data/spec/fixtures/detector/camera.yml +6 -0
  34. data/spec/fixtures/detector/car_browser.yml +1 -0
  35. data/spec/fixtures/detector/console.yml +14 -0
  36. data/spec/fixtures/detector/desktop.yml +308 -48
  37. data/spec/fixtures/detector/feature_phone.yml +63 -0
  38. data/spec/fixtures/detector/feed_reader.yml +50 -26
  39. data/spec/fixtures/detector/mediaplayer.yml +28 -6
  40. data/spec/fixtures/detector/mobile_apps.yml +117 -0
  41. data/spec/fixtures/detector/phablet.yml +1066 -183
  42. data/spec/fixtures/detector/portable_media_player.yml +8 -0
  43. data/spec/fixtures/detector/smart_display.yml +39 -0
  44. data/spec/fixtures/detector/smartphone-1.yml +9469 -0
  45. data/spec/fixtures/detector/smartphone-2.yml +9414 -0
  46. data/spec/fixtures/detector/smartphone-3.yml +9396 -0
  47. data/spec/fixtures/detector/smartphone-4.yml +5742 -0
  48. data/spec/fixtures/detector/smartphone.yml +3446 -23680
  49. data/spec/fixtures/detector/tablet-1.yml +9495 -0
  50. data/spec/fixtures/detector/tablet-2.yml +248 -0
  51. data/spec/fixtures/detector/tablet.yml +2786 -7961
  52. data/spec/fixtures/detector/tv.yml +1376 -210
  53. data/spec/fixtures/detector/unknown.yml +205 -509
  54. data/spec/fixtures/device/console.yml +6 -0
  55. data/spec/fixtures/parser/oss.yml +194 -1
  56. data/spec/fixtures/parser/vendorfragments.yml +6 -0
  57. metadata +16 -6
  58. data/spec/fixtures/parser/bots.yml +0 -2055
data/spec/fixtures/client/feed_reader.yml CHANGED
@@ -177,4 +177,11 @@
177
177
  client:
178
178
  type: feed reader
179
179
  name: RSSOwl
180
- version: "2.2.1.201312301316"
180
+ version: "2.2.1.201312301316"
181
+
182
+ -
183
+ user_agent: Stringer (https://github.com/swanson/stringer)
184
+ client:
185
+ type: feed reader
186
+ name: Stringer
187
+ version: ""
data/spec/fixtures/client/library.yml CHANGED
@@ -75,4 +75,10 @@
75
75
  client:
76
76
  type: library
77
77
  name: Perl
78
- version: "1.0"
78
+ version: "1.0"
79
+ -
80
+ user_agent: Guzzle/3.9.3 curl/7.38.0 PHP/5.6.14-0+deb8u1
81
+ client:
82
+ type: library
83
+ name: Guzzle (PHP HTTP Client)
84
+ version: "3.9.3"
data/spec/fixtures/client/mediaplayer.yml CHANGED
@@ -28,6 +28,12 @@
28
28
  type: mediaplayer
29
29
  name: VLC
30
30
  version: "2.1.0"
31
+ -
32
+ user_agent: LibVLC/2.2.3 (LIVE555 Streaming Media v2015.10.12)
33
+ client:
34
+ type: mediaplayer
35
+ name: VLC
36
+ version: "2.2.3"
31
37
  -
32
38
  user_agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; FunWebProducts; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; Windows-Media-Player/10.00.00.3990)
33
39
  client:
@@ -160,4 +166,3 @@
160
166
  type: mediaplayer
161
167
  name: Instacast
162
168
  version: "4.1.2"
163
-
data/spec/fixtures/client/mobile_app.yml CHANGED
@@ -22,3 +22,9 @@
22
22
  type: mobile app
23
23
  name: FeedR
24
24
  version: "1.0"
25
+ -
26
+ user_agent: Pinterest/3.3.3 CFNetwork/609.1.4 Darwin/13.0.0
27
+ client:
28
+ type: mobile app
29
+ name: Pinterest
30
+ version: "3.3.3"
data/spec/fixtures/client/pim.yml CHANGED
@@ -76,6 +76,12 @@
76
76
  type: pim
77
77
  name: Postbox
78
78
  version: "1.0"
79
+ -
80
+ user_agent: MailBar/1.3.2 (Mac OS X Version 10.11.1 (Build 15B42))
81
+ client:
82
+ type: pim
83
+ name: MailBar
84
+ version: 1.3.2
79
85
  -
80
86
  user_agent: The Bat! 4.0.0.22
81
87
  client:
data/spec/fixtures/detector/bots.yml CHANGED
@@ -1,723 +1,2418 @@
1
- -
1
+ ---
2
+ -
3
+ user_agent: 360spider-image
4
+ bot:
5
+ name: 360Spider
6
+ category: Search bot
7
+ url: http://www.so.com/help/help_3_2.html
8
+ producer:
9
+ name: Online Media Group, Inc.
10
+ url:
11
+ -
12
+ user_agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MDDR; .NET4.0C; .NET4.0E; .NET CLR 1.1.4322; Tablet PC 2.0); 360Spider
13
+ bot:
14
+ name: 360Spider
15
+ category: Search bot
16
+ url: http://www.so.com/help/help_3_2.html
17
+ producer:
18
+ name: Online Media Group, Inc.
19
+ url:
20
+ -
2
21
  user_agent: Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider
3
- name: '360Spider'
4
- -
22
+ bot:
23
+ name: 360Spider
24
+ category: Search bot
25
+ url: http://www.so.com/help/help_3_2.html
26
+ producer:
27
+ name: Online Media Group, Inc.
28
+ url:
29
+ -
5
30
  user_agent: Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider
6
- name: '360Spider'
7
- -
8
- user_agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MDDR; .NET4.0C; .NET4.0E; .NET CLR 1.1.4322; Tablet PC 2.0); 360Spider
9
- name: '360Spider'
10
- -
11
- user_agent: 360spider-image
12
- name: '360Spider'
13
- -
31
+ bot:
32
+ name: 360Spider
33
+ category: Search bot
34
+ url: http://www.so.com/help/help_3_2.html
35
+ producer:
36
+ name: Online Media Group, Inc.
37
+ url:
38
+ -
39
+ user_agent: 'ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com'
40
+ bot:
41
+ name: ADMantX
42
+ -
14
43
  user_agent: Aboundex/0.3 (http://www.aboundex.com/crawler/)
15
- name: 'Aboundexbot'
16
- -
44
+ bot:
45
+ name: Aboundexbot
46
+ category: Search bot
47
+ url: http://www.aboundex.com/crawler/
48
+ producer:
49
+ name: Aboundex.com
50
+ url: http://www.aboundex.com
51
+ -
17
52
  user_agent: Mozilla/5.0 (compatible; AcoonBot/4.11.1; +http://www.acoon.de/robot.asp)
18
- name: 'Acoon'
19
- -
53
+ bot:
54
+ name: Acoon
55
+ category: Search bot
56
+ url: http://www.acoon.de/robot.asp
57
+ producer:
58
+ name: Acoon GmbH
59
+ url: http://www.acoon.de
60
+ -
20
61
  user_agent: AddThis.com robot tech.support@clearspring.com
21
- name: 'AddThis.com'
22
- -
23
- user_agent: Mozilla/5.0 (compatible; AhrefsBot/3.1; +http://ahrefs.com/robot/
24
- name: 'aHrefs Bot'
25
- -
26
- user_agent: Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)
27
- name: 'aHrefs Bot'
28
- -
29
- user_agent: 'Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/) AppEngine-Google; (+http://code.google.com/appengine; appid: s~proxyfile1-hrd)'
30
- name: 'aHrefs Bot'
31
- -
32
- user_agent: AhrefsBot.Feeds v0.1; http://ahrefs.com/
33
- name: 'aHrefs Bot'
34
- -
62
+ bot:
63
+ name: AddThis.com
64
+ category: Social Media Agent
65
+ url:
66
+ producer:
67
+ name: Clearspring Technologies, Inc.
68
+ url: http://www.clearspring.com
69
+ -
35
70
  user_agent: ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)
36
- name: 'Alexa Crawler'
37
- -
71
+ bot:
72
+ name: Alexa Crawler
73
+ category: Search bot
74
+ url: https://alexa.zendesk.com/hc/en-us/sections/200100794-Crawlers
75
+ producer:
76
+ name: Alexa Internet
77
+ url: http://www.alexa.com
78
+ -
38
79
  user_agent: AmorankSpider/0.1; +http://amorank.com/webcrawler.html
39
- name: 'Amorank Spider'
40
- -
41
- user_agent: Curious George - www.analyticsseo.com/crawler
42
- name: 'Analytics SEO Crawler'
43
- -
44
- user_agent: Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)
45
- name: 'archive.org bot'
46
- -
47
- user_agent: Mozilla/5.0 (compatible; special_archiver/3.2.0 +http://www.loc.gov/webarchiving/notice_to_webmasters.html)
48
- name: 'archive.org bot'
49
- -
50
- user_agent: Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)
51
- name: 'archive.org bot'
52
- -
53
- user_agent: Mozilla/5.0 (compatible; archive.org_bot/3.3.0 +http://pandora.nla.gov.au/crawl.html)
54
- name: 'archive.org bot'
55
- -
56
- user_agent: Mozilla/5.0 (compatible; archive.org_bot; Archive-It; +http://archive-it.org/files/site-owners.html)
57
- name: 'archive.org bot'
58
- -
59
- user_agent: Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)
60
- name: 'archive.org bot'
61
- -
62
- user_agent: Mozilla/5.0 (compatible; heritrix/3.1.2-SNAPSHOT-20131029-0036 +http://archive.org/details/archive.org_bot)
63
- name: 'archive.org bot'
64
- -
80
+ bot:
81
+ name: Amorank Spider
82
+ category: Crawler
83
+ url: http://amorank.com/webcrawler.html
84
+ producer:
85
+ name: Amorank
86
+ url: http://www.amorank.com
87
+ -
88
+ user_agent: 'Curious George - www.analyticsseo.com/crawler'
89
+ bot:
90
+ name: Analytics SEO Crawler
91
+ category: Crawler
92
+ url: http://www.analyticsseo.com/crawler
93
+ producer:
94
+ name: Analytics SEO
95
+ url: http://www.analyticsseo.com
96
+ -
97
+ user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10 _1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)
98
+ bot:
99
+ name: Applebot
100
+ category: Crawler
101
+ url: http://www.apple.com/go/applebot
102
+ producer:
103
+ name: Apple Inc
104
+ url: http://www.apple.com
105
+ -
106
+ user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)
107
+ bot:
108
+ name: Applebot
109
+ category: Crawler
110
+ url: http://www.apple.com/go/applebot
111
+ producer:
112
+ name: Apple Inc
113
+ url: http://www.apple.com
114
+ -
65
115
  user_agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
66
- name: 'Ask Jeeves'
67
- -
116
+ bot:
117
+ name: Ask Jeeves
118
+ category: Search bot
119
+ url:
120
+ producer:
121
+ name: Ask Jeeves Inc.
122
+ url: http://www.ask.com
123
+ -
124
+ user_agent: Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)
125
+ bot:
126
+ name: BLEXBot Crawler
127
+ category: Crawler
128
+ url: http://webmeup-crawler.com
129
+ producer:
130
+ name: WebMeUp
131
+ url: http://webmeup.com
132
+ -
133
+ user_agent: BUbiNG (+http://law.di.unimi.it/BUbiNG.html)
134
+ bot:
135
+ name: BUbiNG
136
+ category: Crawler
137
+ url: http://law.di.unimi.it/BUbiNG.html
138
+ producer:
139
+ name:
140
+ url:
141
+ -
68
142
  user_agent: Backlink-Ceck.de (+http://www.backlink-check.de/bot.html)
69
- name: 'Backlink-Ceck.de'
70
- -
143
+ bot:
144
+ name: Backlink-Ceck.de
145
+ category: Crawler
146
+ url: http://www.backlink-check.de/bot.html
147
+ producer:
148
+ name: Mediagreen Medienservice
149
+ url: http://www.backlink-check.de
150
+ -
71
151
  user_agent: BacklinkCrawler (http://www.backlinktest.com/crawler.html)
72
- name: 'BacklinkCrawler'
73
- -
152
+ bot:
153
+ name: BacklinkCrawler
154
+ category: Crawler
155
+ url: http://www.backlinktest.com/crawler.html
156
+ producer:
157
+ name: 2.0Promotion GbR
158
+ url: http://www.backlinktest.com
159
+ -
160
+ user_agent: Baiduspider+(+http://www.baidu.com/search/spider.htm)
161
+ bot:
162
+ name: Baidu Spider
163
+ category: Search bot
164
+ url: http://www.baidu.com/search/spider.htm
165
+ producer:
166
+ name: Baidu
167
+ url: http://www.baidu.com
168
+ -
169
+ user_agent: Baiduspider-image+(+http://www.baidu.com/search/spider.htm)
170
+ bot:
171
+ name: Baidu Spider
172
+ category: Search bot
173
+ url: http://www.baidu.com/search/spider.htm
174
+ producer:
175
+ name: Baidu
176
+ url: http://www.baidu.com
177
+ -
178
+ user_agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; baidu Transcoder;)
179
+ bot:
180
+ name: Baidu Spider
181
+ category: Search bot
182
+ url: http://www.baidu.com/search/spider.htm
183
+ producer:
184
+ name: Baidu
185
+ url: http://www.baidu.com
186
+ -
74
187
  user_agent: Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search
75
- name: 'Baidu Spider'
76
- -
188
+ bot:
189
+ name: Baidu Spider
190
+ category: Search bot
191
+ url: http://www.baidu.com/search/spider.htm
192
+ producer:
193
+ name: Baidu
194
+ url: http://www.baidu.com
195
+ -
196
+ user_agent: Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
197
+ bot:
198
+ name: Baidu Spider
199
+ category: Search bot
200
+ url: http://www.baidu.com/search/spider.htm
201
+ producer:
202
+ name: Baidu
203
+ url: http://www.baidu.com
204
+ -
77
205
  user_agent: Mozilla/5.0 (Linux;u;Android 2.3.7;zh-cn;) AppleWebKit/533.1 (KHTML,like Gecko) Version/4.0 Mobile Safari/533.1 (compatible; +http://www.baidu.com/search/spider.html)
78
- name: 'Baidu Spider'
79
- -
206
+ bot:
207
+ name: Baidu Spider
208
+ category: Search bot
209
+ url: http://www.baidu.com/search/spider.htm
210
+ producer:
211
+ name: Baidu
212
+ url: http://www.baidu.com
213
+ -
80
214
  user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8;baidu Transcoder) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729)
81
- name: 'Baidu Spider'
82
- -
83
- user_agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; baidu Transcoder;)
84
- name: 'Baidu Spider'
85
- -
86
- user_agent: Baiduspider-image+(+http://www.baidu.com/search/spider.htm)
87
- name: 'Baidu Spider'
88
- -
89
- user_agent: Baiduspider+(+http://www.baidu.com/search/spider.htm)
90
- name: 'Baidu Spider'
91
- -
92
- user_agent: Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
93
- name: 'Baidu Spider'
94
- -
95
- user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b
96
- name: 'BingBot'
97
- -
215
+ bot:
216
+ name: Baidu Spider
217
+ category: Search bot
218
+ url: http://www.baidu.com/search/spider.htm
219
+ producer:
220
+ name: Baidu
221
+ url: http://www.baidu.com
222
+ -
223
+ user_agent: Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 2 subscribers)
224
+ bot:
225
+ name: BazQux Reader
226
+ url: https://bazqux.com/fetcher
227
+ category: Feed Fetcher
228
+ producer:
229
+ name:
230
+ url:
231
+ -
98
232
  user_agent: Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
99
- name: 'BingBot'
100
- -
233
+ bot:
234
+ name: BingBot
235
+ category: Search bot
236
+ url: http://search.msn.com/msnbot.htmn
237
+ producer:
238
+ name: Microsoft Corporation
239
+ url: http://www.microsoft.com
240
+ -
241
+ user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b
242
+ bot:
243
+ name: BingBot
244
+ category: Search bot
245
+ url: http://search.msn.com/msnbot.htmn
246
+ producer:
247
+ name: Microsoft Corporation
248
+ url: http://www.microsoft.com
249
+ -
101
250
  user_agent: msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)
102
- name: 'BingBot'
103
- -
104
- user_agent: MSNBot/Nutch-1.5.1
105
- name: 'BingBot'
106
- -
107
- user_agent: msnbot/2.0b v1394720947.6703
108
- name: 'BingBot'
109
- -
251
+ bot:
252
+ name: BingBot
253
+ category: Search bot
254
+ url: http://search.msn.com/msnbot.htmn
255
+ producer:
256
+ name: Microsoft Corporation
257
+ url: http://www.microsoft.com
258
+ -
259
+ user_agent: msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)
260
+ bot:
261
+ name: BingBot
262
+ category: Search bot
263
+ url: http://search.msn.com/msnbot.htmn
264
+ producer:
265
+ name: Microsoft Corporation
266
+ url: http://www.microsoft.com
267
+ -
110
268
  user_agent: msnbot/1.1 (+http://search.msn.com/msnbot.htm)
111
- name: 'BingBot'
112
- -
269
+ bot:
270
+ name: BingBot
271
+ category: Search bot
272
+ url: http://search.msn.com/msnbot.htmn
273
+ producer:
274
+ name: Microsoft Corporation
275
+ url: http://www.microsoft.com
276
+ -
113
277
  user_agent: msnbot/2.0b (+http://search.msn.com/msnbot.htm)
114
- name: 'BingBot'
115
- -
116
- user_agent: msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)
117
- name: 'BingBot'
118
- -
278
+ bot:
279
+ name: BingBot
280
+ category: Search bot
281
+ url: http://search.msn.com/msnbot.htmn
282
+ producer:
283
+ name: Microsoft Corporation
284
+ url: http://www.microsoft.com
285
+ -
286
+ user_agent: msnbot/2.0b v1394720947.6703
287
+ bot:
288
+ name: BingBot
289
+ category: Search bot
290
+ url: http://search.msn.com/msnbot.htmn
291
+ producer:
292
+ name: Microsoft Corporation
293
+ url: http://www.microsoft.com
294
+ -
295
+ user_agent: MSNBot/Nutch-1.5.1
296
+ bot:
297
+ name: BingBot
298
+ category: Search bot
299
+ url: http://search.msn.com/msnbot.htmn
300
+ producer:
301
+ name: Microsoft Corporation
302
+ url: http://www.microsoft.com
303
+ -
119
304
  user_agent: Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)
120
- name: 'Blekkobot'
121
- -
122
- user_agent: Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)
123
- name: 'BLEXBot Crawler'
124
- -
305
+ bot:
306
+ name: Blekkobot
307
+ category: Search bot
308
+ url: http://blekko.com/about/blekkobot
309
+ producer:
310
+ name: Blekko
311
+ url: http://blekko.com
312
+ -
313
+ user_agent: Bloglovin/1.0 (http://www.bloglovin.com; 1 subscribers)
314
+ bot:
315
+ name: Bloglovin
316
+ url: http://www.bloglovin.com
317
+ category: Feed Fetcher
318
+ producer:
319
+ name:
320
+ url:
321
+ -
322
+ user_agent: Blogtrottr/2.0
323
+ bot:
324
+ name: Blogtrottr
325
+ url:
326
+ category: Feed Fetcher
327
+ producer:
328
+ name: Blogtrottr Ltd
329
+ url: https://blogtrottr.com/
330
+ -
125
331
  user_agent: Mozilla/5.0 (compatible; BountiiBot/1.1; +http://bountii.com/contact.php)
126
- name: 'Bountii Bot'
127
- -
128
- user_agent: Mozilla/5.0 (Macintosh; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
129
- name: 'Butterfly Robot'
130
- -
131
- user_agent: Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
132
- name: 'Butterfly Robot'
133
- -
332
+ bot:
333
+ name: Bountii Bot
334
+ category: Search bot
335
+ url: http://bountii.com/contact.php
336
+ producer:
337
+ name: Bountii Inc.
338
+ url: http://bountii.com
339
+ -
134
340
  user_agent: Browsershots
135
- name: 'Browsershots'
136
- -
341
+ bot:
342
+ name: Browsershots
343
+ category: Service Agent
344
+ url: http://browsershots.org/faq
345
+ producer:
346
+ name: Browsershots.org
347
+ url: http://browsershots.org
348
+ -
349
+ user_agent: Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
350
+ bot:
351
+ name: Butterfly Robot
352
+ category: Search bot
353
+ url: http://labs.topsy.com/butterfly
354
+ producer:
355
+ name: Topsy Labs
356
+ url: http://labs.topsy.com
357
+ -
358
+ user_agent: Mozilla/5.0 (Macintosh; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
359
+ bot:
360
+ name: Butterfly Robot
361
+ category: Search bot
362
+ url: http://labs.topsy.com/butterfly
363
+ producer:
364
+ name: Topsy Labs
365
+ url: http://labs.topsy.com
366
+ -
137
367
  user_agent: Mozilla/5.0 (compatible; CareerBot/1.1; +http://www.career-x.de/bot.html)
138
- name: 'CareerBot'
139
- -
140
- user_agent: CCBot/2.0 (http://commoncrawl.org/faq/)
141
- name: 'ccBot crawler'
142
- -
368
+ bot:
369
+ name: CareerBot
370
+ category: Crawler
371
+ url: http://www.career-x.de/bot.html
372
+ producer:
373
+ name: career-x GmbH
374
+ url: http://www.career-x.de
375
+ -
143
376
  user_agent: Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)
144
- name: 'Cliqzbot'
145
- -
377
+ bot:
378
+ name: Cliqzbot
379
+ category: Crawler
380
+ url: http://cliqz.com/company/cliqzbot
381
+ producer:
382
+ name: 10betterpages GmbH
383
+ url: http://cliqz.com
384
+ -
146
385
  user_agent: Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34
147
- name: 'CloudFlare Always Online'
148
- -
386
+ bot:
387
+ name: CloudFlare Always Online
388
+ category: Site Monitor
389
+ url: http://www.cloudflare.com/always-online
390
+ producer:
391
+ name: CloudFlare
392
+ url: http://www.cloudflare.com
393
+ -
394
+ user_agent: CommaFeed/1.0 (http://www.commafeed.com)
395
+ bot:
396
+ name: CommaFeed
397
+ url: http://www.commafeed.com
398
+ category: Feed Fetcher
399
+ producer:
400
+ name:
401
+ url:
402
+ -
403
+ user_agent: Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)
404
+ bot:
405
+ name: Cốc Cốc Bot
406
+ url: http://help.coccoc.com/
407
+ category: Search bot
408
+ producer:
409
+ name: Cốc Cốc
410
+ url: http://coccoc.com/
411
+ -
412
+ user_agent: Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0;
413
+ bot:
414
+ name: Daum
415
+ category: Search bot
416
+ url: http://tab.search.daum.net/aboutWebSearch_en.html
417
+ producer:
418
+ name: Daum Communications Corp.
419
+ url: http://www.kakaocorp.com/main
420
+ -
421
+ user_agent: Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1
422
+ bot:
423
+ name: Daum
424
+ category: Search bot
425
+ url: http://tab.search.daum.net/aboutWebSearch_en.html
426
+ producer:
427
+ name: Daum Communications Corp.
428
+ url: http://www.kakaocorp.com/main
429
+ -
149
430
  user_agent: Mozilla/5.0 (compatible; Dazoobot/0.1; +http://dazoo.fr)
150
- name: 'Dazoobot'
151
- -
431
+ bot:
432
+ name: Dazoobot
433
+ category: Search bot
434
+ url:
435
+ producer:
436
+ name: DAZOO.FR
437
+ url: http://dazoo.fr
438
+ -
152
439
  user_agent: Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)
153
- name: 'Discobot'
154
- -
440
+ bot:
441
+ name: Discobot
442
+ category: Search bot
443
+ url: http://discoveryengine.com/discobot.html
444
+ producer:
445
+ name: Discovery Engine
446
+ url: http://discoveryengine.com
447
+ -
448
+ user_agent: 'Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com'
449
+ bot:
450
+ name: Domain Re-Animator Bot
451
+ category: Crawler
452
+ url:
453
+ producer:
454
+ name: Domain Re-Animator, LLC
455
+ url: http://domainreanimator.com
456
+ -
457
+ user_agent: support@domainreanimator.com
458
+ bot:
459
+ name: Domain Re-Animator Bot
460
+ category: Crawler
461
+ url:
462
+ producer:
463
+ name: Domain Re-Animator, LLC
464
+ url: http://domainreanimator.com
465
+ -
155
466
  user_agent: dotbot
156
- name: 'DotBot'
157
- -
467
+ bot:
468
+ name: DotBot
469
+ category: Crawler
470
+ url: http://www.opensiteexplorer.org/dotbot
471
+ producer:
472
+ name: SEOmoz, Inc.
473
+ url: http://moz.com/
474
+ -
475
+ user_agent: EMail Exractor
476
+ bot:
477
+ name: EMail Exractor
478
+ category: Crawler
479
+ url:
480
+ producer:
481
+ name:
482
+ url:
483
+ -
158
484
  user_agent: Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)
159
- name: 'Easou Spider'
160
- -
485
+ bot:
486
+ name: Easou Spider
487
+ category: Search bot
488
+ url: http://www.easou.com/search/spider.html
489
+ producer:
490
+ name: easou ICP
491
+ url: http://www.easou.com
492
+ -
161
493
  user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us; EasouSpider; +http://www.easou.com/search/spider.html)
162
- name: 'Easou Spider'
163
- -
164
- user_agent: EMail Exractor
165
- name: 'EMail Exractor'
166
- -
167
- user_agent: Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)
168
- name: 'ExaBot'
169
- -
494
+ bot:
495
+ name: Easou Spider
496
+ category: Search bot
497
+ url: http://www.easou.com/search/spider.html
498
+ producer:
499
+ name: easou ICP
500
+ url: http://www.easou.com
501
+ -
170
502
  user_agent: Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)
171
- name: 'ExaBot'
172
- -
503
+ bot:
504
+ name: ExaBot
505
+ category: Crawler
506
+ url: http://www.exabot.com/go/robot
507
+ producer:
508
+ name: Dassault Systèmes
509
+ url: http://www.3ds.com
510
+ -
511
+ user_agent: Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)
512
+ bot:
513
+ name: ExaBot
514
+ category: Crawler
515
+ url: http://www.exabot.com/go/robot
516
+ producer:
517
+ name: Dassault Systèmes
518
+ url: http://www.3ds.com
519
+ -
173
520
  user_agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)
174
- name: 'ExaBot'
175
- -
521
+ bot:
522
+ name: ExaBot
523
+ category: Crawler
524
+ url: http://www.exabot.com/go/robot
525
+ producer:
526
+ name: Dassault Systèmes
527
+ url: http://www.3ds.com
528
+ -
176
529
  user_agent: ExactSeek Crawler (nutch 1.4)/Nutch-1.4 (ExactSeek Crawler; http://www.exactseek.com)
177
- name: 'ExactSeek Crawler'
178
- -
530
+ bot:
531
+ name: ExactSeek Crawler
532
+ category: Search bot
533
+ url: http://www.exactseek.com
534
+ producer:
535
+ name: Jayde Online, Inc.
536
+ url: http://www.jaydeonlineinc.com
537
+ -
179
538
  user_agent: Mozilla/5.0 (compatible; Ezooms/1.0; help@moz.com)
180
- name: 'Ezooms'
181
- -
539
+ bot:
540
+ name: Ezooms
541
+ category: Crawler
542
+ url:
543
+ producer:
544
+ name: SEOmoz, Inc.
545
+ url: http://moz.com/
546
+ -
182
547
  user_agent: facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)
183
- name: 'Facebook External Hit'
184
- -
548
+ bot:
549
+ name: Facebook External Hit
550
+ category: Social Media Agent
551
+ url: https://www.facebook.com/externalhit_uatext.php
552
+ producer:
553
+ name: Facebook
554
+ url: http://www.facebook.com
555
+ -
185
556
  user_agent: facebookplatform/1.0 (+http://developers.facebook.com)
186
- name: 'Facebook External Hit'
187
- -
557
+ bot:
558
+ name: Facebook External Hit
559
+ category: Social Media Agent
560
+ url: https://www.facebook.com/externalhit_uatext.php
561
+ producer:
562
+ name: Facebook
563
+ url: http://www.facebook.com
564
+ -
565
+ user_agent: Feed Wrangler/1.0 (3 subscribers; feed-id=248559; http://feedwrangler.net; Allow like Gecko)
566
+ bot:
567
+ name: Feed Wrangler
568
+ url: https://feedwrangler.net/
569
+ category: Feed Fetcher
570
+ producer:
571
+ name: 'David Smith & Developing Perspective, LLC'
572
+ url: https://david-smith.org
573
+ -
574
+ user_agent: FeedBurner/1.0 (http://www.FeedBurner.com)
575
+ bot:
576
+ name: FeedBurner
577
+ url: http://www.feedburner.com
578
+ category: Feed Fetcher
579
+ producer:
580
+ name:
581
+ url:
582
+ -
583
+ user_agent: 'Feedbin - 9 subscribers'
584
+ bot:
585
+ name: Feedbin
586
+ url: http://feedbin.com/
587
+ category: Feed Fetcher
588
+ producer:
589
+ name:
590
+ url:
591
+ -
592
+ user_agent: Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)
593
+ bot:
594
+ name: Feedly
595
+ url: http://www.feedly.com
596
+ category: Feed Fetcher
597
+ producer:
598
+ name:
599
+ url:
600
+ -
601
+ user_agent: FeedlyApp/1.0 (http://www.feedly.com)
602
+ bot:
603
+ name: Feedly
604
+ url: http://www.feedly.com
605
+ category: Feed Fetcher
606
+ producer:
607
+ name:
608
+ url:
609
+ -
610
+ user_agent: FeedlyBot/1.0 (http://feedly.com)
611
+ bot:
612
+ name: Feedly
613
+ url: http://www.feedly.com
614
+ category: Feed Fetcher
615
+ producer:
616
+ name:
617
+ url:
618
+ -
619
+ user_agent: MetaFeedly/1.0 (http://www.feedly.com)
620
+ bot:
621
+ name: Feedly
622
+ url: http://www.feedly.com
623
+ category: Feed Fetcher
624
+ producer:
625
+ name:
626
+ url:
627
+ -
628
+ user_agent: Feedspot http://www.feedspot.com
629
+ bot:
630
+ name: Feedspot
631
+ url: http://www.feedspot.com
632
+ category: Feed Fetcher
633
+ producer:
634
+ name:
635
+ url:
636
+ -
637
+ user_agent: Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)
638
+ bot:
639
+ name: Fever
640
+ url: http://feedafever.com/
641
+ category: Feed Fetcher
642
+ producer:
643
+ name:
644
+ url:
645
+ -
646
+ user_agent: niki-bot
647
+ bot:
648
+ name: Generic Bot
649
+ -
650
+ user_agent: StatusCake Error Detector Crawler V0.01
651
+ bot:
652
+ name: Generic Bot
653
+ -
188
654
  user_agent: Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)
189
- name: 'Genieo Web filter'
190
- -
655
+ bot:
656
+ name: Genieo Web filter
657
+ category:
658
+ url: http://www.genieo.com/webfilter.html
659
+ producer:
660
+ name: Genieo
661
+ url: http://www.genieo.com
662
+ -
663
+ user_agent: Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)
664
+ bot:
665
+ name: Gluten Free Crawler
666
+ category: Crawler
667
+ url: http://glutenfreepleasure.com/
668
+ producer:
669
+ name:
670
+ url:
671
+ -
672
+ user_agent: Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)
673
+ bot:
674
+ name: Gmail Image Proxy
675
+ category: Crawler
676
+ url:
677
+ producer:
678
+ name: Google Inc.
679
+ url: http://www.google.com
680
+ -
191
681
  user_agent: DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)
192
- name: 'Goo'
193
- -
682
+ bot:
683
+ name: Goo
684
+ category: Search bot
685
+ url: http://search.goo.ne.jp/option/use/sub4/sub4-1
686
+ producer:
687
+ name: NTT Resonant
688
+ url: http://goo.ne.jp
689
+ -
194
690
  user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0_1 like Mac OS X; en-us) AppleWebKit/537.4 (KHTML, like Gecko; Google Page Speed Insights) Version/4.0.5 Mobile/8A306 Safari/6531.22.7
195
- name: 'Google PageSpeed Insights'
196
- -
691
+ bot:
692
+ name: Google PageSpeed Insights
693
+ category: Site Monitor
694
+ url: http://developers.google.com/speed/pagespeed/insights/
695
+ producer:
696
+ name: Google Inc.
697
+ url: http://www.google.com
698
+ -
699
+ user_agent: Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1 google_partner_monitoring FWSzVTDDBz14547302713138T
700
+ bot:
701
+ name: Google Partner Monitoring
702
+ category: Site Monitor
703
+ url:
704
+ producer:
705
+ name: Google Inc.
706
+ url: http://www.google.com
707
+ -
708
+ user_agent: AdsBot-Google (+http://www.google.com/adsbot.html)
709
+ bot:
710
+ name: Googlebot
711
+ category: Search bot
712
+ url: http://www.google.com/bot.html
713
+ producer:
714
+ name: Google Inc.
715
+ url: http://www.google.com
716
+ -
717
+ user_agent: AdsBot-Google-Mobile (+http://www.google.com/mobile/adsbot.html) Mozilla (iPhone; U; CPU iPhone OS 3 0 like Mac OS X) AppleWebKit (KHTML, like Gecko) Mobile Safari
718
+ bot:
719
+ name: Googlebot
720
+ category: Search bot
721
+ url: http://www.google.com/bot.html
722
+ producer:
723
+ name: Google Inc.
724
+ url: http://www.google.com
725
+ -
726
+ user_agent: DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
727
+ bot:
728
+ name: Googlebot
729
+ category: Search bot
730
+ url: http://www.google.com/bot.html
731
+ producer:
732
+ name: Google Inc.
733
+ url: http://www.google.com
734
+ -
735
+ user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 19 subscribers; feed-id=13965549748850348809)
736
+ bot:
737
+ name: Googlebot
738
+ category: Search bot
739
+ url: http://www.google.com/bot.html
740
+ producer:
741
+ name: Google Inc.
742
+ url: http://www.google.com
743
+ -
744
+ user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 2 subscribers; feed-id=17860707833818568603)
745
+ bot:
746
+ name: Googlebot
747
+ category: Search bot
748
+ url: http://www.google.com/bot.html
749
+ producer:
750
+ name: Google Inc.
751
+ url: http://www.google.com
752
+ -
753
+ user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 375 subscribers; feed-id=15381863289700640853)
754
+ bot:
755
+ name: Googlebot
756
+ category: Search bot
757
+ url: http://www.google.com/bot.html
758
+ producer:
759
+ name: Google Inc.
760
+ url: http://www.google.com
761
+ -
762
+ user_agent: Googlebot (gocrawl v0.4)
763
+ bot:
764
+ name: Googlebot
765
+ category: Search bot
766
+ url: http://www.google.com/bot.html
767
+ producer:
768
+ name: Google Inc.
769
+ url: http://www.google.com
770
+ -
771
+ user_agent: Googlebot-Image/1.0
772
+ bot:
773
+ name: Googlebot
774
+ category: Search bot
775
+ url: http://www.google.com/bot.html
776
+ producer:
777
+ name: Google Inc.
778
+ url: http://www.google.com
779
+ -
780
+ user_agent: Googlebot-News (2.3.3, ruby 1.9.3 (2013-11-22))
781
+ bot:
782
+ name: Googlebot
783
+ category: Search bot
784
+ url: http://www.google.com/bot.html
785
+ producer:
786
+ name: Google Inc.
787
+ url: http://www.google.com
788
+ -
789
+ user_agent: Googlebot-Video/1.0
790
+ bot:
791
+ name: Googlebot
792
+ category: Search bot
793
+ url: http://www.google.com/bot.html
794
+ producer:
795
+ name: Google Inc.
796
+ url: http://www.google.com
797
+ -
197
798
  user_agent: Googlebot/2.1 (http://www.googlebot.com/bot.html)
198
- name: 'Googlebot'
199
- -
799
+ bot:
800
+ name: Googlebot
801
+ category: Search bot
802
+ url: http://www.google.com/bot.html
803
+ producer:
804
+ name: Google Inc.
805
+ url: http://www.google.com
806
+ -
200
807
  user_agent: Googlebot/Nutch-1.7
201
- name: 'Googlebot'
202
- -
203
- user_agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36
204
- name: 'Googlebot'
205
- -
808
+ bot:
809
+ name: Googlebot
810
+ category: Search bot
811
+ url: http://www.google.com/bot.html
812
+ producer:
813
+ name: Google Inc.
814
+ url: http://www.google.com
815
+ -
816
+ user_agent: GoogleProducer; (+http://goo.gl/7y4SX)
817
+ bot:
818
+ name: Googlebot
819
+ category: Search bot
820
+ url: http://www.google.com/bot.html
821
+ producer:
822
+ name: Google Inc.
823
+ url: http://www.google.com
824
+ -
825
+ user_agent: Mediapartners-Google
826
+ bot:
827
+ name: Googlebot
828
+ category: Search bot
829
+ url: http://www.google.com/bot.html
830
+ producer:
831
+ name: Google Inc.
832
+ url: http://www.google.com
833
+ -
834
+ user_agent: Mozilla/5.0 (compatible) Feedfetcher-Google;(+http://www.google.com/feedfetcher.html)
835
+ bot:
836
+ name: Googlebot
837
+ category: Search bot
838
+ url: http://www.google.com/bot.html
839
+ producer:
840
+ name: Google Inc.
841
+ url: http://www.google.com
842
+ -
206
843
  user_agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
207
- name: 'Googlebot'
208
- -
844
+ bot:
845
+ name: Googlebot
846
+ category: Search bot
847
+ url: http://www.google.com/bot.html
848
+ producer:
849
+ name: Google Inc.
850
+ url: http://www.google.com
851
+ -
209
852
  user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
210
- name: 'Googlebot'
211
- -
212
- user_agent: Googlebot-Image/1.0
213
- name: 'Googlebot'
214
- -
215
- user_agent: Googlebot-Video/1.0
216
- name: 'Googlebot'
217
- -
218
- user_agent: Googlebot (gocrawl v0.4)
219
- name: 'Googlebot'
220
- -
221
- user_agent: Googlebot-News (2.3.3, ruby 1.9.3 (2013-11-22))
222
- name: 'Googlebot'
223
- -
224
- user_agent: DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
225
- name: 'Googlebot'
226
- -
227
- user_agent: SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
228
- name: 'Googlebot'
229
- -
230
- user_agent: Mozilla/5.0 (compatible) Feedfetcher-Google;(+http://www.google.com/feedfetcher.html)
231
- name: 'Googlebot'
232
- -
233
- user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 2 subscribers; feed-id=17860707833818568603)
234
- name: 'Googlebot'
235
- -
236
- user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 19 subscribers; feed-id=13965549748850348809)
237
- name: 'Googlebot'
238
- -
239
- user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 375 subscribers; feed-id=15381863289700640853)
240
- name: 'Googlebot'
241
- -
853
+ bot:
854
+ name: Googlebot
855
+ category: Search bot
856
+ url: http://www.google.com/bot.html
857
+ producer:
858
+ name: Google Inc.
859
+ url: http://www.google.com
860
+ -
861
+ user_agent: Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; Google-Publisher-Plugin) Chrome/27.0.1453 Mobile Safari/537.36
862
+ bot:
863
+ name: Googlebot
864
+ category: Search bot
865
+ url: http://www.google.com/bot.html
866
+ producer:
867
+ name: Google Inc.
868
+ url: http://www.google.com
869
+ -
242
870
  user_agent: Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0 Google (+https://developers.google.com/+/web/snippet/)
243
- name: 'Googlebot'
244
- -
245
- user_agent: GoogleProducer; (+http://goo.gl/7y4SX)
246
- name: 'Googlebot'
247
- -
871
+ bot:
872
+ name: Googlebot
873
+ category: Search bot
874
+ url: http://www.google.com/bot.html
875
+ producer:
876
+ name: Google Inc.
877
+ url: http://www.google.com
878
+ -
248
879
  user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36
249
- name: 'Googlebot'
250
- -
251
- user_agent: AdsBot-Google-Mobile (+http://www.google.com/mobile/adsbot.html) Mozilla (iPhone; U; CPU iPhone OS 3 0 like Mac OS X) AppleWebKit (KHTML, like Gecko) Mobile Safari
252
- name: 'Googlebot'
253
- -
254
- user_agent: AdsBot-Google (+http://www.google.com/adsbot.html)
255
- name: 'Googlebot'
256
- -
257
- user_agent: Mediapartners-Google
258
- name: 'Googlebot'
259
- -
880
+ bot:
881
+ name: Googlebot
882
+ category: Search bot
883
+ url: http://www.google.com/bot.html
884
+ producer:
885
+ name: Google Inc.
886
+ url: http://www.google.com
887
+ -
888
+ user_agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36
889
+ bot:
890
+ name: Googlebot
891
+ category: Search bot
892
+ url: http://www.google.com/bot.html
893
+ producer:
894
+ name: Google Inc.
895
+ url: http://www.google.com
896
+ -
897
+ user_agent: SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
898
+ bot:
899
+ name: Googlebot
900
+ category: Search bot
901
+ url: http://www.google.com/bot.html
902
+ producer:
903
+ name: Google Inc.
904
+ url: http://www.google.com
905
+ -
906
+ user_agent: HTTPMon/1.0b (http://www.httpmon.com)
907
+ bot:
908
+ name: HTTPMon
909
+ category: Site Monitor
910
+ url: http://www.httpmon.com
911
+ producer:
912
+ name: towards GmbH
913
+ url: http://www.towards.ch/
914
+ -
915
+ user_agent: 'flieder - neofonie heritrix/1.14.3 (+http://spider.neofonie.de)'
916
+ bot:
917
+ name: Heritrix
918
+ category: Crawler
919
+ url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
920
+ producer:
921
+ name: The Internet Archive
922
+ url: http://www.archive.org
923
+ -
924
+ user_agent: Mozilla/5.0 (compatible; heritrix/1.14.4 +http://webarhive.nlc.gov.cn)
925
+ bot:
926
+ name: Heritrix
927
+ category: Crawler
928
+ url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
929
+ producer:
930
+ name: The Internet Archive
931
+ url: http://www.archive.org
932
+ -
260
933
  user_agent: Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.dla-marbach.de)
261
- name: 'Heritrix'
262
- -
263
- user_agent: Mozilla/5.0 (compatible; worio bot heritrix/1.10.0 +http://worio.com)
264
- name: 'Heritrix'
265
- -
266
- user_agent: Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.baidu.com)
267
- name: 'Heritrix'
268
- -
269
- user_agent: flieder - neofonie heritrix/1.14.3 (+http://spider.neofonie.de)
270
- name: 'Heritrix'
271
- -
934
+ bot:
935
+ name: Heritrix
936
+ category: Crawler
937
+ url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
938
+ producer:
939
+ name: The Internet Archive
940
+ url: http://www.archive.org
941
+ -
272
942
  user_agent: mozilla/5.0 (compatible; heritrix/1.3.0 http://archive.crawler.org)
273
- name: 'Heritrix'
274
- -
275
- user_agent: Mozilla/5.0 (compatible; heritrix/3.1.2-SNAPSHOT-20130207.001528 +http://webarchiv.cz/kontakty/)
276
- name: 'Heritrix'
277
- -
278
- user_agent: Mozilla/5.0 (compatible; heritrix/1.14.4 +http://webarhive.nlc.gov.cn)
279
- name: 'Heritrix'
280
- -
943
+ bot:
944
+ name: Heritrix
945
+ category: Crawler
946
+ url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
947
+ producer:
948
+ name: The Internet Archive
949
+ url: http://www.archive.org
950
+ -
951
+ user_agent: Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.baidu.com)
952
+ bot:
953
+ name: Heritrix
954
+ category: Crawler
955
+ url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
956
+ producer:
957
+ name: The Internet Archive
958
+ url: http://www.archive.org
959
+ -
281
960
  user_agent: Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.run4dom.com)
282
- name: 'Heritrix'
283
- -
284
- user_agent: HTTPMon/1.0b (http://www.httpmon.com)
285
- name: 'HTTPMon'
286
- -
961
+ bot:
962
+ name: Heritrix
963
+ category: Crawler
964
+ url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
965
+ producer:
966
+ name: The Internet Archive
967
+ url: http://www.archive.org
968
+ -
969
+ user_agent: Mozilla/5.0 (compatible; heritrix/3.1.2-SNAPSHOT-20130207.001528 +http://webarchiv.cz/kontakty/)
970
+ bot:
971
+ name: Heritrix
972
+ category: Crawler
973
+ url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
974
+ producer:
975
+ name: The Internet Archive
976
+ url: http://www.archive.org
977
+ -
978
+ user_agent: Mozilla/5.0 (compatible; worio bot heritrix/1.10.0 +http://worio.com)
979
+ bot:
980
+ name: Heritrix
981
+ category: Crawler
982
+ url: https://webarchive.jira.com/wiki/display/Heritrix/Heritrix
983
+ producer:
984
+ name: The Internet Archive
985
+ url: http://www.archive.org
986
+ -
987
+ user_agent: HubPages V0.2.2 (http://hubpages.com/help/crawlingpolicy)
988
+ bot:
989
+ name: HubPages
990
+ category: Crawler
991
+ url: http://hubpages.com/help/crawlingpolicy
992
+ producer:
993
+ name: HubPages
994
+ url: http://hubpages.com/
995
+ -
996
+ user_agent: ICC-Crawler/2.0 (Mozilla-compatible; ; http://www.nict.go.jp/en/univ-com/plan/crawl.html)
997
+ bot:
998
+ name: ICC-Crawler
999
+ category: Crawler
1000
+ url: http://www.nict.go.jp/en/univ-com/plan/crawl.html
1001
+ producer:
1002
+ name:
1003
+ url:
1004
+ -
287
1005
  user_agent: iisbot/1.0 (+http://www.iis.net/iisbot.html)
288
- name: 'IIS Site Analysis'
289
- -
290
- user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; linkdexbot-mobile/2.1; +http://www.linkdex.com/about/bots/)
291
- name: 'Linkdex Bot'
292
- -
293
- user_agent: Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)
294
- name: 'Linkdex Bot'
295
- -
1006
+ bot:
1007
+ name: IIS Site Analysis
1008
+ category: Crawler
1009
+ url: http://www.iis.net/iisbot.html
1010
+ producer:
1011
+ name: Microsoft Corporation
1012
+ url: http://www.microsoft.com
1013
+ -
1014
+ user_agent: kouio.com RSS reader
1015
+ bot:
1016
+ name: Kouio
1017
+ url: http://kouio.com/
1018
+ category: Feed Fetcher
1019
+ producer:
1020
+ name:
1021
+ url:
1022
+ -
1023
+ user_agent: 'kouio.com RSS reader - 6 subscribers'
1024
+ bot:
1025
+ name: Kouio
1026
+ url: http://kouio.com/
1027
+ category: Feed Fetcher
1028
+ producer:
1029
+ name:
1030
+ url:
1031
+ -
1032
+ user_agent: 'ltx71 - (http://ltx71.com/)'
1033
+ bot:
1034
+ name: LTX71
1035
+ url: http://ltx71.com/
1036
+ producer:
1037
+ name:
1038
+ url:
1039
+ -
1040
+ user_agent: larbin_2.6.3 larbin2.6.3@unspecified.mail
1041
+ bot:
1042
+ name: Larbin web crawler
1043
+ category: Crawler
1044
+ url: http://larbin.sourceforge.net
1045
+ producer:
1046
+ name:
1047
+ url:
1048
+ -
296
1049
  user_agent: linkdex.com/v2.0 and linkdex.com/v2.1
297
- name: 'Linkdex Bot'
298
- -
1050
+ bot:
1051
+ name: Linkdex Bot
1052
+ category: Search bot
1053
+ url: http://www.linkdex.com/bots
1054
+ producer:
1055
+ name: Mojeek Ltd.
1056
+ url: http://www.mojeek.com
1057
+ -
1058
+ user_agent: Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)
1059
+ bot:
1060
+ name: Linkdex Bot
1061
+ category: Search bot
1062
+ url: http://www.linkdex.com/bots
1063
+ producer:
1064
+ name: Mojeek Ltd.
1065
+ url: http://www.mojeek.com
1066
+ -
1067
+ user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; linkdexbot-mobile/2.1; +http://www.linkdex.com/about/bots/)
1068
+ bot:
1069
+ name: Linkdex Bot
1070
+ category: Search bot
1071
+ url: http://www.linkdex.com/bots
1072
+ producer:
1073
+ name: Mojeek Ltd.
1074
+ url: http://www.mojeek.com
1075
+ -
299
1076
  user_agent: LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)
300
- name: 'LinkedIn Bot'
301
- -
1077
+ bot:
1078
+ name: LinkedIn Bot
1079
+ category: Social Media Agent
1080
+ url: http://www.linkedin.com
1081
+ producer:
1082
+ name: LinkedIn
1083
+ url: http://www.linkedin.com
1084
+ -
1085
+ user_agent: Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)
1086
+ bot:
1087
+ name: MJ12 Bot
1088
+ category: Search bot
1089
+ url: http://majestic12.co.uk/bot.php
1090
+ producer:
1091
+ name: Majestic-12
1092
+ url: http://majestic12.co.uk
1093
+ -
1094
+ user_agent: magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)
1095
+ bot:
1096
+ name: Magpie-Crawler
1097
+ category: Social Media Agent
1098
+ url: http://www.brandwatch.com/magpie-crawler/
1099
+ producer:
1100
+ name: Brandwatch
1101
+ url: http://www.brandwatch.com
1102
+ -
1103
+ user_agent: MagpieRSS/0.72 (+http://magpierss.sf.net)
1104
+ bot:
1105
+ name: MagpieRSS
1106
+ url: http://magpierss.sourceforge.net/
1107
+ category: Feed Parser
1108
+ producer:
1109
+ name:
1110
+ url:
1111
+ -
302
1112
  user_agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
303
- name: 'Mail.Ru Bot'
304
- -
1113
+ bot:
1114
+ name: Mail.Ru Bot
1115
+ category: Search bot
1116
+ url: http://help.mail.ru/webmaster/indexing/robots/types_robots
1117
+ producer:
1118
+ name: Mail.Ru Group
1119
+ url: http://corp.mail.ru
1120
+ -
305
1121
  user_agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +http://go.mail.ru/help/robots)
306
- name: 'Mail.Ru Bot'
307
- -
308
- user_agent: magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)
309
- name: 'Magpie-Crawler'
310
- -
1122
+ bot:
1123
+ name: Mail.Ru Bot
1124
+ category: Search bot
1125
+ url: http://help.mail.ru/webmaster/indexing/robots/types_robots
1126
+ producer:
1127
+ name: Mail.Ru Group
1128
+ url: http://corp.mail.ru
1129
+ -
311
1130
  user_agent: Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)
312
- name: 'Meanpath Bot'
313
- -
1131
+ bot:
1132
+ name: Meanpath Bot
1133
+ category: Search bot
1134
+ url: http://www.meanpath.com/meanpathbot.html
1135
+ producer:
1136
+ name: Meanpath
1137
+ url: http://www.meanpath.com
1138
+ -
314
1139
  user_agent: Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)
315
- name: 'Mixrank Bot'
316
- -
317
- user_agent: Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)
318
- name: 'MJ12 Bot'
319
- -
1140
+ bot:
1141
+ name: Mixrank Bot
1142
+ category: Crawler
1143
+ url: http://mixrank.com
1144
+ producer:
1145
+ name: Online Media Group, Inc.
1146
+ url:
1147
+ -
320
1148
  user_agent: Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)
321
- name: 'MojeekBot'
322
- -
1149
+ bot:
1150
+ name: MojeekBot
1151
+ category: Search bot
1152
+ url: http://www.mojeek.com/bot.html
1153
+ producer:
1154
+ name: Mojeek Ltd.
1155
+ url: http://www.mojeek.com
1156
+ -
1157
+ user_agent: 'Mozilla/5.0 (compatible; www.monitor.us - free monitoring service; http://www.monitor.us)'
1158
+ bot:
1159
+ name: Monitor.Us
1160
+ category: Site Monitor
1161
+ url: http://www.monitor.us
1162
+ producer:
1163
+ name: Monitor.Us
1164
+ url: http://www.monitor.us
1165
+ -
1166
+ user_agent: nlcrawler/1.0 (+http://northernlight.com/)
1167
+ bot:
1168
+ name: NLCrawler
1169
+ category: Crawler
1170
+ url:
1171
+ producer:
1172
+ name: Northern Light
1173
+ url: http://northernlight.com
1174
+ -
323
1175
  user_agent: NalezenCzBot/1.0 (http://www.nalezen.cz/about-crawler)
324
- name: 'NalezenCzBot'
325
- -
1176
+ bot:
1177
+ name: NalezenCzBot
1178
+ category: Crawler
1179
+ url: http://www.nalezen.cz/about-crawler
1180
+ producer:
1181
+ name: Jaroslav Kuboš
1182
+ url:
1183
+ -
1184
+ user_agent: NetLyzer FastProbe
1185
+ bot:
1186
+ name: NetLyzer FastProbe
1187
+ -
326
1188
  user_agent: Mozilla/4.0 (compatible; Netcraft Web Server Survey)
327
- name: 'Netcraft Survey Bot'
328
- -
329
- user_agent: nlcrawler/1.0 (+http://northernlight.com/)
330
- name: 'NLCrawler'
331
- -
1189
+ bot:
1190
+ name: Netcraft Survey Bot
1191
+ category: Search bot
1192
+ url:
1193
+ producer:
1194
+ name: Netcraft
1195
+ url: http://www.netcraft.com
1196
+ -
1197
+ user_agent: Netvibes (http://www.netvibes.com)
1198
+ bot:
1199
+ name: Netvibes
1200
+ url: http://www.netvibes.com/
1201
+ category: Feed Fetcher
1202
+ producer:
1203
+ name:
1204
+ url:
1205
+ -
1206
+ user_agent: 'Netvibes (http://www.netvibes.com/; 8 subscribers; feedID: 2244192)'
1207
+ bot:
1208
+ name: Netvibes
1209
+ url: http://www.netvibes.com/
1210
+ category: Feed Fetcher
1211
+ producer:
1212
+ name:
1213
+ url:
1214
+ -
1215
+ user_agent: 'NewsBlur Favicon Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
1216
+ bot:
1217
+ name: NewsBlur
1218
+ url: http://www.newsblur.com
1219
+ category: Feed Fetcher
1220
+ producer:
1221
+ name:
1222
+ url:
1223
+ -
1224
+ user_agent: 'NewsBlur Feed Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
1225
+ bot:
1226
+ name: NewsBlur
1227
+ url: http://www.newsblur.com
1228
+ category: Feed Fetcher
1229
+ producer:
1230
+ name:
1231
+ url:
1232
+ -
1233
+ user_agent: NewsBlur Feed Finder (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
1234
+ bot:
1235
+ name: NewsBlur
1236
+ url: http://www.newsblur.com
1237
+ category: Feed Fetcher
1238
+ producer:
1239
+ name:
1240
+ url:
1241
+ -
1242
+ user_agent: 'NewsBlur Page Fetcher - 7 subscribers - http://www.newsblur.com/site/3966817/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
1243
+ bot:
1244
+ name: NewsBlur
1245
+ url: http://www.newsblur.com
1246
+ category: Feed Fetcher
1247
+ producer:
1248
+ name:
1249
+ url:
1250
+ -
1251
+ user_agent: NewsGatorOnline/2.0 (http://www.newsgator.com; 2 subscribers)
1252
+ bot:
1253
+ name: NewsGator
1254
+ url: http://www.newsgator.com
1255
+ category: Feed Fetcher
1256
+ producer:
1257
+ name:
1258
+ url:
1259
+ -
1260
+ user_agent: 5acd734a-1ed3-4a17-ad9d-0b09ae7bc2d2/Nutch-1.7
1261
+ bot:
1262
+ name: Nutch-based Bot
1263
+ category: Crawler
1264
+ url: https://nutch.apache.org
1265
+ producer:
1266
+ name: The Apache Software Foundation
1267
+ url: http://www.apache.org/foundation/
1268
+ -
1269
+ user_agent: CB/Nutch-1.7
1270
+ bot:
1271
+ name: Nutch-based Bot
1272
+ category: Crawler
1273
+ url: https://nutch.apache.org
1274
+ producer:
1275
+ name: The Apache Software Foundation
1276
+ url: http://www.apache.org/foundation/
1277
+ -
1278
+ user_agent: Mozilla/5.0 (compatible; summers;)/Nutch-1.7
1279
+ bot:
1280
+ name: Nutch-based Bot
1281
+ category: Crawler
1282
+ url: https://nutch.apache.org
1283
+ producer:
1284
+ name: The Apache Software Foundation
1285
+ url: http://www.apache.org/foundation/
1286
+ -
1287
+ user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8/Nutch-1.7
1288
+ bot:
1289
+ name: Nutch-based Bot
1290
+ category: Crawler
1291
+ url: https://nutch.apache.org
1292
+ producer:
1293
+ name: The Apache Software Foundation
1294
+ url: http://www.apache.org/foundation/
1295
+ -
1296
+ user_agent: My Nutch Spider/Nutch-1.6
1297
+ bot:
1298
+ name: Nutch-based Bot
1299
+ category: Crawler
1300
+ url: https://nutch.apache.org
1301
+ producer:
1302
+ name: The Apache Software Foundation
1303
+ url: http://www.apache.org/foundation/
1304
+ -
1305
+ user_agent: MySpider/Nutch-2.2
1306
+ bot:
1307
+ name: Nutch-based Bot
1308
+ category: Crawler
1309
+ url: https://nutch.apache.org
1310
+ producer:
1311
+ name: The Apache Software Foundation
1312
+ url: http://www.apache.org/foundation/
1313
+ -
1314
+ user_agent: Nutch12/Nutch-1.2
1315
+ bot:
1316
+ name: Nutch-based Bot
1317
+ category: Crawler
1318
+ url: https://nutch.apache.org
1319
+ producer:
1320
+ name: The Apache Software Foundation
1321
+ url: http://www.apache.org/foundation/
1322
+ -
1323
+ user_agent: NutchSpider/Nutch-1.4
1324
+ bot:
1325
+ name: Nutch-based Bot
1326
+ category: Crawler
1327
+ url: https://nutch.apache.org
1328
+ producer:
1329
+ name: The Apache Software Foundation
1330
+ url: http://www.apache.org/foundation/
1331
+ -
1332
+ user_agent: spider/Nutch-1.5.1 (spider; http://www.xxx.com)
1333
+ bot:
1334
+ name: Nutch-based Bot
1335
+ category: Crawler
1336
+ url: https://nutch.apache.org
1337
+ producer:
1338
+ name: The Apache Software Foundation
1339
+ url: http://www.apache.org/foundation/
1340
+ -
1341
+ user_agent: Spider/Nutch-2.3-SNAPSHOT (Webcrawler)
1342
+ bot:
1343
+ name: Nutch-based Bot
1344
+ category: Crawler
1345
+ url: https://nutch.apache.org
1346
+ producer:
1347
+ name: The Apache Software Foundation
1348
+ url: http://www.apache.org/foundation/
1349
+ -
1350
+ user_agent: testnutch/Nutch-1.8
1351
+ bot:
1352
+ name: Nutch-based Bot
1353
+ category: Crawler
1354
+ url: https://nutch.apache.org
1355
+ producer:
1356
+ name: The Apache Software Foundation
1357
+ url: http://www.apache.org/foundation/
1358
+ -
1359
+ user_agent: Your Nutch Spider/Nutch-2.2.1
1360
+ bot:
1361
+ name: Nutch-based Bot
1362
+ category: Crawler
1363
+ url: https://nutch.apache.org
1364
+ producer:
1365
+ name: The Apache Software Foundation
1366
+ url: http://www.apache.org/foundation/
1367
+ -
1368
+ user_agent: your sipder name/Nutch-1.7
1369
+ bot:
1370
+ name: Nutch-based Bot
1371
+ category: Crawler
1372
+ url: https://nutch.apache.org
1373
+ producer:
1374
+ name: The Apache Software Foundation
1375
+ url: http://www.apache.org/foundation/
1376
+ -
332
1377
  user_agent: omgilibot/0.3 +http://www.omgili.com/Crawler.html
333
- name: 'Omgili bot'
334
- -
335
- user_agent: Mozilla/5.0 (compatible; OpenindexSpider; +http://www.openindex.io/en/webmasters/spider.html)
336
- name: 'Openindex Spider'
337
- -
1378
+ bot:
1379
+ name: Omgili bot
1380
+ category: Search bot
1381
+ url: http://www.omgili.com/Crawler.html
1382
+ producer:
1383
+ name: Omgili
1384
+ url: http://www.omgili.com
1385
+ -
338
1386
  user_agent: Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )
339
- name: 'OpenLinkProfiler'
340
- -
1387
+ bot:
1388
+ name: OpenLinkProfiler
1389
+ category: Crawler
1390
+ url: http://openlinkprofiler.org/bot
1391
+ producer:
1392
+ name: Axandra GmbH
1393
+ url: http://www.axandra.com
1394
+ -
341
1395
  user_agent: OpenWebSpider v0.1.4 (http://www.openwebspider.org/)
342
- name: 'OpenWebSpider'
343
- -
1396
+ bot:
1397
+ name: OpenWebSpider
1398
+ category: Crawler
1399
+ url: http://www.openwebspider.org
1400
+ producer:
1401
+ name: OpenWebSpider Lab
1402
+ url: http://lab.openwebspider.org
1403
+ -
1404
+ user_agent: Mozilla/5.0 (compatible; OpenindexSpider; +http://www.openindex.io/en/webmasters/spider.html)
1405
+ bot:
1406
+ name: Openindex Spider
1407
+ category: Search bot
1408
+ url: http://www.openindex.io/en/webmasters/spider.html
1409
+ producer:
1410
+ name: Openindex B.V.
1411
+ url: http://www.openindex.io
1412
+ -
1413
+ user_agent: Mozilla/5.0 (compatible; OrangeBot-Collector/2.0; support.orangebot@orange.com)
1414
+ bot:
1415
+ name: Orange Bot
1416
+ category: Search bot
1417
+ url: http://lemoteur.orange.fr
1418
+ producer:
1419
+ name: Orange
1420
+ url: http://www.orange.fr
1421
+ -
1422
+ user_agent: Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com)
1423
+ bot:
1424
+ name: Orange Bot
1425
+ category: Search bot
1426
+ url: http://lemoteur.orange.fr
1427
+ producer:
1428
+ name: Orange
1429
+ url: http://www.orange.fr
1430
+ -
1431
+ user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)
1432
+ bot:
1433
+ name: Orange Bot
1434
+ category: Search bot
1435
+ url: http://lemoteur.orange.fr
1436
+ producer:
1437
+ name: Orange
1438
+ url: http://www.orange.fr
1439
+ -
1440
+ user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1;fr;rv:1.8.1) VoilaBotCollector BETA 0.1 (http://www.voila.com/)
1441
+ bot:
1442
+ name: Orange Bot
1443
+ category: Search bot
1444
+ url: http://lemoteur.orange.fr
1445
+ producer:
1446
+ name: Orange
1447
+ url: http://www.orange.fr
1448
+ -
1449
+ user_agent: Mozilla/5.0 (Java) outbrain
1450
+ bot:
1451
+ name: Outbrain
1452
+ category: Crawler
1453
+ url:
1454
+ producer:
1455
+ name: Outbrain
1456
+ url: http://www.outbrain.com/
1457
+ -
1458
+ user_agent: Mozilla/5.0 (compatible; phpservermon/3.1.1; +http://www.phpservermonitor.org)
1459
+ bot:
1460
+ name: PHP Server Monitor
1461
+ category: Site Monitor
1462
+ url: https://github.com/phpservermon/phpservermon
1463
+ producer:
1464
+ name: PHP Server Monitor
1465
+ url: http://www.phpservermonitor.org/
1466
+ -
344
1467
  user_agent: Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)
345
- name: 'PaperLiBot'
346
- -
347
- user_agent: psbot/0.1 (+http://www.picsearch.com/bot.html)
348
- name: 'Picsearch bot'
349
- -
1468
+ bot:
1469
+ name: PaperLiBot
1470
+ category: Search bot
1471
+ url: http://support.paper.li/entries/20023257-what-is-paper-li
1472
+ producer:
1473
+ name: Smallrivers SA
1474
+ url: http://www.paper.li
1475
+ -
350
1476
  user_agent: psbot-page (+http://www.picsearch.com/bot.html)
351
- name: 'Picsearch bot'
352
- -
1477
+ bot:
1478
+ name: Picsearch bot
1479
+ category: Search bot
1480
+ url: http://www.picsearch.com/bot.html
1481
+ producer:
1482
+ name: Picsearch
1483
+ url: http://www.picsearch.com
1484
+ -
1485
+ user_agent: psbot/0.1 (+http://www.picsearch.com/bot.html)
1486
+ bot:
1487
+ name: Picsearch bot
1488
+ category: Search bot
1489
+ url: http://www.picsearch.com/bot.html
1490
+ producer:
1491
+ name: Picsearch
1492
+ url: http://www.picsearch.com
1493
+ -
353
1494
  user_agent: Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)
354
- name: 'Pingdom Bot'
355
- -
1495
+ bot:
1496
+ name: Pingdom Bot
1497
+ category: Site Monitor
1498
+ url:
1499
+ producer:
1500
+ name: Pingdom AB
1501
+ url: https://www.pingdom.com
1502
+ -
1503
+ user_agent: Pinterest/0.2 (+http://www.pinterest.com/)
1504
+ bot:
1505
+ name: Pinterest
1506
+ url:
1507
+ category: Crawler
1508
+ producer:
1509
+ name: Pinterest
1510
+ url: http://www.pinterest.com/
1511
+ -
356
1512
  user_agent: QuerySeekerSpider ( http://queryseeker.com/bot.html )
357
- name: 'QuerySeekerSpider'
358
- -
1513
+ bot:
1514
+ name: QuerySeekerSpider
1515
+ category: Crawler
1516
+ url: http://queryseeker.com/bot.html
1517
+ producer:
1518
+ name: QueryEye Inc.
1519
+ url: http://queryeye.com
1520
+ -
1521
+ user_agent: ROI Hunter; https://api-dev.roihunter.com
1522
+ bot:
1523
+ name: ROI Hunter
1524
+ category: Crawler
1525
+ url:
1526
+ producer:
1527
+ name: Roihunter a.s.
1528
+ url: http://roihunter.com/
1529
+ -
359
1530
  user_agent: Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)
360
- name: 'Reddit Bot'
361
- -
1531
+ bot:
1532
+ name: Reddit Bot
1533
+ category: Social Media Agent
1534
+ url: http://www.reddit.com/feedback
1535
+ producer:
1536
+ name: reddit inc.
1537
+ url: http://www.reddit.com
1538
+ -
362
1539
  user_agent: Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)
363
- name: 'Rogerbot'
364
- -
1540
+ bot:
1541
+ name: Rogerbot
1542
+ category: Crawler
1543
+ url: http://moz.com/help/pro/what-is-rogerbot-
1544
+ producer:
1545
+ name: SEOmoz, Inc.
1546
+ url: http://moz.com/
1547
+ -
365
1548
  user_agent: rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)
366
- name: 'Rogerbot'
367
- -
1549
+ bot:
1550
+ name: Rogerbot
1551
+ category: Crawler
1552
+ url: http://moz.com/help/pro/what-is-rogerbot-
1553
+ producer:
1554
+ name: SEOmoz, Inc.
1555
+ url: http://moz.com/
1556
+ -
368
1557
  user_agent: rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)
369
- name: 'Rogerbot'
370
- -
1558
+ bot:
1559
+ name: Rogerbot
1560
+ category: Crawler
1561
+ url: http://moz.com/help/pro/what-is-rogerbot-
1562
+ producer:
1563
+ name: SEOmoz, Inc.
1564
+ url: http://moz.com/
1565
+ -
1566
+ user_agent: SEOENGWorldBot/1.0 (+http://www.seoengine.com/seoengbot.htm)
1567
+ bot:
1568
+ name: SEOENGBot
1569
+ category: Crawler
1570
+ url: http://www.seoengine.com/seoengbot.htm
1571
+ producer:
1572
+ name: SEO Engine
1573
+ url: http://www.seoengine.com
1574
+ -
1575
+ user_agent: Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)
1576
+ bot:
1577
+ name: SISTRIX Crawler
1578
+ category: Crawler
1579
+ url: http://crawler.sistrix.net
1580
+ producer:
1581
+ name: SISTRIX GmbH
1582
+ url: http://www.sistrix.de
1583
+ -
1584
+ user_agent: Scrapy/1.0.3.post6+g2d688cd (+http://scrapy.org)
1585
+ bot:
1586
+ name: Scrapy
1587
+ category: Crawler
1588
+ url: http://scrapy.org
1589
+ -
371
1590
  user_agent: Screaming Frog SEO Spider/2.22
372
- name: 'Screaming Frog SEO Spider'
373
- -
1591
+ bot:
1592
+ name: Screaming Frog SEO Spider
1593
+ category: Crawler
1594
+ url: http://www.screamingfrog.co.uk/seo-spider
1595
+ producer:
1596
+ name: Screaming Frog Ltd
1597
+ url: http://www.screamingfrog.co.uk
1598
+ -
374
1599
  user_agent: ScreenerBot Crawler Beta 2.0 (+http://www.ScreenerBot.com)
375
- name: 'ScreenerBot'
376
- -
1600
+ bot:
1601
+ name: ScreenerBot
1602
+ category: Crawler
1603
+ url: http://www.screenerbot.com
1604
+ producer:
1605
+ name:
1606
+ url:
1607
+ -
377
1608
  user_agent: Mozilla/5.0 (compatible; SemrushBot/0.97; +http://www.semrush.com/bot.html)
378
- name: 'Semrush Bot'
379
- -
1609
+ bot:
1610
+ name: Semrush Bot
1611
+ category: Crawler
1612
+ url: http://www.semrush.com/bot.html
1613
+ producer:
1614
+ name: SEMrush
1615
+ url: http://www.semrush.com
1616
+ -
380
1617
  user_agent: SensikaBot/x.33 (+http://sensika.com)
381
- name: 'Sensika Bot'
382
- -
383
- user_agent: SEOENGWorldBot/1.0 (+http://www.seoengine.com/seoengbot.htm)
384
- name: 'SEOENGBot'
385
- -
1618
+ bot:
1619
+ name: Sensika Bot
1620
+ category:
1621
+ url:
1622
+ producer:
1623
+ name: Sensika
1624
+ url: http://sensika.com
1625
+ -
1626
+ user_agent: Server Density Service Monitoring v2
1627
+ bot:
1628
+ name: Server Density
1629
+ -
386
1630
  user_agent: Mozilla/5.0 (compatible; SeznamBot/3.1-test1; +http://fulltext.sblog.cz/)
387
- name: 'Seznam Bot'
388
- -
1631
+ bot:
1632
+ name: Seznam Bot
1633
+ category: Search bot
1634
+ url: http://www.mapy.cz/cz/seznambot.html
1635
+ producer:
1636
+ name: Seznam.cz, a.s.
1637
+ url: http://www.seznam.cz/
1638
+ -
389
1639
  user_agent: Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://fulltext.sblog.cz/)
390
- name: 'Seznam Bot'
391
- -
392
- user_agent: SeznamBot/3.0 (+http://fulltext.sblog.cz/)
393
- name: 'Seznam Bot'
394
- -
1640
+ bot:
1641
+ name: Seznam Bot
1642
+ category: Search bot
1643
+ url: http://www.mapy.cz/cz/seznambot.html
1644
+ producer:
1645
+ name: Seznam.cz, a.s.
1646
+ url: http://www.seznam.cz/
1647
+ -
395
1648
  user_agent: Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)
396
- name: 'Seznam Bot'
397
- -
1649
+ bot:
1650
+ name: Seznam Bot
1651
+ category: Search bot
1652
+ url: http://www.mapy.cz/cz/seznambot.html
1653
+ producer:
1654
+ name: Seznam.cz, a.s.
1655
+ url: http://www.seznam.cz/
1656
+ -
1657
+ user_agent: SeznamBot/3.0 (+http://fulltext.sblog.cz/)
1658
+ bot:
1659
+ name: Seznam Bot
1660
+ category: Search bot
1661
+ url: http://www.mapy.cz/cz/seznambot.html
1662
+ producer:
1663
+ name: Seznam.cz, a.s.
1664
+ url: http://www.seznam.cz/
1665
+ -
398
1666
  user_agent: ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)
399
- name: 'ShopWiki'
400
- -
401
- user_agent: Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)
402
- name: 'SISTRIX Crawler'
403
- -
404
- user_agent: Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
405
- name: 'Sogou Spider'
406
- -
407
- user_agent: Sogou inst spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
408
- name: 'Sogou Spider'
409
- -
410
- user_agent: Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
411
- name: 'Sogou Spider'
412
- -
1667
+ bot:
1668
+ name: ShopWiki
1669
+ category: Search tools
1670
+ url: http://www.shopwiki.com/wiki/Help:Bot
1671
+ producer:
1672
+ name: ShopWiki Corp.
1673
+ url: http://www.shopwiki.com
1674
+ -
1675
+ user_agent: SilverReader/1.0; http://silverreader.com
1676
+ bot:
1677
+ name: SilverReader
1678
+ url: http://silverreader.com
1679
+ category: Feed Fetcher
1680
+ producer:
1681
+ name:
1682
+ url:
1683
+ -
1684
+ user_agent: freshrss/0.8-dev (Linux; http://freshrss.org) SimplePie/1.4-dev-FreshRSS
1685
+ bot:
1686
+ name: SimplePie
1687
+ url: http://www.simplepie.org
1688
+ category: Feed Parser
1689
+ producer:
1690
+ name:
1691
+ url:
1692
+ -
1693
+ user_agent: SimplePie/1.2.1-dev (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20130514092120
1694
+ bot:
1695
+ name: SimplePie
1696
+ url: http://www.simplepie.org
1697
+ category: Feed Parser
1698
+ producer:
1699
+ name:
1700
+ url:
1701
+ -
1702
+ user_agent: SimplePie/1.3.1 (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20121030175911
1703
+ bot:
1704
+ name: SimplePie
1705
+ url: http://www.simplepie.org
1706
+ category: Feed Parser
1707
+ producer:
1708
+ name:
1709
+ url:
1710
+ -
1711
+ user_agent: Site24x7
1712
+ bot:
1713
+ name: Site24x7 Website Monitoring
1714
+ category: Site Monitor
1715
+ url: https://www.site24x7.com/site24x7-faq.html
1716
+ producer:
1717
+ name: Site24x7
1718
+ url: https://www.site24x7.com
1719
+ -
1720
+ user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5
1721
+ bot:
1722
+ name: Skype URI Preview
1723
+ category: Service Agent
1724
+ url:
1725
+ producer:
1726
+ name: Skype Communications S.à.r.l.
1727
+ url: https://www.skype.com
1728
+ -
1729
+ user_agent: Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)
1730
+ bot:
1731
+ name: Slackbot
1732
+ category: Crawler
1733
+ url: https://api.slack.com/robots
1734
+ producer:
1735
+ name: Slack Technologies
1736
+ url: http://slack.com
1737
+ -
413
1738
  user_agent: New-Sogou-Spider/1.0 (compatible; MSIE 5.5; Windows 98)
414
- name: 'Sogou Spider'
415
- -
1739
+ bot:
1740
+ name: Sogou Spider
1741
+ category: Search bot
1742
+ url: http://www.sogou.com/docs/help/webmasters.htm
1743
+ producer:
1744
+ name: Sohu, Inc.
1745
+ url: http://www.sogou.com
1746
+ -
1747
+ user_agent: Sogou inst spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm
1748
+ bot:
1749
+ name: Sogou Spider
1750
+ category: Search bot
1751
+ url: http://www.sogou.com/docs/help/webmasters.htm
1752
+ producer:
1753
+ name: Sohu, Inc.
1754
+ url: http://www.sogou.com
1755
+ -
1756
+ user_agent: Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm
1757
+ bot:
1758
+ name: Sogou Spider
1759
+ category: Search bot
1760
+ url: http://www.sogou.com/docs/help/webmasters.htm
1761
+ producer:
1762
+ name: Sohu, Inc.
1763
+ url: http://www.sogou.com
1764
+ -
1765
+ user_agent: Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm
1766
+ bot:
1767
+ name: Sogou Spider
1768
+ category: Search bot
1769
+ url: http://www.sogou.com/docs/help/webmasters.htm
1770
+ producer:
1771
+ name: Sohu, Inc.
1772
+ url: http://www.sogou.com
1773
+ -
416
1774
  user_agent: Sosospider+(+http://help.soso.com/webspider.htm)
417
- name: 'Soso Spider'
418
- -
419
- user_agent: 'Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch!'
420
- name: 'Superfeedr Bot'
421
- -
1775
+ bot:
1776
+ name: Soso Spider
1777
+ category: Search bot
1778
+ url: http://help.soso.com/webspider.htm
1779
+ producer:
1780
+ name: Tencent Holdings
1781
+ url: http://www.soso.com
1782
+ -
422
1783
  user_agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.19; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/2010040121 Firefox/3.0.19
423
- name: 'Spinn3r'
424
- -
1784
+ bot:
1785
+ name: Spinn3r
1786
+ category: Crawler
1787
+ url: http://spinn3r.com/robot
1788
+ producer:
1789
+ name: Tailrank Inc
1790
+ url: http://spinn3r.com
1791
+ -
425
1792
  user_agent: Mozilla/5.0 (compatible; SputnikBot/2.2)
426
- name: 'Sputnik Bot'
427
- -
1793
+ bot:
1794
+ name: Sputnik Bot
1795
+ category:
1796
+ url:
1797
+ producer:
1798
+ name:
1799
+ url:
1800
+ -
428
1801
  user_agent: Mozilla/5.0 (compatible; SputnikImageBot/2.2)
429
- name: 'Sputnik Bot'
430
- -
1802
+ bot:
1803
+ name: Sputnik Bot
1804
+ category:
1805
+ url:
1806
+ producer:
1807
+ name:
1808
+ url:
1809
+ -
1810
+ user_agent: 'Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch!'
1811
+ bot:
1812
+ name: Superfeedr Bot
1813
+ category: Feed Fetcher
1814
+ url:
1815
+ producer:
1816
+ name: Superfeedr
1817
+ url: https://superfeedr.com/
1818
+ -
431
1819
  user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)
432
- name: 'Survey Bot'
433
- -
1820
+ bot:
1821
+ name: Survey Bot
1822
+ category: Search bot
1823
+ url: http://www.domaintools.com/webmasters/surveybot.php
1824
+ producer:
1825
+ name: Domain Tools
1826
+ url: http://www.domaintools.com
1827
+ -
1828
+ user_agent: TelegramBot (like TwitterBot)
1829
+ bot:
1830
+ name: TelgramBot
1831
+ url: https://telegram.org/blog/bot-revolution
1832
+ -
434
1833
  user_agent: TinEye-bot/0.02 (see http://www.tineye.com/crawler.html)
435
- name: 'TinEye Crawler'
436
- -
1834
+ bot:
1835
+ name: TinEye Crawler
1836
+ category: Search bot
1837
+ url: http://www.tineye.com/crawler.html
1838
+ producer:
1839
+ name: Idée Inc.
1840
+ url: http://ideeinc.com
1841
+ -
1842
+ user_agent: Tiny Tiny RSS/1.10 (http://tt-rss.org/)
1843
+ bot:
1844
+ name: Tiny Tiny RSS
1845
+ url: http://tt-rss.org
1846
+ category: Feed Fetcher
1847
+ producer:
1848
+ name:
1849
+ url:
1850
+ -
1851
+ user_agent: Tiny Tiny RSS/1.11.4c63934 (http://tt-rss.org/)
1852
+ bot:
1853
+ name: Tiny Tiny RSS
1854
+ url: http://tt-rss.org
1855
+ category: Feed Fetcher
1856
+ producer:
1857
+ name:
1858
+ url:
1859
+ -
437
1860
  user_agent: TurnitinBot/3.0 (http://www.turnitin.com/robot/crawlerinfo.html)
438
- name: 'TurnitinBot'
439
- -
440
- user_agent: TweetedTimes Bot/1.0 (Mozilla/5.0 Compatible, +http://tweetedtimes.com)
441
- name: 'TweetedTimes Bot'
442
- -
1861
+ bot:
1862
+ name: TurnitinBot
1863
+ category: Crawler
1864
+ url: http://www.turnitin.com/robot/crawlerinfo.html
1865
+ producer:
1866
+ name: iParadigms, LLC.
1867
+ url: http://www.turnitin.com
1868
+ -
443
1869
  user_agent: Mozilla/5.0 (compatible; TweetedTimes Bot/1.0; +http://tweetedtimes.com)
444
- name: 'TweetedTimes Bot'
445
- -
1870
+ bot:
1871
+ name: TweetedTimes Bot
1872
+ category: Crawler
1873
+ url: http://tweetedtimes.com
1874
+ producer:
1875
+ name: TweetedTimes
1876
+ url: http://tweetedtimes.com/
1877
+ -
1878
+ user_agent: TweetedTimes Bot/1.0 (Mozilla/5.0 Compatible, +http://tweetedtimes.com)
1879
+ bot:
1880
+ name: TweetedTimes Bot
1881
+ category: Crawler
1882
+ url: http://tweetedtimes.com
1883
+ producer:
1884
+ name: TweetedTimes
1885
+ url: http://tweetedtimes.com/
1886
+ -
446
1887
  user_agent: Mozilla/5.0 (compatible; TweetmemeBot/3.0; +http://tweetmeme.com/)
447
- name: 'Tweetmeme Bot'
448
- -
1888
+ bot:
1889
+ name: Tweetmeme Bot
1890
+ category: Crawler
1891
+ url: http://tweetmeme.com/
1892
+ producer:
1893
+ name: Mediasift
1894
+ url:
1895
+ -
449
1896
  user_agent: Twitterbot/1.0
450
- name: 'Twitterbot'
451
- -
452
- user_agent: Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)
453
- name: 'Uptime Robot'
454
- -
1897
+ bot:
1898
+ name: Twitterbot
1899
+ category: Social Media Agent
1900
+ url: https://dev.twitter.com/docs/cards/getting-started
1901
+ producer:
1902
+ name: Twitter
1903
+ url: http://www.twitter.com
1904
+ -
455
1905
  user_agent: Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)
456
- name: 'URLAppendBot'
457
- -
1906
+ bot:
1907
+ name: URLAppendBot
1908
+ category: Crawler
1909
+ url: http://www.profound.net/urlappendbot.html
1910
+ producer:
1911
+ name: Profound Networks
1912
+ url: http://www.profound.net
1913
+ -
1914
+ user_agent: Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)
1915
+ bot:
1916
+ name: Uptime Robot
1917
+ category: Site Monitor
1918
+ url:
1919
+ producer:
1920
+ name: Uptime Robot
1921
+ url: http://uptimerobot.com
1922
+ -
458
1923
  user_agent: Mozilla/5.0 (compatible; VSMCrawler; http://www.visualsitemapper.com/crawler/)
459
- name: 'Visual Site Mapper Crawler'
460
- -
461
- user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1;fr;rv:1.8.1) VoilaBotCollector BETA 0.1 (http://www.voila.com/)
462
- name: 'Voila Bot'
463
- -
464
- user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)
465
- name: 'Voila Bot'
466
- -
467
- user_agent: Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0 (See <http://validator.w3.org/services>)
468
- name: 'W3C CSS Validator'
469
- -
1924
+ bot:
1925
+ name: Visual Site Mapper Crawler
1926
+ category: Crawler
1927
+ url: http://www.visualsitemapper.com/crawler
1928
+ producer:
1929
+ name: Alentum Software Ltd.
1930
+ url: http://www.alentum.com
1931
+ -
1932
+ user_agent: 'Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0 (See <http://validator.w3.org/services>)'
1933
+ bot:
1934
+ name: W3C CSS Validator
1935
+ category: Validator
1936
+ url: http://jigsaw.w3.org/css-validator
1937
+ producer:
1938
+ name: W3C
1939
+ url: http://www.w3.org
1940
+ -
470
1941
  user_agent: W3C_I18n-Checker/1.0 (http://validator.w3.org/services)
471
- name: 'W3C I18N Checker'
472
- -
473
- user_agent: W3C-checklink/4.0 [4.4] libwww-perl/5.803
474
- name: 'W3C Link Checker'
475
- -
1942
+ bot:
1943
+ name: W3C I18N Checker
1944
+ category: Validator
1945
+ url: http://validator.w3.org/i18n-checker
1946
+ producer:
1947
+ name: W3C
1948
+ url: http://www.w3.org
1949
+ -
1950
+ user_agent: 'W3C-checklink/4.0 [4.4] libwww-perl/5.803'
1951
+ bot:
1952
+ name: W3C Link Checker
1953
+ category: Validator
1954
+ url: http://validator.w3.org/checklink
1955
+ producer:
1956
+ name: W3C
1957
+ url: http://www.w3.org
1958
+ -
476
1959
  user_agent: W3C-checklink/4.81 libwww-perl/5.836
477
- name: 'W3C Link Checker'
478
- -
1960
+ bot:
1961
+ name: W3C Link Checker
1962
+ category: Validator
1963
+ url: http://validator.w3.org/checklink
1964
+ producer:
1965
+ name: W3C
1966
+ url: http://www.w3.org
1967
+ -
479
1968
  user_agent: W3C_Validator/1.3 http://validator.w3.org/services
480
- name: 'W3C Markup Validation Service'
481
- -
1969
+ bot:
1970
+ name: W3C Markup Validation Service
1971
+ category: Validator
1972
+ url: http://validator.w3.org/services
1973
+ producer:
1974
+ name: W3C
1975
+ url: http://www.w3.org
1976
+ -
482
1977
  user_agent: W3C_Validator/1.767
483
- name: 'W3C Markup Validation Service'
484
- -
1978
+ bot:
1979
+ name: W3C Markup Validation Service
1980
+ category: Validator
1981
+ url: http://validator.w3.org/services
1982
+ producer:
1983
+ name: W3C
1984
+ url: http://www.w3.org
1985
+ -
485
1986
  user_agent: W3C-mobileOK/DDC-1.0 (see http://www.w3.org/2006/07/mobileok-ddc)
486
- name: 'W3C MobileOK Checker'
487
- -
1987
+ bot:
1988
+ name: W3C MobileOK Checker
1989
+ category: Validator
1990
+ url: http://validator.w3.org/mobile
1991
+ producer:
1992
+ name: W3C
1993
+ url: http://www.w3.org
1994
+ -
488
1995
  user_agent: W3C_Unicorn/1.0 (http://validator.w3.org/services)
489
- name: 'W3C Unified Validator'
490
- -
491
- user_agent: WebbCrawler 1.0 ( http://badcheese.com/crawler.html )
492
- name: 'WebbCrawler'
493
- -
1996
+ bot:
1997
+ name: W3C Unified Validator
1998
+ category: Validator
1999
+ url: http://validator.w3.org/unicorn
2000
+ producer:
2001
+ name: W3C
2002
+ url: http://www.w3.org
2003
+ -
494
2004
  user_agent: WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)
495
- name: 'WeSEE:Search'
496
- -
2005
+ bot:
2006
+ name: WeSEE:Search
2007
+ category: Search bot
2008
+ url: http://www.wesee.com/bot
2009
+ producer:
2010
+ name: WeSEE Ltd
2011
+ url: http://www.wesee.com
2012
+ -
2013
+ user_agent: websitepulse checker/1.1 (compatible; MSIE 5.5; Netscape 4.75; Linux)
2014
+ bot:
2015
+ name: WebSitePulse
2016
+ category: Site Monitor
2017
+ url: http://badcheese.com/crawler.html
2018
+ producer:
2019
+ name: WebSitePulse
2020
+ url: http://www.websitepulse.com/
2021
+ -
2022
+ user_agent: WebbCrawler 1.0 ( http://badcheese.com/crawler.html )
2023
+ bot:
2024
+ name: WebbCrawler
2025
+ category: Crawler
2026
+ url: http://badcheese.com/crawler.html
2027
+ producer:
2028
+ name: Steve Webb
2029
+ url: http://badcheese.com
2030
+ -
497
2031
  user_agent: Wotbox/2.01 (+http://www.wotbox.com/bot/)
498
- name: 'Wotbox'
499
- -
500
- user_agent: yacybot (freeworld/global; x86 Windows XP 5.1; java 1.7.0_21; GMT+04:00/ru) http://yacy.net/bot.html
501
- name: 'YaCy'
502
- -
2032
+ bot:
2033
+ name: Wotbox
2034
+ category: Search bot
2035
+ url: http://www.wotbox.com/bot/
2036
+ producer:
2037
+ name: Wotbox
2038
+ url: http://www.wotbox.com
2039
+ -
503
2040
  user_agent: yacybot (freeworld/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
504
- name: 'YaCy'
505
- -
2041
+ bot:
2042
+ name: YaCy
2043
+ category: Search bot
2044
+ url: http://yacy.net/bot.html
2045
+ producer:
2046
+ name: YaCy
2047
+ url: http://yacy.net
2048
+ -
2049
+ user_agent: yacybot (freeworld/global; x86 Windows XP 5.1; java 1.7.0_21; GMT+04:00/ru) http://yacy.net/bot.html
2050
+ bot:
2051
+ name: YaCy
2052
+ category: Search bot
2053
+ url: http://yacy.net/bot.html
2054
+ producer:
2055
+ name: YaCy
2056
+ url: http://yacy.net
2057
+ -
2058
+ user_agent: Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)
2059
+ bot:
2060
+ name: Yahoo Gemini
2061
+ category: Crawler
2062
+ url: https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html
2063
+ producer:
2064
+ name: Yahoo! Inc.
2065
+ url: http://www.yahoo.com
2066
+ -
2067
+ user_agent: Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)
2068
+ bot:
2069
+ name: Yahoo! Link Preview
2070
+ category: Crawler
2071
+ url: https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html
2072
+ producer:
2073
+ name: Yahoo! Inc.
2074
+ url: http://www.yahoo.com
2075
+ -
2076
+ user_agent: Yahoo:LinkExpander:Slingstone
2077
+ bot:
2078
+ name: Yahoo! Link Preview
2079
+ category: Crawler
2080
+ url: https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html
2081
+ producer:
2082
+ name: Yahoo! Inc.
2083
+ url: http://www.yahoo.com
2084
+ -
506
2085
  user_agent: Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp) NOT Firefox/3.5
507
- name: 'Yahoo! Slurp'
508
- -
2086
+ bot:
2087
+ name: Yahoo! Slurp
2088
+ category: Search bot
2089
+ url: http://help.yahoo.com/ysearch/slurp
2090
+ producer:
2091
+ name: Yahoo! Inc.
2092
+ url: http://www.yahoo.com
2093
+ -
509
2094
  user_agent: Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
510
- name: 'Yahoo! Slurp'
511
- -
512
- user_agent: Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)
513
- name: 'Yandex Bot'
514
- -
2095
+ bot:
2096
+ name: Yahoo! Slurp
2097
+ category: Search bot
2098
+ url: http://help.yahoo.com/ysearch/slurp
2099
+ producer:
2100
+ name: Yahoo! Inc.
2101
+ url: http://www.yahoo.com
2102
+ -
2103
+ user_agent: Mozilla/5.0 (compatible; Linux i686; Yandex.Gazeta Bot/1.0; +http://gazeta.yandex.ru)
2104
+ bot:
2105
+ name: Yandex Bot
2106
+ category: Search bot
2107
+ url: http://www.yandex.com/bots
2108
+ producer:
2109
+ name: Yandex LLC
2110
+ url: http://company.yandex.com
2111
+ -
2112
+ user_agent: Mozilla/5.0 (compatible; YaDirectFetcher/1.0; +http://yandex.com/bots)
2113
+ bot:
2114
+ name: Yandex Bot
2115
+ category: Search bot
2116
+ url: http://www.yandex.com/bots
2117
+ producer:
2118
+ name: Yandex LLC
2119
+ url: http://company.yandex.com
2120
+ -
515
2121
  user_agent: Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)
516
- name: 'Yandex Bot'
517
- -
518
- user_agent: Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)
519
- name: 'Yandex Bot'
520
- -
521
- user_agent: Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
522
- name: 'Yandex Bot'
523
- -
524
- user_agent: Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; B; +http://yandex.com/bots)
525
- name: 'Yandex Bot'
526
- -
527
- user_agent: Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)
528
- name: 'Yandex Bot'
529
- -
530
- user_agent: Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)
531
- name: 'Yandex Bot'
532
- -
2122
+ bot:
2123
+ name: Yandex Bot
2124
+ category: Search bot
2125
+ url: http://www.yandex.com/bots
2126
+ producer:
2127
+ name: Yandex LLC
2128
+ url: http://company.yandex.com
2129
+ -
533
2130
  user_agent: Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)
534
- name: 'Yandex Bot'
535
- -
2131
+ bot:
2132
+ name: Yandex Bot
2133
+ category: Search bot
2134
+ url: http://www.yandex.com/bots
2135
+ producer:
2136
+ name: Yandex LLC
2137
+ url: http://company.yandex.com
2138
+ -
2139
+ user_agent: Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; B; +http://yandex.com/bots)
2140
+ bot:
2141
+ name: Yandex Bot
2142
+ category: Search bot
2143
+ url: http://www.yandex.com/bots
2144
+ producer:
2145
+ name: Yandex LLC
2146
+ url: http://company.yandex.com
2147
+ -
536
2148
  user_agent: Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; B; +http://yandex.com/bots)
537
- name: 'Yandex Bot'
538
- -
2149
+ bot:
2150
+ name: Yandex Bot
2151
+ category: Search bot
2152
+ url: http://www.yandex.com/bots
2153
+ producer:
2154
+ name: Yandex LLC
2155
+ url: http://company.yandex.com
2156
+ -
2157
+ user_agent: Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
2158
+ bot:
2159
+ name: Yandex Bot
2160
+ category: Search bot
2161
+ url: http://www.yandex.com/bots
2162
+ producer:
2163
+ name: Yandex LLC
2164
+ url: http://company.yandex.com
2165
+ -
2166
+ user_agent: Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)
2167
+ bot:
2168
+ name: Yandex Bot
2169
+ category: Search bot
2170
+ url: http://www.yandex.com/bots
2171
+ producer:
2172
+ name: Yandex LLC
2173
+ url: http://company.yandex.com
2174
+ -
539
2175
  user_agent: Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)
540
- name: 'Yandex Bot'
541
- -
542
- user_agent: Mozilla/5.0 (compatible; YandexNewslinks; +http://yandex.com/bots)
543
- name: 'Yandex Bot'
544
- -
2176
+ bot:
2177
+ name: Yandex Bot
2178
+ category: Search bot
2179
+ url: http://www.yandex.com/bots
2180
+ producer:
2181
+ name: Yandex LLC
2182
+ url: http://company.yandex.com
2183
+ -
2184
+ user_agent: Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)
2185
+ bot:
2186
+ name: Yandex Bot
2187
+ category: Search bot
2188
+ url: http://www.yandex.com/bots
2189
+ producer:
2190
+ name: Yandex LLC
2191
+ url: http://company.yandex.com
2192
+ -
2193
+ user_agent: Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)
2194
+ bot:
2195
+ name: Yandex Bot
2196
+ category: Search bot
2197
+ url: http://www.yandex.com/bots
2198
+ producer:
2199
+ name: Yandex LLC
2200
+ url: http://company.yandex.com
2201
+ -
2202
+ user_agent: Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)
2203
+ bot:
2204
+ name: Yandex Bot
2205
+ category: Search bot
2206
+ url: http://www.yandex.com/bots
2207
+ producer:
2208
+ name: Yandex LLC
2209
+ url: http://company.yandex.com
2210
+ -
545
2211
  user_agent: Mozilla/5.0 (compatible; YandexNews/3.0; +http://yandex.com/bots)
546
- name: 'Yandex Bot'
547
- -
548
- user_agent: Mozilla/5.0 (compatible; Linux i686; Yandex.Gazeta Bot/1.0; +http://gazeta.yandex.ru)
549
- name: 'Yandex Bot'
550
- -
2212
+ bot:
2213
+ name: Yandex Bot
2214
+ category: Search bot
2215
+ url: http://www.yandex.com/bots
2216
+ producer:
2217
+ name: Yandex LLC
2218
+ url: http://company.yandex.com
2219
+ -
2220
+ user_agent: Mozilla/5.0 (compatible; YandexNewslinks; +http://yandex.com/bots)
2221
+ bot:
2222
+ name: Yandex Bot
2223
+ category: Search bot
2224
+ url: http://www.yandex.com/bots
2225
+ producer:
2226
+ name: Yandex LLC
2227
+ url: http://company.yandex.com
2228
+ -
551
2229
  user_agent: Yeti/1.1 (Naver Corp.; http://help.naver.com/robots/)
552
- name: 'Yeti/Naverbot'
553
- -
2230
+ bot:
2231
+ name: Yeti/Naverbot
2232
+ category: Search bot
2233
+ url: http://help.naver.com/robots/
2234
+ producer:
2235
+ name: Naver
2236
+ url: http://www.naver.com
2237
+ -
554
2238
  user_agent: Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )
555
- name: 'Youdao Bot'
556
- -
557
- user_agent: Mozilla/5.0 (compatible; YRSpider; +http://www.yunyun.com/SiteInfo.php?r=about)
558
- name: 'Yunyun Bot'
559
- -
2239
+ bot:
2240
+ name: Youdao Bot
2241
+ category: Search bot
2242
+ url: http://www.youdao.com/help/webmaster/spider
2243
+ producer:
2244
+ name: NetEase, Inc.
2245
+ url: http://corp.163.com
2246
+ -
2247
+ user_agent: YOURLS v1.5.1 +http://yourls.org/ (running on http://fhort.com)
2248
+ bot:
2249
+ name: Yourls
2250
+ category: Crawler
2251
+ url: http://yourls.org
2252
+ -
560
2253
  user_agent: Mozilla/5.0 (compatible; YRSpider; +http://www.yunrang.com/yrspider.html)
561
- name: 'Yunyun Bot'
562
- -
2254
+ bot:
2255
+ name: Yunyun Bot
2256
+ category: Search bot
2257
+ url: http://www.yunyun.com/SiteInfo.php?r=about
2258
+ producer:
2259
+ name: YunYun
2260
+ url: http://www.yunyun.com
2261
+ -
2262
+ user_agent: Mozilla/5.0 (compatible; YRSpider; +http://www.yunyun.com/SiteInfo.php?r=about)
2263
+ bot:
2264
+ name: Yunyun Bot
2265
+ category: Search bot
2266
+ url: http://www.yunyun.com/SiteInfo.php?r=about
2267
+ producer:
2268
+ name: YunYun
2269
+ url: http://www.yunyun.com
2270
+ -
563
2271
  user_agent: Mozilla/5.0 (compatible; YYSpider; +http://www.yunyun.com/spider.html)
564
- name: 'Yunyun Bot'
565
- -
2272
+ bot:
2273
+ name: Yunyun Bot
2274
+ category: Search bot
2275
+ url: http://www.yunyun.com/SiteInfo.php?r=about
2276
+ producer:
2277
+ name: YunYun
2278
+ url: http://www.yunyun.com
2279
+ -
2280
+ user_agent: Zao/0.1 (http://www.kototol.org/zao)
2281
+ bot:
2282
+ name: Zao
2283
+ category: Crawler
2284
+ -
566
2285
  user_agent: Zookabot/2.5;++http://zookabot.com
567
- name: 'Zookabot'
568
- -
2286
+ bot:
2287
+ name: Zookabot
2288
+ category: Crawler
2289
+ url: http://zookabot.com
2290
+ producer:
2291
+ name: Hwacha ApS
2292
+ url: http://hwacha.dk
2293
+ -
569
2294
  user_agent: Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)
570
- name: 'ZumBot'
571
- -
2295
+ bot:
2296
+ name: ZumBot
2297
+ category: Search bot
2298
+ url: http://help.zum.com/inquiry
2299
+ producer:
2300
+ name: ZUM internet
2301
+ url: http://www.zuminternet.com/
2302
+ -
572
2303
  user_agent: ZumBot/1.0 (ZUM Search; http://help.zum.com/inquiry)
573
- name: 'ZumBot'
574
- -
575
- user_agent: Your Nutch Spider/Nutch-2.2.1
576
- name: 'Nutch-based Bot'
577
- -
578
- user_agent: Nutch12/Nutch-1.2
579
- name: 'Nutch-based Bot'
580
- -
581
- user_agent: spider/Nutch-1.5.1 (spider; http://www.xxx.com)
582
- name: 'Nutch-based Bot'
583
- -
584
- user_agent: MySpider/Nutch-2.2
585
- name: 'Nutch-based Bot'
586
- -
587
- user_agent: My Nutch Spider/Nutch-1.6
588
- name: 'Nutch-based Bot'
589
- -
590
- user_agent: NutchSpider/Nutch-1.4
591
- name: 'Nutch-based Bot'
592
- -
593
- user_agent: testnutch/Nutch-1.8
594
- name: 'Nutch-based Bot'
595
- -
596
- user_agent: Spider/Nutch-2.3-SNAPSHOT (Webcrawler)
597
- name: 'Nutch-based Bot'
598
- -
599
- user_agent: your sipder name/Nutch-1.7
600
- name: 'Nutch-based Bot'
601
- -
602
- user_agent: 5acd734a-1ed3-4a17-ad9d-0b09ae7bc2d2/Nutch-1.7
603
- name: 'Nutch-based Bot'
604
- -
605
- user_agent: Mozilla/5.0 (compatible; summers;)/Nutch-1.7
606
- name: 'Nutch-based Bot'
607
- -
608
- user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8) Firefox/3.6.8/Nutch-1.7
609
- name: 'Nutch-based Bot'
610
- -
611
- user_agent: CB/Nutch-1.7
612
- name: 'Nutch-based Bot'
613
- -
614
- user_agent: StatusCake Error Detector Crawler V0.01
615
- name: 'Generic Bot'
616
- -
617
- user_agent: niki-bot
618
- name: 'Generic Bot'
619
-
620
-
621
- -
622
- user_agent: Bloglovin/1.0 (http://www.bloglovin.com; 1 subscribers)
623
- name: 'Bloglovin'
624
- -
625
- user_agent: CommaFeed/1.0 (http://www.commafeed.com)
626
- name: 'CommaFeed'
627
- -
628
- user_agent: FeedBurner/1.0 (http://www.FeedBurner.com)
629
- name: 'FeedBurner'
630
- -
631
- user_agent: Feedbin - 9 subscribers
632
- name: 'Feedbin'
633
- -
634
- user_agent: Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)
635
- name: 'Feedly'
636
- -
637
- user_agent: FeedlyBot/1.0 (http://feedly.com)
638
- name: 'Feedly'
639
- -
640
- user_agent: FeedlyApp/1.0 (http://www.feedly.com)
641
- name: 'Feedly'
642
- -
643
- user_agent: MetaFeedly/1.0 (http://www.feedly.com)
644
- name: 'Feedly'
645
- -
646
- user_agent: Feedspot http://www.feedspot.com
647
- name: 'Feedspot'
648
- -
649
- user_agent: Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)
650
- name: 'Fever'
651
- -
652
- user_agent: kouio.com RSS reader
653
- name: 'Kouio'
654
- -
655
- user_agent: kouio.com RSS reader - 6 subscribers
656
- name: 'Kouio'
657
- -
658
- user_agent: MagpieRSS/0.72 (+http://magpierss.sf.net)
659
- name: 'MagpieRSS'
660
- -
661
- user_agent: Netvibes (http://www.netvibes.com)
662
- name: 'Netvibes'
663
- -
664
- user_agent: 'Netvibes (http://www.netvibes.com/; 8 subscribers; feedID: 2244192)'
665
- name: 'Netvibes'
666
- -
667
- user_agent: NewsBlur Feed Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
668
- name: 'NewsBlur'
669
- -
670
- user_agent: NewsBlur Feed Finder (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
671
- name: 'NewsBlur'
672
- -
673
- user_agent: NewsBlur Page Fetcher - 7 subscribers - http://www.newsblur.com/site/3966817/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
674
- name: 'NewsBlur'
675
- -
676
- user_agent: NewsBlur Favicon Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
677
- name: 'NewsBlur'
678
- -
679
- user_agent: NewsGatorOnline/2.0 (http://www.newsgator.com; 2 subscribers)
680
- name: 'NewsGator'
681
- -
682
- user_agent: SilverReader/1.0; http://silverreader.com
683
- name: 'SilverReader'
684
- -
685
- user_agent: SimplePie/1.3.1 (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20121030175911
686
- name: 'SimplePie'
687
- -
688
- user_agent: freshrss/0.8-dev (Linux; http://freshrss.org) SimplePie/1.4-dev-FreshRSS
689
- name: 'SimplePie'
690
- -
691
- user_agent: SimplePie/1.2.1-dev (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20130514092120
692
- name: 'SimplePie'
693
- -
694
- user_agent: 'Tiny Tiny RSS/1.10 (http://tt-rss.org/)'
695
- name: 'Tiny Tiny RSS'
696
- -
697
- user_agent: 'Tiny Tiny RSS/1.11.4c63934 (http://tt-rss.org/)'
698
- name: 'Tiny Tiny RSS'
699
- -
700
- user_agent: 'Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)'
701
- name: 'Yahoo Gemini'
702
- -
703
- user_agent: 'Mozilla/5.0 (Java) outbrain'
704
- name: 'Outbrain'
705
- -
706
- user_agent: 'HubPages V0.2.2 (http://hubpages.com/help/crawlingpolicy)'
707
- name: 'HubPages'
708
- -
709
- user_agent: 'ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com'
710
- name: 'ADMantX'
711
- -
712
- user_agent: 'Pinterest/0.2 (+http://www.pinterest.com/)'
713
- name: 'Pinterest'
714
- -
715
- user_agent: 'Server Density Service Monitoring v2'
716
- name: 'Server Density'
717
- -
718
- user_agent: 'Site24x7'
719
- name: 'Site24x7 Website Monitoring'
720
- -
721
- user_agent: 'Mozilla/5.0 (compatible; www.monitor.us - free monitoring service; http://www.monitor.us)'
722
- name: 'Monitor.Us'
723
-
2304
+ bot:
2305
+ name: ZumBot
2306
+ category: Search bot
2307
+ url: http://help.zum.com/inquiry
2308
+ producer:
2309
+ name: ZUM internet
2310
+ url: http://www.zuminternet.com/
2311
+ -
2312
+ user_agent: AhrefsBot.Feeds v0.1; http://ahrefs.com/
2313
+ bot:
2314
+ name: aHrefs Bot
2315
+ category: Crawler
2316
+ url: http://ahrefs.com/robot
2317
+ producer:
2318
+ name: Ahrefs Pte Ltd
2319
+ url: http://ahrefs.com/robot
2320
+ -
2321
+ user_agent: Mozilla/5.0 (compatible; AhrefsBot/3.1; +http://ahrefs.com/robot/
2322
+ bot:
2323
+ name: aHrefs Bot
2324
+ category: Crawler
2325
+ url: http://ahrefs.com/robot
2326
+ producer:
2327
+ name: Ahrefs Pte Ltd
2328
+ url: http://ahrefs.com/robot
2329
+ -
2330
+ user_agent: Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)
2331
+ bot:
2332
+ name: aHrefs Bot
2333
+ category: Crawler
2334
+ url: http://ahrefs.com/robot
2335
+ producer:
2336
+ name: Ahrefs Pte Ltd
2337
+ url: http://ahrefs.com/robot
2338
+ -
2339
+ user_agent: 'Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/) AppEngine-Google; (+http://code.google.com/appengine; appid: s~proxyfile1-hrd)'
2340
+ bot:
2341
+ name: aHrefs Bot
2342
+ category: Crawler
2343
+ url: http://ahrefs.com/robot
2344
+ producer:
2345
+ name: Ahrefs Pte Ltd
2346
+ url: http://ahrefs.com/robot
2347
+ -
2348
+ user_agent: Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)
2349
+ bot:
2350
+ name: archive.org bot
2351
+ category: Crawler
2352
+ url: http://www.archive.org/details/archive.org_bot
2353
+ producer:
2354
+ name: The Internet Archive
2355
+ url: http://www.archive.org
2356
+ -
2357
+ user_agent: Mozilla/5.0 (compatible; archive.org_bot/3.3.0 +http://pandora.nla.gov.au/crawl.html)
2358
+ bot:
2359
+ name: archive.org bot
2360
+ category: Crawler
2361
+ url: http://www.archive.org/details/archive.org_bot
2362
+ producer:
2363
+ name: The Internet Archive
2364
+ url: http://www.archive.org
2365
+ -
2366
+ user_agent: Mozilla/5.0 (compatible; archive.org_bot; Archive-It; +http://archive-it.org/files/site-owners.html)
2367
+ bot:
2368
+ name: archive.org bot
2369
+ category: Crawler
2370
+ url: http://www.archive.org/details/archive.org_bot
2371
+ producer:
2372
+ name: The Internet Archive
2373
+ url: http://www.archive.org
2374
+ -
2375
+ user_agent: Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)
2376
+ bot:
2377
+ name: archive.org bot
2378
+ category: Crawler
2379
+ url: http://www.archive.org/details/archive.org_bot
2380
+ producer:
2381
+ name: The Internet Archive
2382
+ url: http://www.archive.org
2383
+ -
2384
+ user_agent: Mozilla/5.0 (compatible; heritrix/3.1.2-SNAPSHOT-20131029-0036 +http://archive.org/details/archive.org_bot)
2385
+ bot:
2386
+ name: archive.org bot
2387
+ category: Crawler
2388
+ url: http://www.archive.org/details/archive.org_bot
2389
+ producer:
2390
+ name: The Internet Archive
2391
+ url: http://www.archive.org
2392
+ -
2393
+ user_agent: Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)
2394
+ bot:
2395
+ name: archive.org bot
2396
+ category: Crawler
2397
+ url: http://www.archive.org/details/archive.org_bot
2398
+ producer:
2399
+ name: The Internet Archive
2400
+ url: http://www.archive.org
2401
+ -
2402
+ user_agent: Mozilla/5.0 (compatible; special_archiver/3.2.0 +http://www.loc.gov/webarchiving/notice_to_webmasters.html)
2403
+ bot:
2404
+ name: archive.org bot
2405
+ category: Crawler
2406
+ url: http://www.archive.org/details/archive.org_bot
2407
+ producer:
2408
+ name: The Internet Archive
2409
+ url: http://www.archive.org
2410
+ -
2411
+ user_agent: CCBot/2.0 (http://commoncrawl.org/faq/)
2412
+ bot:
2413
+ name: ccBot crawler
2414
+ category: Crawler
2415
+ url: http://commoncrawl.org/faq/
2416
+ producer:
2417
+ name: reddit inc.
2418
+ url: http://www.reddit.com