device_detector 1.0.0 → 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. checksums.yaml +5 -5
  2. data/.rubocop.yml +49 -0
  3. data/.ruby-version +1 -0
  4. data/.travis.yml +5 -9
  5. data/CHANGELOG.md +18 -5
  6. data/README.md +7 -9
  7. data/Rakefile +20 -13
  8. data/device_detector.gemspec +1 -0
  9. data/lib/device_detector.rb +32 -28
  10. data/lib/device_detector/bot.rb +2 -2
  11. data/lib/device_detector/client.rb +3 -2
  12. data/lib/device_detector/device.rb +46 -20
  13. data/lib/device_detector/memory_cache.rb +26 -19
  14. data/lib/device_detector/metadata_extractor.rb +7 -8
  15. data/lib/device_detector/model_extractor.rb +3 -3
  16. data/lib/device_detector/name_extractor.rb +2 -2
  17. data/lib/device_detector/os.rb +121 -111
  18. data/lib/device_detector/parser.rb +22 -9
  19. data/lib/device_detector/version.rb +3 -1
  20. data/lib/device_detector/version_extractor.rb +2 -3
  21. data/regexes/bots.yml +895 -21
  22. data/regexes/client/browser_engine.yml +11 -2
  23. data/regexes/client/browsers.yml +1025 -205
  24. data/regexes/client/feed_readers.yml +38 -2
  25. data/regexes/client/libraries.yml +81 -2
  26. data/regexes/client/mediaplayers.yml +25 -5
  27. data/regexes/client/mobile_apps.yml +213 -2
  28. data/regexes/client/pim.yml +10 -1
  29. data/regexes/device/cameras.yml +1 -1
  30. data/regexes/device/car_browsers.yml +7 -3
  31. data/regexes/device/consoles.yml +3 -3
  32. data/regexes/device/mobiles.yml +12768 -571
  33. data/regexes/device/notebooks.yml +114 -0
  34. data/regexes/device/portable_media_player.yml +4 -6
  35. data/regexes/device/televisions.yml +33 -5
  36. data/regexes/oss.yml +138 -49
  37. data/regexes/vendorfragments.yml +6 -2
  38. data/spec/device_detector/concrete_user_agent_spec.rb +16 -17
  39. data/spec/device_detector/detector_fixtures_spec.rb +59 -24
  40. data/spec/device_detector/device_spec.rb +28 -48
  41. data/spec/device_detector/memory_cache_spec.rb +60 -28
  42. data/spec/device_detector/model_extractor_spec.rb +3 -3
  43. data/spec/device_detector/version_extractor_spec.rb +5 -6
  44. data/spec/device_detector_spec.rb +60 -69
  45. data/spec/fixtures/client/browser.yml +1810 -260
  46. data/spec/fixtures/client/feed_reader.yml +47 -35
  47. data/spec/fixtures/client/library.yml +118 -3
  48. data/spec/fixtures/client/mediaplayer.yml +32 -37
  49. data/spec/fixtures/client/mobile_app.yml +225 -5
  50. data/spec/fixtures/client/pim.yml +37 -18
  51. data/spec/fixtures/detector/bots.yml +1531 -136
  52. data/spec/fixtures/detector/camera.yml +36 -10
  53. data/spec/fixtures/detector/car_browser.yml +64 -3
  54. data/spec/fixtures/detector/console.yml +80 -26
  55. data/spec/fixtures/detector/desktop.yml +3865 -2238
  56. data/spec/fixtures/detector/feature_phone.yml +151 -42
  57. data/spec/fixtures/detector/feed_reader.yml +186 -121
  58. data/spec/fixtures/detector/mediaplayer.yml +113 -39
  59. data/spec/fixtures/detector/mobile_apps.yml +451 -21
  60. data/spec/fixtures/detector/phablet.yml +3847 -660
  61. data/spec/fixtures/detector/portable_media_player.yml +81 -16
  62. data/spec/fixtures/detector/smart_display.yml +8 -5
  63. data/spec/fixtures/detector/smart_speaker.yml +55 -0
  64. data/spec/fixtures/detector/smartphone-1.yml +5385 -4948
  65. data/spec/fixtures/detector/smartphone-10.yml +9973 -0
  66. data/spec/fixtures/detector/smartphone-11.yml +10015 -0
  67. data/spec/fixtures/detector/smartphone-12.yml +9897 -0
  68. data/spec/fixtures/detector/smartphone-13.yml +9912 -0
  69. data/spec/fixtures/detector/smartphone-14.yml +9935 -0
  70. data/spec/fixtures/detector/smartphone-15.yml +6595 -0
  71. data/spec/fixtures/detector/smartphone-16.yml +10021 -0
  72. data/spec/fixtures/detector/smartphone-17.yml +9408 -0
  73. data/spec/fixtures/detector/smartphone-2.yml +5221 -4646
  74. data/spec/fixtures/detector/smartphone-3.yml +5047 -4494
  75. data/spec/fixtures/detector/smartphone-4.yml +6958 -2761
  76. data/spec/fixtures/detector/smartphone-5.yml +9937 -0
  77. data/spec/fixtures/detector/smartphone-6.yml +10114 -0
  78. data/spec/fixtures/detector/smartphone-7.yml +9975 -0
  79. data/spec/fixtures/detector/smartphone-8.yml +9897 -0
  80. data/spec/fixtures/detector/smartphone-9.yml +9880 -0
  81. data/spec/fixtures/detector/smartphone.yml +5229 -4649
  82. data/spec/fixtures/detector/tablet-1.yml +4640 -4140
  83. data/spec/fixtures/detector/tablet-2.yml +9806 -71
  84. data/spec/fixtures/detector/tablet-3.yml +9968 -0
  85. data/spec/fixtures/detector/tablet-4.yml +7113 -0
  86. data/spec/fixtures/detector/tablet.yml +4619 -4132
  87. data/spec/fixtures/detector/tv.yml +4093 -1342
  88. data/spec/fixtures/detector/unknown.yml +1042 -1006
  89. data/spec/fixtures/detector/wearable.yml +61 -0
  90. data/spec/fixtures/device/camera.yml +4 -3
  91. data/spec/fixtures/device/car_browser.yml +9 -2
  92. data/spec/fixtures/device/console.yml +15 -14
  93. data/spec/fixtures/device/notebook.yml +7 -0
  94. data/spec/fixtures/parser/oss.yml +291 -2
  95. data/spec/fixtures/parser/vendorfragments.yml +8 -2
  96. metadata +59 -7
data/spec/fixtures/client/mobile_app.yml CHANGED
@@ -1,30 +1,250 @@
1
- -
1
+ ---
2
+ -
2
3
  user_agent: Pulse/4.0.5 (iPhone; iOS 7.0.6; Scale/2.00)
3
4
  client:
4
5
  type: mobile app
5
6
  name: Pulse
6
7
  version: "4.0.5"
7
- -
8
+ -
8
9
  user_agent: Mozilla/5.0 (Linux; Android 4.4.4; SM-G7509 Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 MicroMessenger/6.0.2.56_r958800.520 NetType/WIFI
9
10
  client:
10
11
  type: mobile app
11
12
  name: WeChat
12
13
  version: "6.0.2.56.r958800.520"
13
14
  -
15
+ user_agent: Mozilla/5.0 (iPad; CPU OS 11_2_6 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Mobile/15D100 AliApp(DingTalk/5.1.12) com.laiwang.DingTalk/13441885 Channel/201200 Pad/iPad language/zh-Hans-CN UT4Aplus/0.0.6 WK
16
+ client:
17
+ type: mobile app
18
+ name: DingTalk
19
+ version: 5.1.12
20
+ -
14
21
  user_agent: Mozilla/5.0 (Linux; U; Android 4.0.4; en-us; ZTE U807N Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 Weibo (ZTE-ZTE U807N__weibo__5.0.0__android__android4.0.4)
15
22
  client:
16
23
  type: mobile app
17
24
  name: Sina Weibo
18
25
  version: "5.0.0"
19
- -
26
+ -
20
27
  user_agent: FeedR/1.0 (Android)
21
28
  client:
22
29
  type: mobile app
23
30
  name: FeedR
24
31
  version: "1.0"
25
- -
32
+ -
26
33
  user_agent: Pinterest/3.3.3 CFNetwork/609.1.4 Darwin/13.0.0
27
34
  client:
28
35
  type: mobile app
29
36
  name: Pinterest
30
- version: "3.3.3"
37
+ version: "3.3.3"
38
+ -
39
+ user_agent: Instacast/5.0a8 CFNetwork/711.1.11 Darwin/13.4.0
40
+ client:
41
+ type: mobile app
42
+ name: Instacast
43
+ version: "5.0a8"
44
+ -
45
+ user_agent: InstacastHD/1.1.2 CFNetwork/711.0.6 Darwin/14.0.0
46
+ client:
47
+ type: mobile app
48
+ name: Instacast
49
+ version: "1.1.2"
50
+ -
51
+ user_agent: Podcasts/2.1.2
52
+ client:
53
+ type: mobile app
54
+ name: Podcasts
55
+ version: "2.1.2"
56
+ -
57
+ user_agent: Shifty Jelly Pocket Casts, Android v4.4.3.1
58
+ client:
59
+ type: mobile app
60
+ name: Pocket Casts
61
+ version: "4.4.3.1"
62
+ -
63
+ user_agent: Pocket Casts
64
+ client:
65
+ type: mobile app
66
+ name: Pocket Casts
67
+ version: ""
68
+ -
69
+ user_agent: Mozilla/5.0 (Linux; U; en-us; BeyondPod)
70
+ client:
71
+ type: mobile app
72
+ name: BeyondPod
73
+ version: ""
74
+ -
75
+ user_agent: AntennaPod/0.9.9.1
76
+ client:
77
+ type: mobile app
78
+ name: AntennaPod
79
+ version: "0.9.9.1"
80
+ -
81
+ user_agent: AntennaPod/
82
+ client:
83
+ type: mobile app
84
+ name: AntennaPod
85
+ version: ""
86
+ -
87
+ user_agent: Overcast/1.0 (+http://overcast.fm/; iOS podcast app)
88
+ client:
89
+ type: mobile app
90
+ name: Overcast
91
+ version: "1.0"
92
+ -
93
+ user_agent: Podkicker/1.9.4
94
+ client:
95
+ type: mobile app
96
+ name: Podkicker
97
+ version: "1.9.4"
98
+ -
99
+ user_agent: Podkicker Pro/1.9.4
100
+ client:
101
+ type: mobile app
102
+ name: Podkicker
103
+ version: "1.9.4"
104
+ -
105
+ user_agent: Castro/64 CFNetwork/672.1.15 Darwin/14.0.0
106
+ client:
107
+ type: mobile app
108
+ name: Castro
109
+ version: "64"
110
+ -
111
+ user_agent: Mozilla/5.0 (Linux; U; Windows NT 6.1; en-us; dream) DoggCatcher
112
+ client:
113
+ type: mobile app
114
+ name: DoggCatcher
115
+ version: ""
116
+ -
117
+ user_agent: Mozilla/5.0 (Linux; U; Android 4.2.2; ja-jp; SOL22 Build/10.3.1.D.0.257) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 YJApp-ANDROID jp.co.yahoo.android.yjtop/3.15.1
118
+ client:
119
+ type: mobile app
120
+ name: Yahoo! Japan
121
+ version: "3.15.1"
122
+ -
123
+ user_agent: RSSRadio/9220 (iPad;iOS;12.3.1)
124
+ client:
125
+ type: mobile app
126
+ name: RSSRadio
127
+ version: "9220"
128
+ -
129
+ user_agent: RSSRadio/9252
130
+ client:
131
+ type: mobile app
132
+ name: RSSRadio
133
+ version: "9252"
134
+ -
135
+ user_agent: RSSRadio/
136
+ client:
137
+ type: mobile app
138
+ name: RSSRadio
139
+ version: ""
140
+ -
141
+ user_agent: Mozilla/5.0 (Linux; Android 7.0; JMM-AL00 Build/HONORJMM-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36 SogouSearch Android1.0 version3.0 AppVersion/6002
142
+ client:
143
+ type: mobile app
144
+ name: SogouSearch App
145
+ version: "3.0"
146
+ -
147
+ user_agent: Dalvik/2.1.0 (Linux; U; Android 6.0.1; MI 4LTE MIUI/V7.3.2.0.MXDCNDD) NewsArticle/5.1.3
148
+ client:
149
+ type: mobile app
150
+ name: NewsArticle App
151
+ version: "5.1.3"
152
+ -
153
+ user_agent: Mozilla/5.0 (Linux; Android 5.1; m2 note) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 baidubrowser/5.3.4.0 (Baidu; P1 4.3.1) tieba/6.7.2 BMW
154
+ client:
155
+ type: mobile app
156
+ name: tieba
157
+ version: "6.7.2"
158
+ -
159
+ user_agent: com.douban.group/2.2.0(220) Mozilla/5.0 (Linux; U; Android 4.1.1; zh-cn; MI 2A Build/JRO03L) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30
160
+ client:
161
+ type: mobile app
162
+ name: douban App
163
+ version: "2.2.0"
164
+ -
165
+ user_agent: Mozilla/5.0 (Linux; Android 7.0; VIE-L09 Build/HUAWEIVIE-L09; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/59.0.3071.125 Mobile Safari/537.36 BingWeb/6.9.25207603
166
+ client:
167
+ type: mobile app
168
+ name: BingWebApp
169
+ version: "6.9.25207603"
170
+ -
171
+ user_agent: Mozilla/5.0 (Linux; U; Android 4.1.2; fr-ca; LT30a Build/9.1.A.0.489) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 GSA/3.3.11.1069658.arm
172
+ client:
173
+ type: mobile app
174
+ name: Google Search App
175
+ version: "3.3.11.1069658"
176
+ -
177
+ user_agent: Mozilla/5.0 (Linux; U; Android 2.3.4; ru-ru; WT19i Build/4.0.2.A.0.58) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1 Flipboard/2.1.2/854,2.1.2.854,2013-12-05 11:59, +0300
178
+ client:
179
+ type: mobile app
180
+ name: Flipboard App
181
+ version: "2.1.2"
182
+ -
183
+ user_agent: Mozilla/5.0 (Linux; Android 6.0.1; ASUS_Z011D Build/MMB29P; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36 Instagram 53.0.0.13.84 Android (23/6.0.1; 480dpi; 1080x1920; asus; ASUS_Z011D; ASUS_Z011; qcom; fr_FR; 116756948)
184
+ client:
185
+ type: mobile app
186
+ name: Instagram App
187
+ version: "53.0.0.13.84"
188
+ -
189
+ user_agent: TCL J706T_TD/1.0 Linux/3.4.5 Android/4.1.2 Release/08.27.2013 Browser/AppleWebKit534.30 Profile/MIDP-2.0 Configuration/CLDC-1.1 baiduboxapp/4.2 (Baidu; P1 4.1.2)
190
+ client:
191
+ type: mobile app
192
+ name: Baidu Box App
193
+ version: "4.2"
194
+ -
195
+ user_agent: Mozilla/5.0 (Linux; Android 4.4.4; GT-I9195I Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Crosswalk/14.43.343.17 Mobile Safari/537.36
196
+ client:
197
+ type: mobile app
198
+ name: CrosswalkApp
199
+ version: "14.43.343.17"
200
+ -
201
+ user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.2.10 (KHTML, like Gecko) Mobile/15A372 Twitter for iPhone/7.52
202
+ client:
203
+ type: mobile app
204
+ name: Twitter
205
+ version: "7.52"
206
+ -
207
+ user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X; en-us) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone
208
+ client:
209
+ type: mobile app
210
+ name: Twitter
211
+ version: ""
212
+ -
213
+ user_agent: Mozilla/5.0 (Linux; Android 5.1; HTC Desire 628 Build/LMY47D; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.108 Mobile Safari/537.36 JsSdk/2 TopBuzz/10.2.1 NetType/3G
214
+ client:
215
+ type: mobile app
216
+ name: TopBuzz
217
+ version: 10.2.1
218
+ -
219
+ user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Mobile/15E148 Snapchat/10.77.0.54 (like Safari/604.1)
220
+ client:
221
+ type: mobile app
222
+ name: Snapchat
223
+ version: 10.77.0.54
224
+
225
+ -
226
+ user_agent: Siri/1 CFNetwork/1128.0.1 Darwin/19.6.0
227
+ client:
228
+ type: mobile app
229
+ name: Siri
230
+ version: "1.0"
231
+
232
+ -
233
+ user_agent: LinkedIn/9.7.1681 CFNetwork/808.1.4 Darwin/16.1.0
234
+ client:
235
+ type: mobile app
236
+ name: LinkedIn
237
+ version: "9.7.1681"
238
+
239
+ -
240
+ user_agent: Twitter/7.57 CFNetwork/978.0.7 Darwin/18.7.0
241
+ client:
242
+ type: mobile app
243
+ name: Twitter
244
+ version: "7.57"
245
+ -
246
+ user_agent: Mozilla/5.0 (Linux; Android 9; SM-G950F Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/80.0.3987.117 Mobile Safari/537.36 TwitterAndroid
247
+ client:
248
+ type: mobile app
249
+ name: Twitter
250
+ version: ""
data/spec/fixtures/client/pim.yml CHANGED
@@ -1,96 +1,115 @@
1
- -
1
+ ---
2
+ -
2
3
  user_agent: Outlook-Express/7.0 (MSIE 7.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; AskTbORJ/5.15.9.29495; .NET4.0E; TmstmpExt)
3
4
  client:
4
5
  type: pim
5
6
  name: Outlook Express
6
7
  version: "7.0"
7
- -
8
+ -
8
9
  user_agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Win64; x64; Trident/7.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3; .NET CLR 1.1.4322; FDM; Tablet PC 2.0; .NET4.0E; Microsoft Outlook 14.0.7113; ms-office; MSOffice 14)
9
10
  client:
10
11
  type: pim
11
12
  name: Microsoft Outlook
12
13
  version: "14.0.7113"
13
- -
14
+ -
14
15
  user_agent: Mozilla/5.0 (X11; Linux i686; rv:17.0) Gecko/20130330 Thunderbird/17.0.5
15
16
  client:
16
17
  type: pim
17
18
  name: Thunderbird
18
19
  version: "17.0.5"
19
- -
20
+ -
20
21
  user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Thunderbird/24.4.0 Lightning/2.6.4
21
22
  client:
22
23
  type: pim
23
24
  name: Thunderbird
24
25
  version: "24.4.0"
25
- -
26
+ -
26
27
  user_agent: Airmail 1.4 rv:238 (Macintosh; Mac OS X 10.9.2; hr_HR)
27
28
  client:
28
29
  type: pim
29
30
  name: Airmail
30
31
  version: "1.4"
31
- -
32
+ -
32
33
  user_agent: Airmail 1.3.3 rv:237 (Macintosh; Mac OS X 10.9.2; en_US)
33
34
  client:
34
35
  type: pim
35
36
  name: Airmail
36
37
  version: "1.3.3"
37
- -
38
+ -
38
39
  user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Icedove/24.4.0
39
40
  client:
40
41
  type: pim
41
42
  name: Thunderbird
42
43
  version: "24.4.0"
43
- -
44
+ -
44
45
  user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Icedove/24.4.0 Lightning/2.6.5
45
46
  client:
46
47
  type: pim
47
48
  name: Thunderbird
48
49
  version: "24.4.0"
49
- -
50
+ -
50
51
  user_agent: Mozilla/4.0 (compatible; Lotus-Notes/6.0; Windows-NT)
51
52
  client:
52
53
  type: pim
53
54
  name: Lotus Notes
54
55
  version: "6.0"
55
- -
56
+ -
56
57
  user_agent: Barca/2.8.4400
57
58
  client:
58
59
  type: pim
59
60
  name: Barca
60
61
  version: "2.8.4400"
61
- -
62
+ -
62
63
  user_agent: BarcaPro/1.4 L.1001
63
64
  client:
64
65
  type: pim
65
66
  name: Barca
66
67
  version: "1.4"
67
- -
68
+ -
68
69
  user_agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.1.8) Gecko/20100317 Postbox/1.1.3
69
70
  client:
70
71
  type: pim
71
72
  name: Postbox
72
73
  version: "1.1.3"
73
- -
74
+ -
74
75
  user_agent: Postbox 1.0b14 (Windows/2009072715)
75
76
  client:
76
77
  type: pim
77
78
  name: Postbox
78
79
  version: "1.0"
79
- -
80
+ -
80
81
  user_agent: MailBar/1.3.2 (Mac OS X Version 10.11.1 (Build 15B42))
81
82
  client:
82
83
  type: pim
83
84
  name: MailBar
84
- version: 1.3.2
85
- -
85
+ version: "1.3.2"
86
+ -
86
87
  user_agent: The Bat! 4.0.0.22
87
88
  client:
88
89
  type: pim
89
90
  name: The Bat!
90
91
  version: "4.0.0.22"
91
- -
92
+ -
92
93
  user_agent: The Bat! Voyager 4.0.18.4
93
94
  client:
94
95
  type: pim
95
96
  name: The Bat!
96
- version: "4.0.18.4"
97
+ version: "4.0.18.4"
98
+ -
99
+ user_agent: DAVdroid/1.6.2-ose (2017/06/23; dav4android; okhttp3) Android/7.0
100
+ client:
101
+ type: pim
102
+ name: DAVdroid
103
+ version: "1.6.2"
104
+ -
105
+ user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:10.0.12) Gecko/20130823 Firefox/10.0.11esrpre Iceape/2.7.12
106
+ client:
107
+ type: pim
108
+ name: SeaMonkey
109
+ version: "2.7.12"
110
+ -
111
+ user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:29.0) Gecko/20100101 Firefox/29.0 SeaMonkey/2.26a1 Lightning/3.1a1
112
+ client:
113
+ type: pim
114
+ name: SeaMonkey
115
+ version: "2.26"
data/spec/fixtures/detector/bots.yml CHANGED
@@ -7,7 +7,7 @@
7
7
  url: http://www.so.com/help/help_3_2.html
8
8
  producer:
9
9
  name: Online Media Group, Inc.
10
- url:
10
+ url: ""
11
11
  -
12
12
  user_agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MDDR; .NET4.0C; .NET4.0E; .NET CLR 1.1.4322; Tablet PC 2.0); 360Spider
13
13
  bot:
@@ -16,7 +16,7 @@
16
16
  url: http://www.so.com/help/help_3_2.html
17
17
  producer:
18
18
  name: Online Media Group, Inc.
19
- url:
19
+ url: ""
20
20
  -
21
21
  user_agent: Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider
22
22
  bot:
@@ -25,7 +25,7 @@
25
25
  url: http://www.so.com/help/help_3_2.html
26
26
  producer:
27
27
  name: Online Media Group, Inc.
28
- url:
28
+ url: ""
29
29
  -
30
30
  user_agent: Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider
31
31
  bot:
@@ -34,7 +34,7 @@
34
34
  url: http://www.so.com/help/help_3_2.html
35
35
  producer:
36
36
  name: Online Media Group, Inc.
37
- url:
37
+ url: ""
38
38
  -
39
39
  user_agent: 'ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com'
40
40
  bot:
@@ -62,7 +62,7 @@
62
62
  bot:
63
63
  name: AddThis.com
64
64
  category: Social Media Agent
65
- url:
65
+ url: ""
66
66
  producer:
67
67
  name: Clearspring Technologies, Inc.
68
68
  url: http://www.clearspring.com
@@ -75,6 +75,31 @@
75
75
  producer:
76
76
  name: Alexa Internet
77
77
  url: http://www.alexa.com
78
+ -
79
+ user_agent: Mozilla/5.0 (compatible; alexa site audit/1.0; http://www.alexa.com/help/webmasters; )
80
+ bot:
81
+ name: Alexa Site Audit
82
+ category: Site Monitor
83
+ url: http://www.alexa.com/help/webmasters
84
+ producer:
85
+ name: Alexa Internet
86
+ url: http://www.alexa.com
87
+ -
88
+ user_agent: Amazon Route 53 Health Check Service; ref:xxxxxxxx-xxxx-xxxxxxxxx-xxxxxxxxxxxx; report http://amzn.to/xxxxxxx
89
+ bot:
90
+ name: Amazon Route53 Health Check
91
+ category: Service Agent
92
+ producer:
93
+ name: Amazon Web Services
94
+ url: https://aws.amazon.com/
95
+ -
96
+ user_agent: Amazon-Route53-Health-Check-Service (ref b0eb04d5-cb5e-40e7-839b-558e52fc3f0d; report http://amzn.to/1vsZADi)
97
+ bot:
98
+ name: Amazon Route53 Health Check
99
+ category: Service Agent
100
+ producer:
101
+ name: Amazon Web Services
102
+ url: https://aws.amazon.com/
78
103
  -
79
104
  user_agent: AmorankSpider/0.1; +http://amorank.com/webcrawler.html
80
105
  bot:
@@ -93,6 +118,15 @@
93
118
  producer:
94
119
  name: Analytics SEO
95
120
  url: http://www.analyticsseo.com
121
+ -
122
+ user_agent: ApacheBench/2.3
123
+ bot:
124
+ name: ApacheBench
125
+ category: Benchmark
126
+ url: https://httpd.apache.org/docs/2.4/programs/ab.html
127
+ producer:
128
+ name: The Apache Software Foundation
129
+ url: http://www.apache.org/foundation/
96
130
  -
97
131
  user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10 _1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)
98
132
  bot:
@@ -111,15 +145,60 @@
111
145
  producer:
112
146
  name: Apple Inc
113
147
  url: http://www.apple.com
148
+ -
149
+ user_agent: Arachni/v1.5.1
150
+ bot:
151
+ name: Arachni
152
+ category: Security Checker
153
+ url: http://www.arachni-scanner.com
154
+ producer:
155
+ name: Sarosys LLC
156
+ url: http://www.sarosys.com/
114
157
  -
115
158
  user_agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
116
159
  bot:
117
160
  name: Ask Jeeves
118
161
  category: Search bot
119
- url:
162
+ url: ""
120
163
  producer:
121
164
  name: Ask Jeeves Inc.
122
165
  url: http://www.ask.com
166
+ -
167
+ user_agent: Mozilla/5.0 (compatible;AspiegelBot)
168
+ bot:
169
+ name: AspiegelBot
170
+ category: Crawler
171
+ url: https://aspiegel.com/
172
+ producer:
173
+ name: Huawei
174
+ url: https://www.huawei.com/
175
+ -
176
+ user_agent: Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)
177
+ bot:
178
+ name: AspiegelBot
179
+ category: Crawler
180
+ url: https://aspiegel.com/
181
+ producer:
182
+ name: Huawei
183
+ url: https://www.huawei.com/
184
+ -
185
+ user_agent: AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)
186
+ bot:
187
+ name: Awario
188
+ category: Feed Fetcher
189
+ url: https://awario.com/bots.html
190
+ producer:
191
+ name: Awario
192
+ url: https://awario.com/
193
+ -
194
+ user_agent: AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)
195
+ bot:
196
+ name: Awario
197
+ category: Search bot
198
+ url: https://awario.com/bots.html
199
+ producer:
200
+ name: Awario
201
+ url: https://awario.com/
123
202
  -
124
203
  user_agent: Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)
125
204
  bot:
@@ -136,12 +215,12 @@
136
215
  category: Crawler
137
216
  url: http://law.di.unimi.it/BUbiNG.html
138
217
  producer:
139
- name:
140
- url:
218
+ name: The Laboratory for Web Algorithmics (LAW)
219
+ url: 'http://law.di.unimi.it/software.php#buging'
141
220
  -
142
- user_agent: Backlink-Ceck.de (+http://www.backlink-check.de/bot.html)
221
+ user_agent: Backlink-Check.de (+http://www.backlink-check.de/bot.html)
143
222
  bot:
144
- name: Backlink-Ceck.de
223
+ name: Backlink-Check.de
145
224
  category: Crawler
146
225
  url: http://www.backlink-check.de/bot.html
147
226
  producer:
@@ -226,8 +305,8 @@
226
305
  url: https://bazqux.com/fetcher
227
306
  category: Feed Fetcher
228
307
  producer:
229
- name:
230
- url:
308
+ name: ""
309
+ url: ""
231
310
  -
232
311
  user_agent: Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
233
312
  bot:
@@ -300,6 +379,15 @@
300
379
  producer:
301
380
  name: Microsoft Corporation
302
381
  url: http://www.microsoft.com
382
+ -
383
+ user_agent: bitlybot/3.0
384
+ bot:
385
+ name: BitlyBot
386
+ category: Crawler
387
+ url: https://bitly.com
388
+ producer:
389
+ name: Bitly, Inc.
390
+ url: https://bitly.com
303
391
  -
304
392
  user_agent: Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)
305
393
  bot:
@@ -316,17 +404,34 @@
316
404
  url: http://www.bloglovin.com
317
405
  category: Feed Fetcher
318
406
  producer:
319
- name:
320
- url:
407
+ name: ""
408
+ url: ""
321
409
  -
322
410
  user_agent: Blogtrottr/2.0
323
411
  bot:
324
412
  name: Blogtrottr
325
- url:
413
+ url: ""
326
414
  category: Feed Fetcher
327
415
  producer:
328
416
  name: Blogtrottr Ltd
329
417
  url: https://blogtrottr.com/
418
+ -
419
+ user_agent: BoardReader Favicon Fetcher /1.0 info@boardreader.com
420
+ bot:
421
+ name: BoardReader
422
+ category: Search bot
423
+ url: http://boardreader.com/
424
+ producer:
425
+ name: Effyis Inc
426
+ url: http://boardreader.com/
427
+ -
428
+ user_agent: BoardReader Blog Indexer(http://boardreader.com)
429
+ bot:
430
+ name: BoardReader Blog Indexer
431
+ category: Crawler
432
+ producer:
433
+ name: BoardReader
434
+ url: http://boardreader.com/
330
435
  -
331
436
  user_agent: Mozilla/5.0 (compatible; BountiiBot/1.1; +http://bountii.com/contact.php)
332
437
  bot:
@@ -336,6 +441,15 @@
336
441
  producer:
337
442
  name: Bountii Inc.
338
443
  url: http://bountii.com
444
+ -
445
+ user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)
446
+ bot:
447
+ name: BrandVerity
448
+ category: Crawler
449
+ url: https://www.brandverity.com/why-is-brandverity-visiting-me
450
+ producer:
451
+ name: BrandVerity, Inc.
452
+ url: https://www.brandverity.com/
339
453
  -
340
454
  user_agent: Browsershots
341
455
  bot:
@@ -345,6 +459,15 @@
345
459
  producer:
346
460
  name: Browsershots.org
347
461
  url: http://browsershots.org
462
+ -
463
+ user_agent: Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)
464
+ bot:
465
+ name: Buck
466
+ category: Search bot
467
+ url: https://hypefactors.com/
468
+ producer:
469
+ name: Hypefactors A/S
470
+ url: https://hypefactors.com/
348
471
  -
349
472
  user_agent: Mozilla/5.0 (compatible; Butterfly/1.0; +http://labs.topsy.com/butterfly/) Gecko/2009032608 Firefox/3.0.8
350
473
  bot:
@@ -363,6 +486,24 @@
363
486
  producer:
364
487
  name: Topsy Labs
365
488
  url: http://labs.topsy.com
489
+ -
490
+ user_agent: Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.5668.1446 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com
491
+ bot:
492
+ name: Bytespider
493
+ category: Search bot
494
+ url: https://bytedance.com/
495
+ producer:
496
+ name: ByteDance Ltd.
497
+ url: https://bytedance.com/
498
+ -
499
+ user_agent: CATExplorador/1.0beta (sistemes at domini dot cat; http://domini.cat/catexplorador.html)
500
+ bot:
501
+ name: CATExplorador
502
+ category: Search bot
503
+ url: https://fundacio.cat/ca/domini/
504
+ producer:
505
+ name: Fundació puntCAT
506
+ url: https://fundacio.cat/ca/domini/
366
507
  -
367
508
  user_agent: Mozilla/5.0 (compatible; CareerBot/1.1; +http://www.career-x.de/bot.html)
368
509
  bot:
@@ -372,6 +513,15 @@
372
513
  producer:
373
514
  name: career-x GmbH
374
515
  url: http://www.career-x.de
516
+ -
517
+ user_agent: Castro 2, Episode Duration Lookup
518
+ bot:
519
+ name: Castro 2
520
+ category: Service Agent
521
+ url: http://supertop.co/castro/
522
+ producer:
523
+ name: Supertop
524
+ url: http://supertop.co
375
525
  -
376
526
  user_agent: Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)
377
527
  bot:
@@ -381,6 +531,15 @@
381
531
  producer:
382
532
  name: 10betterpages GmbH
383
533
  url: http://cliqz.com
534
+ -
535
+ user_agent: Mozilla/5.0 (compatible; Cloudflare-AMP/1.0; +https://amp.cloudflare.com/doc/fetcher.html) AppleWebKit/534.34
536
+ bot:
537
+ name: CloudFlare AMP Fetcher
538
+ category: Crawler
539
+ url: https://amp.cloudflare.com/doc/fetcher.html
540
+ producer:
541
+ name: CloudFlare
542
+ url: http://www.cloudflare.com
384
543
  -
385
544
  user_agent: Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34
386
545
  bot:
@@ -390,6 +549,15 @@
390
549
  producer:
391
550
  name: CloudFlare
392
551
  url: http://www.cloudflare.com
552
+ -
553
+ user_agent: collectd/5.5.1
554
+ bot:
555
+ name: Collectd
556
+ url: https://collectd.org/
557
+ category: Site Monitor
558
+ producer:
559
+ name: Collectd
560
+ url: https://collectd.org/
393
561
  -
394
562
  user_agent: CommaFeed/1.0 (http://www.commafeed.com)
395
563
  bot:
@@ -397,17 +565,107 @@
397
565
  url: http://www.commafeed.com
398
566
  category: Feed Fetcher
399
567
  producer:
400
- name:
401
- url:
568
+ name: ""
569
+ url: ""
570
+ -
571
+ user_agent: coccocbot-web/1.0 (+http://help.coccoc.com/searchengine)
572
+ bot:
573
+ name: Cốc Cốc Bot
574
+ category: Search bot
575
+ url: https://help.coccoc.com/en/search-engine/coccoc-robots
576
+ producer:
577
+ name: Cốc Cốc
578
+ url: https://coccoc.com/
402
579
  -
403
580
  user_agent: Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)
404
581
  bot:
405
582
  name: Cốc Cốc Bot
406
- url: http://help.coccoc.com/
407
583
  category: Search bot
584
+ url: https://help.coccoc.com/en/search-engine/coccoc-robots
585
+ producer:
586
+ name: Cốc Cốc
587
+ url: https://coccoc.com/
588
+ -
589
+ user_agent: Mozilla/5.0 (compatible; coccocbot-ads/1.0; +http://help.coccoc.com/searchengine)
590
+ bot:
591
+ name: Cốc Cốc Bot
592
+ category: Search bot
593
+ url: https://help.coccoc.com/en/search-engine/coccoc-robots
594
+ producer:
595
+ name: Cốc Cốc
596
+ url: https://coccoc.com/
597
+ -
598
+ user_agent: Mozilla/5.0 (compatible; coccocbot-fast/1.0; +http://help.coccoc.com/searchengine)
599
+ bot:
600
+ name: Cốc Cốc Bot
601
+ category: Search bot
602
+ url: https://help.coccoc.com/en/search-engine/coccoc-robots
603
+ producer:
604
+ name: Cốc Cốc
605
+ url: https://coccoc.com/
606
+ -
607
+ user_agent: Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)
608
+ bot:
609
+ name: Cốc Cốc Bot
610
+ category: Search bot
611
+ url: https://help.coccoc.com/en/search-engine/coccoc-robots
612
+ producer:
613
+ name: Cốc Cốc
614
+ url: https://coccoc.com/
615
+ -
616
+ user_agent: Mozilla/5.0 (compatible; coccocbot-shopping/1.0; +http://help.coccoc.com/searchengine)
617
+ bot:
618
+ name: Cốc Cốc Bot
619
+ category: Search bot
620
+ url: https://help.coccoc.com/en/search-engine/coccoc-robots
621
+ producer:
622
+ name: Cốc Cốc
623
+ url: https://coccoc.com/
624
+ -
625
+ user_agent: Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)
626
+ bot:
627
+ name: Cốc Cốc Bot
628
+ category: Search bot
629
+ url: https://help.coccoc.com/en/search-engine/coccoc-robots
630
+ producer:
631
+ name: Cốc Cốc
632
+ url: https://coccoc.com/
633
+ -
634
+ user_agent: Mozilla/5.0 (compatible; coccocbot/1.0; +http://help.coccoc.com/searchengine)
635
+ bot:
636
+ name: Cốc Cốc Bot
637
+ category: Search bot
638
+ url: https://help.coccoc.com/en/search-engine/coccoc-robots
408
639
  producer:
409
640
  name: Cốc Cốc
410
- url: http://coccoc.com/
641
+ url: https://coccoc.com/
642
+ -
643
+ user_agent: Datadog Agent/5.10.1
644
+ bot:
645
+ name: Datadog Agent
646
+ url: https://github.com/DataDog/dd-agent
647
+ category: Site Monitor
648
+ producer:
649
+ name: Datadog
650
+ url: https://www.datadoghq.com/
651
+ -
652
+ user_agent: Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36
653
+ bot:
654
+ name: Datanyze
655
+ url: ""
656
+ category: Crawler
657
+ producer:
658
+ name: Datanyze
659
+ url: https://www.datanyze.com
660
+ -
661
+ user_agent: Mozilla/5.0 (compatible; Dataprovider/6.92; +https://www.dataprovider.com/)
662
+ bot:
663
+ name: Dataprovider
664
+ category: Crawler
665
+ url: ""
666
+ producer:
667
+ name: Dataprovider B.V.
668
+ url: https://www.dataprovider.com/
411
669
  -
412
670
  user_agent: Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0;
413
671
  bot:
@@ -431,7 +689,7 @@
431
689
  bot:
432
690
  name: Dazoobot
433
691
  category: Search bot
434
- url:
692
+ url: ""
435
693
  producer:
436
694
  name: DAZOO.FR
437
695
  url: http://dazoo.fr
@@ -449,7 +707,7 @@
449
707
  bot:
450
708
  name: Domain Re-Animator Bot
451
709
  category: Crawler
452
- url:
710
+ url: ""
453
711
  producer:
454
712
  name: Domain Re-Animator, LLC
455
713
  url: http://domainreanimator.com
@@ -458,7 +716,7 @@
458
716
  bot:
459
717
  name: Domain Re-Animator Bot
460
718
  category: Crawler
461
- url:
719
+ url: ""
462
720
  producer:
463
721
  name: Domain Re-Animator, LLC
464
722
  url: http://domainreanimator.com
@@ -471,15 +729,33 @@
471
729
  producer:
472
730
  name: SEOmoz, Inc.
473
731
  url: http://moz.com/
732
+ -
733
+ user_agent: DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)
734
+ bot:
735
+ name: DuckDuckGo Bot
736
+ category: Search bot
737
+ url: https://duckduckgo.com/duckduckbot
738
+ producer:
739
+ name: DuckDuckGo
740
+ url: https://duckduckgo.com/
741
+ -
742
+ user_agent: Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)
743
+ bot:
744
+ name: DuckDuckGo Bot
745
+ category: Search bot
746
+ url: https://duckduckgo.com/duckduckbot
747
+ producer:
748
+ name: DuckDuckGo
749
+ url: https://duckduckgo.com/
474
750
  -
475
751
  user_agent: EMail Exractor
476
752
  bot:
477
753
  name: EMail Exractor
478
754
  category: Crawler
479
- url:
755
+ url: ""
480
756
  producer:
481
- name:
482
- url:
757
+ name: ""
758
+ url: ""
483
759
  -
484
760
  user_agent: Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)
485
761
  bot:
@@ -498,6 +774,15 @@
498
774
  producer:
499
775
  name: easou ICP
500
776
  url: http://www.easou.com
777
+ -
778
+ user_agent: Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)
779
+ bot:
780
+ name: Embedly
781
+ category: Crawler
782
+ url: https://support.embed.ly/hc/en-us
783
+ producer:
784
+ name: A Medium, Corp.
785
+ url: https://medium.com/
501
786
  -
502
787
  user_agent: Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)
503
788
  bot:
@@ -539,7 +824,7 @@
539
824
  bot:
540
825
  name: Ezooms
541
826
  category: Crawler
542
- url:
827
+ url: ""
543
828
  producer:
544
829
  name: SEOmoz, Inc.
545
830
  url: http://moz.com/
@@ -552,6 +837,15 @@
552
837
  producer:
553
838
  name: Facebook
554
839
  url: http://www.facebook.com
840
+ -
841
+ user_agent: facebookexternalua
842
+ bot:
843
+ name: Facebook External Hit
844
+ category: Social Media Agent
845
+ url: https://www.facebook.com/externalhit_uatext.php
846
+ producer:
847
+ name: Facebook
848
+ url: http://www.facebook.com
555
849
  -
556
850
  user_agent: facebookplatform/1.0 (+http://developers.facebook.com)
557
851
  bot:
@@ -577,8 +871,8 @@
577
871
  url: http://www.feedburner.com
578
872
  category: Feed Fetcher
579
873
  producer:
580
- name:
581
- url:
874
+ name: ""
875
+ url: ""
582
876
  -
583
877
  user_agent: 'Feedbin - 9 subscribers'
584
878
  bot:
@@ -586,8 +880,8 @@
586
880
  url: http://feedbin.com/
587
881
  category: Feed Fetcher
588
882
  producer:
589
- name:
590
- url:
883
+ name: ""
884
+ url: ""
591
885
  -
592
886
  user_agent: Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)
593
887
  bot:
@@ -595,8 +889,8 @@
595
889
  url: http://www.feedly.com
596
890
  category: Feed Fetcher
597
891
  producer:
598
- name:
599
- url:
892
+ name: ""
893
+ url: ""
600
894
  -
601
895
  user_agent: FeedlyApp/1.0 (http://www.feedly.com)
602
896
  bot:
@@ -604,8 +898,8 @@
604
898
  url: http://www.feedly.com
605
899
  category: Feed Fetcher
606
900
  producer:
607
- name:
608
- url:
901
+ name: ""
902
+ url: ""
609
903
  -
610
904
  user_agent: FeedlyBot/1.0 (http://feedly.com)
611
905
  bot:
@@ -613,8 +907,8 @@
613
907
  url: http://www.feedly.com
614
908
  category: Feed Fetcher
615
909
  producer:
616
- name:
617
- url:
910
+ name: ""
911
+ url: ""
618
912
  -
619
913
  user_agent: MetaFeedly/1.0 (http://www.feedly.com)
620
914
  bot:
@@ -622,8 +916,8 @@
622
916
  url: http://www.feedly.com
623
917
  category: Feed Fetcher
624
918
  producer:
625
- name:
626
- url:
919
+ name: ""
920
+ url: ""
627
921
  -
628
922
  user_agent: Feedspot http://www.feedspot.com
629
923
  bot:
@@ -631,8 +925,8 @@
631
925
  url: http://www.feedspot.com
632
926
  category: Feed Fetcher
633
927
  producer:
634
- name:
635
- url:
928
+ name: ""
929
+ url: ""
636
930
  -
637
931
  user_agent: Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)
638
932
  bot:
@@ -640,25 +934,119 @@
640
934
  url: http://feedafever.com/
641
935
  category: Feed Fetcher
642
936
  producer:
643
- name:
644
- url:
937
+ name: ""
938
+ url: ""
939
+ -
940
+ user_agent: Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)
941
+ bot:
942
+ name: Findxbot
943
+ category: Crawler
944
+ url: http://www.findxbot.com
945
+ -
946
+ user_agent: Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)
947
+ bot:
948
+ name: Flipboard
949
+ url: http://flipboard.com/browserproxy
950
+ category: Feed Fetcher
951
+ producer:
952
+ name: Flipboard
953
+ url: http://flipboard.com/
954
+ -
955
+ user_agent: Mozilla/5.0 (compatible; FlipboardRSS/1.2; +http://flipboard.com/browserproxy)
956
+ bot:
957
+ name: Flipboard
958
+ url: http://flipboard.com/browserproxy
959
+ category: Feed Fetcher
960
+ producer:
961
+ name: Flipboard
962
+ url: http://flipboard.com/
963
+ -
964
+ user_agent: freshrss/0.8-dev (Linux; http://freshrss.org) SimplePie/1.4-dev-FreshRSS
965
+ bot:
966
+ name: FreshRSS
967
+ category: Feed Fetcher
968
+ url: https://freshrss.org/
969
+ -
970
+ user_agent: FreshRSS/1.12.0 (Linux; https://freshrss.org)
971
+ bot:
972
+ name: FreshRSS
973
+ category: Feed Fetcher
974
+ url: https://freshrss.org/
975
+ -
976
+ user_agent: Mozilla/5.0 (X11; Linux x86_64; GTmetrix https://gtmetrix.com/) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36
977
+ bot:
978
+ name: GTmetrix
979
+ category: Crawler
980
+ url: https://gtmetrix.com/
981
+ producer:
982
+ name: Carbon60 Operating Co. Ltd.
983
+ url: https://www.carbon60.com/
984
+ -
985
+ user_agent: 7Siters/1.07 (+https://7ooo.ru/siters/)
986
+ bot:
987
+ name: Generic Bot
988
+ -
989
+ user_agent: COMODO SSL Checker
990
+ bot:
991
+ name: Generic Bot
992
+ -
993
+ user_agent: help@dataminr.com
994
+ bot:
995
+ name: Generic Bot
996
+ -
997
+ user_agent: 'HybridBot (hybrid.ru/about. If our bot caused problems please contact us. Contact email: m.lyashkov@targetix.net)'
998
+ bot:
999
+ name: Generic Bot
1000
+ -
1001
+ user_agent: 'Mozilla/5.0 (compatible; +centuryb.o.t9[at]gmail.com)'
1002
+ bot:
1003
+ name: Generic Bot
1004
+ -
1005
+ user_agent: Mozilla/5.0 (compatible; +http://tweetedtimes.com)
1006
+ bot:
1007
+ name: Generic Bot
1008
+ -
1009
+ user_agent: Mozilla/5.0 (compatible; Shareaholicbot/2.0; +http://www.shareaholic.com/bot)
1010
+ bot:
1011
+ name: Generic Bot
1012
+ -
1013
+ user_agent: Mozilla/5.0 (compatible; TrendsmapResolver/0.1)
1014
+ bot:
1015
+ name: Generic Bot
645
1016
  -
646
1017
  user_agent: niki-bot
647
1018
  bot:
648
1019
  name: Generic Bot
649
1020
  -
650
- user_agent: StatusCake Error Detector Crawler V0.01
1021
+ user_agent: PHPCrawl
1022
+ bot:
1023
+ name: Generic Bot
1024
+ -
1025
+ user_agent: robots
1026
+ bot:
1027
+ name: Generic Bot
1028
+ -
1029
+ user_agent: SeopultContentAnalyzer/1.0
651
1030
  bot:
652
1031
  name: Generic Bot
653
1032
  -
654
1033
  user_agent: Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)
655
1034
  bot:
656
1035
  name: Genieo Web filter
657
- category:
1036
+ category: ""
658
1037
  url: http://www.genieo.com/webfilter.html
659
1038
  producer:
660
1039
  name: Genieo
661
1040
  url: http://www.genieo.com
1041
+ -
1042
+ user_agent: GigablastOpenSource/1.0
1043
+ bot:
1044
+ name: Gigablast
1045
+ category: Search bot
1046
+ url: https://github.com/gigablast/open-source-search-engine
1047
+ producer:
1048
+ name: Matt Wells
1049
+ url: http://www.gigablast.com/faq.html
662
1050
  -
663
1051
  user_agent: Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)
664
1052
  bot:
@@ -666,14 +1054,14 @@
666
1054
  category: Crawler
667
1055
  url: http://glutenfreepleasure.com/
668
1056
  producer:
669
- name:
670
- url:
1057
+ name: ""
1058
+ url: ""
671
1059
  -
672
1060
  user_agent: Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)
673
1061
  bot:
674
1062
  name: Gmail Image Proxy
675
1063
  category: Crawler
676
- url:
1064
+ url: ""
677
1065
  producer:
678
1066
  name: Google Inc.
679
1067
  url: http://www.google.com
@@ -686,6 +1074,11 @@
686
1074
  producer:
687
1075
  name: NTT Resonant
688
1076
  url: http://goo.ne.jp
1077
+ -
1078
+ user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon
1079
+ bot:
1080
+ name: Google Favicon
1081
+ category: Crawler
689
1082
  -
690
1083
  user_agent: Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0_1 like Mac OS X; en-us) AppleWebKit/537.4 (KHTML, like Gecko; Google Page Speed Insights) Version/4.0.5 Mobile/8A306 Safari/6531.22.7
691
1084
  bot:
@@ -700,7 +1093,25 @@
700
1093
  bot:
701
1094
  name: Google Partner Monitoring
702
1095
  category: Site Monitor
703
- url:
1096
+ url: ""
1097
+ producer:
1098
+ name: Google Inc.
1099
+ url: http://www.google.com
1100
+ -
1101
+ user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Search Console) Chrome/41.0.2272.118 Safari/537.36
1102
+ bot:
1103
+ name: Google Search Console
1104
+ category: Crawler
1105
+ url: https://search.google.com/search-console/about
1106
+ producer:
1107
+ name: Google Inc.
1108
+ url: http://www.google.com
1109
+ -
1110
+ user_agent: Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)
1111
+ bot:
1112
+ name: Google Structured Data Testing Tool
1113
+ category: Validator
1114
+ url: https://search.google.com/structured-data/testing-tool
704
1115
  producer:
705
1116
  name: Google Inc.
706
1117
  url: http://www.google.com
@@ -723,7 +1134,7 @@
723
1134
  name: Google Inc.
724
1135
  url: http://www.google.com
725
1136
  -
726
- user_agent: DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
1137
+ user_agent: APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)
727
1138
  bot:
728
1139
  name: Googlebot
729
1140
  category: Search bot
@@ -732,7 +1143,7 @@
732
1143
  name: Google Inc.
733
1144
  url: http://www.google.com
734
1145
  -
735
- user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 19 subscribers; feed-id=13965549748850348809)
1146
+ user_agent: DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
736
1147
  bot:
737
1148
  name: Googlebot
738
1149
  category: Search bot
@@ -741,7 +1152,7 @@
741
1152
  name: Google Inc.
742
1153
  url: http://www.google.com
743
1154
  -
744
- user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 2 subscribers; feed-id=17860707833818568603)
1155
+ user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 19 subscribers; feed-id=13965549748850348809)
745
1156
  bot:
746
1157
  name: Googlebot
747
1158
  category: Search bot
@@ -750,7 +1161,7 @@
750
1161
  name: Google Inc.
751
1162
  url: http://www.google.com
752
1163
  -
753
- user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 375 subscribers; feed-id=15381863289700640853)
1164
+ user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 2 subscribers; feed-id=17860707833818568603)
754
1165
  bot:
755
1166
  name: Googlebot
756
1167
  category: Search bot
@@ -759,7 +1170,7 @@
759
1170
  name: Google Inc.
760
1171
  url: http://www.google.com
761
1172
  -
762
- user_agent: Googlebot (gocrawl v0.4)
1173
+ user_agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 375 subscribers; feed-id=15381863289700640853)
763
1174
  bot:
764
1175
  name: Googlebot
765
1176
  category: Search bot
@@ -768,7 +1179,7 @@
768
1179
  name: Google Inc.
769
1180
  url: http://www.google.com
770
1181
  -
771
- user_agent: Googlebot-Image/1.0
1182
+ user_agent: Google-AdWords-Express
772
1183
  bot:
773
1184
  name: Googlebot
774
1185
  category: Search bot
@@ -777,7 +1188,7 @@
777
1188
  name: Google Inc.
778
1189
  url: http://www.google.com
779
1190
  -
780
- user_agent: Googlebot-News (2.3.3, ruby 1.9.3 (2013-11-22))
1191
+ user_agent: Google-Adwords-Instant (+http://www.google.com/adsbot.html)
781
1192
  bot:
782
1193
  name: Googlebot
783
1194
  category: Search bot
@@ -786,7 +1197,7 @@
786
1197
  name: Google Inc.
787
1198
  url: http://www.google.com
788
1199
  -
789
- user_agent: Googlebot-Video/1.0
1200
+ user_agent: Google-speakr
790
1201
  bot:
791
1202
  name: Googlebot
792
1203
  category: Search bot
@@ -795,7 +1206,7 @@
795
1206
  name: Google Inc.
796
1207
  url: http://www.google.com
797
1208
  -
798
- user_agent: Googlebot/2.1 (http://www.googlebot.com/bot.html)
1209
+ user_agent: Googlebot (gocrawl v0.4)
799
1210
  bot:
800
1211
  name: Googlebot
801
1212
  category: Search bot
@@ -804,7 +1215,7 @@
804
1215
  name: Google Inc.
805
1216
  url: http://www.google.com
806
1217
  -
807
- user_agent: Googlebot/Nutch-1.7
1218
+ user_agent: Googlebot-Image/1.0
808
1219
  bot:
809
1220
  name: Googlebot
810
1221
  category: Search bot
@@ -813,7 +1224,7 @@
813
1224
  name: Google Inc.
814
1225
  url: http://www.google.com
815
1226
  -
816
- user_agent: GoogleProducer; (+http://goo.gl/7y4SX)
1227
+ user_agent: Googlebot-News (2.3.3, ruby 1.9.3 (2013-11-22))
817
1228
  bot:
818
1229
  name: Googlebot
819
1230
  category: Search bot
@@ -822,7 +1233,7 @@
822
1233
  name: Google Inc.
823
1234
  url: http://www.google.com
824
1235
  -
825
- user_agent: Mediapartners-Google
1236
+ user_agent: Googlebot-Video/1.0
826
1237
  bot:
827
1238
  name: Googlebot
828
1239
  category: Search bot
@@ -831,7 +1242,7 @@
831
1242
  name: Google Inc.
832
1243
  url: http://www.google.com
833
1244
  -
834
- user_agent: Mozilla/5.0 (compatible) Feedfetcher-Google;(+http://www.google.com/feedfetcher.html)
1245
+ user_agent: Googlebot/2.1 (http://www.googlebot.com/bot.html)
835
1246
  bot:
836
1247
  name: Googlebot
837
1248
  category: Search bot
@@ -840,7 +1251,7 @@
840
1251
  name: Google Inc.
841
1252
  url: http://www.google.com
842
1253
  -
843
- user_agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
1254
+ user_agent: Googlebot/Nutch-1.7
844
1255
  bot:
845
1256
  name: Googlebot
846
1257
  category: Search bot
@@ -849,7 +1260,7 @@
849
1260
  name: Google Inc.
850
1261
  url: http://www.google.com
851
1262
  -
852
- user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
1263
+ user_agent: GoogleProducer; (+http://goo.gl/7y4SX)
853
1264
  bot:
854
1265
  name: Googlebot
855
1266
  category: Search bot
@@ -858,7 +1269,7 @@
858
1269
  name: Google Inc.
859
1270
  url: http://www.google.com
860
1271
  -
861
- user_agent: Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; Google-Publisher-Plugin) Chrome/27.0.1453 Mobile Safari/537.36
1272
+ user_agent: Mediapartners-Google
862
1273
  bot:
863
1274
  name: Googlebot
864
1275
  category: Search bot
@@ -867,7 +1278,7 @@
867
1278
  name: Google Inc.
868
1279
  url: http://www.google.com
869
1280
  -
870
- user_agent: Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0 Google (+https://developers.google.com/+/web/snippet/)
1281
+ user_agent: Mozilla/5.0 (compatible) Feedfetcher-Google;(+http://www.google.com/feedfetcher.html)
871
1282
  bot:
872
1283
  name: Googlebot
873
1284
  category: Search bot
@@ -876,7 +1287,115 @@
876
1287
  name: Google Inc.
877
1288
  url: http://www.google.com
878
1289
  -
879
- user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36
1290
+ user_agent: Mozilla/5.0 (compatible; Google-Youtube-Links)
1291
+ bot:
1292
+ name: Googlebot
1293
+ category: Search bot
1294
+ url: http://www.google.com/bot.html
1295
+ producer:
1296
+ name: Google Inc.
1297
+ url: http://www.google.com
1298
+ -
1299
+ user_agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
1300
+ bot:
1301
+ name: Googlebot
1302
+ category: Search bot
1303
+ url: http://www.google.com/bot.html
1304
+ producer:
1305
+ name: Google Inc.
1306
+ url: http://www.google.com
1307
+ -
1308
+ user_agent: Mozilla/5.0 (en-US) AppleWebKit/537.36 (KHTML, like Gecko; Google-Assess) Chrome/34.0.1847.116 Safari/537.36
1309
+ bot:
1310
+ name: Googlebot
1311
+ category: Search bot
1312
+ url: http://www.google.com/bot.html
1313
+ producer:
1314
+ name: Google Inc.
1315
+ url: http://www.google.com
1316
+ -
1317
+ user_agent: Mozilla/5.0 (en-us) AppleWebKit/537.36(KHTML, like Gecko; Google-Adwords-DisplayAds-WebRender;) Chrome/41.0.2272.118Safari/537.36
1318
+ bot:
1319
+ name: Googlebot
1320
+ category: Search bot
1321
+ url: http://www.google.com/bot.html
1322
+ producer:
1323
+ name: Google Inc.
1324
+ url: http://www.google.com
1325
+ -
1326
+ user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)
1327
+ bot:
1328
+ name: Googlebot
1329
+ category: Search bot
1330
+ url: http://www.google.com/bot.html
1331
+ producer:
1332
+ name: Google Inc.
1333
+ url: http://www.google.com
1334
+ -
1335
+ user_agent: Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/537.36 (KHTML, like Gecko; Google-Publisher-Plugin) Chrome/27.0.1453 Mobile Safari/537.36
1336
+ bot:
1337
+ name: Googlebot
1338
+ category: Search bot
1339
+ url: http://www.google.com/bot.html
1340
+ producer:
1341
+ name: Google Inc.
1342
+ url: http://www.google.com
1343
+ -
1344
+ user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Google-AMPHTML)
1345
+ bot:
1346
+ name: Googlebot
1347
+ category: Search bot
1348
+ url: http://www.google.com/bot.html
1349
+ producer:
1350
+ name: Google Inc.
1351
+ url: http://www.google.com
1352
+ -
1353
+ user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943)
1354
+ bot:
1355
+ name: Googlebot
1356
+ category: Search bot
1357
+ url: http://www.google.com/bot.html
1358
+ producer:
1359
+ name: Google Inc.
1360
+ url: http://www.google.com
1361
+ -
1362
+ user_agent: Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; DuplexWeb-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Mobile Safari/537.36
1363
+ bot:
1364
+ name: Googlebot
1365
+ category: Search bot
1366
+ url: http://www.google.com/bot.html
1367
+ producer:
1368
+ name: Google Inc.
1369
+ url: http://www.google.com
1370
+ -
1371
+ user_agent: Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0 Google (+https://developers.google.com/+/web/snippet/)
1372
+ bot:
1373
+ name: Googlebot
1374
+ category: Search bot
1375
+ url: http://www.google.com/bot.html
1376
+ producer:
1377
+ name: Google Inc.
1378
+ url: http://www.google.com
1379
+ -
1380
+ user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Safari/537.36 (compatible; Google-HotelAdsVerifier/2.0)
1381
+ bot:
1382
+ name: Googlebot
1383
+ category: Search bot
1384
+ url: http://www.google.com/bot.html
1385
+ producer:
1386
+ name: Google Inc.
1387
+ url: http://www.google.com
1388
+ -
1389
+ user_agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36
1390
+ bot:
1391
+ name: Googlebot
1392
+ category: Search bot
1393
+ url: http://www.google.com/bot.html
1394
+ producer:
1395
+ name: Google Inc.
1396
+ url: http://www.google.com
1397
+ -
1398
+ user_agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Shopping-Quality +http://www.google.com/merchants/tos/extend/DE/tos.html)
880
1399
  bot:
881
1400
  name: Googlebot
882
1401
  category: Search bot
@@ -902,6 +1421,24 @@
902
1421
  producer:
903
1422
  name: Google Inc.
904
1423
  url: http://www.google.com
1424
+ -
1425
+ user_agent: Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +https://www.grapeshot.com/crawler/)
1426
+ bot:
1427
+ name: Grapeshot
1428
+ category: Crawler
1429
+ url: https://www.grapeshot.com/crawler
1430
+ producer:
1431
+ name: Grapeshot
1432
+ url: https://www.grapeshot.com
1433
+ -
1434
+ user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1. 4 (compatible; GrapeshotCrawler/2.0; +https://www.grapeshot.com/crawler/)
1435
+ bot:
1436
+ name: Grapeshot
1437
+ category: Crawler
1438
+ url: https://www.grapeshot.com/crawler
1439
+ producer:
1440
+ name: Grapeshot
1441
+ url: https://www.grapeshot.com
905
1442
  -
906
1443
  user_agent: HTTPMon/1.0b (http://www.httpmon.com)
907
1444
  bot:
@@ -983,6 +1520,15 @@
983
1520
  producer:
984
1521
  name: The Internet Archive
985
1522
  url: http://www.archive.org
1523
+ -
1524
+ user_agent: Heurekabot-Feed/1.0 (+https://sluzby.heureka.cz/napoveda/heurekabot/)
1525
+ bot:
1526
+ name: Heureka Feed
1527
+ category: Crawler
1528
+ url: https://sluzby.heureka.cz/napoveda/heurekabot/
1529
+ producer:
1530
+ name: Heureka.cz, a.s.
1531
+ url: https://www.heureka.cz/
986
1532
  -
987
1533
  user_agent: HubPages V0.2.2 (http://hubpages.com/help/crawlingpolicy)
988
1534
  bot:
@@ -992,6 +1538,14 @@
992
1538
  producer:
993
1539
  name: HubPages
994
1540
  url: http://hubpages.com/
1541
+ -
1542
+ user_agent: HubSpot Website Grader (web-crawlers@hubspot.com)
1543
+ bot:
1544
+ name: HubSpot
1545
+ category: Crawler
1546
+ producer:
1547
+ name: HubSpot Inc.
1548
+ url: https://www.hubspot.com
995
1549
  -
996
1550
  user_agent: ICC-Crawler/2.0 (Mozilla-compatible; ; http://www.nict.go.jp/en/univ-com/plan/crawl.html)
997
1551
  bot:
@@ -999,8 +1553,17 @@
999
1553
  category: Crawler
1000
1554
  url: http://www.nict.go.jp/en/univ-com/plan/crawl.html
1001
1555
  producer:
1002
- name:
1003
- url:
1556
+ name: ""
1557
+ url: ""
1558
+ -
1559
+ user_agent: IDG/IT (http://spaziodati.eu/)
1560
+ bot:
1561
+ name: IDG/IT
1562
+ category: Search bot
1563
+ url: https://spaziodati.eu/
1564
+ producer:
1565
+ name: SpazioDati S.r.l.
1566
+ url: https://spaziodati.eu/
1004
1567
  -
1005
1568
  user_agent: iisbot/1.0 (+http://www.iis.net/iisbot.html)
1006
1569
  bot:
@@ -1010,6 +1573,32 @@
1010
1573
  producer:
1011
1574
  name: Microsoft Corporation
1012
1575
  url: http://www.microsoft.com
1576
+ -
1577
+ user_agent: IP-Guide.com Crawler/1.0 (https://ip-guide.com)
1578
+ bot:
1579
+ name: IP-Guide Crawler
1580
+ category: Crawler
1581
+ url: ""
1582
+ producer:
1583
+ name: ""
1584
+ url: https://ip-guide.com
1585
+ -
1586
+ user_agent: Mozilla/5.0 (compatible; ips-agent)
1587
+ bot:
1588
+ name: IPS Agent
1589
+ category: crawler
1590
+ producer:
1591
+ name: VeriSign, Inc
1592
+ url: http://www.verisign.com/
1593
+ -
1594
+ user_agent: Kaspersky Lab CFR link resolver cfradmins@kaspersky.com
1595
+ bot:
1596
+ name: Kaspersky
1597
+ category: Security Checker
1598
+ url: https://www.kaspersky.com/
1599
+ producer:
1600
+ name: AO Kaspersky Lab
1601
+ url: https://www.kaspersky.com/
1013
1602
  -
1014
1603
  user_agent: kouio.com RSS reader
1015
1604
  bot:
@@ -1017,8 +1606,8 @@
1017
1606
  url: http://kouio.com/
1018
1607
  category: Feed Fetcher
1019
1608
  producer:
1020
- name:
1021
- url:
1609
+ name: ""
1610
+ url: ""
1022
1611
  -
1023
1612
  user_agent: 'kouio.com RSS reader - 6 subscribers'
1024
1613
  bot:
@@ -1026,16 +1615,25 @@
1026
1615
  url: http://kouio.com/
1027
1616
  category: Feed Fetcher
1028
1617
  producer:
1029
- name:
1030
- url:
1618
+ name: ""
1619
+ url: ""
1620
+ -
1621
+ user_agent: LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)
1622
+ bot:
1623
+ name: LCC
1624
+ category: Search bot
1625
+ url: https://corpora.uni-leipzig.de/crawler_faq.html
1626
+ producer:
1627
+ name: Universität Leipzig
1628
+ url: https://www.uni-leipzig.de/
1031
1629
  -
1032
1630
  user_agent: 'ltx71 - (http://ltx71.com/)'
1033
1631
  bot:
1034
1632
  name: LTX71
1035
1633
  url: http://ltx71.com/
1036
1634
  producer:
1037
- name:
1038
- url:
1635
+ name: ""
1636
+ url: ""
1039
1637
  -
1040
1638
  user_agent: larbin_2.6.3 larbin2.6.3@unspecified.mail
1041
1639
  bot:
@@ -1043,8 +1641,26 @@
1043
1641
  category: Crawler
1044
1642
  url: http://larbin.sourceforge.net
1045
1643
  producer:
1046
- name:
1047
- url:
1644
+ name: ""
1645
+ url: ""
1646
+ -
1647
+ user_agent: "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)"
1648
+ bot:
1649
+ name: "Let's Encrypt Validation"
1650
+ category: Service Agent
1651
+ url: https://letsencrypt.org/how-it-works/
1652
+ producer:
1653
+ name: "Let's Encrypt"
1654
+ url: https://letsencrypt.org
1655
+ -
1656
+ user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/61.0.3116.0 Mobile Safari/537.36 Chrome-Lighthouse
1657
+ bot:
1658
+ name: Lighthouse
1659
+ category: Site Monitor
1660
+ url: https://developers.google.com/web/tools/lighthouse
1661
+ producer:
1662
+ name: Lighthouse
1663
+ url: https://developers.google.com/web/tools/lighthouse
1048
1664
  -
1049
1665
  user_agent: linkdex.com/v2.0 and linkdex.com/v2.1
1050
1666
  bot:
@@ -1106,8 +1722,8 @@
1106
1722
  url: http://magpierss.sourceforge.net/
1107
1723
  category: Feed Parser
1108
1724
  producer:
1109
- name:
1110
- url:
1725
+ name: ""
1726
+ url: ""
1111
1727
  -
1112
1728
  user_agent: Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)
1113
1729
  bot:
@@ -1126,6 +1742,16 @@
1126
1742
  producer:
1127
1743
  name: Mail.Ru Group
1128
1744
  url: http://corp.mail.ru
1745
+ -
1746
+ user_agent: http.rb/2.2.2 (Mastodon/1.6.1; +https://mathtod.online/)
1747
+ bot:
1748
+ name: Mastodon Bot
1749
+ category: Social Media Agent
1750
+ -
1751
+ user_agent: http.rb/3.2.0 (Mastodon/2.4.3; +https://uwu.social/)
1752
+ bot:
1753
+ name: Mastodon Bot
1754
+ category: Social Media Agent
1129
1755
  -
1130
1756
  user_agent: Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)
1131
1757
  bot:
@@ -1135,6 +1761,21 @@
1135
1761
  producer:
1136
1762
  name: Meanpath
1137
1763
  url: http://www.meanpath.com
1764
+ -
1765
+ user_agent: MetaInspector/5.4.0 (+https://github.com/jaimeiniesta/metainspector)
1766
+ bot:
1767
+ name: MetaInspector
1768
+ category: Crawler
1769
+ url: https://github.com/jaimeiniesta/metainspector
1770
+ -
1771
+ user_agent: Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)
1772
+ bot:
1773
+ name: MetaJobBot
1774
+ category: Crawler
1775
+ url: http://www.metajob.at/the/crawler
1776
+ producer:
1777
+ name: MetaJob
1778
+ url: http://www.metajob.at
1138
1779
  -
1139
1780
  user_agent: Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)
1140
1781
  bot:
@@ -1143,7 +1784,16 @@
1143
1784
  url: http://mixrank.com
1144
1785
  producer:
1145
1786
  name: Online Media Group, Inc.
1146
- url:
1787
+ url: ""
1788
+ -
1789
+ user_agent: Mnogosearch-3.1.21
1790
+ bot:
1791
+ name: Mnogosearch
1792
+ category: Search bot
1793
+ url: http://www.mnogosearch.org/
1794
+ producer:
1795
+ name: Lavtech.Com Corp.
1796
+ url: ""
1147
1797
  -
1148
1798
  user_agent: Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)
1149
1799
  bot:
@@ -1162,15 +1812,42 @@
1162
1812
  producer:
1163
1813
  name: Monitor.Us
1164
1814
  url: http://www.monitor.us
1815
+ -
1816
+ user_agent: munin/2.0.30-1 (libwww-perl/6.15)
1817
+ bot:
1818
+ name: Munin
1819
+ category: Site Monitor
1820
+ url: http://munin-monitoring.org/
1821
+ producer:
1822
+ name: Munin
1823
+ url: http://munin-monitoring.org/
1824
+ -
1825
+ user_agent: munin/http_loadtime
1826
+ bot:
1827
+ name: Munin
1828
+ category: Site Monitor
1829
+ url: http://munin-monitoring.org/
1830
+ producer:
1831
+ name: Munin
1832
+ url: http://munin-monitoring.org/
1165
1833
  -
1166
1834
  user_agent: nlcrawler/1.0 (+http://northernlight.com/)
1167
1835
  bot:
1168
1836
  name: NLCrawler
1169
1837
  category: Crawler
1170
- url:
1838
+ url: ""
1171
1839
  producer:
1172
1840
  name: Northern Light
1173
1841
  url: http://northernlight.com
1842
+ -
1843
+ user_agent: check_http/v1.5 (nagios-plugins 1.5)
1844
+ bot:
1845
+ name: Nagios check_http
1846
+ category: Site Monitor
1847
+ url: https://nagios.org
1848
+ producer:
1849
+ name: Nagios Plugins Development Team
1850
+ url: https://nagios.org
1174
1851
  -
1175
1852
  user_agent: NalezenCzBot/1.0 (http://www.nalezen.cz/about-crawler)
1176
1853
  bot:
@@ -1179,7 +1856,7 @@
1179
1856
  url: http://www.nalezen.cz/about-crawler
1180
1857
  producer:
1181
1858
  name: Jaroslav Kuboš
1182
- url:
1859
+ url: ""
1183
1860
  -
1184
1861
  user_agent: NetLyzer FastProbe
1185
1862
  bot:
@@ -1189,7 +1866,25 @@
1189
1866
  bot:
1190
1867
  name: Netcraft Survey Bot
1191
1868
  category: Search bot
1192
- url:
1869
+ url: ""
1870
+ producer:
1871
+ name: Netcraft
1872
+ url: http://www.netcraft.com
1873
+ -
1874
+ user_agent: Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)
1875
+ bot:
1876
+ name: Netcraft Survey Bot
1877
+ category: Search bot
1878
+ url: ""
1879
+ producer:
1880
+ name: Netcraft
1881
+ url: http://www.netcraft.com
1882
+ -
1883
+ user_agent: 'Netcraft SSL Server Survey - contact info@netcraft.com'
1884
+ bot:
1885
+ name: Netcraft Survey Bot
1886
+ category: Search bot
1887
+ url: ""
1193
1888
  producer:
1194
1889
  name: Netcraft
1195
1890
  url: http://www.netcraft.com
@@ -1200,8 +1895,8 @@
1200
1895
  url: http://www.netvibes.com/
1201
1896
  category: Feed Fetcher
1202
1897
  producer:
1203
- name:
1204
- url:
1898
+ name: ""
1899
+ url: ""
1205
1900
  -
1206
1901
  user_agent: 'Netvibes (http://www.netvibes.com/; 8 subscribers; feedID: 2244192)'
1207
1902
  bot:
@@ -1209,8 +1904,8 @@
1209
1904
  url: http://www.netvibes.com/
1210
1905
  category: Feed Fetcher
1211
1906
  producer:
1212
- name:
1213
- url:
1907
+ name: ""
1908
+ url: ""
1214
1909
  -
1215
1910
  user_agent: 'NewsBlur Favicon Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
1216
1911
  bot:
@@ -1218,8 +1913,8 @@
1218
1913
  url: http://www.newsblur.com
1219
1914
  category: Feed Fetcher
1220
1915
  producer:
1221
- name:
1222
- url:
1916
+ name: ""
1917
+ url: ""
1223
1918
  -
1224
1919
  user_agent: 'NewsBlur Feed Fetcher - 7 subscribers - http://www.newsblur.com/site/1948420/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
1225
1920
  bot:
@@ -1227,8 +1922,8 @@
1227
1922
  url: http://www.newsblur.com
1228
1923
  category: Feed Fetcher
1229
1924
  producer:
1230
- name:
1231
- url:
1925
+ name: ""
1926
+ url: ""
1232
1927
  -
1233
1928
  user_agent: NewsBlur Feed Finder (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)
1234
1929
  bot:
@@ -1236,8 +1931,8 @@
1236
1931
  url: http://www.newsblur.com
1237
1932
  category: Feed Fetcher
1238
1933
  producer:
1239
- name:
1240
- url:
1934
+ name: ""
1935
+ url: ""
1241
1936
  -
1242
1937
  user_agent: 'NewsBlur Page Fetcher - 7 subscribers - http://www.newsblur.com/site/3966817/analytics-piwik (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_1) AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3)'
1243
1938
  bot:
@@ -1245,8 +1940,8 @@
1245
1940
  url: http://www.newsblur.com
1246
1941
  category: Feed Fetcher
1247
1942
  producer:
1248
- name:
1249
- url:
1943
+ name: ""
1944
+ url: ""
1250
1945
  -
1251
1946
  user_agent: NewsGatorOnline/2.0 (http://www.newsgator.com; 2 subscribers)
1252
1947
  bot:
@@ -1254,8 +1949,17 @@
1254
1949
  url: http://www.newsgator.com
1255
1950
  category: Feed Fetcher
1256
1951
  producer:
1257
- name:
1258
- url:
1952
+ name: ""
1953
+ url: ""
1954
+ -
1955
+ user_agent: Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)
1956
+ bot:
1957
+ name: Nmap
1958
+ category: Security Checker
1959
+ url: https://nmap.org/book/nse.html
1960
+ producer:
1961
+ name: Nmap
1962
+ url: https://nmap.org/
1259
1963
  -
1260
1964
  user_agent: 5acd734a-1ed3-4a17-ad9d-0b09ae7bc2d2/Nutch-1.7
1261
1965
  bot:
@@ -1373,6 +2077,27 @@
1373
2077
  producer:
1374
2078
  name: The Apache Software Foundation
1375
2079
  url: http://www.apache.org/foundation/
2080
+ -
2081
+ user_agent: Nuzzel
2082
+ bot:
2083
+ name: Nuzzel
2084
+ category: Crawler
2085
+ producer:
2086
+ name: Nuzzel
2087
+ url: https://www.nuzzel.com/
2088
+ -
2089
+ user_agent: Octopus 1.0.2
2090
+ bot:
2091
+ name: Octopus
2092
+ -
2093
+ user_agent: omgili/0.5 +http://omgili.com
2094
+ bot:
2095
+ name: Omgili bot
2096
+ category: Search bot
2097
+ url: http://www.omgili.com/Crawler.html
2098
+ producer:
2099
+ name: Omgili
2100
+ url: http://www.omgili.com
1376
2101
  -
1377
2102
  user_agent: omgilibot/0.3 +http://www.omgili.com/Crawler.html
1378
2103
  bot:
@@ -1450,7 +2175,7 @@
1450
2175
  bot:
1451
2176
  name: Outbrain
1452
2177
  category: Crawler
1453
- url:
2178
+ url: ""
1454
2179
  producer:
1455
2180
  name: Outbrain
1456
2181
  url: http://www.outbrain.com/
@@ -1472,6 +2197,12 @@
1472
2197
  producer:
1473
2198
  name: Smallrivers SA
1474
2199
  url: http://www.paper.li
2200
+ -
2201
+ user_agent: phantomas/1.11.0 (PhantomJS/1.9.8; linux x64)
2202
+ bot:
2203
+ name: Phantomas
2204
+ category: Site Monitor
2205
+ url: https://github.com/macbre/phantomas
1475
2206
  -
1476
2207
  user_agent: psbot-page (+http://www.picsearch.com/bot.html)
1477
2208
  bot:
@@ -1495,19 +2226,46 @@
1495
2226
  bot:
1496
2227
  name: Pingdom Bot
1497
2228
  category: Site Monitor
1498
- url:
2229
+ url: ""
1499
2230
  producer:
1500
2231
  name: Pingdom AB
1501
2232
  url: https://www.pingdom.com
2233
+ -
2234
+ user_agent: Mozilla/5.0 (compatible; Pinterestbot/1.0; http://www.pinterest.com/bot.html)
2235
+ bot:
2236
+ name: Pinterest
2237
+ url: http://www.pinterest.com/bot.html
2238
+ category: Crawler
2239
+ producer:
2240
+ name: Pinterest
2241
+ url: http://www.pinterest.com/
1502
2242
  -
1503
2243
  user_agent: Pinterest/0.2 (+http://www.pinterest.com/)
1504
2244
  bot:
1505
2245
  name: Pinterest
1506
- url:
2246
+ url: http://www.pinterest.com/bot.html
1507
2247
  category: Crawler
1508
2248
  producer:
1509
2249
  name: Pinterest
1510
2250
  url: http://www.pinterest.com/
2251
+ -
2252
+ user_agent: PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)
2253
+ bot:
2254
+ name: PocketParser
2255
+ category: Read-it-later Service
2256
+ url: https://getpocket.com/pocketparser_ua
2257
+ producer:
2258
+ name: Pocket
2259
+ url: https://getpocket.com/
2260
+ -
2261
+ user_agent: PritTorrent/1.0
2262
+ bot:
2263
+ name: PritTorrent
2264
+ category: Crawler
2265
+ url: https://github.com/astro/prittorrent
2266
+ producer:
2267
+ name: Bitlove
2268
+ url: http://bitlove.org/
1511
2269
  -
1512
2270
  user_agent: QuerySeekerSpider ( http://queryseeker.com/bot.html )
1513
2271
  bot:
@@ -1517,15 +2275,52 @@
1517
2275
  producer:
1518
2276
  name: QueryEye Inc.
1519
2277
  url: http://queryeye.com
2278
+ -
2279
+ user_agent: Quora Link Preview/1.0 (http://www.quora.com)
2280
+ bot:
2281
+ name: Quora Link Preview
2282
+ category: Crawler
2283
+ url: ""
2284
+ producer:
2285
+ name: Quora
2286
+ url: http://www.quora.com
2287
+ -
2288
+ user_agent: 'Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*'
2289
+ bot:
2290
+ name: Qwantify
2291
+ category: Crawler
2292
+ url: https://www.qwant.com/
2293
+ producer:
2294
+ name: Qwant Corporation
2295
+ url: https://www.qwant.com/
1520
2296
  -
1521
2297
  user_agent: ROI Hunter; https://api-dev.roihunter.com
1522
2298
  bot:
1523
2299
  name: ROI Hunter
1524
2300
  category: Crawler
1525
- url:
2301
+ url: ""
1526
2302
  producer:
1527
2303
  name: Roihunter a.s.
1528
2304
  url: http://roihunter.com/
2305
+ -
2306
+ user_agent: RSSRadio (Push Notification Scanner;support@dorada.co.uk)
2307
+ bot:
2308
+ name: RSSRadio Bot
2309
+ -
2310
+ user_agent: Rainmeter WebParser plugin
2311
+ bot:
2312
+ name: Rainmeter
2313
+ category: Crawler
2314
+ url: https://www.rainmeter.net
2315
+ -
2316
+ user_agent: RamblerMail/6.0 (incompatible; ImageProxy/6.0)
2317
+ bot:
2318
+ name: RamblerMail Image Proxy
2319
+ category: Crawler
2320
+ url: ""
2321
+ producer:
2322
+ name: 'Rambler&Co'
2323
+ url: https://rambler-co.ru/
1529
2324
  -
1530
2325
  user_agent: Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)
1531
2326
  bot:
@@ -1571,6 +2366,15 @@
1571
2366
  producer:
1572
2367
  name: SEO Engine
1573
2368
  url: http://www.seoengine.com
2369
+ -
2370
+ user_agent: Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)
2371
+ bot:
2372
+ name: SEOkicks-Robot
2373
+ category: Crawler
2374
+ url: http://www.seokicks.de/robot.html
2375
+ producer:
2376
+ name: SEOkicks
2377
+ url: https://www.seokicks.de/
1574
2378
  -
1575
2379
  user_agent: Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)
1576
2380
  bot:
@@ -1580,6 +2384,60 @@
1580
2384
  producer:
1581
2385
  name: SISTRIX GmbH
1582
2386
  url: http://www.sistrix.de
2387
+ -
2388
+ user_agent: Mozilla/5.0 (compatible; Optimizer)
2389
+ bot:
2390
+ name: SISTRIX Optimizer
2391
+ category: Crawler
2392
+ url: https://optimizer.sistrix.com
2393
+ producer:
2394
+ name: SISTRIX GmbH
2395
+ url: http://www.sistrix.de
2396
+ -
2397
+ user_agent: Mozilla/5.0 (compatible; SISTRIX Optimizer)
2398
+ bot:
2399
+ name: SISTRIX Optimizer
2400
+ category: Crawler
2401
+ url: https://optimizer.sistrix.com
2402
+ producer:
2403
+ name: SISTRIX GmbH
2404
+ url: http://www.sistrix.de
2405
+ -
2406
+ user_agent: Mozilla/5.0 (compatible; SISTRIX Optimizer; Uptime; +https://www.sistrix.com/faq/uptime)
2407
+ bot:
2408
+ name: SISTRIX Optimizer
2409
+ category: Crawler
2410
+ url: https://optimizer.sistrix.com
2411
+ producer:
2412
+ name: SISTRIX GmbH
2413
+ url: http://www.sistrix.de
2414
+ -
2415
+ user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)
2416
+ bot:
2417
+ name: SMTBot
2418
+ category: Search bot
2419
+ url: https://www.similartech.com/smtbot
2420
+ producer:
2421
+ name: SimilarTech Ltd.
2422
+ url: https://www.similartech.com/
2423
+ -
2424
+ user_agent: SSL Labs (https://www.ssllabs.com/about/assessment.html)
2425
+ bot:
2426
+ name: SSL Labs
2427
+ category: Validator
2428
+ url: https://www.ssllabs.com/about/assessment.html
2429
+ producer:
2430
+ name: SSL Labs
2431
+ url: https://www.ssllabs.com/about/assessment.html
2432
+ -
2433
+ user_agent: SafeDNSBot (https://www.safedns.com/searchbot)
2434
+ bot:
2435
+ name: SafeDNSBot
2436
+ category: Crawler
2437
+ url: https://www.safedns.com/searchbot
2438
+ producer:
2439
+ name: SafeDNS, Inc.
2440
+ url: https://www.safedns.com/
1583
2441
  -
1584
2442
  user_agent: Scrapy/1.0.3.post6+g2d688cd (+http://scrapy.org)
1585
2443
  bot:
@@ -1602,8 +2460,8 @@
1602
2460
  category: Crawler
1603
2461
  url: http://www.screenerbot.com
1604
2462
  producer:
1605
- name:
1606
- url:
2463
+ name: ""
2464
+ url: ""
1607
2465
  -
1608
2466
  user_agent: Mozilla/5.0 (compatible; SemrushBot/0.97; +http://www.semrush.com/bot.html)
1609
2467
  bot:
@@ -1617,11 +2475,24 @@
1617
2475
  user_agent: SensikaBot/x.33 (+http://sensika.com)
1618
2476
  bot:
1619
2477
  name: Sensika Bot
1620
- category:
1621
- url:
2478
+ category: ""
2479
+ url: ""
1622
2480
  producer:
1623
2481
  name: Sensika
1624
2482
  url: http://sensika.com
2483
+ -
2484
+ user_agent: sentry/8.6.0 (https://getsentry.com)
2485
+ bot:
2486
+ name: Sentry Bot
2487
+ producer:
2488
+ name: Sentry
2489
+ url: https://sentry.io
2490
+ -
2491
+ user_agent: Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)
2492
+ bot:
2493
+ name: Seoscanners.net
2494
+ category: Crawler
2495
+ url: ""
1625
2496
  -
1626
2497
  user_agent: Server Density Service Monitoring v2
1627
2498
  bot:
@@ -1662,6 +2533,33 @@
1662
2533
  producer:
1663
2534
  name: Seznam.cz, a.s.
1664
2535
  url: http://www.seznam.cz/
2536
+ -
2537
+ user_agent: Mozilla/5.0 SeznamEmailProxy/2.0.174
2538
+ bot:
2539
+ name: Seznam Email Proxy
2540
+ category: Crawler
2541
+ url: ""
2542
+ producer:
2543
+ name: Seznam.cz, a.s.
2544
+ url: http://www.seznam.cz/
2545
+ -
2546
+ user_agent: Seznam-Zbozi-robot/3.0
2547
+ bot:
2548
+ name: Seznam Zbozi.cz
2549
+ category: Crawler
2550
+ url: ""
2551
+ producer:
2552
+ name: Seznam.cz, a.s.
2553
+ url: https://www.zbozi.cz/
2554
+ -
2555
+ user_agent: Mozilla/5.0 (ShopAlike; LadenZeile) FeedBot
2556
+ bot:
2557
+ name: ShopAlike
2558
+ category: Crawler
2559
+ url: ""
2560
+ producer:
2561
+ name: Visual Meta
2562
+ url: https://www.shopalike.cz/
1665
2563
  -
1666
2564
  user_agent: ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)
1667
2565
  bot:
@@ -1671,6 +2569,15 @@
1671
2569
  producer:
1672
2570
  name: ShopWiki Corp.
1673
2571
  url: http://www.shopwiki.com
2572
+ -
2573
+ user_agent: shopify-partner-homepage-scraper
2574
+ bot:
2575
+ name: Shopify Partner
2576
+ category: Crawler
2577
+ url: https://www.shopify.com/partners
2578
+ producer:
2579
+ name: Shopify
2580
+ url: https://www.shopify.com/
1674
2581
  -
1675
2582
  user_agent: SilverReader/1.0; http://silverreader.com
1676
2583
  bot:
@@ -1678,17 +2585,8 @@
1678
2585
  url: http://silverreader.com
1679
2586
  category: Feed Fetcher
1680
2587
  producer:
1681
- name:
1682
- url:
1683
- -
1684
- user_agent: freshrss/0.8-dev (Linux; http://freshrss.org) SimplePie/1.4-dev-FreshRSS
1685
- bot:
1686
- name: SimplePie
1687
- url: http://www.simplepie.org
1688
- category: Feed Parser
1689
- producer:
1690
- name:
1691
- url:
2588
+ name: ""
2589
+ url: ""
1692
2590
  -
1693
2591
  user_agent: SimplePie/1.2.1-dev (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20130514092120
1694
2592
  bot:
@@ -1696,8 +2594,8 @@
1696
2594
  url: http://www.simplepie.org
1697
2595
  category: Feed Parser
1698
2596
  producer:
1699
- name:
1700
- url:
2597
+ name: ""
2598
+ url: ""
1701
2599
  -
1702
2600
  user_agent: SimplePie/1.3.1 (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20121030175911
1703
2601
  bot:
@@ -1705,8 +2603,8 @@
1705
2603
  url: http://www.simplepie.org
1706
2604
  category: Feed Parser
1707
2605
  producer:
1708
- name:
1709
- url:
2606
+ name: ""
2607
+ url: ""
1710
2608
  -
1711
2609
  user_agent: Site24x7
1712
2610
  bot:
@@ -1716,12 +2614,45 @@
1716
2614
  producer:
1717
2615
  name: Site24x7
1718
2616
  url: https://www.site24x7.com
2617
+ -
2618
+ user_agent: SiteSucker for macOS/2.10.5
2619
+ bot:
2620
+ name: SiteSucker
2621
+ category: Crawler
2622
+ url: http://ricks-apps.com/osx/sitesucker/
2623
+ -
2624
+ user_agent: Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com
2625
+ bot:
2626
+ name: Siteimprove
2627
+ category: Search bot
2628
+ url: https://siteimprove.com/
2629
+ producer:
2630
+ name: Siteimprove GmbH
2631
+ url: https://siteimprove.com/
2632
+ -
2633
+ user_agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
2634
+ bot:
2635
+ name: Siteimprove
2636
+ category: Search bot
2637
+ url: https://siteimprove.com/
2638
+ producer:
2639
+ name: Siteimprove GmbH
2640
+ url: https://siteimprove.com/
2641
+ -
2642
+ user_agent: sixy.ch/1.0
2643
+ bot:
2644
+ name: Sixy.ch
2645
+ category: Site Monitor
2646
+ url: http://sixy.ch
2647
+ producer:
2648
+ name: Manuel Kasper
2649
+ url: https://neon1.net/
1719
2650
  -
1720
2651
  user_agent: Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5
1721
2652
  bot:
1722
2653
  name: Skype URI Preview
1723
2654
  category: Service Agent
1724
- url:
2655
+ url: ""
1725
2656
  producer:
1726
2657
  name: Skype Communications S.à.r.l.
1727
2658
  url: https://www.skype.com
@@ -1734,6 +2665,15 @@
1734
2665
  producer:
1735
2666
  name: Slack Technologies
1736
2667
  url: http://slack.com
2668
+ -
2669
+ user_agent: 'AppEngine-Google; (+http://code.google.com/appengine; appid: s~snapchat-proxy)'
2670
+ bot:
2671
+ name: Snapchat Proxy
2672
+ category: Crawler
2673
+ url: https://www.snapchat.com
2674
+ producer:
2675
+ name: Snapchat Inc.
2676
+ url: https://www.snapchat.com
1737
2677
  -
1738
2678
  user_agent: New-Sogou-Spider/1.0 (compatible; MSIE 5.5; Windows 98)
1739
2679
  bot:
@@ -1779,6 +2719,12 @@
1779
2719
  producer:
1780
2720
  name: Tencent Holdings
1781
2721
  url: http://www.soso.com
2722
+ -
2723
+ user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Sparkler/0.2.0-SNAPSHOT
2724
+ bot:
2725
+ name: Sparkler
2726
+ category: Crawler
2727
+ url: https://github.com/USCDataScience/sparkler
1782
2728
  -
1783
2729
  user_agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.19; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/2010040121 Firefox/3.0.19
1784
2730
  bot:
@@ -1788,30 +2734,55 @@
1788
2734
  producer:
1789
2735
  name: Tailrank Inc
1790
2736
  url: http://spinn3r.com
2737
+ -
2738
+ user_agent: Spotify/1.0
2739
+ bot:
2740
+ name: Spotify
2741
+ producer:
2742
+ name: Spotify
2743
+ url: https://www.spotify.com
1791
2744
  -
1792
2745
  user_agent: Mozilla/5.0 (compatible; SputnikBot/2.2)
1793
2746
  bot:
1794
2747
  name: Sputnik Bot
1795
- category:
1796
- url:
2748
+ category: ""
2749
+ url: ""
1797
2750
  producer:
1798
- name:
1799
- url:
2751
+ name: ""
2752
+ url: ""
1800
2753
  -
1801
2754
  user_agent: Mozilla/5.0 (compatible; SputnikImageBot/2.2)
1802
2755
  bot:
1803
2756
  name: Sputnik Bot
1804
- category:
1805
- url:
2757
+ category: ""
2758
+ url: ""
1806
2759
  producer:
1807
- name:
1808
- url:
2760
+ name: ""
2761
+ url: ""
2762
+ -
2763
+ user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/536.30.1 (KHTML, like Gecko) (compatible; Startpagina-Linkchecker/1.0; +https://www.startpagina.nl/linkchecker)
2764
+ bot:
2765
+ name: Startpagina Linkchecker
2766
+ category: Search bot
2767
+ url: https://www.startpagina.nl/linkchecker
2768
+ producer:
2769
+ name: Startpagina B.V.
2770
+ url: https://www.startpagina.nl/
2771
+ -
2772
+ user_agent: Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/98 Safari/537.4 (StatusCake)
2773
+ bot:
2774
+ name: StatusCake
2775
+ category: Site Monitor
2776
+ url: https://www.statuscake.com
2777
+ producer:
2778
+ name: StatusCake
2779
+ url: https://www.statuscake.com
1809
2780
  -
1810
2781
  user_agent: 'Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch!'
1811
2782
  bot:
1812
2783
  name: Superfeedr Bot
1813
2784
  category: Feed Fetcher
1814
- url:
2785
+ url: ""
1815
2786
  producer:
1816
2787
  name: Superfeedr
1817
2788
  url: https://superfeedr.com/
@@ -1824,11 +2795,31 @@
1824
2795
  producer:
1825
2796
  name: Domain Tools
1826
2797
  url: http://www.domaintools.com
2798
+ -
2799
+ user_agent: TLSProbe/1.0 (+https://scan.trustnet.venafi.com/)
2800
+ bot:
2801
+ name: TLSProbe
2802
+ url: https://scan.trustnet.venafi.com/
2803
+ category: Security search bot
2804
+ producer:
2805
+ name: Venafi TrustNet
2806
+ url: https://www.venafi.com
2807
+ -
2808
+ user_agent: Tarmot Gezgin/1.0 (compatible; TarmotGezgin/1.1; +http://www.tarmot.com/gezgin)
2809
+ bot:
2810
+ name: Tarmot Gezgin
2811
+ url: http://www.tarmot.com/gezgin/
2812
+ category: Search bot
1827
2813
  -
1828
2814
  user_agent: TelegramBot (like TwitterBot)
1829
2815
  bot:
1830
- name: TelgramBot
2816
+ name: TelegramBot
1831
2817
  url: https://telegram.org/blog/bot-revolution
2818
+ -
2819
+ user_agent: The Knowledge AI
2820
+ bot:
2821
+ name: The Knowledge AI
2822
+ category: Crawler
1832
2823
  -
1833
2824
  user_agent: TinEye-bot/0.02 (see http://www.tineye.com/crawler.html)
1834
2825
  bot:
@@ -1845,8 +2836,8 @@
1845
2836
  url: http://tt-rss.org
1846
2837
  category: Feed Fetcher
1847
2838
  producer:
1848
- name:
1849
- url:
2839
+ name: ""
2840
+ url: ""
1850
2841
  -
1851
2842
  user_agent: Tiny Tiny RSS/1.11.4c63934 (http://tt-rss.org/)
1852
2843
  bot:
@@ -1854,8 +2845,26 @@
1854
2845
  url: http://tt-rss.org
1855
2846
  category: Feed Fetcher
1856
2847
  producer:
1857
- name:
1858
- url:
2848
+ name: ""
2849
+ url: ""
2850
+ -
2851
+ user_agent: Mozilla/5.0 (compatible; tracemyfile/1.0)
2852
+ bot:
2853
+ name: TraceMyFile
2854
+ category: Search bot
2855
+ url: https://www.tracemyfile.com/
2856
+ producer:
2857
+ name: Idee Inc.
2858
+ url: http://ideeinc.com/
2859
+ -
2860
+ user_agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11
2861
+ bot:
2862
+ name: Trendiction Bot
2863
+ category: Crawler
2864
+ url: http://www.trendiction.de/bot
2865
+ producer:
2866
+ name: Talkwalker Inc.
2867
+ url: http://www.talkwalker.com
1859
2868
  -
1860
2869
  user_agent: TurnitinBot/3.0 (http://www.turnitin.com/robot/crawlerinfo.html)
1861
2870
  bot:
@@ -1891,7 +2900,15 @@
1891
2900
  url: http://tweetmeme.com/
1892
2901
  producer:
1893
2902
  name: Mediasift
1894
- url:
2903
+ url: ""
2904
+ -
2905
+ user_agent: Mozilla/5.0 (compatible; Twingly Recon; twingly.com)
2906
+ bot:
2907
+ name: Twingly Recon
2908
+ category: Crawler
2909
+ producer:
2910
+ name: Twingly
2911
+ url: https://www.twingly.com
1895
2912
  -
1896
2913
  user_agent: Twitterbot/1.0
1897
2914
  bot:
@@ -1910,15 +2927,60 @@
1910
2927
  producer:
1911
2928
  name: Profound Networks
1912
2929
  url: http://www.profound.net
2930
+ -
2931
+ user_agent: 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 (via secureurl.fwdcdn.com - mail.ukr.net proxy)'
2932
+ bot:
2933
+ name: UkrNet Mail Proxy
2934
+ category: Crawler
2935
+ url: ""
2936
+ producer:
2937
+ name: UkrNet Ltd
2938
+ url: https://www.ukr.net/
2939
+ -
2940
+ user_agent: UniversalFeedParser/5.2.1 +https://code.google.com/p/feedparser/
2941
+ bot:
2942
+ name: UniversalFeedParser
2943
+ category: Feed Fetcher
2944
+ url: https://github.com/kurtmckee/feedparser
2945
+ producer:
2946
+ name: Kurt McKee
2947
+ url: https://github.com/kurtmckee
1913
2948
  -
1914
2949
  user_agent: Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)
1915
2950
  bot:
1916
2951
  name: Uptime Robot
1917
2952
  category: Site Monitor
1918
- url:
2953
+ url: ""
1919
2954
  producer:
1920
2955
  name: Uptime Robot
1921
2956
  url: http://uptimerobot.com
2957
+ -
2958
+ user_agent: Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)
2959
+ bot:
2960
+ name: Uptimebot
2961
+ category: Site Monitor
2962
+ url: https://uptime.com/uptimebot
2963
+ producer:
2964
+ name: Uptime
2965
+ url: https://uptime.com
2966
+ -
2967
+ user_agent: Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)
2968
+ bot:
2969
+ name: VK Share Button
2970
+ category: Crawler
2971
+ url: http://vk.com/dev/Share
2972
+ producer:
2973
+ name: VK
2974
+ url: http://vk.com/
2975
+ -
2976
+ user_agent: 'Mozilla/4.0 (compatible; Vagabondo/4.0; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)'
2977
+ bot:
2978
+ name: Vagabondo
2979
+ category: Crawler
2980
+ url: ""
2981
+ producer:
2982
+ name: WiseGuys
2983
+ url: http://www.wise-guys.nl/
1922
2984
  -
1923
2985
  user_agent: Mozilla/5.0 (compatible; VSMCrawler; http://www.visualsitemapper.com/crawler/)
1924
2986
  bot:
@@ -1964,6 +3026,15 @@
1964
3026
  producer:
1965
3027
  name: W3C
1966
3028
  url: http://www.w3.org
3029
+ -
3030
+ user_agent: Validator.nu/LV http://validator.w3.org/services
3031
+ bot:
3032
+ name: W3C Markup Validation Service
3033
+ category: Validator
3034
+ url: http://validator.w3.org/services
3035
+ producer:
3036
+ name: W3C
3037
+ url: http://www.w3.org
1967
3038
  -
1968
3039
  user_agent: W3C_Validator/1.3 http://validator.w3.org/services
1969
3040
  bot:
@@ -2000,6 +3071,14 @@
2000
3071
  producer:
2001
3072
  name: W3C
2002
3073
  url: http://www.w3.org
3074
+ -
3075
+ user_agent: Mozilla/5.0 (compatible; Wappalyzer; +https://github.com/AliasIO/Wappalyzer)
3076
+ bot:
3077
+ name: Wappalyzer
3078
+ url: https://github.com/AliasIO/Wappalyzer
3079
+ producer:
3080
+ name: AliasIO
3081
+ url: https://github.com/AliasIO
2003
3082
  -
2004
3083
  user_agent: WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)
2005
3084
  bot:
@@ -2009,6 +3088,12 @@
2009
3088
  producer:
2010
3089
  name: WeSEE Ltd
2011
3090
  url: http://www.wesee.com
3091
+ -
3092
+ user_agent: Mozilla/5.0 (Linux; Android 6.0.1; Moto G (4) Build/MPJ24.139-64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Mobile Safari/537.36 PTST/180829.190838
3093
+ bot:
3094
+ name: WebPageTest
3095
+ category: Site Monitor
3096
+ url: https://www.webpagetest.org
2012
3097
  -
2013
3098
  user_agent: websitepulse checker/1.1 (compatible; MSIE 5.5; Netscape 4.75; Linux)
2014
3099
  bot:
@@ -2027,6 +3112,51 @@
2027
3112
  producer:
2028
3113
  name: Steve Webb
2029
3114
  url: http://badcheese.com
3115
+ -
3116
+ user_agent: weborama-fetcher (+http://www.weborama.com)
3117
+ bot:
3118
+ name: Weborama
3119
+ category: Search bot
3120
+ url: https://weborama.com/
3121
+ producer:
3122
+ name: Weborama SA
3123
+ url: https://weborama.com/
3124
+ -
3125
+ user_agent: WikiDo/1.1 (http://wikido.com; crawler@wikido.com)
3126
+ bot:
3127
+ name: WikiDo
3128
+ category: Search bot
3129
+ url: https://www.wikido.com/
3130
+ producer:
3131
+ name: Fotolitografie Fiorentine di Becchi Antonio s.n.c.
3132
+ url: https://www.wikido.com/
3133
+ -
3134
+ user_agent: Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)
3135
+ bot:
3136
+ name: WooRank
3137
+ category: Search bot
3138
+ url: https://www.woorank.com/
3139
+ producer:
3140
+ name: WooRank sprl
3141
+ url: https://www.woorank.com/
3142
+ -
3143
+ user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)
3144
+ bot:
3145
+ name: WooRank
3146
+ category: Search bot
3147
+ url: https://www.woorank.com/
3148
+ producer:
3149
+ name: WooRank sprl
3150
+ url: https://www.woorank.com/
3151
+ -
3152
+ user_agent: WordPress/4.7.2; https://example.com
3153
+ bot:
3154
+ name: WordPress
3155
+ category: Service Agent
3156
+ url: https://wordpress.org/
3157
+ producer:
3158
+ name: Wordpress.org
3159
+ url: https://wordpress.org/
2030
3160
  -
2031
3161
  user_agent: Wotbox/2.01 (+http://www.wotbox.com/bot/)
2032
3162
  bot:
@@ -2036,6 +3166,15 @@
2036
3166
  producer:
2037
3167
  name: Wotbox
2038
3168
  url: http://www.wotbox.com
3169
+ -
3170
+ user_agent: XenForo/2.x (https://www.example.com)
3171
+ bot:
3172
+ name: XenForo
3173
+ category: Service Agent
3174
+ url: https://xenforo.com/
3175
+ producer:
3176
+ name: XenForo Ltd.
3177
+ url: https://xenforo.com/
2039
3178
  -
2040
3179
  user_agent: yacybot (freeworld/global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_25; Europe/en) http://yacy.net/bot.html
2041
3180
  bot:
@@ -2063,6 +3202,15 @@
2063
3202
  producer:
2064
3203
  name: Yahoo! Inc.
2065
3204
  url: http://www.yahoo.com
3205
+ -
3206
+ user_agent: Y!J-BRW/1.0 (https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716)
3207
+ bot:
3208
+ name: Yahoo! Japan BRW
3209
+ category: Crawler
3210
+ url: https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/ウェブページにアクセスするシステムのユーザーエージェントについて
3211
+ producer:
3212
+ name: Yahoo! Japan Corp.
3213
+ url: https://www.yahoo.co.jp/
2066
3214
  -
2067
3215
  user_agent: Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)
2068
3216
  bot:
@@ -2207,6 +3355,15 @@
2207
3355
  producer:
2208
3356
  name: Yandex LLC
2209
3357
  url: http://company.yandex.com
3358
+ -
3359
+ user_agent: Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)
3360
+ bot:
3361
+ name: Yandex Bot
3362
+ category: Search bot
3363
+ url: http://www.yandex.com/bots
3364
+ producer:
3365
+ name: Yandex LLC
3366
+ url: http://company.yandex.com
2210
3367
  -
2211
3368
  user_agent: Mozilla/5.0 (compatible; YandexNews/3.0; +http://yandex.com/bots)
2212
3369
  bot:
@@ -2225,6 +3382,87 @@
2225
3382
  producer:
2226
3383
  name: Yandex LLC
2227
3384
  url: http://company.yandex.com
3385
+ -
3386
+ user_agent: Mozilla/5.0 (compatible; YandexOntoDB/1.0; +http://yandex.com/bots)
3387
+ bot:
3388
+ name: Yandex Bot
3389
+ category: Search bot
3390
+ url: http://www.yandex.com/bots
3391
+ producer:
3392
+ name: Yandex LLC
3393
+ url: http://company.yandex.com
3394
+ -
3395
+ user_agent: Mozilla/5.0 (compatible; YandexOntoDBAPI/1.0; +http://yandex.com/bots)
3396
+ bot:
3397
+ name: Yandex Bot
3398
+ category: Search bot
3399
+ url: http://www.yandex.com/bots
3400
+ producer:
3401
+ name: Yandex LLC
3402
+ url: http://company.yandex.com
3403
+ -
3404
+ user_agent: Mozilla/5.0 (compatible; YandexPartner/3.0; +http://yandex.com/bots)
3405
+ bot:
3406
+ name: Yandex Bot
3407
+ category: Search bot
3408
+ url: http://www.yandex.com/bots
3409
+ producer:
3410
+ name: Yandex LLC
3411
+ url: http://company.yandex.com
3412
+ -
3413
+ user_agent: Mozilla/5.0 (compatible; YandexRCA/1.0; +http://yandex.com/bots)
3414
+ bot:
3415
+ name: Yandex Bot
3416
+ category: Search bot
3417
+ url: http://www.yandex.com/bots
3418
+ producer:
3419
+ name: Yandex LLC
3420
+ url: http://company.yandex.com
3421
+ -
3422
+ user_agent: Mozilla/5.0 (compatible; YandexSearchShop/1.0; +http://yandex.com/bots)
3423
+ bot:
3424
+ name: Yandex Bot
3425
+ category: Search bot
3426
+ url: http://www.yandex.com/bots
3427
+ producer:
3428
+ name: Yandex LLC
3429
+ url: http://company.yandex.com
3430
+ -
3431
+ user_agent: Mozilla/5.0 (compatible; YandexTracker/1.0; +http://yandex.com/bots)
3432
+ bot:
3433
+ name: Yandex Bot
3434
+ category: Search bot
3435
+ url: http://www.yandex.com/bots
3436
+ producer:
3437
+ name: Yandex LLC
3438
+ url: http://company.yandex.com
3439
+ -
3440
+ user_agent: Mozilla/5.0 (compatible; YandexTurbo/1.0; +http://yandex.com/bots)
3441
+ bot:
3442
+ name: Yandex Bot
3443
+ category: Search bot
3444
+ url: http://www.yandex.com/bots
3445
+ producer:
3446
+ name: Yandex LLC
3447
+ url: http://company.yandex.com
3448
+ -
3449
+ user_agent: Mozilla/5.0 (compatible; YandexVerticals/1.0; http://yandex.com/bots)
3450
+ bot:
3451
+ name: Yandex Bot
3452
+ category: Search bot
3453
+ url: http://www.yandex.com/bots
3454
+ producer:
3455
+ name: Yandex LLC
3456
+ url: http://company.yandex.com
3457
+ -
3458
+ user_agent: Mozilla/5.0 (compatible; NaverJapan/1.0; +http://corp.naver.jp/)
3459
+ bot:
3460
+ name: Yeti/Naverbot
3461
+ category: Search bot
3462
+ url: http://help.naver.com/robots/
3463
+ producer:
3464
+ name: Naver
3465
+ url: http://www.naver.com
2228
3466
  -
2229
3467
  user_agent: Yeti/1.1 (Naver Corp.; http://help.naver.com/robots/)
2230
3468
  bot:
@@ -2281,6 +3519,15 @@
2281
3519
  bot:
2282
3520
  name: Zao
2283
3521
  category: Crawler
3522
+ -
3523
+ user_agent: zelist.ro feed parser (+http://www.zelist.ro)
3524
+ bot:
3525
+ name: Ze List
3526
+ category: Feed Fetcher
3527
+ url: https://www.zelist.ro/
3528
+ producer:
3529
+ name: Treeworks SRL
3530
+ url: https://www.tree.ro/
2284
3531
  -
2285
3532
  user_agent: Zookabot/2.5;++http://zookabot.com
2286
3533
  bot:
@@ -2416,3 +3663,151 @@
2416
3663
  producer:
2417
3664
  name: reddit inc.
2418
3665
  url: http://www.reddit.com
3666
+ -
3667
+ user_agent: Mozilla/5.0 eCairn-Grabber/1.0 (+http://ecairn.com/grabber)
3668
+ bot:
3669
+ name: eCairn-Grabber
3670
+ category: Crawler
3671
+ producer:
3672
+ name: eCairn
3673
+ url: https://ecairn.com
3674
+ -
3675
+ user_agent: eZ Publish Link Validator
3676
+ bot:
3677
+ name: eZ Publish Link Validator
3678
+ category: Crawler
3679
+ url: https://ez.no/
3680
+ producer:
3681
+ name: eZ Systems AS
3682
+ url: https://ez.no/
3683
+ -
3684
+ user_agent: Mozilla/5.0 (compatible; evc-batch/2.0)
3685
+ bot:
3686
+ name: evc-batch
3687
+ category: Crawler
3688
+ url: ""
3689
+ producer:
3690
+ name: eVenture Capital Partners II, LLC
3691
+ url: http://www.eventures.vc/
3692
+ -
3693
+ user_agent: Mozilla/5.0 (compatible; inoreader.com; 2 subscribers)
3694
+ bot:
3695
+ name: inoreader
3696
+ category: Feed Reader
3697
+ url: https://www.inoreader.com
3698
+ -
3699
+ user_agent: masscan/1.0 (https://github.com/robertdavidgraham/masscan)
3700
+ bot:
3701
+ name: masscan
3702
+ url: https://github.com/robertdavidgraham/masscan
3703
+ category: Crawler
3704
+ producer:
3705
+ name: Robert Graham
3706
+ url: https://github.com/robertdavidgraham
3707
+ -
3708
+ user_agent: 'Mozilla/5.0/Firefox/42.0 - nbertaupete95(at)gmail.com'
3709
+ bot:
3710
+ name: nbertaupete95
3711
+ category: Crawler
3712
+ -
3713
+ user_agent: Mozilla/5.0 (compatible; oBot/2.3.1; http://www.xforce-security.com/crawler/)
3714
+ bot:
3715
+ name: oBot
3716
+ category: Search bot
3717
+ url: http://www.xforce-security.com/crawler/
3718
+ producer:
3719
+ name: 'IBM Germany Research & Development GmbH'
3720
+ url: https://exchange.xforce.ibmcloud.com/
3721
+ -
3722
+ user_agent: 'sqlmap/1.1.8.2#dev (http://sqlmap.org)'
3723
+ bot:
3724
+ name: sqlmap
3725
+ category: Security Checker
3726
+ url: http://sqlmap.org/
3727
+ producer:
3728
+ name: sqlmap
3729
+ url: http://sqlmap.org/
3730
+ -
3731
+ user_agent: Mozilla/5.0 (compatible; theoldreader.com; 1 subscribers; feed-id=aaa)
3732
+ bot:
3733
+ name: theoldreader
3734
+ category: Feed Reader
3735
+ url: https://theoldreader.com
3736
+
3737
+ -
3738
+ user_agent: Seobility
3739
+ bot:
3740
+ name: Seobility
3741
+ category: Crawler
3742
+ url: 'https://www.seobility.net/en/faq/?category=crawling#!aboutourbot'
3743
+
3744
+ -
3745
+ user_agent: Vercelbot (+https://vercel.com)
3746
+ bot:
3747
+ name: Vercel Bot
3748
+ category: Service bot
3749
+ url: https://vercel.com
3750
+
3751
+ -
3752
+ user_agent: Grammarly/1.0 (http://www.grammarly.com)
3753
+ bot:
3754
+ name: Grammarly
3755
+ category: Service bot
3756
+ url: http://www.grammarly.com
3757
+
3758
+ -
3759
+ user_agent: Robozilla/1.0
3760
+ bot:
3761
+ name: Robozilla
3762
+ category: Crawler
3763
+
3764
+ -
3765
+ user_agent: Mozilla/5.0 (compatible; Domains Project/1.1.0; +https://domainsproject.org)
3766
+ bot:
3767
+ name: Domains Project
3768
+ category: Crawler
3769
+ url: https://domainsproject.org
3770
+
3771
+ -
3772
+ user_agent: Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://aspiegel.com/petalbot)
3773
+ bot:
3774
+ name: Petal Bot
3775
+ category: Crawler
3776
+ url: https://aspiegel.com/petalbot
3777
+
3778
+ -
3779
+ user_agent: SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)
3780
+ bot:
3781
+ name: Serendeputy Bot
3782
+ category: Crawler
3783
+ url: http://serendeputy.com/about/serendeputy-bot
3784
+
3785
+ -
3786
+ user_agent: ias-va/3.1 (+https://www.admantx.com/service-fetcher.html)
3787
+ bot:
3788
+ name: ADmantX Service Fetcher
3789
+ category: Service bot
3790
+ url: https://www.admantx.com/service-fetcher.html
3791
+
3792
+ -
3793
+ user_agent: Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)
3794
+ bot:
3795
+ name: Semantic Scholar Bot
3796
+ category: Crawler
3797
+ url: https://www.semanticscholar.org/crawler
3798
+
3799
+ -
3800
+ user_agent: Mozilla/5.0 (compatible; VelenPublicWebCrawler/1.0; +https://velen.io)
3801
+ bot:
3802
+ name: Velen Public Web Crawler
3803
+ category: Crawler
3804
+ url: https://hunter.io/robot
3805
+
3806
+ -
3807
+ user_agent: Barkrowler/0.9 (+http://www.exensa.com/crawl)
3808
+ bot:
3809
+ name: Barkrowler
3810
+ category: Crawler
3811
+ url: http://www.exensa.com/crawl
3812
+
3813
+