voight_kampff 1.1.2 → 1.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/config/crawler-user-agents.json +1448 -338
- data/lib/tasks/voight_kampff.rake +6 -2
- data/lib/voight_kampff/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: aaad7c130bf25186c8a6f0d83b788b37dfd359f5
|
|
4
|
+
data.tar.gz: 65eaafd893c9e7b1043199bdb56b8df525509a6f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 90e012989497795cacb3357e6b3654ab5a0fa0e965e79b51868f7b23d0e9ca0788eb21e22cb8da8943622a654b74a6c1831ef01073c201d4991ca63c7137ae40
|
|
7
|
+
data.tar.gz: cf79235228114f2e1ff3125277823556591c7426058af965f4aff8e0ed85a583ac4cab8b2e7cd58a92fa4cecd9f1cc90ddf1af72d5c973e3b373a4cccd0e906f
|
data/README.md
CHANGED
|
@@ -52,7 +52,7 @@ Also, the gem no longer extends `ActionDispatch::Request` instead it extends `Ra
|
|
|
52
52
|
FAQ
|
|
53
53
|
---
|
|
54
54
|
__Q:__ __What's with the name?__
|
|
55
|
-
__A:__ It's the [machine in Blade Runner](http://en.wikipedia.org/wiki/Blade_Runner#
|
|
55
|
+
__A:__ It's the [machine in Blade Runner](http://en.wikipedia.org/wiki/Blade_Runner#Voigt-Kampff_machine) that is used to test whether someone is a human or a replicant.
|
|
56
56
|
|
|
57
57
|
__Q:__ __I've found a bot that isn't being matched__
|
|
58
58
|
__A:__ The list is being pulled from [github.com/monperrus/crawler-user-agents](https://github.com/monperrus/crawler-user-agents).
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36",
|
|
14
14
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
|
15
15
|
]
|
|
16
|
-
}
|
|
16
|
+
}
|
|
17
|
+
,
|
|
17
18
|
{
|
|
18
19
|
"pattern": "Googlebot-Mobile",
|
|
19
20
|
"instances": [
|
|
@@ -23,32 +24,37 @@
|
|
|
23
24
|
"Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
|
|
24
25
|
"SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"
|
|
25
26
|
]
|
|
26
|
-
}
|
|
27
|
+
}
|
|
28
|
+
,
|
|
27
29
|
{
|
|
28
30
|
"pattern": "Googlebot-Image",
|
|
29
31
|
"instances": [
|
|
30
32
|
"Googlebot-Image/1.0"
|
|
31
33
|
]
|
|
32
|
-
}
|
|
34
|
+
}
|
|
35
|
+
,
|
|
33
36
|
{
|
|
34
37
|
"pattern": "Googlebot-News",
|
|
35
38
|
"instances": [
|
|
36
39
|
"Googlebot-News"
|
|
37
40
|
]
|
|
38
|
-
}
|
|
41
|
+
}
|
|
42
|
+
,
|
|
39
43
|
{
|
|
40
44
|
"pattern": "Googlebot-Video",
|
|
41
45
|
"instances": [
|
|
42
46
|
"Googlebot-Video/1.0"
|
|
43
47
|
]
|
|
44
|
-
}
|
|
48
|
+
}
|
|
49
|
+
,
|
|
45
50
|
{
|
|
46
51
|
"pattern": "AdsBot-Google([^-]|$)",
|
|
47
52
|
"url": "https://support.google.com/webmasters/answer/1061943?hl=en",
|
|
48
53
|
"instances": [
|
|
49
54
|
"AdsBot-Google (+http://www.google.com/adsbot.html)"
|
|
50
55
|
]
|
|
51
|
-
}
|
|
56
|
+
}
|
|
57
|
+
,
|
|
52
58
|
{
|
|
53
59
|
"pattern": "AdsBot-Google-Mobile",
|
|
54
60
|
"addition_date": "2017/08/21",
|
|
@@ -58,7 +64,17 @@
|
|
|
58
64
|
"Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
|
|
59
65
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)"
|
|
60
66
|
]
|
|
61
|
-
}
|
|
67
|
+
}
|
|
68
|
+
,
|
|
69
|
+
{
|
|
70
|
+
"pattern": "Feedfetcher-Google",
|
|
71
|
+
"addition_date": "2018/06/27",
|
|
72
|
+
"url": "https://support.google.com/webmasters/answer/178852",
|
|
73
|
+
"instances": [
|
|
74
|
+
"Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)"
|
|
75
|
+
]
|
|
76
|
+
}
|
|
77
|
+
,
|
|
62
78
|
{
|
|
63
79
|
"pattern": "Mediapartners-Google",
|
|
64
80
|
"url": "https://support.google.com/webmasters/answer/1061943?hl=en",
|
|
@@ -68,13 +84,15 @@
|
|
|
68
84
|
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)",
|
|
69
85
|
"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"
|
|
70
86
|
]
|
|
71
|
-
}
|
|
87
|
+
}
|
|
88
|
+
,
|
|
72
89
|
{
|
|
73
90
|
"pattern": "Mediapartners \\(Googlebot\\)",
|
|
74
91
|
"addition_date": "2017/08/08",
|
|
75
92
|
"url": "https://support.google.com/webmasters/answer/1061943?hl=en",
|
|
76
93
|
"instances": []
|
|
77
|
-
}
|
|
94
|
+
}
|
|
95
|
+
,
|
|
78
96
|
{
|
|
79
97
|
"pattern": "APIs-Google",
|
|
80
98
|
"addition_date": "2017/08/08",
|
|
@@ -82,7 +100,8 @@
|
|
|
82
100
|
"instances": [
|
|
83
101
|
"APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)"
|
|
84
102
|
]
|
|
85
|
-
}
|
|
103
|
+
}
|
|
104
|
+
,
|
|
86
105
|
{
|
|
87
106
|
"pattern": "bingbot",
|
|
88
107
|
"url": "http://www.bing.com/bingbot.htm",
|
|
@@ -100,58 +119,65 @@
|
|
|
100
119
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
|
|
101
120
|
"Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"
|
|
102
121
|
]
|
|
103
|
-
}
|
|
122
|
+
}
|
|
123
|
+
,
|
|
104
124
|
{
|
|
105
|
-
"pattern": "
|
|
125
|
+
"pattern": "Slurp",
|
|
106
126
|
"url": "http://help.yahoo.com/help/us/ysearch/slurp",
|
|
107
127
|
"instances": [
|
|
108
128
|
"Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)",
|
|
109
|
-
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
|
|
110
|
-
|
|
111
|
-
},
|
|
112
|
-
{
|
|
113
|
-
"pattern": "java",
|
|
114
|
-
"instances": [
|
|
115
|
-
"Apache-HttpClient/4.2.3 (java 1.5)",
|
|
116
|
-
"Apache-HttpClient/4.2.5 (java 1.5)",
|
|
117
|
-
"Apache-HttpClient/4.3.1 (java 1.5)",
|
|
118
|
-
"Apache-HttpClient/4.3.3 (java 1.5)",
|
|
119
|
-
"Apache-HttpClient/4.3.5 (java 1.5)",
|
|
120
|
-
"Apache-HttpClient/UNAVAILABLE (java 1.4)",
|
|
121
|
-
"yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
|
|
122
|
-
"yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
|
|
123
|
-
"yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
|
|
124
|
-
"yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html"
|
|
129
|
+
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
|
|
130
|
+
"Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
|
|
125
131
|
]
|
|
126
|
-
}
|
|
132
|
+
}
|
|
133
|
+
,
|
|
127
134
|
{
|
|
128
135
|
"pattern": "[wW]get",
|
|
129
136
|
"instances": [
|
|
130
137
|
"WGETbot/1.0 (+http://wget.alanreed.org)",
|
|
131
138
|
"Wget/1.14 (linux-gnu)"
|
|
132
139
|
]
|
|
133
|
-
}
|
|
140
|
+
}
|
|
141
|
+
,
|
|
134
142
|
{
|
|
135
143
|
"pattern": "curl",
|
|
136
144
|
"instances": [
|
|
137
145
|
"eCairn-Grabber/1.0 (+http://ecairn.com/grabber) curl/7.15"
|
|
138
146
|
]
|
|
139
|
-
}
|
|
147
|
+
}
|
|
148
|
+
,
|
|
140
149
|
{
|
|
141
|
-
"pattern": "
|
|
150
|
+
"pattern": "LinkedInBot",
|
|
142
151
|
"instances": [
|
|
143
152
|
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)",
|
|
144
153
|
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)"
|
|
145
154
|
]
|
|
146
|
-
}
|
|
155
|
+
}
|
|
156
|
+
,
|
|
147
157
|
{
|
|
148
158
|
"pattern": "Python-urllib",
|
|
149
159
|
"instances": [
|
|
150
|
-
"Python-urllib/2.
|
|
151
|
-
"Python-urllib/2.
|
|
152
|
-
"Python-urllib/2.
|
|
160
|
+
"Python-urllib/2.5",
|
|
161
|
+
"Python-urllib/2.5",
|
|
162
|
+
"Python-urllib/2.6",
|
|
163
|
+
"Python-urllib/2.7",
|
|
164
|
+
"Python-urllib/3.1",
|
|
165
|
+
"Python-urllib/3.2",
|
|
166
|
+
"Python-urllib/3.3",
|
|
167
|
+
"Python-urllib/3.4",
|
|
168
|
+
"Python-urllib/3.5",
|
|
169
|
+
"Python-urllib/3.6"
|
|
170
|
+
]
|
|
171
|
+
}
|
|
172
|
+
,
|
|
173
|
+
{
|
|
174
|
+
"pattern": "python-requests",
|
|
175
|
+
"addition_date": "2018/05/27",
|
|
176
|
+
"instances": [
|
|
177
|
+
"python-requests/2.18.4"
|
|
153
178
|
]
|
|
154
|
-
}
|
|
179
|
+
}
|
|
180
|
+
,
|
|
155
181
|
{
|
|
156
182
|
"pattern": "libwww",
|
|
157
183
|
"instances": [
|
|
@@ -168,20 +194,23 @@
|
|
|
168
194
|
"W3C-checklink/4.5 [4.160] libwww-perl/5.823",
|
|
169
195
|
"amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831"
|
|
170
196
|
]
|
|
171
|
-
}
|
|
197
|
+
}
|
|
198
|
+
,
|
|
172
199
|
{
|
|
173
200
|
"pattern": "httpunit",
|
|
174
201
|
"instances": [
|
|
175
202
|
"httpunit/1.x"
|
|
176
203
|
]
|
|
177
|
-
}
|
|
204
|
+
}
|
|
205
|
+
,
|
|
178
206
|
{
|
|
179
207
|
"pattern": "nutch",
|
|
180
208
|
"instances": [
|
|
181
209
|
"NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
|
|
182
210
|
"istellabot-nutch/Nutch-1.10"
|
|
183
211
|
]
|
|
184
|
-
}
|
|
212
|
+
}
|
|
213
|
+
,
|
|
185
214
|
{
|
|
186
215
|
"pattern": "Go-http-client",
|
|
187
216
|
"addition_date": "2016/03/26",
|
|
@@ -189,7 +218,8 @@
|
|
|
189
218
|
"instances": [
|
|
190
219
|
"Go-http-client/1.1"
|
|
191
220
|
]
|
|
192
|
-
}
|
|
221
|
+
}
|
|
222
|
+
,
|
|
193
223
|
{
|
|
194
224
|
"pattern": "phpcrawl",
|
|
195
225
|
"addition_date": "2012-09/17",
|
|
@@ -197,7 +227,8 @@
|
|
|
197
227
|
"instances": [
|
|
198
228
|
"phpcrawl"
|
|
199
229
|
]
|
|
200
|
-
}
|
|
230
|
+
}
|
|
231
|
+
,
|
|
201
232
|
{
|
|
202
233
|
"pattern": "msnbot",
|
|
203
234
|
"url": "http://search.msn.com/msnbot.htm",
|
|
@@ -217,11 +248,13 @@
|
|
|
217
248
|
"msnbot/2.0b (+http://search.msn.com/msnbot.htm).",
|
|
218
249
|
"msnbot/2.0b (+http://search.msn.com/msnbot.htm)._"
|
|
219
250
|
]
|
|
220
|
-
}
|
|
251
|
+
}
|
|
252
|
+
,
|
|
221
253
|
{
|
|
222
254
|
"pattern": "jyxobot",
|
|
223
255
|
"instances": []
|
|
224
|
-
}
|
|
256
|
+
}
|
|
257
|
+
,
|
|
225
258
|
{
|
|
226
259
|
"pattern": "FAST-WebCrawler",
|
|
227
260
|
"instances": [
|
|
@@ -230,20 +263,23 @@
|
|
|
230
263
|
"FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",
|
|
231
264
|
"FAST-WebCrawler/3.8"
|
|
232
265
|
]
|
|
233
|
-
}
|
|
266
|
+
}
|
|
267
|
+
,
|
|
234
268
|
{
|
|
235
269
|
"pattern": "FAST Enterprise Crawler",
|
|
236
270
|
"instances": [
|
|
237
271
|
"FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/",
|
|
238
272
|
"FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)"
|
|
239
273
|
]
|
|
240
|
-
}
|
|
274
|
+
}
|
|
275
|
+
,
|
|
241
276
|
{
|
|
242
277
|
"pattern": "BIGLOTRON",
|
|
243
278
|
"instances": [
|
|
244
279
|
"BIGLOTRON (Beta 2;GNU/Linux)"
|
|
245
280
|
]
|
|
246
|
-
}
|
|
281
|
+
}
|
|
282
|
+
,
|
|
247
283
|
{
|
|
248
284
|
"pattern": "Teoma",
|
|
249
285
|
"instances": [
|
|
@@ -251,21 +287,24 @@
|
|
|
251
287
|
"Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)"
|
|
252
288
|
],
|
|
253
289
|
"url": "http://about.ask.com/en/docs/about/webmasters.shtml"
|
|
254
|
-
}
|
|
290
|
+
}
|
|
291
|
+
,
|
|
255
292
|
{
|
|
256
293
|
"pattern": "convera",
|
|
257
294
|
"instances": [
|
|
258
295
|
"ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)"
|
|
259
296
|
],
|
|
260
297
|
"url": "http://ews.converasearch.com/crawl.htm"
|
|
261
|
-
}
|
|
298
|
+
}
|
|
299
|
+
,
|
|
262
300
|
{
|
|
263
301
|
"pattern": "seekbot",
|
|
264
302
|
"instances": [
|
|
265
303
|
"Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2"
|
|
266
304
|
],
|
|
267
305
|
"url": "http://www.seekbot.net/bot.html"
|
|
268
|
-
}
|
|
306
|
+
}
|
|
307
|
+
,
|
|
269
308
|
{
|
|
270
309
|
"pattern": "Gigabot",
|
|
271
310
|
"instances": [
|
|
@@ -273,14 +312,16 @@
|
|
|
273
312
|
"Gigabot/2.0 (http://www.gigablast.com/spider.html)"
|
|
274
313
|
],
|
|
275
314
|
"url": "http://www.gigablast.com/spider.html"
|
|
276
|
-
}
|
|
315
|
+
}
|
|
316
|
+
,
|
|
277
317
|
{
|
|
278
318
|
"pattern": "Gigablast",
|
|
279
319
|
"instances": [
|
|
280
320
|
"GigablastOpenSource/1.0"
|
|
281
321
|
],
|
|
282
322
|
"url": "https://github.com/gigablast/open-source-search-engine"
|
|
283
|
-
}
|
|
323
|
+
}
|
|
324
|
+
,
|
|
284
325
|
{
|
|
285
326
|
"pattern": "exabot",
|
|
286
327
|
"instances": [
|
|
@@ -290,30 +331,35 @@
|
|
|
290
331
|
"Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)",
|
|
291
332
|
"Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)"
|
|
292
333
|
]
|
|
293
|
-
}
|
|
334
|
+
}
|
|
335
|
+
,
|
|
294
336
|
{
|
|
295
337
|
"pattern": "ia_archiver",
|
|
296
338
|
"instances": [
|
|
297
339
|
"ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",
|
|
298
340
|
"ia_archiver-web.archive.org"
|
|
299
341
|
]
|
|
300
|
-
}
|
|
342
|
+
}
|
|
343
|
+
,
|
|
301
344
|
{
|
|
302
345
|
"pattern": "GingerCrawler",
|
|
303
346
|
"instances": [
|
|
304
347
|
"GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)"
|
|
305
348
|
]
|
|
306
|
-
}
|
|
349
|
+
}
|
|
350
|
+
,
|
|
307
351
|
{
|
|
308
352
|
"pattern": "webmon ",
|
|
309
353
|
"instances": []
|
|
310
|
-
}
|
|
354
|
+
}
|
|
355
|
+
,
|
|
311
356
|
{
|
|
312
357
|
"pattern": "HTTrack",
|
|
313
358
|
"instances": [
|
|
314
359
|
"Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"
|
|
315
360
|
]
|
|
316
|
-
}
|
|
361
|
+
}
|
|
362
|
+
,
|
|
317
363
|
{
|
|
318
364
|
"pattern": "grub.org",
|
|
319
365
|
"instances": [
|
|
@@ -329,19 +375,23 @@
|
|
|
329
375
|
"Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)",
|
|
330
376
|
"Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)"
|
|
331
377
|
]
|
|
332
|
-
}
|
|
378
|
+
}
|
|
379
|
+
,
|
|
333
380
|
{
|
|
334
381
|
"pattern": "UsineNouvelleCrawler",
|
|
335
382
|
"instances": []
|
|
336
|
-
}
|
|
383
|
+
}
|
|
384
|
+
,
|
|
337
385
|
{
|
|
338
386
|
"pattern": "antibot",
|
|
339
387
|
"instances": []
|
|
340
|
-
}
|
|
388
|
+
}
|
|
389
|
+
,
|
|
341
390
|
{
|
|
342
391
|
"pattern": "netresearchserver",
|
|
343
392
|
"instances": []
|
|
344
|
-
}
|
|
393
|
+
}
|
|
394
|
+
,
|
|
345
395
|
{
|
|
346
396
|
"pattern": "speedy",
|
|
347
397
|
"instances": [
|
|
@@ -351,17 +401,20 @@
|
|
|
351
401
|
"Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)",
|
|
352
402
|
"Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)"
|
|
353
403
|
]
|
|
354
|
-
}
|
|
404
|
+
}
|
|
405
|
+
,
|
|
355
406
|
{
|
|
356
407
|
"pattern": "fluffy",
|
|
357
408
|
"instances": []
|
|
358
|
-
}
|
|
409
|
+
}
|
|
410
|
+
,
|
|
359
411
|
{
|
|
360
412
|
"pattern": "bibnum.bnf",
|
|
361
413
|
"instances": [
|
|
362
414
|
"Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)"
|
|
363
415
|
]
|
|
364
|
-
}
|
|
416
|
+
}
|
|
417
|
+
,
|
|
365
418
|
{
|
|
366
419
|
"pattern": "findlink",
|
|
367
420
|
"instances": [
|
|
@@ -389,17 +442,20 @@
|
|
|
389
442
|
"findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",
|
|
390
443
|
"findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)"
|
|
391
444
|
]
|
|
392
|
-
}
|
|
445
|
+
}
|
|
446
|
+
,
|
|
393
447
|
{
|
|
394
448
|
"pattern": "msrbot",
|
|
395
449
|
"instances": []
|
|
396
|
-
}
|
|
450
|
+
}
|
|
451
|
+
,
|
|
397
452
|
{
|
|
398
453
|
"pattern": "panscient",
|
|
399
454
|
"instances": [
|
|
400
455
|
"panscient.com"
|
|
401
456
|
]
|
|
402
|
-
}
|
|
457
|
+
}
|
|
458
|
+
,
|
|
403
459
|
{
|
|
404
460
|
"pattern": "yacybot",
|
|
405
461
|
"instances": [
|
|
@@ -445,17 +501,19 @@
|
|
|
445
501
|
"yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html",
|
|
446
502
|
"yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
|
|
447
503
|
"yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html",
|
|
504
|
+
"yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
|
|
505
|
+
"yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
|
|
506
|
+
"yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
|
|
507
|
+
"yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
|
|
448
508
|
"yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html"
|
|
449
509
|
]
|
|
450
|
-
}
|
|
510
|
+
}
|
|
511
|
+
,
|
|
451
512
|
{
|
|
452
513
|
"pattern": "AISearchBot",
|
|
453
514
|
"instances": []
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
"pattern": "IOI",
|
|
457
|
-
"instances": []
|
|
458
|
-
},
|
|
515
|
+
}
|
|
516
|
+
,
|
|
459
517
|
{
|
|
460
518
|
"pattern": "ips-agent",
|
|
461
519
|
"instances": [
|
|
@@ -465,11 +523,13 @@
|
|
|
465
523
|
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24",
|
|
466
524
|
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1"
|
|
467
525
|
]
|
|
468
|
-
}
|
|
526
|
+
}
|
|
527
|
+
,
|
|
469
528
|
{
|
|
470
529
|
"pattern": "tagoobot",
|
|
471
530
|
"instances": []
|
|
472
|
-
}
|
|
531
|
+
}
|
|
532
|
+
,
|
|
473
533
|
{
|
|
474
534
|
"pattern": "MJ12bot",
|
|
475
535
|
"instances": [
|
|
@@ -490,34 +550,41 @@
|
|
|
490
550
|
"Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)",
|
|
491
551
|
"Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)",
|
|
492
552
|
"Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)",
|
|
493
|
-
"Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)"
|
|
553
|
+
"Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)",
|
|
554
|
+
"Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)",
|
|
555
|
+
"Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)"
|
|
494
556
|
]
|
|
495
|
-
}
|
|
557
|
+
}
|
|
558
|
+
,
|
|
496
559
|
{
|
|
497
560
|
"pattern": "woriobot",
|
|
498
561
|
"instances": [
|
|
499
562
|
"Mozilla/5.0 (compatible; woriobot +http://worio.com)",
|
|
500
563
|
"Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)"
|
|
501
564
|
]
|
|
502
|
-
}
|
|
565
|
+
}
|
|
566
|
+
,
|
|
503
567
|
{
|
|
504
568
|
"pattern": "yanga",
|
|
505
569
|
"instances": [
|
|
506
570
|
"Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)"
|
|
507
571
|
]
|
|
508
|
-
}
|
|
572
|
+
}
|
|
573
|
+
,
|
|
509
574
|
{
|
|
510
575
|
"pattern": "buzzbot",
|
|
511
576
|
"instances": [
|
|
512
577
|
"Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)"
|
|
513
578
|
]
|
|
514
|
-
}
|
|
579
|
+
}
|
|
580
|
+
,
|
|
515
581
|
{
|
|
516
582
|
"pattern": "mlbot",
|
|
517
583
|
"instances": [
|
|
518
584
|
"MLBot (www.metadatalabs.com/mlbot)"
|
|
519
585
|
]
|
|
520
|
-
}
|
|
586
|
+
}
|
|
587
|
+
,
|
|
521
588
|
{
|
|
522
589
|
"pattern": "YandexBot",
|
|
523
590
|
"url": "http://yandex.com/bots",
|
|
@@ -525,20 +592,24 @@
|
|
|
525
592
|
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"
|
|
526
593
|
],
|
|
527
594
|
"addition_date": "2015/04/14"
|
|
528
|
-
}
|
|
595
|
+
}
|
|
596
|
+
,
|
|
529
597
|
{
|
|
530
598
|
"pattern": "yandex.com\\/bots",
|
|
531
599
|
"url": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml#robot-in-logs",
|
|
532
600
|
"instances": [
|
|
533
|
-
"Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)"
|
|
601
|
+
"Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)",
|
|
602
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)"
|
|
534
603
|
],
|
|
535
604
|
"addition_date": "2016/12/01"
|
|
536
|
-
}
|
|
605
|
+
}
|
|
606
|
+
,
|
|
537
607
|
{
|
|
538
608
|
"pattern": "purebot",
|
|
539
609
|
"addition_date": "2010/01/19",
|
|
540
610
|
"instances": []
|
|
541
|
-
}
|
|
611
|
+
}
|
|
612
|
+
,
|
|
542
613
|
{
|
|
543
614
|
"pattern": "Linguee Bot",
|
|
544
615
|
"addition_date": "2010/01/26",
|
|
@@ -547,7 +618,8 @@
|
|
|
547
618
|
"Linguee Bot (http://www.linguee.com/bot)",
|
|
548
619
|
"Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)"
|
|
549
620
|
]
|
|
550
|
-
}
|
|
621
|
+
}
|
|
622
|
+
,
|
|
551
623
|
{
|
|
552
624
|
"pattern": "CyberPatrol",
|
|
553
625
|
"addition_date": "2010/02/11",
|
|
@@ -555,7 +627,8 @@
|
|
|
555
627
|
"instances": [
|
|
556
628
|
"CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)"
|
|
557
629
|
]
|
|
558
|
-
}
|
|
630
|
+
}
|
|
631
|
+
,
|
|
559
632
|
{
|
|
560
633
|
"pattern": "voilabot",
|
|
561
634
|
"addition_date": "2010/05/18",
|
|
@@ -564,7 +637,8 @@
|
|
|
564
637
|
"Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
|
|
565
638
|
"Mozilla/5.0 (compatible; OrangeBot/2.0; support.voilabot@orange.com)"
|
|
566
639
|
]
|
|
567
|
-
}
|
|
640
|
+
}
|
|
641
|
+
,
|
|
568
642
|
{
|
|
569
643
|
"pattern": "Baiduspider",
|
|
570
644
|
"addition_date": "2010/07/15",
|
|
@@ -572,12 +646,14 @@
|
|
|
572
646
|
"instances": [
|
|
573
647
|
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
|
|
574
648
|
]
|
|
575
|
-
}
|
|
649
|
+
}
|
|
650
|
+
,
|
|
576
651
|
{
|
|
577
652
|
"pattern": "citeseerxbot",
|
|
578
653
|
"addition_date": "2010/07/17",
|
|
579
654
|
"instances": []
|
|
580
|
-
}
|
|
655
|
+
}
|
|
656
|
+
,
|
|
581
657
|
{
|
|
582
658
|
"pattern": "spbot",
|
|
583
659
|
"addition_date": "2010/07/31",
|
|
@@ -618,13 +694,15 @@
|
|
|
618
694
|
"Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )",
|
|
619
695
|
"Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )"
|
|
620
696
|
]
|
|
621
|
-
}
|
|
697
|
+
}
|
|
698
|
+
,
|
|
622
699
|
{
|
|
623
700
|
"pattern": "twengabot",
|
|
624
701
|
"addition_date": "2010/08/03",
|
|
625
702
|
"url": "http://www.twenga.com/bot.html",
|
|
626
703
|
"instances": []
|
|
627
|
-
}
|
|
704
|
+
}
|
|
705
|
+
,
|
|
628
706
|
{
|
|
629
707
|
"pattern": "postrank",
|
|
630
708
|
"addition_date": "2010/08/03",
|
|
@@ -633,19 +711,22 @@
|
|
|
633
711
|
"PostRank/2.0 (postrank.com)",
|
|
634
712
|
"PostRank/2.0 (postrank.com; 1 subscribers)"
|
|
635
713
|
]
|
|
636
|
-
}
|
|
714
|
+
}
|
|
715
|
+
,
|
|
637
716
|
{
|
|
638
717
|
"pattern": "turnitinbot",
|
|
639
718
|
"addition_date": "2010/09/26",
|
|
640
719
|
"url": "http://www.turnitin.com",
|
|
641
720
|
"instances": []
|
|
642
|
-
}
|
|
721
|
+
}
|
|
722
|
+
,
|
|
643
723
|
{
|
|
644
724
|
"pattern": "scribdbot",
|
|
645
725
|
"addition_date": "2010/09/28",
|
|
646
726
|
"url": "http://www.scribd.com",
|
|
647
727
|
"instances": []
|
|
648
|
-
}
|
|
728
|
+
}
|
|
729
|
+
,
|
|
649
730
|
{
|
|
650
731
|
"pattern": "page2rss",
|
|
651
732
|
"addition_date": "2010/10/07",
|
|
@@ -653,7 +734,8 @@
|
|
|
653
734
|
"instances": [
|
|
654
735
|
"Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)"
|
|
655
736
|
]
|
|
656
|
-
}
|
|
737
|
+
}
|
|
738
|
+
,
|
|
657
739
|
{
|
|
658
740
|
"pattern": "sitebot",
|
|
659
741
|
"addition_date": "2010/12/15",
|
|
@@ -661,7 +743,8 @@
|
|
|
661
743
|
"instances": [
|
|
662
744
|
"Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)"
|
|
663
745
|
]
|
|
664
|
-
}
|
|
746
|
+
}
|
|
747
|
+
,
|
|
665
748
|
{
|
|
666
749
|
"pattern": "linkdex",
|
|
667
750
|
"addition_date": "2011/01/06",
|
|
@@ -675,19 +758,22 @@
|
|
|
675
758
|
"linkdex.com/v2.0",
|
|
676
759
|
"linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)"
|
|
677
760
|
]
|
|
678
|
-
}
|
|
761
|
+
}
|
|
762
|
+
,
|
|
679
763
|
{
|
|
680
764
|
"pattern": "Adidxbot",
|
|
681
765
|
"url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx",
|
|
682
766
|
"instances": []
|
|
683
|
-
}
|
|
767
|
+
}
|
|
768
|
+
,
|
|
684
769
|
{
|
|
685
770
|
"pattern": "blekkobot",
|
|
686
771
|
"url": "http://blekko.com/about/blekkobot",
|
|
687
772
|
"instances": [
|
|
688
773
|
"Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)"
|
|
689
774
|
]
|
|
690
|
-
}
|
|
775
|
+
}
|
|
776
|
+
,
|
|
691
777
|
{
|
|
692
778
|
"pattern": "ezooms",
|
|
693
779
|
"addition_date": "2011/04/27",
|
|
@@ -695,7 +781,8 @@
|
|
|
695
781
|
"instances": [
|
|
696
782
|
"Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)"
|
|
697
783
|
]
|
|
698
|
-
}
|
|
784
|
+
}
|
|
785
|
+
,
|
|
699
786
|
{
|
|
700
787
|
"pattern": "dotbot",
|
|
701
788
|
"addition_date": "2011/04/27",
|
|
@@ -703,7 +790,8 @@
|
|
|
703
790
|
"Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)",
|
|
704
791
|
"dotbot"
|
|
705
792
|
]
|
|
706
|
-
}
|
|
793
|
+
}
|
|
794
|
+
,
|
|
707
795
|
{
|
|
708
796
|
"pattern": "Mail.RU_Bot",
|
|
709
797
|
"addition_date": "2011/04/27",
|
|
@@ -711,7 +799,8 @@
|
|
|
711
799
|
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/",
|
|
712
800
|
"Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/"
|
|
713
801
|
]
|
|
714
|
-
}
|
|
802
|
+
}
|
|
803
|
+
,
|
|
715
804
|
{
|
|
716
805
|
"pattern": "discobot",
|
|
717
806
|
"addition_date": "2011/05/03",
|
|
@@ -721,7 +810,8 @@
|
|
|
721
810
|
"Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)",
|
|
722
811
|
"mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)"
|
|
723
812
|
]
|
|
724
|
-
}
|
|
813
|
+
}
|
|
814
|
+
,
|
|
725
815
|
{
|
|
726
816
|
"pattern": "heritrix",
|
|
727
817
|
"addition_date": "2011/06/21",
|
|
@@ -753,13 +843,15 @@
|
|
|
753
843
|
"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)",
|
|
754
844
|
"Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)"
|
|
755
845
|
]
|
|
756
|
-
}
|
|
846
|
+
}
|
|
847
|
+
,
|
|
757
848
|
{
|
|
758
849
|
"pattern": "findthatfile",
|
|
759
850
|
"addition_date": "2011/06/21",
|
|
760
851
|
"url": "http://www.findthatfile.com/",
|
|
761
852
|
"instances": []
|
|
762
|
-
}
|
|
853
|
+
}
|
|
854
|
+
,
|
|
763
855
|
{
|
|
764
856
|
"pattern": "europarchive.org",
|
|
765
857
|
"addition_date": "2011/06/21",
|
|
@@ -767,7 +859,8 @@
|
|
|
767
859
|
"instances": [
|
|
768
860
|
"Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)"
|
|
769
861
|
]
|
|
770
|
-
}
|
|
862
|
+
}
|
|
863
|
+
,
|
|
771
864
|
{
|
|
772
865
|
"pattern": "NerdByNature.Bot",
|
|
773
866
|
"addition_date": "2011/07/12",
|
|
@@ -775,19 +868,136 @@
|
|
|
775
868
|
"instances": [
|
|
776
869
|
"Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)"
|
|
777
870
|
]
|
|
778
|
-
}
|
|
871
|
+
}
|
|
872
|
+
,
|
|
779
873
|
{
|
|
780
874
|
"pattern": "sistrix crawler",
|
|
781
875
|
"addition_date": "2011/08/02",
|
|
782
876
|
"instances": []
|
|
783
|
-
}
|
|
877
|
+
}
|
|
878
|
+
,
|
|
784
879
|
{
|
|
785
|
-
"pattern": "
|
|
880
|
+
"pattern": "Ahrefs(Bot|SiteAudit)",
|
|
786
881
|
"addition_date": "2011/08/28",
|
|
787
882
|
"instances": [
|
|
788
|
-
"Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)"
|
|
883
|
+
"Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)",
|
|
884
|
+
"Mozilla/5.0 (compatible; AhrefsSiteAudit/5.2; +http://ahrefs.com/robot/)"
|
|
789
885
|
]
|
|
790
|
-
}
|
|
886
|
+
}
|
|
887
|
+
,
|
|
888
|
+
{
|
|
889
|
+
"pattern": "fuelbot",
|
|
890
|
+
"addition_date": "2018/06/28",
|
|
891
|
+
"instances": [
|
|
892
|
+
"fuelbot"
|
|
893
|
+
]
|
|
894
|
+
}
|
|
895
|
+
,
|
|
896
|
+
{
|
|
897
|
+
"pattern": "CrunchBot",
|
|
898
|
+
"addition_date": "2018/06/28",
|
|
899
|
+
"instances": [
|
|
900
|
+
"CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)"
|
|
901
|
+
]
|
|
902
|
+
}
|
|
903
|
+
,
|
|
904
|
+
{
|
|
905
|
+
"pattern": "centurybot9",
|
|
906
|
+
"addition_date": "2018/06/28",
|
|
907
|
+
"instances": [
|
|
908
|
+
"Mozilla/5.0 (compatible; Go-http-client/1.1; +centurybot9@gmail.com)"
|
|
909
|
+
]
|
|
910
|
+
}
|
|
911
|
+
,
|
|
912
|
+
{
|
|
913
|
+
"pattern": "IndeedBot",
|
|
914
|
+
"addition_date": "2018/06/28",
|
|
915
|
+
"instances": [
|
|
916
|
+
"Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)"
|
|
917
|
+
]
|
|
918
|
+
}
|
|
919
|
+
,
|
|
920
|
+
{
|
|
921
|
+
"pattern": "mappydata",
|
|
922
|
+
"addition_date": "2018/06/28",
|
|
923
|
+
"instances": [
|
|
924
|
+
"Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)"
|
|
925
|
+
]
|
|
926
|
+
}
|
|
927
|
+
,
|
|
928
|
+
{
|
|
929
|
+
"pattern": "woobot",
|
|
930
|
+
"addition_date": "2018/06/28",
|
|
931
|
+
"instances": [
|
|
932
|
+
"woobot"
|
|
933
|
+
]
|
|
934
|
+
}
|
|
935
|
+
,
|
|
936
|
+
{
|
|
937
|
+
"pattern": "ZoominfoBot",
|
|
938
|
+
"addition_date": "2018/06/28",
|
|
939
|
+
"instances": [
|
|
940
|
+
"ZoominfoBot (zoominfobot at zoominfo dot com)"
|
|
941
|
+
]
|
|
942
|
+
}
|
|
943
|
+
,
|
|
944
|
+
{
|
|
945
|
+
"pattern": "PrivacyAwareBot",
|
|
946
|
+
"addition_date": "2018/06/28",
|
|
947
|
+
"instances": [
|
|
948
|
+
"Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)"
|
|
949
|
+
]
|
|
950
|
+
}
|
|
951
|
+
,
|
|
952
|
+
{
|
|
953
|
+
"pattern": "Multiviewbot",
|
|
954
|
+
"addition_date": "2018/06/28",
|
|
955
|
+
"instances": [
|
|
956
|
+
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot"
|
|
957
|
+
]
|
|
958
|
+
}
|
|
959
|
+
,
|
|
960
|
+
{
|
|
961
|
+
"pattern": "SWIMGBot",
|
|
962
|
+
"addition_date": "2018/06/28",
|
|
963
|
+
"instances": [
|
|
964
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot"
|
|
965
|
+
]
|
|
966
|
+
}
|
|
967
|
+
,
|
|
968
|
+
{
|
|
969
|
+
"pattern": "Grobbot",
|
|
970
|
+
"addition_date": "2018/06/28",
|
|
971
|
+
"instances": [
|
|
972
|
+
"Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)"
|
|
973
|
+
]
|
|
974
|
+
}
|
|
975
|
+
,
|
|
976
|
+
{
|
|
977
|
+
"pattern": "eright",
|
|
978
|
+
"addition_date": "2018/06/28",
|
|
979
|
+
"instances": [
|
|
980
|
+
"Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)"
|
|
981
|
+
]
|
|
982
|
+
}
|
|
983
|
+
,
|
|
984
|
+
{
|
|
985
|
+
"pattern": "Apercite",
|
|
986
|
+
"addition_date": "2018/06/28",
|
|
987
|
+
"instances": [
|
|
988
|
+
"Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)"
|
|
989
|
+
]
|
|
990
|
+
}
|
|
991
|
+
,
|
|
992
|
+
{
|
|
993
|
+
"pattern": "semanticbot",
|
|
994
|
+
"addition_date": "2018/06/28",
|
|
995
|
+
"instances": [
|
|
996
|
+
"semanticbot",
|
|
997
|
+
"semanticbot (info@semanticaudience.com)"
|
|
998
|
+
]
|
|
999
|
+
}
|
|
1000
|
+
,
|
|
791
1001
|
{
|
|
792
1002
|
"pattern": "Aboundex",
|
|
793
1003
|
"addition_date": "2011/09/28",
|
|
@@ -796,20 +1006,23 @@
|
|
|
796
1006
|
"Aboundex/0.2 (http://www.aboundex.com/crawler/)",
|
|
797
1007
|
"Aboundex/0.3 (http://www.aboundex.com/crawler/)"
|
|
798
1008
|
]
|
|
799
|
-
}
|
|
1009
|
+
}
|
|
1010
|
+
,
|
|
800
1011
|
{
|
|
801
1012
|
"pattern": "domaincrawler",
|
|
802
1013
|
"addition_date": "2011/10/21",
|
|
803
1014
|
"instances": [
|
|
804
1015
|
"CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)"
|
|
805
1016
|
]
|
|
806
|
-
}
|
|
1017
|
+
}
|
|
1018
|
+
,
|
|
807
1019
|
{
|
|
808
1020
|
"pattern": "wbsearchbot",
|
|
809
1021
|
"addition_date": "2011/12/21",
|
|
810
1022
|
"url": "http://www.warebay.com/bot.html",
|
|
811
1023
|
"instances": []
|
|
812
|
-
}
|
|
1024
|
+
}
|
|
1025
|
+
,
|
|
813
1026
|
{
|
|
814
1027
|
"pattern": "summify",
|
|
815
1028
|
"addition_date": "2012/01/04",
|
|
@@ -817,7 +1030,8 @@
|
|
|
817
1030
|
"instances": [
|
|
818
1031
|
"Summify (Summify/1.0.1; +http://summify.com)"
|
|
819
1032
|
]
|
|
820
|
-
}
|
|
1033
|
+
}
|
|
1034
|
+
,
|
|
821
1035
|
{
|
|
822
1036
|
"pattern": "CCBot",
|
|
823
1037
|
"addition_date": "2012/02/05",
|
|
@@ -825,12 +1039,14 @@
|
|
|
825
1039
|
"instances": [
|
|
826
1040
|
"CCBot/2.0 (http://commoncrawl.org/faq/)"
|
|
827
1041
|
]
|
|
828
|
-
}
|
|
1042
|
+
}
|
|
1043
|
+
,
|
|
829
1044
|
{
|
|
830
1045
|
"pattern": "edisterbot",
|
|
831
1046
|
"addition_date": "2012/02/25",
|
|
832
1047
|
"instances": []
|
|
833
|
-
}
|
|
1048
|
+
}
|
|
1049
|
+
,
|
|
834
1050
|
{
|
|
835
1051
|
"pattern": "seznambot",
|
|
836
1052
|
"addition_date": "2012/03/14",
|
|
@@ -841,29 +1057,36 @@
|
|
|
841
1057
|
"Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)",
|
|
842
1058
|
"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)"
|
|
843
1059
|
]
|
|
844
|
-
}
|
|
1060
|
+
}
|
|
1061
|
+
,
|
|
845
1062
|
{
|
|
846
1063
|
"pattern": "ec2linkfinder",
|
|
847
1064
|
"addition_date": "2012/03/22",
|
|
848
1065
|
"instances": [
|
|
849
1066
|
"ec2linkfinder"
|
|
850
1067
|
]
|
|
851
|
-
}
|
|
1068
|
+
}
|
|
1069
|
+
,
|
|
852
1070
|
{
|
|
853
1071
|
"pattern": "gslfbot",
|
|
854
1072
|
"addition_date": "2012/04/03",
|
|
855
1073
|
"instances": []
|
|
856
|
-
}
|
|
1074
|
+
}
|
|
1075
|
+
,
|
|
857
1076
|
{
|
|
858
|
-
"pattern": "
|
|
1077
|
+
"pattern": "aiHitBot",
|
|
859
1078
|
"addition_date": "2012/04/16",
|
|
860
|
-
"instances": [
|
|
861
|
-
|
|
1079
|
+
"instances": [
|
|
1080
|
+
"Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)"
|
|
1081
|
+
]
|
|
1082
|
+
}
|
|
1083
|
+
,
|
|
862
1084
|
{
|
|
863
1085
|
"pattern": "intelium_bot",
|
|
864
1086
|
"addition_date": "2012/05/07",
|
|
865
1087
|
"instances": []
|
|
866
|
-
}
|
|
1088
|
+
}
|
|
1089
|
+
,
|
|
867
1090
|
{
|
|
868
1091
|
"pattern": "facebookexternalhit",
|
|
869
1092
|
"addition_date": "2012/05/07",
|
|
@@ -872,7 +1095,8 @@
|
|
|
872
1095
|
"facebookexternalhit/1.1",
|
|
873
1096
|
"facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"
|
|
874
1097
|
]
|
|
875
|
-
}
|
|
1098
|
+
}
|
|
1099
|
+
,
|
|
876
1100
|
{
|
|
877
1101
|
"pattern": "Yeti",
|
|
878
1102
|
"addition_date": "2012/05/07",
|
|
@@ -880,19 +1104,22 @@
|
|
|
880
1104
|
"instances": [
|
|
881
1105
|
"Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)"
|
|
882
1106
|
]
|
|
883
|
-
}
|
|
1107
|
+
}
|
|
1108
|
+
,
|
|
884
1109
|
{
|
|
885
1110
|
"pattern": "RetrevoPageAnalyzer",
|
|
886
1111
|
"addition_date": "2012/05/07",
|
|
887
1112
|
"instances": [
|
|
888
1113
|
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)"
|
|
889
1114
|
]
|
|
890
|
-
}
|
|
1115
|
+
}
|
|
1116
|
+
,
|
|
891
1117
|
{
|
|
892
1118
|
"pattern": "lb-spider",
|
|
893
1119
|
"addition_date": "2012/05/07",
|
|
894
1120
|
"instances": []
|
|
895
|
-
}
|
|
1121
|
+
}
|
|
1122
|
+
,
|
|
896
1123
|
{
|
|
897
1124
|
"pattern": "Sogou",
|
|
898
1125
|
"addition_date": "2012/05/13",
|
|
@@ -902,18 +1129,21 @@
|
|
|
902
1129
|
"Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
|
|
903
1130
|
"Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
|
|
904
1131
|
]
|
|
905
|
-
}
|
|
1132
|
+
}
|
|
1133
|
+
,
|
|
906
1134
|
{
|
|
907
1135
|
"pattern": "lssbot",
|
|
908
1136
|
"addition_date": "2012/05/15",
|
|
909
1137
|
"instances": []
|
|
910
|
-
}
|
|
1138
|
+
}
|
|
1139
|
+
,
|
|
911
1140
|
{
|
|
912
1141
|
"pattern": "careerbot",
|
|
913
1142
|
"addition_date": "2012/05/23",
|
|
914
1143
|
"url": "http://www.career-x.de/bot.html",
|
|
915
1144
|
"instances": []
|
|
916
|
-
}
|
|
1145
|
+
}
|
|
1146
|
+
,
|
|
917
1147
|
{
|
|
918
1148
|
"pattern": "wotbox",
|
|
919
1149
|
"addition_date": "2012/06/12",
|
|
@@ -922,13 +1152,15 @@
|
|
|
922
1152
|
"Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)",
|
|
923
1153
|
"Wotbox/2.01 (+http://www.wotbox.com/bot/)"
|
|
924
1154
|
]
|
|
925
|
-
}
|
|
1155
|
+
}
|
|
1156
|
+
,
|
|
926
1157
|
{
|
|
927
1158
|
"pattern": "wocbot",
|
|
928
1159
|
"addition_date": "2012/07/25",
|
|
929
1160
|
"url": "http://www.wocodi.com/crawler",
|
|
930
1161
|
"instances": []
|
|
931
|
-
}
|
|
1162
|
+
}
|
|
1163
|
+
,
|
|
932
1164
|
{
|
|
933
1165
|
"pattern": "ichiro",
|
|
934
1166
|
"addition_date": "2012/08/28",
|
|
@@ -950,7 +1182,8 @@
|
|
|
950
1182
|
"ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)",
|
|
951
1183
|
"ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)"
|
|
952
1184
|
]
|
|
953
|
-
}
|
|
1185
|
+
}
|
|
1186
|
+
,
|
|
954
1187
|
{
|
|
955
1188
|
"pattern": "DuckDuckBot",
|
|
956
1189
|
"addition_date": "2012/09/19",
|
|
@@ -959,12 +1192,14 @@
|
|
|
959
1192
|
"DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
|
|
960
1193
|
"DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)"
|
|
961
1194
|
]
|
|
962
|
-
}
|
|
1195
|
+
}
|
|
1196
|
+
,
|
|
963
1197
|
{
|
|
964
1198
|
"pattern": "lssrocketcrawler",
|
|
965
1199
|
"addition_date": "2012/09/24",
|
|
966
1200
|
"instances": []
|
|
967
|
-
}
|
|
1201
|
+
}
|
|
1202
|
+
,
|
|
968
1203
|
{
|
|
969
1204
|
"pattern": "drupact",
|
|
970
1205
|
"addition_date": "2012/09/27",
|
|
@@ -972,38 +1207,45 @@
|
|
|
972
1207
|
"instances": [
|
|
973
1208
|
"drupact/0.7; http://www.arocom.de/drupact"
|
|
974
1209
|
]
|
|
975
|
-
}
|
|
1210
|
+
}
|
|
1211
|
+
,
|
|
976
1212
|
{
|
|
977
1213
|
"pattern": "webcompanycrawler",
|
|
978
1214
|
"addition_date": "2012/10/03",
|
|
979
1215
|
"instances": []
|
|
980
|
-
}
|
|
1216
|
+
}
|
|
1217
|
+
,
|
|
981
1218
|
{
|
|
982
1219
|
"pattern": "acoonbot",
|
|
983
1220
|
"addition_date": "2012/10/07",
|
|
984
1221
|
"url": "http://www.acoon.de/robot.asp",
|
|
985
1222
|
"instances": []
|
|
986
|
-
}
|
|
1223
|
+
}
|
|
1224
|
+
,
|
|
987
1225
|
{
|
|
988
1226
|
"pattern": "openindexspider",
|
|
989
1227
|
"addition_date": "2012/10/26",
|
|
990
1228
|
"url": "http://www.openindex.io/en/webmasters/spider.html",
|
|
991
1229
|
"instances": []
|
|
992
|
-
}
|
|
1230
|
+
}
|
|
1231
|
+
,
|
|
993
1232
|
{
|
|
994
1233
|
"pattern": "gnam gnam spider",
|
|
995
1234
|
"addition_date": "2012/10/31",
|
|
996
1235
|
"instances": []
|
|
997
|
-
}
|
|
1236
|
+
}
|
|
1237
|
+
,
|
|
998
1238
|
{
|
|
999
1239
|
"pattern": "web-archive-net.com.bot",
|
|
1000
1240
|
"instances": []
|
|
1001
|
-
}
|
|
1241
|
+
}
|
|
1242
|
+
,
|
|
1002
1243
|
{
|
|
1003
1244
|
"pattern": "backlinkcrawler",
|
|
1004
1245
|
"addition_date": "2013/01/04",
|
|
1005
1246
|
"instances": []
|
|
1006
|
-
}
|
|
1247
|
+
}
|
|
1248
|
+
,
|
|
1007
1249
|
{
|
|
1008
1250
|
"pattern": "coccoc",
|
|
1009
1251
|
"addition_date": "2013/01/04",
|
|
@@ -1021,7 +1263,8 @@
|
|
|
1021
1263
|
"coccoc/1.0 (http://help.coccoc.com/)",
|
|
1022
1264
|
"coccoc/1.0 (http://help.coccoc.vn/)"
|
|
1023
1265
|
]
|
|
1024
|
-
}
|
|
1266
|
+
}
|
|
1267
|
+
,
|
|
1025
1268
|
{
|
|
1026
1269
|
"pattern": "integromedb",
|
|
1027
1270
|
"addition_date": "2013/01/10",
|
|
@@ -1029,22 +1272,20 @@
|
|
|
1029
1272
|
"instances": [
|
|
1030
1273
|
"www.integromedb.org/Crawler"
|
|
1031
1274
|
]
|
|
1032
|
-
}
|
|
1275
|
+
}
|
|
1276
|
+
,
|
|
1033
1277
|
{
|
|
1034
1278
|
"pattern": "content crawler spider",
|
|
1035
1279
|
"addition_date": "2013/01/11",
|
|
1036
1280
|
"instances": []
|
|
1037
|
-
}
|
|
1281
|
+
}
|
|
1282
|
+
,
|
|
1038
1283
|
{
|
|
1039
1284
|
"pattern": "toplistbot",
|
|
1040
1285
|
"addition_date": "2013/02/05",
|
|
1041
1286
|
"instances": []
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
"pattern": "seokicks-robot",
|
|
1045
|
-
"addition_date": "2013/02/25",
|
|
1046
|
-
"instances": []
|
|
1047
|
-
},
|
|
1287
|
+
}
|
|
1288
|
+
,
|
|
1048
1289
|
{
|
|
1049
1290
|
"pattern": "it2media-domain-crawler",
|
|
1050
1291
|
"addition_date": "2013/03/12",
|
|
@@ -1052,12 +1293,14 @@
|
|
|
1052
1293
|
"it2media-domain-crawler/1.0 on crawler-prod.it2media.de",
|
|
1053
1294
|
"it2media-domain-crawler/2.0"
|
|
1054
1295
|
]
|
|
1055
|
-
}
|
|
1296
|
+
}
|
|
1297
|
+
,
|
|
1056
1298
|
{
|
|
1057
1299
|
"pattern": "ip-web-crawler.com",
|
|
1058
1300
|
"addition_date": "2013/03/22",
|
|
1059
1301
|
"instances": []
|
|
1060
|
-
}
|
|
1302
|
+
}
|
|
1303
|
+
,
|
|
1061
1304
|
{
|
|
1062
1305
|
"pattern": "siteexplorer.info",
|
|
1063
1306
|
"addition_date": "2013/05/01",
|
|
@@ -1065,12 +1308,14 @@
|
|
|
1065
1308
|
"Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)",
|
|
1066
1309
|
"Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)"
|
|
1067
1310
|
]
|
|
1068
|
-
}
|
|
1311
|
+
}
|
|
1312
|
+
,
|
|
1069
1313
|
{
|
|
1070
1314
|
"pattern": "elisabot",
|
|
1071
1315
|
"addition_date": "2013/06/27",
|
|
1072
1316
|
"instances": []
|
|
1073
|
-
}
|
|
1317
|
+
}
|
|
1318
|
+
,
|
|
1074
1319
|
{
|
|
1075
1320
|
"pattern": "proximic",
|
|
1076
1321
|
"addition_date": "2013/09/12",
|
|
@@ -1079,7 +1324,8 @@
|
|
|
1079
1324
|
"Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)",
|
|
1080
1325
|
"Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)"
|
|
1081
1326
|
]
|
|
1082
|
-
}
|
|
1327
|
+
}
|
|
1328
|
+
,
|
|
1083
1329
|
{
|
|
1084
1330
|
"pattern": "changedetection",
|
|
1085
1331
|
"addition_date": "2013/09/13",
|
|
@@ -1087,18 +1333,14 @@
|
|
|
1087
1333
|
"instances": [
|
|
1088
1334
|
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )"
|
|
1089
1335
|
]
|
|
1090
|
-
}
|
|
1091
|
-
|
|
1092
|
-
"pattern": "blexbot",
|
|
1093
|
-
"addition_date": "2013/10/03",
|
|
1094
|
-
"url": "http://webmeup-crawler.com/",
|
|
1095
|
-
"instances": []
|
|
1096
|
-
},
|
|
1336
|
+
}
|
|
1337
|
+
,
|
|
1097
1338
|
{
|
|
1098
1339
|
"pattern": "arabot",
|
|
1099
1340
|
"addition_date": "2013/10/09",
|
|
1100
1341
|
"instances": []
|
|
1101
|
-
}
|
|
1342
|
+
}
|
|
1343
|
+
,
|
|
1102
1344
|
{
|
|
1103
1345
|
"pattern": "WeSEE:Search",
|
|
1104
1346
|
"addition_date": "2013/11/18",
|
|
@@ -1106,18 +1348,21 @@
|
|
|
1106
1348
|
"WeSEE:Search",
|
|
1107
1349
|
"WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)"
|
|
1108
1350
|
]
|
|
1109
|
-
}
|
|
1351
|
+
}
|
|
1352
|
+
,
|
|
1110
1353
|
{
|
|
1111
1354
|
"pattern": "niki-bot",
|
|
1112
1355
|
"addition_date": "2014/01/01",
|
|
1113
1356
|
"instances": []
|
|
1114
|
-
}
|
|
1357
|
+
}
|
|
1358
|
+
,
|
|
1115
1359
|
{
|
|
1116
1360
|
"pattern": "CrystalSemanticsBot",
|
|
1117
1361
|
"addition_date": "2014/02/17",
|
|
1118
1362
|
"url": "http://www.crystalsemantics.com/user-agent/",
|
|
1119
1363
|
"instances": []
|
|
1120
|
-
}
|
|
1364
|
+
}
|
|
1365
|
+
,
|
|
1121
1366
|
{
|
|
1122
1367
|
"pattern": "rogerbot",
|
|
1123
1368
|
"addition_date": "2014/02/28",
|
|
@@ -1137,7 +1382,8 @@
|
|
|
1137
1382
|
"rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)",
|
|
1138
1383
|
"rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)"
|
|
1139
1384
|
]
|
|
1140
|
-
}
|
|
1385
|
+
}
|
|
1386
|
+
,
|
|
1141
1387
|
{
|
|
1142
1388
|
"pattern": "360Spider",
|
|
1143
1389
|
"addition_date": "2014/03/14",
|
|
@@ -1153,7 +1399,8 @@
|
|
|
1153
1399
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider",
|
|
1154
1400
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)"
|
|
1155
1401
|
]
|
|
1156
|
-
}
|
|
1402
|
+
}
|
|
1403
|
+
,
|
|
1157
1404
|
{
|
|
1158
1405
|
"pattern": "psbot",
|
|
1159
1406
|
"addition_date": "2014/03/31",
|
|
@@ -1163,13 +1410,15 @@
|
|
|
1163
1410
|
"psbot-page (+http://www.picsearch.com/bot.html)",
|
|
1164
1411
|
"psbot/0.1 (+http://www.picsearch.com/bot.html)"
|
|
1165
1412
|
]
|
|
1166
|
-
}
|
|
1413
|
+
}
|
|
1414
|
+
,
|
|
1167
1415
|
{
|
|
1168
1416
|
"pattern": "InterfaxScanBot",
|
|
1169
1417
|
"addition_date": "2014/03/31",
|
|
1170
1418
|
"url": "http://scan-interfax.ru",
|
|
1171
1419
|
"instances": []
|
|
1172
|
-
}
|
|
1420
|
+
}
|
|
1421
|
+
,
|
|
1173
1422
|
{
|
|
1174
1423
|
"pattern": "CC Metadata Scaper",
|
|
1175
1424
|
"addition_date": "2014/04/01",
|
|
@@ -1177,13 +1426,15 @@
|
|
|
1177
1426
|
"instances": [
|
|
1178
1427
|
"CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper"
|
|
1179
1428
|
]
|
|
1180
|
-
}
|
|
1429
|
+
}
|
|
1430
|
+
,
|
|
1181
1431
|
{
|
|
1182
1432
|
"pattern": "g00g1e.net",
|
|
1183
1433
|
"addition_date": "2014/04/01",
|
|
1184
1434
|
"url": "http://www.g00g1e.net/",
|
|
1185
1435
|
"instances": []
|
|
1186
|
-
}
|
|
1436
|
+
}
|
|
1437
|
+
,
|
|
1187
1438
|
{
|
|
1188
1439
|
"pattern": "GrapeshotCrawler",
|
|
1189
1440
|
"addition_date": "2014/04/01",
|
|
@@ -1191,7 +1442,8 @@
|
|
|
1191
1442
|
"instances": [
|
|
1192
1443
|
"Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)"
|
|
1193
1444
|
]
|
|
1194
|
-
}
|
|
1445
|
+
}
|
|
1446
|
+
,
|
|
1195
1447
|
{
|
|
1196
1448
|
"pattern": "urlappendbot",
|
|
1197
1449
|
"addition_date": "2014/05/10",
|
|
@@ -1199,19 +1451,22 @@
|
|
|
1199
1451
|
"instances": [
|
|
1200
1452
|
"Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)"
|
|
1201
1453
|
]
|
|
1202
|
-
}
|
|
1454
|
+
}
|
|
1455
|
+
,
|
|
1203
1456
|
{
|
|
1204
1457
|
"pattern": "brainobot",
|
|
1205
1458
|
"addition_date": "2014/06/24",
|
|
1206
1459
|
"instances": []
|
|
1207
|
-
}
|
|
1460
|
+
}
|
|
1461
|
+
,
|
|
1208
1462
|
{
|
|
1209
1463
|
"pattern": "fr-crawler",
|
|
1210
1464
|
"addition_date": "2014/07/31",
|
|
1211
1465
|
"instances": [
|
|
1212
1466
|
"Mozilla/5.0 (compatible; fr-crawler/1.1)"
|
|
1213
1467
|
]
|
|
1214
|
-
}
|
|
1468
|
+
}
|
|
1469
|
+
,
|
|
1215
1470
|
{
|
|
1216
1471
|
"pattern": "binlar",
|
|
1217
1472
|
"addition_date": "2014/09/12",
|
|
@@ -1222,14 +1477,16 @@
|
|
|
1222
1477
|
"binlar_2.6.3 phanendra_kalapala@McAfee.com",
|
|
1223
1478
|
"binlar_2.6.3 test@mgmt.mic"
|
|
1224
1479
|
]
|
|
1225
|
-
}
|
|
1480
|
+
}
|
|
1481
|
+
,
|
|
1226
1482
|
{
|
|
1227
1483
|
"pattern": "SimpleCrawler",
|
|
1228
1484
|
"addition_date": "2014/09/12",
|
|
1229
1485
|
"instances": [
|
|
1230
1486
|
"SimpleCrawler/0.1"
|
|
1231
1487
|
]
|
|
1232
|
-
}
|
|
1488
|
+
}
|
|
1489
|
+
,
|
|
1233
1490
|
{
|
|
1234
1491
|
"pattern": "Twitterbot",
|
|
1235
1492
|
"addition_date": "2014/09/12",
|
|
@@ -1238,7 +1495,8 @@
|
|
|
1238
1495
|
"Twitterbot/0.1",
|
|
1239
1496
|
"Twitterbot/1.0"
|
|
1240
1497
|
]
|
|
1241
|
-
}
|
|
1498
|
+
}
|
|
1499
|
+
,
|
|
1242
1500
|
{
|
|
1243
1501
|
"pattern": "cXensebot",
|
|
1244
1502
|
"addition_date": "2014/10/05",
|
|
@@ -1246,7 +1504,8 @@
|
|
|
1246
1504
|
"cXensebot/1.1a"
|
|
1247
1505
|
],
|
|
1248
1506
|
"url": "http://www.cxense.com/bot.html"
|
|
1249
|
-
}
|
|
1507
|
+
}
|
|
1508
|
+
,
|
|
1250
1509
|
{
|
|
1251
1510
|
"pattern": "smtbot",
|
|
1252
1511
|
"addition_date": "2014/10/04",
|
|
@@ -1255,7 +1514,8 @@
|
|
|
1255
1514
|
"SMTBot (similartech.com/smtbot)"
|
|
1256
1515
|
],
|
|
1257
1516
|
"url": "http://www.similartech.com/smtbot"
|
|
1258
|
-
}
|
|
1517
|
+
}
|
|
1518
|
+
,
|
|
1259
1519
|
{
|
|
1260
1520
|
"pattern": "bnf.fr_bot",
|
|
1261
1521
|
"addition_date": "2014/11/18",
|
|
@@ -1263,7 +1523,8 @@
|
|
|
1263
1523
|
"instances": [
|
|
1264
1524
|
"Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)"
|
|
1265
1525
|
]
|
|
1266
|
-
}
|
|
1526
|
+
}
|
|
1527
|
+
,
|
|
1267
1528
|
{
|
|
1268
1529
|
"pattern": "A6-Indexer",
|
|
1269
1530
|
"addition_date": "2014/12/05",
|
|
@@ -1271,7 +1532,8 @@
|
|
|
1271
1532
|
"instances": [
|
|
1272
1533
|
"A6-Indexer"
|
|
1273
1534
|
]
|
|
1274
|
-
}
|
|
1535
|
+
}
|
|
1536
|
+
,
|
|
1275
1537
|
{
|
|
1276
1538
|
"pattern": "ADmantX",
|
|
1277
1539
|
"addition_date": "2014/12/05",
|
|
@@ -1279,7 +1541,8 @@
|
|
|
1279
1541
|
"instances": [
|
|
1280
1542
|
"ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com"
|
|
1281
1543
|
]
|
|
1282
|
-
}
|
|
1544
|
+
}
|
|
1545
|
+
,
|
|
1283
1546
|
{
|
|
1284
1547
|
"pattern": "Facebot",
|
|
1285
1548
|
"url": "https://developers.facebook.com/docs/sharing/best-practices#crawl",
|
|
@@ -1287,14 +1550,16 @@
|
|
|
1287
1550
|
"instances": [
|
|
1288
1551
|
"Facebot/1.0"
|
|
1289
1552
|
]
|
|
1290
|
-
}
|
|
1553
|
+
}
|
|
1554
|
+
,
|
|
1291
1555
|
{
|
|
1292
|
-
"pattern": "OrangeBot",
|
|
1556
|
+
"pattern": "OrangeBot\\/",
|
|
1293
1557
|
"instances": [
|
|
1294
1558
|
"Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com"
|
|
1295
1559
|
],
|
|
1296
1560
|
"addition_date": "2015/01/12"
|
|
1297
|
-
}
|
|
1561
|
+
}
|
|
1562
|
+
,
|
|
1298
1563
|
{
|
|
1299
1564
|
"pattern": "memorybot",
|
|
1300
1565
|
"url": "http://mignify.com/bot.htm",
|
|
@@ -1302,7 +1567,8 @@
|
|
|
1302
1567
|
"Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)"
|
|
1303
1568
|
],
|
|
1304
1569
|
"addition_date": "2015/02/01"
|
|
1305
|
-
}
|
|
1570
|
+
}
|
|
1571
|
+
,
|
|
1306
1572
|
{
|
|
1307
1573
|
"pattern": "AdvBot",
|
|
1308
1574
|
"url": "http://advbot.net/bot.html",
|
|
@@ -1310,7 +1576,8 @@
|
|
|
1310
1576
|
"Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)"
|
|
1311
1577
|
],
|
|
1312
1578
|
"addition_date": "2015/02/01"
|
|
1313
|
-
}
|
|
1579
|
+
}
|
|
1580
|
+
,
|
|
1314
1581
|
{
|
|
1315
1582
|
"pattern": "MegaIndex",
|
|
1316
1583
|
"url": "https://www.megaindex.ru/?tab=linkAnalyze",
|
|
@@ -1318,7 +1585,8 @@
|
|
|
1318
1585
|
"Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)"
|
|
1319
1586
|
],
|
|
1320
1587
|
"addition_date": "2015/03/28"
|
|
1321
|
-
}
|
|
1588
|
+
}
|
|
1589
|
+
,
|
|
1322
1590
|
{
|
|
1323
1591
|
"pattern": "SemanticScholarBot",
|
|
1324
1592
|
"url": "http://s2.allenai.org/bot.html",
|
|
@@ -1326,7 +1594,8 @@
|
|
|
1326
1594
|
"SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)"
|
|
1327
1595
|
],
|
|
1328
1596
|
"addition_date": "2015/03/28"
|
|
1329
|
-
}
|
|
1597
|
+
}
|
|
1598
|
+
,
|
|
1330
1599
|
{
|
|
1331
1600
|
"pattern": "ltx71",
|
|
1332
1601
|
"url": "http://ltx71.com/",
|
|
@@ -1334,7 +1603,8 @@
|
|
|
1334
1603
|
"ltx71 - (http://ltx71.com/)"
|
|
1335
1604
|
],
|
|
1336
1605
|
"addition_date": "2015/04/04"
|
|
1337
|
-
}
|
|
1606
|
+
}
|
|
1607
|
+
,
|
|
1338
1608
|
{
|
|
1339
1609
|
"pattern": "nerdybot",
|
|
1340
1610
|
"url": "http://nerdybot.com/",
|
|
@@ -1342,7 +1612,8 @@
|
|
|
1342
1612
|
"nerdybot"
|
|
1343
1613
|
],
|
|
1344
1614
|
"addition_date": "2015/04/05"
|
|
1345
|
-
}
|
|
1615
|
+
}
|
|
1616
|
+
,
|
|
1346
1617
|
{
|
|
1347
1618
|
"pattern": "xovibot",
|
|
1348
1619
|
"url": "http://www.xovibot.net/",
|
|
@@ -1350,7 +1621,8 @@
|
|
|
1350
1621
|
"Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)"
|
|
1351
1622
|
],
|
|
1352
1623
|
"addition_date": "2015/04/05"
|
|
1353
|
-
}
|
|
1624
|
+
}
|
|
1625
|
+
,
|
|
1354
1626
|
{
|
|
1355
1627
|
"pattern": "BUbiNG",
|
|
1356
1628
|
"url": "http://law.di.unimi.it/BUbiNG.html",
|
|
@@ -1358,7 +1630,8 @@
|
|
|
1358
1630
|
"BUbiNG (+http://law.di.unimi.it/BUbiNG.html)"
|
|
1359
1631
|
],
|
|
1360
1632
|
"addition_date": "2015/04/06"
|
|
1361
|
-
}
|
|
1633
|
+
}
|
|
1634
|
+
,
|
|
1362
1635
|
{
|
|
1363
1636
|
"pattern": "Qwantify",
|
|
1364
1637
|
"url": "https://www.qwant.com/",
|
|
@@ -1366,7 +1639,8 @@
|
|
|
1366
1639
|
"Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*"
|
|
1367
1640
|
],
|
|
1368
1641
|
"addition_date": "2015/04/06"
|
|
1369
|
-
}
|
|
1642
|
+
}
|
|
1643
|
+
,
|
|
1370
1644
|
{
|
|
1371
1645
|
"pattern": "archive.org_bot",
|
|
1372
1646
|
"url": "http://www.archive.org/details/archive.org_bot",
|
|
@@ -1374,7 +1648,8 @@
|
|
|
1374
1648
|
"Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)"
|
|
1375
1649
|
],
|
|
1376
1650
|
"addition_date": "2015/04/14"
|
|
1377
|
-
}
|
|
1651
|
+
}
|
|
1652
|
+
,
|
|
1378
1653
|
{
|
|
1379
1654
|
"pattern": "Applebot",
|
|
1380
1655
|
"url": "http://www.apple.com/go/applebot",
|
|
@@ -1386,7 +1661,8 @@
|
|
|
1386
1661
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",
|
|
1387
1662
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)"
|
|
1388
1663
|
]
|
|
1389
|
-
}
|
|
1664
|
+
}
|
|
1665
|
+
,
|
|
1390
1666
|
{
|
|
1391
1667
|
"pattern": "TweetmemeBot",
|
|
1392
1668
|
"url": "http://datasift.com/bot.html",
|
|
@@ -1394,7 +1670,8 @@
|
|
|
1394
1670
|
"Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0"
|
|
1395
1671
|
],
|
|
1396
1672
|
"addition_date": "2015/04/15"
|
|
1397
|
-
}
|
|
1673
|
+
}
|
|
1674
|
+
,
|
|
1398
1675
|
{
|
|
1399
1676
|
"pattern": "crawler4j",
|
|
1400
1677
|
"url": "https://github.com/yasserg/crawler4j",
|
|
@@ -1402,7 +1679,8 @@
|
|
|
1402
1679
|
"crawler4j (http://code.google.com/p/crawler4j/)"
|
|
1403
1680
|
],
|
|
1404
1681
|
"addition_date": "2015/05/07"
|
|
1405
|
-
}
|
|
1682
|
+
}
|
|
1683
|
+
,
|
|
1406
1684
|
{
|
|
1407
1685
|
"pattern": "findxbot",
|
|
1408
1686
|
"url": "http://www.findxbot.com",
|
|
@@ -1410,7 +1688,8 @@
|
|
|
1410
1688
|
"Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)"
|
|
1411
1689
|
],
|
|
1412
1690
|
"addition_date": "2015/05/07"
|
|
1413
|
-
}
|
|
1691
|
+
}
|
|
1692
|
+
,
|
|
1414
1693
|
{
|
|
1415
1694
|
"pattern": "S[eE][mM]rushBot",
|
|
1416
1695
|
"url": "http://www.semrush.com/bot.html",
|
|
@@ -1419,7 +1698,8 @@
|
|
|
1419
1698
|
"SEMrushBot"
|
|
1420
1699
|
],
|
|
1421
1700
|
"addition_date": "2015/05/26"
|
|
1422
|
-
}
|
|
1701
|
+
}
|
|
1702
|
+
,
|
|
1423
1703
|
{
|
|
1424
1704
|
"pattern": "yoozBot",
|
|
1425
1705
|
"url": "http://yooz.ir",
|
|
@@ -1427,7 +1707,8 @@
|
|
|
1427
1707
|
"Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)"
|
|
1428
1708
|
],
|
|
1429
1709
|
"addition_date": "2015/05/26"
|
|
1430
|
-
}
|
|
1710
|
+
}
|
|
1711
|
+
,
|
|
1431
1712
|
{
|
|
1432
1713
|
"pattern": "lipperhey",
|
|
1433
1714
|
"url": "http://www.lipperhey.com/",
|
|
@@ -1438,15 +1719,22 @@
|
|
|
1438
1719
|
"Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)"
|
|
1439
1720
|
],
|
|
1440
1721
|
"addition_date": "2015/08/26"
|
|
1441
|
-
}
|
|
1722
|
+
}
|
|
1723
|
+
,
|
|
1442
1724
|
{
|
|
1443
|
-
"pattern": "Y!J
|
|
1444
|
-
"url": "
|
|
1725
|
+
"pattern": "Y!J",
|
|
1726
|
+
"url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6",
|
|
1445
1727
|
"instances": [
|
|
1446
|
-
"Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)"
|
|
1728
|
+
"Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)",
|
|
1729
|
+
"Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",
|
|
1730
|
+
"Y!J-PSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",
|
|
1731
|
+
"Y!J-BRW/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",
|
|
1732
|
+
"Mozilla/5.0 (iPhone; Y!J-BRY/YATSH crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",
|
|
1733
|
+
"Mozilla/5.0 (compatible; Y!J SearchMonkey/1.0 (Y!J-AGENT; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html))"
|
|
1447
1734
|
],
|
|
1448
1735
|
"addition_date": "2015/05/26"
|
|
1449
|
-
}
|
|
1736
|
+
}
|
|
1737
|
+
,
|
|
1450
1738
|
{
|
|
1451
1739
|
"pattern": "Domain Re-Animator Bot",
|
|
1452
1740
|
"url": "http://domainreanimator.com",
|
|
@@ -1454,7 +1742,8 @@
|
|
|
1454
1742
|
"Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com"
|
|
1455
1743
|
],
|
|
1456
1744
|
"addition_date": "2015/04/14"
|
|
1457
|
-
}
|
|
1745
|
+
}
|
|
1746
|
+
,
|
|
1458
1747
|
{
|
|
1459
1748
|
"pattern": "AddThis",
|
|
1460
1749
|
"url": "https://www.addthis.com",
|
|
@@ -1462,7 +1751,8 @@
|
|
|
1462
1751
|
"AddThis.com robot tech.support@clearspring.com"
|
|
1463
1752
|
],
|
|
1464
1753
|
"addition_date": "2015/06/02"
|
|
1465
|
-
}
|
|
1754
|
+
}
|
|
1755
|
+
,
|
|
1466
1756
|
{
|
|
1467
1757
|
"pattern": "Screaming Frog SEO Spider",
|
|
1468
1758
|
"url": "http://www.screamingfrog.co.uk/seo-spider",
|
|
@@ -1470,7 +1760,8 @@
|
|
|
1470
1760
|
"Screaming Frog SEO Spider/5.1"
|
|
1471
1761
|
],
|
|
1472
1762
|
"addition_date": "2016/01/08"
|
|
1473
|
-
}
|
|
1763
|
+
}
|
|
1764
|
+
,
|
|
1474
1765
|
{
|
|
1475
1766
|
"pattern": "MetaURI",
|
|
1476
1767
|
"url": "http://www.useragentstring.com/MetaURI_id_17683.php",
|
|
@@ -1478,7 +1769,8 @@
|
|
|
1478
1769
|
"MetaURI API/2.0 +metauri.com"
|
|
1479
1770
|
],
|
|
1480
1771
|
"addition_date": "2016/01/02"
|
|
1481
|
-
}
|
|
1772
|
+
}
|
|
1773
|
+
,
|
|
1482
1774
|
{
|
|
1483
1775
|
"pattern": "Scrapy",
|
|
1484
1776
|
"url": "http://scrapy.org/",
|
|
@@ -1486,7 +1778,8 @@
|
|
|
1486
1778
|
"Scrapy/1.0.3 (+http://scrapy.org)"
|
|
1487
1779
|
],
|
|
1488
1780
|
"addition_date": "2016/01/02"
|
|
1489
|
-
}
|
|
1781
|
+
}
|
|
1782
|
+
,
|
|
1490
1783
|
{
|
|
1491
1784
|
"pattern": "Livelap[bB]ot",
|
|
1492
1785
|
"url": "http://site.livelap.com/crawler",
|
|
@@ -1495,7 +1788,8 @@
|
|
|
1495
1788
|
"Livelapbot/0.1"
|
|
1496
1789
|
],
|
|
1497
1790
|
"addition_date": "2016/01/02"
|
|
1498
|
-
}
|
|
1791
|
+
}
|
|
1792
|
+
,
|
|
1499
1793
|
{
|
|
1500
1794
|
"pattern": "OpenHoseBot",
|
|
1501
1795
|
"url": "http://www.openhose.org/bot.html",
|
|
@@ -1503,7 +1797,8 @@
|
|
|
1503
1797
|
"Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)"
|
|
1504
1798
|
],
|
|
1505
1799
|
"addition_date": "2016/01/02"
|
|
1506
|
-
}
|
|
1800
|
+
}
|
|
1801
|
+
,
|
|
1507
1802
|
{
|
|
1508
1803
|
"pattern": "CapsuleChecker",
|
|
1509
1804
|
"url": "http://www.capsulink.com/about",
|
|
@@ -1511,7 +1806,8 @@
|
|
|
1511
1806
|
"CapsuleChecker (http://www.capsulink.com/)"
|
|
1512
1807
|
],
|
|
1513
1808
|
"addition_date": "2016/01/02"
|
|
1514
|
-
}
|
|
1809
|
+
}
|
|
1810
|
+
,
|
|
1515
1811
|
{
|
|
1516
1812
|
"pattern": "collection@infegy.com",
|
|
1517
1813
|
"url": "http://infegy.com/",
|
|
@@ -1519,7 +1815,8 @@
|
|
|
1519
1815
|
"Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com"
|
|
1520
1816
|
],
|
|
1521
1817
|
"addition_date": "2016/01/03"
|
|
1522
|
-
}
|
|
1818
|
+
}
|
|
1819
|
+
,
|
|
1523
1820
|
{
|
|
1524
1821
|
"pattern": "IstellaBot",
|
|
1525
1822
|
"url": "http://www.tiscali.it/",
|
|
@@ -1527,7 +1824,8 @@
|
|
|
1527
1824
|
"Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)"
|
|
1528
1825
|
],
|
|
1529
1826
|
"addition_date": "2016/01/09"
|
|
1530
|
-
}
|
|
1827
|
+
}
|
|
1828
|
+
,
|
|
1531
1829
|
{
|
|
1532
1830
|
"pattern": "DeuSu\\/",
|
|
1533
1831
|
"addition_date": "2016/01/23",
|
|
@@ -1536,12 +1834,14 @@
|
|
|
1536
1834
|
"Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)",
|
|
1537
1835
|
"Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)"
|
|
1538
1836
|
]
|
|
1539
|
-
}
|
|
1837
|
+
}
|
|
1838
|
+
,
|
|
1540
1839
|
{
|
|
1541
1840
|
"pattern": "betaBot",
|
|
1542
1841
|
"addition_date": "2016/01/23",
|
|
1543
1842
|
"instances": []
|
|
1544
|
-
}
|
|
1843
|
+
}
|
|
1844
|
+
,
|
|
1545
1845
|
{
|
|
1546
1846
|
"pattern": "Cliqzbot\\/",
|
|
1547
1847
|
"addition_date": "2016/01/23",
|
|
@@ -1552,7 +1852,8 @@
|
|
|
1552
1852
|
"Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)",
|
|
1553
1853
|
"Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)"
|
|
1554
1854
|
]
|
|
1555
|
-
}
|
|
1855
|
+
}
|
|
1856
|
+
,
|
|
1556
1857
|
{
|
|
1557
1858
|
"pattern": "MojeekBot\\/",
|
|
1558
1859
|
"addition_date": "2016/01/23",
|
|
@@ -1565,7 +1866,8 @@
|
|
|
1565
1866
|
"Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)",
|
|
1566
1867
|
"Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)"
|
|
1567
1868
|
]
|
|
1568
|
-
}
|
|
1869
|
+
}
|
|
1870
|
+
,
|
|
1569
1871
|
{
|
|
1570
1872
|
"pattern": "netEstate NE Crawler",
|
|
1571
1873
|
"addition_date": "2016/01/23",
|
|
@@ -1574,7 +1876,8 @@
|
|
|
1574
1876
|
"netEstate NE Crawler (+http://www.sengine.info/)",
|
|
1575
1877
|
"netEstate NE Crawler (+http://www.website-datenbank.de/)"
|
|
1576
1878
|
]
|
|
1577
|
-
}
|
|
1879
|
+
}
|
|
1880
|
+
,
|
|
1578
1881
|
{
|
|
1579
1882
|
"pattern": "SafeSearch microdata crawler",
|
|
1580
1883
|
"addition_date": "2016/01/23",
|
|
@@ -1582,7 +1885,8 @@
|
|
|
1582
1885
|
"instances": [
|
|
1583
1886
|
"SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)"
|
|
1584
1887
|
]
|
|
1585
|
-
}
|
|
1888
|
+
}
|
|
1889
|
+
,
|
|
1586
1890
|
{
|
|
1587
1891
|
"pattern": "Gluten Free Crawler\\/",
|
|
1588
1892
|
"addition_date": "2016/01/23",
|
|
@@ -1590,7 +1894,8 @@
|
|
|
1590
1894
|
"instances": [
|
|
1591
1895
|
"Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)"
|
|
1592
1896
|
]
|
|
1593
|
-
}
|
|
1897
|
+
}
|
|
1898
|
+
,
|
|
1594
1899
|
{
|
|
1595
1900
|
"pattern": "Sonic",
|
|
1596
1901
|
"addition_date": "2016/02/08",
|
|
@@ -1600,7 +1905,8 @@
|
|
|
1600
1905
|
"Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)",
|
|
1601
1906
|
"Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)"
|
|
1602
1907
|
]
|
|
1603
|
-
}
|
|
1908
|
+
}
|
|
1909
|
+
,
|
|
1604
1910
|
{
|
|
1605
1911
|
"pattern": "Sysomos",
|
|
1606
1912
|
"addition_date": "2016/02/08",
|
|
@@ -1608,13 +1914,15 @@
|
|
|
1608
1914
|
"instances": [
|
|
1609
1915
|
"Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)"
|
|
1610
1916
|
]
|
|
1611
|
-
}
|
|
1917
|
+
}
|
|
1918
|
+
,
|
|
1612
1919
|
{
|
|
1613
1920
|
"pattern": "Trove",
|
|
1614
1921
|
"addition_date": "2016/02/08",
|
|
1615
1922
|
"url": "http://www.trove.com",
|
|
1616
1923
|
"instances": []
|
|
1617
|
-
}
|
|
1924
|
+
}
|
|
1925
|
+
,
|
|
1618
1926
|
{
|
|
1619
1927
|
"pattern": "deadlinkchecker",
|
|
1620
1928
|
"addition_date": "2016/02/08",
|
|
@@ -1624,7 +1932,8 @@
|
|
|
1624
1932
|
"www.deadlinkchecker.com XMLHTTP/1.0",
|
|
1625
1933
|
"www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36"
|
|
1626
1934
|
]
|
|
1627
|
-
}
|
|
1935
|
+
}
|
|
1936
|
+
,
|
|
1628
1937
|
{
|
|
1629
1938
|
"pattern": "Slack-ImgProxy",
|
|
1630
1939
|
"addition_date": "2016/04/25",
|
|
@@ -1637,7 +1946,8 @@
|
|
|
1637
1946
|
"Slack-ImgProxy 1.138 (+https://api.slack.com/robots)",
|
|
1638
1947
|
"Slack-ImgProxy 149 (+https://api.slack.com/robots)"
|
|
1639
1948
|
]
|
|
1640
|
-
}
|
|
1949
|
+
}
|
|
1950
|
+
,
|
|
1641
1951
|
{
|
|
1642
1952
|
"pattern": "Embedly",
|
|
1643
1953
|
"addition_date": "2016/04/25",
|
|
@@ -1647,7 +1957,8 @@
|
|
|
1647
1957
|
"Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)",
|
|
1648
1958
|
"Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)"
|
|
1649
1959
|
]
|
|
1650
|
-
}
|
|
1960
|
+
}
|
|
1961
|
+
,
|
|
1651
1962
|
{
|
|
1652
1963
|
"pattern": "RankActiveLinkBot",
|
|
1653
1964
|
"addition_date": "2016/06/20",
|
|
@@ -1655,7 +1966,8 @@
|
|
|
1655
1966
|
"instances": [
|
|
1656
1967
|
"Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)"
|
|
1657
1968
|
]
|
|
1658
|
-
}
|
|
1969
|
+
}
|
|
1970
|
+
,
|
|
1659
1971
|
{
|
|
1660
1972
|
"pattern": "iskanie",
|
|
1661
1973
|
"addition_date": "2016/09/02",
|
|
@@ -1663,7 +1975,8 @@
|
|
|
1663
1975
|
"instances": [
|
|
1664
1976
|
"iskanie (+http://www.iskanie.com)"
|
|
1665
1977
|
]
|
|
1666
|
-
}
|
|
1978
|
+
}
|
|
1979
|
+
,
|
|
1667
1980
|
{
|
|
1668
1981
|
"pattern": "SafeDNSBot",
|
|
1669
1982
|
"addition_date": "2016/09/10",
|
|
@@ -1671,14 +1984,16 @@
|
|
|
1671
1984
|
"instances": [
|
|
1672
1985
|
"SafeDNSBot (https://www.safedns.com/searchbot)"
|
|
1673
1986
|
]
|
|
1674
|
-
}
|
|
1987
|
+
}
|
|
1988
|
+
,
|
|
1675
1989
|
{
|
|
1676
1990
|
"pattern": "SkypeUriPreview",
|
|
1677
1991
|
"addition_date": "2016/10/10",
|
|
1678
1992
|
"instances": [
|
|
1679
1993
|
"Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5"
|
|
1680
1994
|
]
|
|
1681
|
-
}
|
|
1995
|
+
}
|
|
1996
|
+
,
|
|
1682
1997
|
{
|
|
1683
1998
|
"pattern": "Veoozbot",
|
|
1684
1999
|
"addition_date": "2016/11/03",
|
|
@@ -1686,7 +2001,8 @@
|
|
|
1686
2001
|
"instances": [
|
|
1687
2002
|
"Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)"
|
|
1688
2003
|
]
|
|
1689
|
-
}
|
|
2004
|
+
}
|
|
2005
|
+
,
|
|
1690
2006
|
{
|
|
1691
2007
|
"pattern": "Slackbot",
|
|
1692
2008
|
"addition_date": "2016/11/03",
|
|
@@ -1695,7 +2011,8 @@
|
|
|
1695
2011
|
"Slackbot-LinkExpanding (+https://api.slack.com/robots)",
|
|
1696
2012
|
"Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"
|
|
1697
2013
|
]
|
|
1698
|
-
}
|
|
2014
|
+
}
|
|
2015
|
+
,
|
|
1699
2016
|
{
|
|
1700
2017
|
"pattern": "redditbot",
|
|
1701
2018
|
"addition_date": "2016/11/03",
|
|
@@ -1703,7 +2020,8 @@
|
|
|
1703
2020
|
"instances": [
|
|
1704
2021
|
"Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)"
|
|
1705
2022
|
]
|
|
1706
|
-
}
|
|
2023
|
+
}
|
|
2024
|
+
,
|
|
1707
2025
|
{
|
|
1708
2026
|
"pattern": "datagnionbot",
|
|
1709
2027
|
"addition_date": "2016/11/03",
|
|
@@ -1711,7 +2029,8 @@
|
|
|
1711
2029
|
"instances": [
|
|
1712
2030
|
"datagnionbot (+http://www.datagnion.com/bot.html)"
|
|
1713
2031
|
]
|
|
1714
|
-
}
|
|
2032
|
+
}
|
|
2033
|
+
,
|
|
1715
2034
|
{
|
|
1716
2035
|
"pattern": "Google-Adwords-Instant",
|
|
1717
2036
|
"addition_date": "2016/11/03",
|
|
@@ -1719,7 +2038,8 @@
|
|
|
1719
2038
|
"instances": [
|
|
1720
2039
|
"Google-Adwords-Instant (+http://www.google.com/adsbot.html)"
|
|
1721
2040
|
]
|
|
1722
|
-
}
|
|
2041
|
+
}
|
|
2042
|
+
,
|
|
1723
2043
|
{
|
|
1724
2044
|
"pattern": "adbeat_bot",
|
|
1725
2045
|
"addition_date": "2016/11/04",
|
|
@@ -1727,7 +2047,8 @@
|
|
|
1727
2047
|
"Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)",
|
|
1728
2048
|
"adbeat_bot"
|
|
1729
2049
|
]
|
|
1730
|
-
}
|
|
2050
|
+
}
|
|
2051
|
+
,
|
|
1731
2052
|
{
|
|
1732
2053
|
"pattern": "WhatsApp",
|
|
1733
2054
|
"addition_date": "2016/11/15",
|
|
@@ -1750,14 +2071,16 @@
|
|
|
1750
2071
|
"WhatsApp/2.16.42 A",
|
|
1751
2072
|
"WhatsApp/2.16.57 A"
|
|
1752
2073
|
]
|
|
1753
|
-
}
|
|
2074
|
+
}
|
|
2075
|
+
,
|
|
1754
2076
|
{
|
|
1755
2077
|
"pattern": "contxbot",
|
|
1756
2078
|
"addition_date": "2017/02/25",
|
|
1757
2079
|
"instances": [
|
|
1758
2080
|
"Mozilla/5.0 (compatible;contxbot/1.0)"
|
|
1759
2081
|
]
|
|
1760
|
-
}
|
|
2082
|
+
}
|
|
2083
|
+
,
|
|
1761
2084
|
{
|
|
1762
2085
|
"pattern": "pinterest",
|
|
1763
2086
|
"addition_date": "2017/03/03",
|
|
@@ -1765,7 +2088,8 @@
|
|
|
1765
2088
|
"Pinterest/0.2 (+http://www.pinterest.com/bot.html)"
|
|
1766
2089
|
],
|
|
1767
2090
|
"url": "http://www.pinterest.com/bot.html"
|
|
1768
|
-
}
|
|
2091
|
+
}
|
|
2092
|
+
,
|
|
1769
2093
|
{
|
|
1770
2094
|
"pattern": "electricmonk",
|
|
1771
2095
|
"addition_date": "2017/03/04",
|
|
@@ -1773,7 +2097,8 @@
|
|
|
1773
2097
|
"Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)"
|
|
1774
2098
|
],
|
|
1775
2099
|
"url": "https://www.duedil.com/our-crawler/"
|
|
1776
|
-
}
|
|
2100
|
+
}
|
|
2101
|
+
,
|
|
1777
2102
|
{
|
|
1778
2103
|
"pattern": "GarlikCrawler",
|
|
1779
2104
|
"addition_date": "2017/03/18",
|
|
@@ -1781,7 +2106,8 @@
|
|
|
1781
2106
|
"GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)"
|
|
1782
2107
|
],
|
|
1783
2108
|
"url": "http://garlik.com/"
|
|
1784
|
-
}
|
|
2109
|
+
}
|
|
2110
|
+
,
|
|
1785
2111
|
{
|
|
1786
2112
|
"pattern": "BingPreview\\/",
|
|
1787
2113
|
"addition_date": "2017/04/23",
|
|
@@ -1793,7 +2119,8 @@
|
|
|
1793
2119
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)",
|
|
1794
2120
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b"
|
|
1795
2121
|
]
|
|
1796
|
-
}
|
|
2122
|
+
}
|
|
2123
|
+
,
|
|
1797
2124
|
{
|
|
1798
2125
|
"pattern": "vebidoobot",
|
|
1799
2126
|
"addition_date": "2017/05/08",
|
|
@@ -1801,7 +2128,8 @@
|
|
|
1801
2128
|
"Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/"
|
|
1802
2129
|
],
|
|
1803
2130
|
"url": "https://blog.vebidoo.de/vebidoobot/"
|
|
1804
|
-
}
|
|
2131
|
+
}
|
|
2132
|
+
,
|
|
1805
2133
|
{
|
|
1806
2134
|
"pattern": "FemtosearchBot",
|
|
1807
2135
|
"addition_date": "2017/05/16",
|
|
@@ -1809,7 +2137,8 @@
|
|
|
1809
2137
|
"Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)"
|
|
1810
2138
|
],
|
|
1811
2139
|
"url": "http://femtosearch.com"
|
|
1812
|
-
}
|
|
2140
|
+
}
|
|
2141
|
+
,
|
|
1813
2142
|
{
|
|
1814
2143
|
"pattern": "Yahoo Link Preview",
|
|
1815
2144
|
"addition_date": "2017/06/28",
|
|
@@ -1817,7 +2146,8 @@
|
|
|
1817
2146
|
"Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)"
|
|
1818
2147
|
],
|
|
1819
2148
|
"url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html"
|
|
1820
|
-
}
|
|
2149
|
+
}
|
|
2150
|
+
,
|
|
1821
2151
|
{
|
|
1822
2152
|
"pattern": "MetaJobBot",
|
|
1823
2153
|
"addition_date": "2017/08/16",
|
|
@@ -1825,7 +2155,8 @@
|
|
|
1825
2155
|
"Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)"
|
|
1826
2156
|
],
|
|
1827
2157
|
"url": "http://www.metajob.de/the/crawler"
|
|
1828
|
-
}
|
|
2158
|
+
}
|
|
2159
|
+
,
|
|
1829
2160
|
{
|
|
1830
2161
|
"pattern": "DomainStatsBot",
|
|
1831
2162
|
"addition_date": "2017/08/16",
|
|
@@ -1833,7 +2164,8 @@
|
|
|
1833
2164
|
"DomainStatsBot/1.0 (http://domainstats.io/our-bot)"
|
|
1834
2165
|
],
|
|
1835
2166
|
"url": "http://domainstats.io/our-bot"
|
|
1836
|
-
}
|
|
2167
|
+
}
|
|
2168
|
+
,
|
|
1837
2169
|
{
|
|
1838
2170
|
"pattern": "mindUpBot",
|
|
1839
2171
|
"addition_date": "2017/08/16",
|
|
@@ -1841,15 +2173,17 @@
|
|
|
1841
2173
|
"mindUpBot (datenbutler.de)"
|
|
1842
2174
|
],
|
|
1843
2175
|
"url": "http://www.datenbutler.de/"
|
|
1844
|
-
}
|
|
2176
|
+
}
|
|
2177
|
+
,
|
|
1845
2178
|
{
|
|
1846
|
-
"pattern": "Daum",
|
|
2179
|
+
"pattern": "Daum\\/",
|
|
1847
2180
|
"addition_date": "2017/08/16",
|
|
1848
2181
|
"instances": [
|
|
1849
2182
|
"Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)"
|
|
1850
2183
|
],
|
|
1851
2184
|
"url": "http://cs.daum.net/faq/15/4118.html?faqId=28966"
|
|
1852
|
-
}
|
|
2185
|
+
}
|
|
2186
|
+
,
|
|
1853
2187
|
{
|
|
1854
2188
|
"pattern": "Jugendschutzprogramm-Crawler",
|
|
1855
2189
|
"addition_date": "2017/08/16",
|
|
@@ -1857,7 +2191,8 @@
|
|
|
1857
2191
|
"Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de"
|
|
1858
2192
|
],
|
|
1859
2193
|
"url": "http://www.jugendschutzprogramm.de"
|
|
1860
|
-
}
|
|
2194
|
+
}
|
|
2195
|
+
,
|
|
1861
2196
|
{
|
|
1862
2197
|
"pattern": "Xenu Link Sleuth",
|
|
1863
2198
|
"addition_date": "2017/08/19",
|
|
@@ -1865,7 +2200,8 @@
|
|
|
1865
2200
|
"Xenu Link Sleuth/1.3.8"
|
|
1866
2201
|
],
|
|
1867
2202
|
"url": "http://home.snafu.de/tilman/xenulink.html"
|
|
1868
|
-
}
|
|
2203
|
+
}
|
|
2204
|
+
,
|
|
1869
2205
|
{
|
|
1870
2206
|
"pattern": "Pcore-HTTP",
|
|
1871
2207
|
"addition_date": "2017/08/19",
|
|
@@ -1873,7 +2209,8 @@
|
|
|
1873
2209
|
"Pcore-HTTP/v0.40.3"
|
|
1874
2210
|
],
|
|
1875
2211
|
"url": "https://bitbucket.org/softvisio/pcore/overview"
|
|
1876
|
-
}
|
|
2212
|
+
}
|
|
2213
|
+
,
|
|
1877
2214
|
{
|
|
1878
2215
|
"pattern": "moatbot",
|
|
1879
2216
|
"addition_date": "2017/09/16",
|
|
@@ -1882,7 +2219,8 @@
|
|
|
1882
2219
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot"
|
|
1883
2220
|
],
|
|
1884
2221
|
"url": "https://moat.com"
|
|
1885
|
-
}
|
|
2222
|
+
}
|
|
2223
|
+
,
|
|
1886
2224
|
{
|
|
1887
2225
|
"pattern": "KosmioBot",
|
|
1888
2226
|
"addition_date": "2017/09/16",
|
|
@@ -1890,15 +2228,18 @@
|
|
|
1890
2228
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)"
|
|
1891
2229
|
],
|
|
1892
2230
|
"url": "http://kosm.io/bot.html"
|
|
1893
|
-
}
|
|
2231
|
+
}
|
|
2232
|
+
,
|
|
1894
2233
|
{
|
|
1895
|
-
"pattern": "
|
|
2234
|
+
"pattern": "pingdom",
|
|
1896
2235
|
"addition_date": "2017/09/16",
|
|
1897
2236
|
"instances": [
|
|
1898
|
-
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)"
|
|
2237
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)",
|
|
2238
|
+
"Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)"
|
|
1899
2239
|
],
|
|
1900
2240
|
"url": "http://www.pingdom.com"
|
|
1901
|
-
}
|
|
2241
|
+
}
|
|
2242
|
+
,
|
|
1902
2243
|
{
|
|
1903
2244
|
"pattern": "PhantomJS",
|
|
1904
2245
|
"addition_date": "2017/09/18",
|
|
@@ -1906,7 +2247,8 @@
|
|
|
1906
2247
|
"Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)"
|
|
1907
2248
|
],
|
|
1908
2249
|
"url": "http://phantomjs.org/"
|
|
1909
|
-
}
|
|
2250
|
+
}
|
|
2251
|
+
,
|
|
1910
2252
|
{
|
|
1911
2253
|
"pattern": "Gowikibot",
|
|
1912
2254
|
"addition_date": "2017/10/26",
|
|
@@ -1914,7 +2256,8 @@
|
|
|
1914
2256
|
"Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)"
|
|
1915
2257
|
],
|
|
1916
2258
|
"url": "http://www.gowikibot.com"
|
|
1917
|
-
}
|
|
2259
|
+
}
|
|
2260
|
+
,
|
|
1918
2261
|
{
|
|
1919
2262
|
"pattern": "PiplBot",
|
|
1920
2263
|
"addition_date": "2017/10/30",
|
|
@@ -1922,7 +2265,8 @@
|
|
|
1922
2265
|
"Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)"
|
|
1923
2266
|
],
|
|
1924
2267
|
"url": "http://www.pipl.com/bot/"
|
|
1925
|
-
}
|
|
2268
|
+
}
|
|
2269
|
+
,
|
|
1926
2270
|
{
|
|
1927
2271
|
"pattern": "Discordbot",
|
|
1928
2272
|
"addition_date": "2017/09/22",
|
|
@@ -1930,21 +2274,16 @@
|
|
|
1930
2274
|
"instances": [
|
|
1931
2275
|
"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"
|
|
1932
2276
|
]
|
|
1933
|
-
}
|
|
2277
|
+
}
|
|
2278
|
+
,
|
|
1934
2279
|
{
|
|
1935
2280
|
"pattern": "TelegramBot",
|
|
1936
2281
|
"addition_date": "2017/10/01",
|
|
1937
2282
|
"instances": [
|
|
1938
2283
|
"TelegramBot (like TwitterBot)"
|
|
1939
2284
|
]
|
|
1940
|
-
}
|
|
1941
|
-
|
|
1942
|
-
"pattern": "InfoPath.2",
|
|
1943
|
-
"addition_date": "2017/10/07",
|
|
1944
|
-
"instances": [
|
|
1945
|
-
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; InfoPath.2)"
|
|
1946
|
-
]
|
|
1947
|
-
},
|
|
2285
|
+
}
|
|
2286
|
+
,
|
|
1948
2287
|
{
|
|
1949
2288
|
"pattern": "Jetslide",
|
|
1950
2289
|
"addition_date": "2017/09/27",
|
|
@@ -1952,7 +2291,8 @@
|
|
|
1952
2291
|
"instances": [
|
|
1953
2292
|
"Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)"
|
|
1954
2293
|
]
|
|
1955
|
-
}
|
|
2294
|
+
}
|
|
2295
|
+
,
|
|
1956
2296
|
{
|
|
1957
2297
|
"pattern": "newsharecounts",
|
|
1958
2298
|
"addition_date": "2017/09/30",
|
|
@@ -1960,7 +2300,8 @@
|
|
|
1960
2300
|
"instances": [
|
|
1961
2301
|
"Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)"
|
|
1962
2302
|
]
|
|
1963
|
-
}
|
|
2303
|
+
}
|
|
2304
|
+
,
|
|
1964
2305
|
{
|
|
1965
2306
|
"pattern": "James BOT",
|
|
1966
2307
|
"addition_date": "2017/10/12",
|
|
@@ -1968,23 +2309,28 @@
|
|
|
1968
2309
|
"instances": [
|
|
1969
2310
|
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html"
|
|
1970
2311
|
]
|
|
1971
|
-
}
|
|
2312
|
+
}
|
|
2313
|
+
,
|
|
1972
2314
|
{
|
|
1973
2315
|
"pattern": "Barkrowler",
|
|
1974
2316
|
"addition_date": "2017/10/09",
|
|
1975
2317
|
"url": "http://www.exensa.com/crawl",
|
|
1976
2318
|
"instances": [
|
|
1977
|
-
"Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG"
|
|
2319
|
+
"Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG",
|
|
2320
|
+
"Barkrowler/0.7 (+http://www.exensa.com/crawl)"
|
|
1978
2321
|
]
|
|
1979
|
-
}
|
|
2322
|
+
}
|
|
2323
|
+
,
|
|
1980
2324
|
{
|
|
1981
|
-
"pattern": "TinEye
|
|
2325
|
+
"pattern": "TinEye",
|
|
1982
2326
|
"addition_date": "2017/10/14",
|
|
1983
2327
|
"url": "http://www.tineye.com/crawler.html",
|
|
1984
2328
|
"instances": [
|
|
1985
|
-
"Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)"
|
|
2329
|
+
"Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)",
|
|
2330
|
+
"TinEye/1.1 (http://tineye.com/crawler.html)"
|
|
1986
2331
|
]
|
|
1987
|
-
}
|
|
2332
|
+
}
|
|
2333
|
+
,
|
|
1988
2334
|
{
|
|
1989
2335
|
"pattern": "SocialRankIOBot",
|
|
1990
2336
|
"addition_date": "2017/10/19",
|
|
@@ -1992,7 +2338,8 @@
|
|
|
1992
2338
|
"instances": [
|
|
1993
2339
|
"SocialRankIOBot; http://socialrank.io/about"
|
|
1994
2340
|
]
|
|
1995
|
-
}
|
|
2341
|
+
}
|
|
2342
|
+
,
|
|
1996
2343
|
{
|
|
1997
2344
|
"pattern": "trendictionbot",
|
|
1998
2345
|
"addition_date": "2017/10/30",
|
|
@@ -2000,14 +2347,16 @@
|
|
|
2000
2347
|
"instances": [
|
|
2001
2348
|
"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11"
|
|
2002
2349
|
]
|
|
2003
|
-
}
|
|
2350
|
+
}
|
|
2351
|
+
,
|
|
2004
2352
|
{
|
|
2005
2353
|
"pattern": "Ocarinabot",
|
|
2006
2354
|
"addition_date": "2017/09/27",
|
|
2007
2355
|
"instances": [
|
|
2008
2356
|
"Ocarinabot"
|
|
2009
2357
|
]
|
|
2010
|
-
}
|
|
2358
|
+
}
|
|
2359
|
+
,
|
|
2011
2360
|
{
|
|
2012
2361
|
"pattern": "epicbot",
|
|
2013
2362
|
"addition_date": "2017/10/31",
|
|
@@ -2015,7 +2364,8 @@
|
|
|
2015
2364
|
"instances": [
|
|
2016
2365
|
"Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)"
|
|
2017
2366
|
]
|
|
2018
|
-
}
|
|
2367
|
+
}
|
|
2368
|
+
,
|
|
2019
2369
|
{
|
|
2020
2370
|
"pattern": "Primalbot",
|
|
2021
2371
|
"addition_date": "2017/09/27",
|
|
@@ -2023,7 +2373,8 @@
|
|
|
2023
2373
|
"instances": [
|
|
2024
2374
|
"Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)"
|
|
2025
2375
|
]
|
|
2026
|
-
}
|
|
2376
|
+
}
|
|
2377
|
+
,
|
|
2027
2378
|
{
|
|
2028
2379
|
"pattern": "DuckDuckGo-Favicons-Bot",
|
|
2029
2380
|
"addition_date": "2017/10/06",
|
|
@@ -2031,7 +2382,8 @@
|
|
|
2031
2382
|
"instances": [
|
|
2032
2383
|
"Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)"
|
|
2033
2384
|
]
|
|
2034
|
-
}
|
|
2385
|
+
}
|
|
2386
|
+
,
|
|
2035
2387
|
{
|
|
2036
2388
|
"pattern": "GnowitNewsbot",
|
|
2037
2389
|
"addition_date": "2017/10/30",
|
|
@@ -2039,7 +2391,8 @@
|
|
|
2039
2391
|
"instances": [
|
|
2040
2392
|
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com"
|
|
2041
2393
|
]
|
|
2042
|
-
}
|
|
2394
|
+
}
|
|
2395
|
+
,
|
|
2043
2396
|
{
|
|
2044
2397
|
"pattern": "Leikibot",
|
|
2045
2398
|
"addition_date": "2017/09/24",
|
|
@@ -2047,22 +2400,25 @@
|
|
|
2047
2400
|
"instances": [
|
|
2048
2401
|
"Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)"
|
|
2049
2402
|
]
|
|
2050
|
-
}
|
|
2403
|
+
}
|
|
2404
|
+
,
|
|
2051
2405
|
{
|
|
2052
2406
|
"pattern": "LinkArchiver",
|
|
2053
2407
|
"addition_date": "2017/09/24",
|
|
2054
2408
|
"instances": [
|
|
2055
2409
|
"@LinkArchiver twitter bot"
|
|
2056
2410
|
]
|
|
2057
|
-
}
|
|
2411
|
+
}
|
|
2412
|
+
,
|
|
2058
2413
|
{
|
|
2059
|
-
"pattern": "YaK",
|
|
2414
|
+
"pattern": "YaK\\/",
|
|
2060
2415
|
"addition_date": "2017/09/25",
|
|
2061
2416
|
"url": "http://linkfluence.com",
|
|
2062
2417
|
"instances": [
|
|
2063
2418
|
"Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)"
|
|
2064
2419
|
]
|
|
2065
|
-
}
|
|
2420
|
+
}
|
|
2421
|
+
,
|
|
2066
2422
|
{
|
|
2067
2423
|
"pattern": "PaperLiBot",
|
|
2068
2424
|
"addition_date": "2017/09/25",
|
|
@@ -2070,7 +2426,8 @@
|
|
|
2070
2426
|
"instances": [
|
|
2071
2427
|
"Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)"
|
|
2072
2428
|
]
|
|
2073
|
-
}
|
|
2429
|
+
}
|
|
2430
|
+
,
|
|
2074
2431
|
{
|
|
2075
2432
|
"pattern": "Digg Deeper",
|
|
2076
2433
|
"addition_date": "2017/09/26",
|
|
@@ -2078,14 +2435,16 @@
|
|
|
2078
2435
|
"instances": [
|
|
2079
2436
|
"Digg Deeper/v1 (http://digg.com/about)"
|
|
2080
2437
|
]
|
|
2081
|
-
}
|
|
2438
|
+
}
|
|
2439
|
+
,
|
|
2082
2440
|
{
|
|
2083
2441
|
"pattern": "dcrawl",
|
|
2084
2442
|
"addition_date": "2017/09/22",
|
|
2085
2443
|
"instances": [
|
|
2086
2444
|
"dcrawl/1.0"
|
|
2087
2445
|
]
|
|
2088
|
-
}
|
|
2446
|
+
}
|
|
2447
|
+
,
|
|
2089
2448
|
{
|
|
2090
2449
|
"pattern": "Snacktory",
|
|
2091
2450
|
"addition_date": "2017/09/23",
|
|
@@ -2093,7 +2452,8 @@
|
|
|
2093
2452
|
"instances": [
|
|
2094
2453
|
"Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)"
|
|
2095
2454
|
]
|
|
2096
|
-
}
|
|
2455
|
+
}
|
|
2456
|
+
,
|
|
2097
2457
|
{
|
|
2098
2458
|
"pattern": "AndersPinkBot",
|
|
2099
2459
|
"addition_date": "2017/09/24",
|
|
@@ -2101,14 +2461,16 @@
|
|
|
2101
2461
|
"instances": [
|
|
2102
2462
|
"Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)"
|
|
2103
2463
|
]
|
|
2104
|
-
}
|
|
2464
|
+
}
|
|
2465
|
+
,
|
|
2105
2466
|
{
|
|
2106
2467
|
"pattern": "Fyrebot",
|
|
2107
2468
|
"addition_date": "2017/09/22",
|
|
2108
2469
|
"instances": [
|
|
2109
2470
|
"Fyrebot/1.0"
|
|
2110
|
-
|
|
2111
|
-
}
|
|
2471
|
+
]
|
|
2472
|
+
}
|
|
2473
|
+
,
|
|
2112
2474
|
{
|
|
2113
2475
|
"pattern": "EveryoneSocialBot",
|
|
2114
2476
|
"addition_date": "2017/09/22",
|
|
@@ -2116,7 +2478,8 @@
|
|
|
2116
2478
|
"instances": [
|
|
2117
2479
|
"Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)"
|
|
2118
2480
|
]
|
|
2119
|
-
}
|
|
2481
|
+
}
|
|
2482
|
+
,
|
|
2120
2483
|
{
|
|
2121
2484
|
"pattern": "Mediatoolkitbot",
|
|
2122
2485
|
"addition_date": "2017/10/06",
|
|
@@ -2124,14 +2487,16 @@
|
|
|
2124
2487
|
"instances": [
|
|
2125
2488
|
"Mediatoolkitbot (complaints@mediatoolkit.com)"
|
|
2126
2489
|
]
|
|
2127
|
-
}
|
|
2490
|
+
}
|
|
2491
|
+
,
|
|
2128
2492
|
{
|
|
2129
2493
|
"pattern": "Luminator-robots",
|
|
2130
2494
|
"addition_date": "2017/09/22",
|
|
2131
2495
|
"instances": [
|
|
2132
2496
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0"
|
|
2133
2497
|
]
|
|
2134
|
-
}
|
|
2498
|
+
}
|
|
2499
|
+
,
|
|
2135
2500
|
{
|
|
2136
2501
|
"pattern": "ExtLinksBot",
|
|
2137
2502
|
"addition_date": "2017/11/02",
|
|
@@ -2139,35 +2504,43 @@
|
|
|
2139
2504
|
"instances": [
|
|
2140
2505
|
"Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)"
|
|
2141
2506
|
]
|
|
2142
|
-
}
|
|
2507
|
+
}
|
|
2508
|
+
,
|
|
2143
2509
|
{
|
|
2144
2510
|
"pattern": "SurveyBot",
|
|
2145
2511
|
"addition_date": "2017/11/02",
|
|
2146
2512
|
"instances": [
|
|
2147
2513
|
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)"
|
|
2148
2514
|
]
|
|
2149
|
-
}
|
|
2515
|
+
}
|
|
2516
|
+
,
|
|
2150
2517
|
{
|
|
2151
|
-
"pattern": "NING",
|
|
2518
|
+
"pattern": "NING\\/",
|
|
2152
2519
|
"addition_date": "2017/11/02",
|
|
2153
2520
|
"instances": [
|
|
2154
2521
|
"NING/1.0"
|
|
2155
2522
|
]
|
|
2156
|
-
}
|
|
2523
|
+
}
|
|
2524
|
+
,
|
|
2157
2525
|
{
|
|
2158
2526
|
"pattern": "okhttp",
|
|
2159
2527
|
"addition_date": "2017/11/02",
|
|
2160
2528
|
"instances": [
|
|
2161
|
-
"okhttp/2.5.0"
|
|
2529
|
+
"okhttp/2.5.0",
|
|
2530
|
+
"okhttp/2.7.5",
|
|
2531
|
+
"okhttp/3.2.0",
|
|
2532
|
+
"okhttp/3.5.0"
|
|
2162
2533
|
]
|
|
2163
|
-
}
|
|
2534
|
+
}
|
|
2535
|
+
,
|
|
2164
2536
|
{
|
|
2165
2537
|
"pattern": "Nuzzel",
|
|
2166
2538
|
"addition_date": "2017/11/02",
|
|
2167
2539
|
"instances": [
|
|
2168
2540
|
"Nuzzel"
|
|
2169
2541
|
]
|
|
2170
|
-
}
|
|
2542
|
+
}
|
|
2543
|
+
,
|
|
2171
2544
|
{
|
|
2172
2545
|
"pattern": "omgili",
|
|
2173
2546
|
"addition_date": "2017/11/02",
|
|
@@ -2175,7 +2548,8 @@
|
|
|
2175
2548
|
"instances": [
|
|
2176
2549
|
"omgili/0.5 +http://omgili.com"
|
|
2177
2550
|
]
|
|
2178
|
-
}
|
|
2551
|
+
}
|
|
2552
|
+
,
|
|
2179
2553
|
{
|
|
2180
2554
|
"pattern": "PocketParser",
|
|
2181
2555
|
"addition_date": "2017/11/02",
|
|
@@ -2183,21 +2557,24 @@
|
|
|
2183
2557
|
"instances": [
|
|
2184
2558
|
"PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)"
|
|
2185
2559
|
]
|
|
2186
|
-
}
|
|
2560
|
+
}
|
|
2561
|
+
,
|
|
2187
2562
|
{
|
|
2188
2563
|
"pattern": "YisouSpider",
|
|
2189
2564
|
"addition_date": "2017/11/02",
|
|
2190
2565
|
"instances": [
|
|
2191
2566
|
"YisouSpider"
|
|
2192
2567
|
]
|
|
2193
|
-
}
|
|
2568
|
+
}
|
|
2569
|
+
,
|
|
2194
2570
|
{
|
|
2195
2571
|
"pattern": "um-LN",
|
|
2196
2572
|
"addition_date": "2017/11/02",
|
|
2197
2573
|
"instances": [
|
|
2198
2574
|
"Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)"
|
|
2199
2575
|
]
|
|
2200
|
-
}
|
|
2576
|
+
}
|
|
2577
|
+
,
|
|
2201
2578
|
{
|
|
2202
2579
|
"pattern": "ToutiaoSpider",
|
|
2203
2580
|
"addition_date": "2017/11/02",
|
|
@@ -2205,7 +2582,8 @@
|
|
|
2205
2582
|
"instances": [
|
|
2206
2583
|
"Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)"
|
|
2207
2584
|
]
|
|
2208
|
-
}
|
|
2585
|
+
}
|
|
2586
|
+
,
|
|
2209
2587
|
{
|
|
2210
2588
|
"pattern": "MuckRack",
|
|
2211
2589
|
"addition_date": "2017/11/02",
|
|
@@ -2213,7 +2591,8 @@
|
|
|
2213
2591
|
"instances": [
|
|
2214
2592
|
"Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)"
|
|
2215
2593
|
]
|
|
2216
|
-
}
|
|
2594
|
+
}
|
|
2595
|
+
,
|
|
2217
2596
|
{
|
|
2218
2597
|
"pattern": "Jamie's Spider",
|
|
2219
2598
|
"addition_date": "2017/11/02",
|
|
@@ -2221,56 +2600,70 @@
|
|
|
2221
2600
|
"instances": [
|
|
2222
2601
|
"Jamie's Spider (http://jamiembrown.com/)"
|
|
2223
2602
|
]
|
|
2224
|
-
}
|
|
2603
|
+
}
|
|
2604
|
+
,
|
|
2225
2605
|
{
|
|
2226
|
-
"pattern": "AHC",
|
|
2606
|
+
"pattern": "AHC\\/",
|
|
2227
2607
|
"addition_date": "2017/11/02",
|
|
2228
2608
|
"instances": [
|
|
2229
2609
|
"AHC/2.0"
|
|
2230
2610
|
]
|
|
2231
|
-
}
|
|
2611
|
+
}
|
|
2612
|
+
,
|
|
2232
2613
|
{
|
|
2233
2614
|
"pattern": "NetcraftSurveyAgent",
|
|
2234
2615
|
"addition_date": "2017/11/02",
|
|
2235
2616
|
"instances": [
|
|
2236
2617
|
"Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)"
|
|
2237
2618
|
]
|
|
2238
|
-
}
|
|
2619
|
+
}
|
|
2620
|
+
,
|
|
2239
2621
|
{
|
|
2240
2622
|
"pattern": "Laserlikebot",
|
|
2241
2623
|
"addition_date": "2017/11/02",
|
|
2242
2624
|
"instances": [
|
|
2243
2625
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)"
|
|
2244
2626
|
]
|
|
2245
|
-
}
|
|
2627
|
+
}
|
|
2628
|
+
,
|
|
2246
2629
|
{
|
|
2247
2630
|
"pattern": "Apache-HttpClient",
|
|
2248
2631
|
"addition_date": "2017/11/02",
|
|
2249
2632
|
"instances": [
|
|
2250
|
-
"Apache-HttpClient/4.
|
|
2633
|
+
"Apache-HttpClient/4.2.3 (java 1.5)",
|
|
2634
|
+
"Apache-HttpClient/4.2.5 (java 1.5)",
|
|
2635
|
+
"Apache-HttpClient/4.3.1 (java 1.5)",
|
|
2636
|
+
"Apache-HttpClient/4.3.3 (java 1.5)",
|
|
2637
|
+
"Apache-HttpClient/4.3.5 (java 1.5)",
|
|
2638
|
+
"Apache-HttpClient/4.4.1 (Java/1.8.0_65)",
|
|
2639
|
+
"Apache-HttpClient/4.5.3 (Java/1.8.0_121)"
|
|
2251
2640
|
]
|
|
2252
|
-
}
|
|
2641
|
+
}
|
|
2642
|
+
,
|
|
2253
2643
|
{
|
|
2254
2644
|
"pattern": "AppEngine-Google",
|
|
2255
2645
|
"addition_date": "2017/11/02",
|
|
2256
2646
|
"instances": [
|
|
2257
2647
|
"AppEngine-Google; (+http://code.google.com/appengine; appid: example)"
|
|
2258
2648
|
]
|
|
2259
|
-
}
|
|
2649
|
+
}
|
|
2650
|
+
,
|
|
2260
2651
|
{
|
|
2261
2652
|
"pattern": "Jetty",
|
|
2262
2653
|
"addition_date": "2017/11/02",
|
|
2263
2654
|
"instances": [
|
|
2264
2655
|
"Jetty/9.3.z-SNAPSHOT"
|
|
2265
2656
|
]
|
|
2266
|
-
}
|
|
2657
|
+
}
|
|
2658
|
+
,
|
|
2267
2659
|
{
|
|
2268
2660
|
"pattern": "Upflow",
|
|
2269
2661
|
"addition_date": "2017/11/02",
|
|
2270
2662
|
"instances": [
|
|
2271
2663
|
"Upflow/1.0"
|
|
2272
2664
|
]
|
|
2273
|
-
}
|
|
2665
|
+
}
|
|
2666
|
+
,
|
|
2274
2667
|
{
|
|
2275
2668
|
"pattern": "Thinklab",
|
|
2276
2669
|
"addition_date": "2017/11/02",
|
|
@@ -2278,7 +2671,8 @@
|
|
|
2278
2671
|
"instances": [
|
|
2279
2672
|
"Thinklab (thinklab.com)"
|
|
2280
2673
|
]
|
|
2281
|
-
}
|
|
2674
|
+
}
|
|
2675
|
+
,
|
|
2282
2676
|
{
|
|
2283
2677
|
"pattern": "Traackr.com",
|
|
2284
2678
|
"addition_date": "2017/11/02",
|
|
@@ -2286,7 +2680,8 @@
|
|
|
2286
2680
|
"instances": [
|
|
2287
2681
|
"Traackr.com"
|
|
2288
2682
|
]
|
|
2289
|
-
}
|
|
2683
|
+
}
|
|
2684
|
+
,
|
|
2290
2685
|
{
|
|
2291
2686
|
"pattern": "Twurly",
|
|
2292
2687
|
"addition_date": "2017/11/02",
|
|
@@ -2294,33 +2689,748 @@
|
|
|
2294
2689
|
"instances": [
|
|
2295
2690
|
"Ruby, Twurly v1.1 (http://twurly.org)"
|
|
2296
2691
|
]
|
|
2297
|
-
}
|
|
2692
|
+
}
|
|
2693
|
+
,
|
|
2298
2694
|
{
|
|
2299
2695
|
"pattern": "Mastodon",
|
|
2300
2696
|
"addition_date": "2017/11/02",
|
|
2301
2697
|
"instances": [
|
|
2302
2698
|
"http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)"
|
|
2303
2699
|
]
|
|
2304
|
-
}
|
|
2700
|
+
}
|
|
2701
|
+
,
|
|
2305
2702
|
{
|
|
2306
2703
|
"pattern": "http_get",
|
|
2307
2704
|
"addition_date": "2017/11/02",
|
|
2308
2705
|
"instances": [
|
|
2309
2706
|
"http_get"
|
|
2310
2707
|
]
|
|
2311
|
-
}
|
|
2708
|
+
}
|
|
2709
|
+
,
|
|
2312
2710
|
{
|
|
2313
2711
|
"pattern": "DnyzBot",
|
|
2314
2712
|
"addition_date": "2017/11/20",
|
|
2315
2713
|
"instances": [
|
|
2316
|
-
"Mozilla/5.0 (compatible; DnyzBot/1.0)"
|
|
2714
|
+
"Mozilla/5.0 (compatible; DnyzBot/1.0)",
|
|
2715
|
+
"Mozilla/5.0 (compatible; DnyzBot/1.0) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/64.0.3282.167 Safari/537.36",
|
|
2716
|
+
"Mozilla/5.0 (compatible; DnyzBot/1.0) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/64.0.3264.0 Safari/537.36"
|
|
2317
2717
|
]
|
|
2318
|
-
}
|
|
2718
|
+
}
|
|
2719
|
+
,
|
|
2319
2720
|
{
|
|
2320
|
-
"pattern": "
|
|
2721
|
+
"pattern": "botify",
|
|
2321
2722
|
"addition_date": "2018/02/01",
|
|
2322
2723
|
"instances": [
|
|
2323
2724
|
"Mozilla/5.0 (compatible; botify; http://botify.com)"
|
|
2324
2725
|
]
|
|
2325
2726
|
}
|
|
2727
|
+
,
|
|
2728
|
+
{
|
|
2729
|
+
"pattern": "007ac9 Crawler",
|
|
2730
|
+
"addition_date": "2018/02/09",
|
|
2731
|
+
"instances": [
|
|
2732
|
+
"Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)"
|
|
2733
|
+
]
|
|
2734
|
+
}
|
|
2735
|
+
,
|
|
2736
|
+
{
|
|
2737
|
+
"pattern": "BehloolBot",
|
|
2738
|
+
"addition_date": "2018/02/09",
|
|
2739
|
+
"instances": [
|
|
2740
|
+
"Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)"
|
|
2741
|
+
]
|
|
2742
|
+
}
|
|
2743
|
+
,
|
|
2744
|
+
{
|
|
2745
|
+
"pattern": "BrandVerity",
|
|
2746
|
+
"addition_date": "2018/02/27",
|
|
2747
|
+
"instances": [
|
|
2748
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)"
|
|
2749
|
+
]
|
|
2750
|
+
}
|
|
2751
|
+
,
|
|
2752
|
+
{
|
|
2753
|
+
"pattern": "check_http",
|
|
2754
|
+
"addition_date": "2018/02/09",
|
|
2755
|
+
"instances": [
|
|
2756
|
+
"check_http/v2.2.1 (nagios-plugins 2.2.1)"
|
|
2757
|
+
]
|
|
2758
|
+
}
|
|
2759
|
+
,
|
|
2760
|
+
{
|
|
2761
|
+
"pattern": "BDCbot",
|
|
2762
|
+
"addition_date": "2018/02/09",
|
|
2763
|
+
"instances": [
|
|
2764
|
+
"Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36"
|
|
2765
|
+
]
|
|
2766
|
+
}
|
|
2767
|
+
,
|
|
2768
|
+
{
|
|
2769
|
+
"pattern": "ZumBot",
|
|
2770
|
+
"addition_date": "2018/02/09",
|
|
2771
|
+
"instances": [
|
|
2772
|
+
"Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)"
|
|
2773
|
+
]
|
|
2774
|
+
}
|
|
2775
|
+
,
|
|
2776
|
+
{
|
|
2777
|
+
"pattern": "EZID",
|
|
2778
|
+
"addition_date": "2018/02/09",
|
|
2779
|
+
"instances": [
|
|
2780
|
+
"EZID (EZID link checker; https://ezid.cdlib.org/)"
|
|
2781
|
+
]
|
|
2782
|
+
}
|
|
2783
|
+
,
|
|
2784
|
+
{
|
|
2785
|
+
"pattern": "ICC-Crawler",
|
|
2786
|
+
"addition_date": "2018/02/28",
|
|
2787
|
+
"instances": [
|
|
2788
|
+
"ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)"
|
|
2789
|
+
],
|
|
2790
|
+
"url": "http://ucri.nict.go.jp/en/icccrawler.html"
|
|
2791
|
+
}
|
|
2792
|
+
,
|
|
2793
|
+
{
|
|
2794
|
+
"pattern": "ArchiveBot",
|
|
2795
|
+
"addition_date": "2018/02/28",
|
|
2796
|
+
"instances": [
|
|
2797
|
+
"ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)"
|
|
2798
|
+
],
|
|
2799
|
+
"url": "https://github.com/ArchiveTeam/ArchiveBot"
|
|
2800
|
+
}
|
|
2801
|
+
,
|
|
2802
|
+
{
|
|
2803
|
+
"pattern": "^LCC ",
|
|
2804
|
+
"addition_date": "2018/02/28",
|
|
2805
|
+
"instances": [
|
|
2806
|
+
"LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)"
|
|
2807
|
+
],
|
|
2808
|
+
"url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html"
|
|
2809
|
+
}
|
|
2810
|
+
,
|
|
2811
|
+
{
|
|
2812
|
+
"pattern": "filterdb.iss.net\\/crawler",
|
|
2813
|
+
"addition_date": "2018/03/16",
|
|
2814
|
+
"instances": [
|
|
2815
|
+
"Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)"
|
|
2816
|
+
],
|
|
2817
|
+
"url": "http://filterdb.iss.net/crawler/"
|
|
2818
|
+
}
|
|
2819
|
+
,
|
|
2820
|
+
{
|
|
2821
|
+
"pattern": "BLP_bbot",
|
|
2822
|
+
"addition_date": "2018/03/27",
|
|
2823
|
+
"instances": [
|
|
2824
|
+
"BLP_bbot/0.1"
|
|
2825
|
+
]
|
|
2826
|
+
}
|
|
2827
|
+
,
|
|
2828
|
+
{
|
|
2829
|
+
"pattern": "BomboraBot",
|
|
2830
|
+
"addition_date": "2018/03/27",
|
|
2831
|
+
"instances": [
|
|
2832
|
+
"Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)"
|
|
2833
|
+
],
|
|
2834
|
+
"url": "http://www.bombora.com/bot"
|
|
2835
|
+
}
|
|
2836
|
+
,
|
|
2837
|
+
{
|
|
2838
|
+
"pattern": "Buck\\/",
|
|
2839
|
+
"addition_date": "2018/03/27",
|
|
2840
|
+
"instances": [
|
|
2841
|
+
"Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)"
|
|
2842
|
+
],
|
|
2843
|
+
"url": "https://app.hypefactors.com/media-monitoring/about.html"
|
|
2844
|
+
}
|
|
2845
|
+
,
|
|
2846
|
+
{
|
|
2847
|
+
"pattern": "Companybook-Crawler",
|
|
2848
|
+
"addition_date": "2018/03/27",
|
|
2849
|
+
"instances": [
|
|
2850
|
+
"Companybook-Crawler (+https://www.companybooknetworking.com/)"
|
|
2851
|
+
],
|
|
2852
|
+
"url": "https://www.companybooknetworking.com/"
|
|
2853
|
+
}
|
|
2854
|
+
,
|
|
2855
|
+
{
|
|
2856
|
+
"pattern": "Genieo",
|
|
2857
|
+
"addition_date": "2018/03/27",
|
|
2858
|
+
"instances": [
|
|
2859
|
+
"Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)"
|
|
2860
|
+
],
|
|
2861
|
+
"url": "http://www.genieo.com/webfilter.html"
|
|
2862
|
+
}
|
|
2863
|
+
,
|
|
2864
|
+
{
|
|
2865
|
+
"pattern": "magpie-crawler",
|
|
2866
|
+
"addition_date": "2018/03/27",
|
|
2867
|
+
"instances": [
|
|
2868
|
+
"magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)"
|
|
2869
|
+
],
|
|
2870
|
+
"url": "http://www.brandwatch.net"
|
|
2871
|
+
}
|
|
2872
|
+
,
|
|
2873
|
+
{
|
|
2874
|
+
"pattern": "MeltwaterNews",
|
|
2875
|
+
"addition_date": "2018/03/27",
|
|
2876
|
+
"instances": [
|
|
2877
|
+
"MeltwaterNews www.meltwater.com"
|
|
2878
|
+
],
|
|
2879
|
+
"url": "http://www.meltwater.com"
|
|
2880
|
+
}
|
|
2881
|
+
,
|
|
2882
|
+
{
|
|
2883
|
+
"pattern": "Moreover",
|
|
2884
|
+
"addition_date": "2018/03/27",
|
|
2885
|
+
"instances": [
|
|
2886
|
+
"Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)"
|
|
2887
|
+
],
|
|
2888
|
+
"url": "http://www.moreover.com"
|
|
2889
|
+
}
|
|
2890
|
+
,
|
|
2891
|
+
{
|
|
2892
|
+
"pattern": "newspaper\\/",
|
|
2893
|
+
"addition_date": "2018/03/27",
|
|
2894
|
+
"instances": [
|
|
2895
|
+
"newspaper/0.2.5",
|
|
2896
|
+
"newspaper/0.2.6",
|
|
2897
|
+
"newspaper/0.1.0.7"
|
|
2898
|
+
]
|
|
2899
|
+
}
|
|
2900
|
+
,
|
|
2901
|
+
{
|
|
2902
|
+
"pattern": "ScoutJet",
|
|
2903
|
+
"addition_date": "2018/03/27",
|
|
2904
|
+
"instances": [
|
|
2905
|
+
"Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)"
|
|
2906
|
+
],
|
|
2907
|
+
"url": "http://www.scoutjet.com/"
|
|
2908
|
+
}
|
|
2909
|
+
,
|
|
2910
|
+
{
|
|
2911
|
+
"pattern": "(^| )sentry\\/",
|
|
2912
|
+
"addition_date": "2018/03/27",
|
|
2913
|
+
"instances": [
|
|
2914
|
+
"sentry/8.22.0 (https://sentry.io)"
|
|
2915
|
+
],
|
|
2916
|
+
"url": "https://sentry.io"
|
|
2917
|
+
}
|
|
2918
|
+
,
|
|
2919
|
+
{
|
|
2920
|
+
"pattern": "StorygizeBot",
|
|
2921
|
+
"addition_date": "2018/03/27",
|
|
2922
|
+
"instances": [
|
|
2923
|
+
"Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)"
|
|
2924
|
+
],
|
|
2925
|
+
"url": "http://www.storygize.com"
|
|
2926
|
+
}
|
|
2927
|
+
,
|
|
2928
|
+
{
|
|
2929
|
+
"pattern": "UptimeRobot",
|
|
2930
|
+
"addition_date": "2018/03/27",
|
|
2931
|
+
"instances": [
|
|
2932
|
+
"Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)"
|
|
2933
|
+
],
|
|
2934
|
+
"url": "http://www.uptimerobot.com/"
|
|
2935
|
+
}
|
|
2936
|
+
,
|
|
2937
|
+
{
|
|
2938
|
+
"pattern": "OutclicksBot",
|
|
2939
|
+
"addition_date": "2018/04/21",
|
|
2940
|
+
"instances": [
|
|
2941
|
+
"OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR",
|
|
2942
|
+
"OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe",
|
|
2943
|
+
"OutclicksBot/2 +https://www.outclicks.net/agent/PryJzTl8POCRHfvEUlRN5FKtZoWDQOBEvFJ2wh6KH5J",
|
|
2944
|
+
"OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h"
|
|
2945
|
+
],
|
|
2946
|
+
"url": "https://www.outclicks.net"
|
|
2947
|
+
}
|
|
2948
|
+
,
|
|
2949
|
+
{
|
|
2950
|
+
"pattern": "seoscanners",
|
|
2951
|
+
"addition_date": "2018/05/27",
|
|
2952
|
+
"instances": [
|
|
2953
|
+
"Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)"
|
|
2954
|
+
],
|
|
2955
|
+
"url": "http://www.seoscanners.net/"
|
|
2956
|
+
}
|
|
2957
|
+
,
|
|
2958
|
+
{
|
|
2959
|
+
"pattern": "Hatena",
|
|
2960
|
+
"addition_date": "2018/05/29",
|
|
2961
|
+
"instances": [
|
|
2962
|
+
"Hatena Antenna/0.3",
|
|
2963
|
+
"Hatena::Russia::Crawler/0.01"
|
|
2964
|
+
]
|
|
2965
|
+
}
|
|
2966
|
+
,
|
|
2967
|
+
{
|
|
2968
|
+
"pattern": "Google Web Preview",
|
|
2969
|
+
"addition_date": "2018/05/31",
|
|
2970
|
+
"instances": [
|
|
2971
|
+
"Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36"
|
|
2972
|
+
]
|
|
2973
|
+
}
|
|
2974
|
+
,
|
|
2975
|
+
{
|
|
2976
|
+
"pattern": "MauiBot",
|
|
2977
|
+
"addition_date": "2018/06/06",
|
|
2978
|
+
"instances": [
|
|
2979
|
+
"MauiBot (crawler.feedback+wc@gmail.com)"
|
|
2980
|
+
]
|
|
2981
|
+
}
|
|
2982
|
+
,
|
|
2983
|
+
{
|
|
2984
|
+
"pattern": "AlphaBot",
|
|
2985
|
+
"addition_date": "2018/05/27",
|
|
2986
|
+
"instances": [
|
|
2987
|
+
"Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)"
|
|
2988
|
+
],
|
|
2989
|
+
"url": "http://alphaseobot.com/bot.html"
|
|
2990
|
+
}
|
|
2991
|
+
,
|
|
2992
|
+
{
|
|
2993
|
+
"pattern": "SBL-BOT",
|
|
2994
|
+
"addition_date": "2018/06/06",
|
|
2995
|
+
"instances": [
|
|
2996
|
+
"SBL-BOT (http://sbl.net)"
|
|
2997
|
+
],
|
|
2998
|
+
"url": "http://sbl.net",
|
|
2999
|
+
"description" : "Bot of SoftByte BlackWidow"
|
|
3000
|
+
}
|
|
3001
|
+
,
|
|
3002
|
+
{
|
|
3003
|
+
"pattern": "IAS crawler",
|
|
3004
|
+
"addition_date": "2018/06/06",
|
|
3005
|
+
"instances": [
|
|
3006
|
+
"IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)"
|
|
3007
|
+
],
|
|
3008
|
+
"url": "http://integralads.com/site-indexing-policy/",
|
|
3009
|
+
"description" : "Bot of Integral Ad Science, Inc."
|
|
3010
|
+
}
|
|
3011
|
+
,
|
|
3012
|
+
{
|
|
3013
|
+
"pattern": "adscanner",
|
|
3014
|
+
"addition_date": "2018/06/24",
|
|
3015
|
+
"instances": [
|
|
3016
|
+
"Mozilla/5.0 (compatible; adscanner/)"
|
|
3017
|
+
]
|
|
3018
|
+
}
|
|
3019
|
+
,
|
|
3020
|
+
{
|
|
3021
|
+
"pattern": "Netvibes",
|
|
3022
|
+
"addition_date": "2018/06/24",
|
|
3023
|
+
"instances": [
|
|
3024
|
+
"Netvibes (crawler/bot; http://www.netvibes.com"
|
|
3025
|
+
],
|
|
3026
|
+
"url": "http://www.netvibes.com"
|
|
3027
|
+
}
|
|
3028
|
+
,
|
|
3029
|
+
{
|
|
3030
|
+
"pattern": "acapbot",
|
|
3031
|
+
"addition_date": "2018/06/27",
|
|
3032
|
+
"instances": [
|
|
3033
|
+
"Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)",
|
|
3034
|
+
"Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)"
|
|
3035
|
+
]
|
|
3036
|
+
}
|
|
3037
|
+
,
|
|
3038
|
+
{
|
|
3039
|
+
"pattern": "Baidu-YunGuanCe",
|
|
3040
|
+
"addition_date": "2018/06/27",
|
|
3041
|
+
"instances": [
|
|
3042
|
+
"Baidu-YunGuanCe-Bot(ce.baidu.com)",
|
|
3043
|
+
"Baidu-YunGuanCe-SLABot(ce.baidu.com)",
|
|
3044
|
+
"Baidu-YunGuanCe-ScanBot(ce.baidu.com)",
|
|
3045
|
+
"Baidu-YunGuanCe-PerfBot(ce.baidu.com)",
|
|
3046
|
+
"Baidu-YunGuanCe-VSBot(ce.baidu.com)"
|
|
3047
|
+
],
|
|
3048
|
+
"url": "https://ce.baidu.com/topic/topic20150908",
|
|
3049
|
+
"description": "Baidu Cloud Watch"
|
|
3050
|
+
}
|
|
3051
|
+
,
|
|
3052
|
+
{
|
|
3053
|
+
"pattern": "bitlybot",
|
|
3054
|
+
"addition_date": "2018/06/27",
|
|
3055
|
+
"instances": [
|
|
3056
|
+
"bitlybot/3.0 (+http://bit.ly/)",
|
|
3057
|
+
"bitlybot/2.0",
|
|
3058
|
+
"bitlybot"
|
|
3059
|
+
],
|
|
3060
|
+
"url": "http://bit.ly/"
|
|
3061
|
+
}
|
|
3062
|
+
,
|
|
3063
|
+
{
|
|
3064
|
+
"pattern": "blogmuraBot",
|
|
3065
|
+
"addition_date": "2018/06/27",
|
|
3066
|
+
"instances": [
|
|
3067
|
+
"blogmuraBot (+http://www.blogmura.com)"
|
|
3068
|
+
],
|
|
3069
|
+
"url": "http://www.blogmura.com",
|
|
3070
|
+
"description": "A blog ranking site which links to blogs on just about every theme possible."
|
|
3071
|
+
}
|
|
3072
|
+
,
|
|
3073
|
+
{
|
|
3074
|
+
"pattern": "Bot.AraTurka.com",
|
|
3075
|
+
"addition_date": "2018/06/27",
|
|
3076
|
+
"instances": [
|
|
3077
|
+
"Bot.AraTurka.com/0.0.1"
|
|
3078
|
+
],
|
|
3079
|
+
"url": "http://www.araturka.com"
|
|
3080
|
+
}
|
|
3081
|
+
,
|
|
3082
|
+
{
|
|
3083
|
+
"pattern": "bot-pge.chlooe.com",
|
|
3084
|
+
"addition_date": "2018/06/27",
|
|
3085
|
+
"instances": [
|
|
3086
|
+
"bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)"
|
|
3087
|
+
]
|
|
3088
|
+
}
|
|
3089
|
+
,
|
|
3090
|
+
{
|
|
3091
|
+
"pattern": "BoxcarBot",
|
|
3092
|
+
"addition_date": "2018/06/27",
|
|
3093
|
+
"instances": [
|
|
3094
|
+
"Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)"
|
|
3095
|
+
],
|
|
3096
|
+
"url": "https://boxcar.io/"
|
|
3097
|
+
}
|
|
3098
|
+
,
|
|
3099
|
+
{
|
|
3100
|
+
"pattern": "BTWebClient",
|
|
3101
|
+
"addition_date": "2018/06/27",
|
|
3102
|
+
"instances": [
|
|
3103
|
+
"BTWebClient/180B(9704)"
|
|
3104
|
+
],
|
|
3105
|
+
"url": "http://www.utorrent.com/",
|
|
3106
|
+
"description": "µTorrent BitTorrent Client"
|
|
3107
|
+
}
|
|
3108
|
+
,
|
|
3109
|
+
{
|
|
3110
|
+
"pattern": "ContextAd Bot",
|
|
3111
|
+
"addition_date": "2018/06/27",
|
|
3112
|
+
"instances": [
|
|
3113
|
+
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)",
|
|
3114
|
+
"ContextAd Bot 1.0"
|
|
3115
|
+
]
|
|
3116
|
+
}
|
|
3117
|
+
,
|
|
3118
|
+
{
|
|
3119
|
+
"pattern": "Digincore bot",
|
|
3120
|
+
"addition_date": "2018/06/27",
|
|
3121
|
+
"instances": [
|
|
3122
|
+
"Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)"
|
|
3123
|
+
],
|
|
3124
|
+
"url": "http://www.digincore.com/crawler.html"
|
|
3125
|
+
}
|
|
3126
|
+
,
|
|
3127
|
+
{
|
|
3128
|
+
"pattern": "Disqus",
|
|
3129
|
+
"addition_date": "2018/06/27",
|
|
3130
|
+
"instances": [
|
|
3131
|
+
"Disqus/1.0"
|
|
3132
|
+
],
|
|
3133
|
+
"url": "https://disqus.com/",
|
|
3134
|
+
"description": "validate and quality check pages."
|
|
3135
|
+
}
|
|
3136
|
+
,
|
|
3137
|
+
{
|
|
3138
|
+
"pattern": "Feedly",
|
|
3139
|
+
"addition_date": "2018/06/27",
|
|
3140
|
+
"instances": [
|
|
3141
|
+
"Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)",
|
|
3142
|
+
"FeedlyBot/1.0 (http://feedly.com)"
|
|
3143
|
+
],
|
|
3144
|
+
"url": "https://www.feedly.com/fetcher.html",
|
|
3145
|
+
"description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud."
|
|
3146
|
+
}
|
|
3147
|
+
,
|
|
3148
|
+
{
|
|
3149
|
+
"pattern": "Fetch\\/",
|
|
3150
|
+
"addition_date": "2018/06/27",
|
|
3151
|
+
"instances": [
|
|
3152
|
+
"Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)"
|
|
3153
|
+
]
|
|
3154
|
+
}
|
|
3155
|
+
,
|
|
3156
|
+
{
|
|
3157
|
+
"pattern": "Fever",
|
|
3158
|
+
"addition_date": "2018/06/27",
|
|
3159
|
+
"instances": [
|
|
3160
|
+
"Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)"
|
|
3161
|
+
],
|
|
3162
|
+
"url": "http://feedafever.com"
|
|
3163
|
+
}
|
|
3164
|
+
,
|
|
3165
|
+
{
|
|
3166
|
+
"pattern": "Flamingo_SearchEngine",
|
|
3167
|
+
"addition_date": "2018/06/27",
|
|
3168
|
+
"instances": [
|
|
3169
|
+
"Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)"
|
|
3170
|
+
]
|
|
3171
|
+
}
|
|
3172
|
+
,
|
|
3173
|
+
{
|
|
3174
|
+
"pattern": "FlipboardProxy",
|
|
3175
|
+
"addition_date": "2018/06/27",
|
|
3176
|
+
"instances": [
|
|
3177
|
+
"Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)",
|
|
3178
|
+
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)",
|
|
3179
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)"
|
|
3180
|
+
],
|
|
3181
|
+
"url": "https://about.flipboard.com/browserproxy/",
|
|
3182
|
+
"description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application"
|
|
3183
|
+
}
|
|
3184
|
+
,
|
|
3185
|
+
{
|
|
3186
|
+
"pattern": "g2reader-bot",
|
|
3187
|
+
"addition_date": "2018/06/27",
|
|
3188
|
+
"instances": [
|
|
3189
|
+
"g2reader-bot/1.0 (+http://www.g2reader.com/)"
|
|
3190
|
+
],
|
|
3191
|
+
"url": "http://www.g2reader.com/"
|
|
3192
|
+
}
|
|
3193
|
+
,
|
|
3194
|
+
{
|
|
3195
|
+
"pattern": "imrbot",
|
|
3196
|
+
"addition_date": "2018/06/27",
|
|
3197
|
+
"instances": [
|
|
3198
|
+
"Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)"
|
|
3199
|
+
],
|
|
3200
|
+
"url": "http://www.mignify.com"
|
|
3201
|
+
}
|
|
3202
|
+
,
|
|
3203
|
+
{
|
|
3204
|
+
"pattern": "K7MLWCBot",
|
|
3205
|
+
"addition_date": "2018/06/27",
|
|
3206
|
+
"instances": [
|
|
3207
|
+
"K7MLWCBot/1.0 (+http://www.k7computing.com)"
|
|
3208
|
+
],
|
|
3209
|
+
"url": "http://www.k7computing.com",
|
|
3210
|
+
"description": "Virus scanner"
|
|
3211
|
+
}
|
|
3212
|
+
,
|
|
3213
|
+
{
|
|
3214
|
+
"pattern": "Kemvibot",
|
|
3215
|
+
"addition_date": "2018/06/27",
|
|
3216
|
+
"instances": [
|
|
3217
|
+
"Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)"
|
|
3218
|
+
],
|
|
3219
|
+
"url": "http://kemvi.com"
|
|
3220
|
+
}
|
|
3221
|
+
,
|
|
3222
|
+
{
|
|
3223
|
+
"pattern": "Landau-Media-Spider",
|
|
3224
|
+
"addition_date": "2018/06/27",
|
|
3225
|
+
"instances": [
|
|
3226
|
+
"Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)"
|
|
3227
|
+
],
|
|
3228
|
+
"url": "http://bots.landaumedia.de/bot.html"
|
|
3229
|
+
}
|
|
3230
|
+
,
|
|
3231
|
+
{
|
|
3232
|
+
"pattern": "linkapediabot",
|
|
3233
|
+
"addition_date": "2018/06/27",
|
|
3234
|
+
"instances": [
|
|
3235
|
+
"linkapediabot (+http://www.linkapedia.com)"
|
|
3236
|
+
],
|
|
3237
|
+
"url": "http://www.linkapedia.com"
|
|
3238
|
+
}
|
|
3239
|
+
,
|
|
3240
|
+
{
|
|
3241
|
+
"pattern": "vkShare",
|
|
3242
|
+
"addition_date": "2018/07/02",
|
|
3243
|
+
"instances": [
|
|
3244
|
+
"Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)"
|
|
3245
|
+
],
|
|
3246
|
+
"url": "http://vk.com/dev/Share"
|
|
3247
|
+
}
|
|
3248
|
+
,
|
|
3249
|
+
{
|
|
3250
|
+
"pattern": "Siteimprove.com",
|
|
3251
|
+
"addition_date": "2018/06/22",
|
|
3252
|
+
"instances": [
|
|
3253
|
+
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com",
|
|
3254
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com",
|
|
3255
|
+
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com",
|
|
3256
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com"
|
|
3257
|
+
]
|
|
3258
|
+
}
|
|
3259
|
+
,
|
|
3260
|
+
{
|
|
3261
|
+
"pattern": "BLEXBot\\/",
|
|
3262
|
+
"addition_date": "2018/07/07",
|
|
3263
|
+
"instances": [
|
|
3264
|
+
"Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)"
|
|
3265
|
+
],
|
|
3266
|
+
"url": "http://webmeup-crawler.com"
|
|
3267
|
+
}
|
|
3268
|
+
,
|
|
3269
|
+
{
|
|
3270
|
+
"pattern": "DareBoost",
|
|
3271
|
+
"addition_date": "2018/07/07",
|
|
3272
|
+
"instances": [
|
|
3273
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost"
|
|
3274
|
+
],
|
|
3275
|
+
"url": "https://www.dareboost.com/",
|
|
3276
|
+
"description": "Bot to test, Analyze and Optimize website"
|
|
3277
|
+
}
|
|
3278
|
+
,
|
|
3279
|
+
{
|
|
3280
|
+
"pattern": "ZuperlistBot\\/",
|
|
3281
|
+
"addition_date": "2018/07/07",
|
|
3282
|
+
"instances": [
|
|
3283
|
+
"Mozilla/5.0 (compatible; ZuperlistBot/1.0)"
|
|
3284
|
+
]
|
|
3285
|
+
}
|
|
3286
|
+
,
|
|
3287
|
+
{
|
|
3288
|
+
"pattern": "Miniflux\\/",
|
|
3289
|
+
"addition_date": "2018/07/07",
|
|
3290
|
+
"instances": [
|
|
3291
|
+
"Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)"
|
|
3292
|
+
],
|
|
3293
|
+
"url": "https://miniflux.net",
|
|
3294
|
+
"description": "Miniflux is a minimalist and opinionated feed reader."
|
|
3295
|
+
}
|
|
3296
|
+
,
|
|
3297
|
+
{
|
|
3298
|
+
"pattern": "Feedspotbot\\/",
|
|
3299
|
+
"addition_date": "2018/07/07",
|
|
3300
|
+
"instances": [
|
|
3301
|
+
"Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)"
|
|
3302
|
+
],
|
|
3303
|
+
"url": "http://www.feedspot.com/fs/bot"
|
|
3304
|
+
}
|
|
3305
|
+
,
|
|
3306
|
+
{
|
|
3307
|
+
"pattern": "Diffbot\\/",
|
|
3308
|
+
"addition_date": "2018/07/07",
|
|
3309
|
+
"instances": [
|
|
3310
|
+
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)"
|
|
3311
|
+
],
|
|
3312
|
+
"url": "http://www.diffbot.com"
|
|
3313
|
+
}
|
|
3314
|
+
,
|
|
3315
|
+
{
|
|
3316
|
+
"pattern": "SEOkicks",
|
|
3317
|
+
"addition_date": "2018/08/22",
|
|
3318
|
+
"instances": [
|
|
3319
|
+
"Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)"
|
|
3320
|
+
],
|
|
3321
|
+
"url": "https://www.seokicks.de/robot.html"
|
|
3322
|
+
}
|
|
3323
|
+
,
|
|
3324
|
+
{
|
|
3325
|
+
"pattern": "tracemyfile",
|
|
3326
|
+
"addition_date": "2018/08/23",
|
|
3327
|
+
"instances": [
|
|
3328
|
+
"Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)"
|
|
3329
|
+
]
|
|
3330
|
+
}
|
|
3331
|
+
,
|
|
3332
|
+
{
|
|
3333
|
+
"pattern": "Nimbostratus-Bot",
|
|
3334
|
+
"addition_date": "2018/08/29",
|
|
3335
|
+
"instances": [
|
|
3336
|
+
"Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)"
|
|
3337
|
+
]
|
|
3338
|
+
}
|
|
3339
|
+
,
|
|
3340
|
+
{
|
|
3341
|
+
"pattern": "zgrab",
|
|
3342
|
+
"addition_date": "2018/08/30",
|
|
3343
|
+
"instances": [
|
|
3344
|
+
"Mozilla/5.0 zgrab/0.x"
|
|
3345
|
+
],
|
|
3346
|
+
"url": "https://zmap.io/"
|
|
3347
|
+
}
|
|
3348
|
+
,
|
|
3349
|
+
{
|
|
3350
|
+
"pattern": "PR-CY.RU",
|
|
3351
|
+
"addition_date": "2018/08/30",
|
|
3352
|
+
"instances": [
|
|
3353
|
+
"Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)"
|
|
3354
|
+
],
|
|
3355
|
+
"url": "https://a.pr-cy.ru/"
|
|
3356
|
+
}
|
|
3357
|
+
,
|
|
3358
|
+
{
|
|
3359
|
+
"pattern": "AdsTxtCrawler",
|
|
3360
|
+
"addition_date": "2018/08/30",
|
|
3361
|
+
"instances": [
|
|
3362
|
+
"AdsTxtCrawler/1.0"
|
|
3363
|
+
]
|
|
3364
|
+
},
|
|
3365
|
+
{
|
|
3366
|
+
"pattern": "Datafeedwatch",
|
|
3367
|
+
"addition_date": "2018/09/05",
|
|
3368
|
+
"instances": [
|
|
3369
|
+
"Datafeedwatch/2.1.x"
|
|
3370
|
+
],
|
|
3371
|
+
"url": "https://www.datafeedwatch.com/"
|
|
3372
|
+
}
|
|
3373
|
+
,
|
|
3374
|
+
{
|
|
3375
|
+
"pattern": "Zabbix",
|
|
3376
|
+
"addition_date": "2018/09/05",
|
|
3377
|
+
"instances": [
|
|
3378
|
+
"Zabbix"
|
|
3379
|
+
],
|
|
3380
|
+
"url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring"
|
|
3381
|
+
}
|
|
3382
|
+
,
|
|
3383
|
+
{
|
|
3384
|
+
"pattern": "TangibleeBot",
|
|
3385
|
+
"addition_date": "2018/09/05",
|
|
3386
|
+
"instances": [
|
|
3387
|
+
"TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)"
|
|
3388
|
+
],
|
|
3389
|
+
"url": "http://tangiblee.com/bot"
|
|
3390
|
+
}
|
|
3391
|
+
,
|
|
3392
|
+
{
|
|
3393
|
+
"pattern": "google-xrawler",
|
|
3394
|
+
"addition_date": "2018/09/05",
|
|
3395
|
+
"instances": [
|
|
3396
|
+
"google-xrawler"
|
|
3397
|
+
],
|
|
3398
|
+
"url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for"
|
|
3399
|
+
}
|
|
3400
|
+
,
|
|
3401
|
+
{
|
|
3402
|
+
"pattern": "axios",
|
|
3403
|
+
"addition_date": "2018/09/06",
|
|
3404
|
+
"instances": [
|
|
3405
|
+
"axios/0.18.0"
|
|
3406
|
+
],
|
|
3407
|
+
"url": "https://github.com/axios/axios"
|
|
3408
|
+
}
|
|
3409
|
+
,
|
|
3410
|
+
{
|
|
3411
|
+
"pattern": "Amazon CloudFront",
|
|
3412
|
+
"addition_date": "2018/09/07",
|
|
3413
|
+
"instances": [
|
|
3414
|
+
"Amazon CloudFront"
|
|
3415
|
+
],
|
|
3416
|
+
"url": "https://aws.amazon.com/cloudfront/"
|
|
3417
|
+
}
|
|
3418
|
+
,
|
|
3419
|
+
{
|
|
3420
|
+
"pattern": "Pulsepoint",
|
|
3421
|
+
"addition_date": "2018/09/24",
|
|
3422
|
+
"instances": [
|
|
3423
|
+
"Pulsepoint XT3 web scraper"
|
|
3424
|
+
]
|
|
3425
|
+
}
|
|
3426
|
+
,
|
|
3427
|
+
{
|
|
3428
|
+
"pattern": "CloudFlare-AlwaysOnline",
|
|
3429
|
+
"addition_date": "2018/09/27",
|
|
3430
|
+
"instances": [
|
|
3431
|
+
"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34",
|
|
3432
|
+
"Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34"
|
|
3433
|
+
],
|
|
3434
|
+
"url" : "https://www.cloudflare.com/always-online/"
|
|
3435
|
+
}
|
|
2326
3436
|
]
|