voight_kampff 1.1.1 → 1.1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (4) hide show
  1. checksums.yaml +4 -4
  2. data/config/crawler-user-agents.json +1653 -266
  3. data/lib/voight_kampff/version.rb +1 -1
  4. metadata +3 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d84469d1aed5ee0786d045d4a064d96a9c96c6a1
4
- data.tar.gz: 14e7e52f8fd161842dfd36eefc003dc4acf68c06
3
+ metadata.gz: 51a3721f69a6b32a7c619496d3369fbe3a8121e7
4
+ data.tar.gz: 59596dc12c149da92840f78c8d96d2e891deb056
5
5
  SHA512:
6
- metadata.gz: b20c3443891cc19f27c0fe8f278330f2e7077a023a237f728b2ef8803a423d730378b6b312eced348c61cbf0d5e8f95b18006502c39c9171812984f02baffd43
7
- data.tar.gz: 7fa0cfba7a668b9cdd760aeb04f2073b997908e429236aaf6e2c8bab2563448b14972b413eb5878abb7da132ee7e4cc419fc5dc574947067ed52fe94ddaf5816
6
+ metadata.gz: 4c9dd899fd331df778fd8f7592144532ee15a4f7964ddaffccc1c3fdefd58c1326e8645a782511737b527729f509c31d9edcdbfeb7fc2a22a561a6886834cee5
7
+ data.tar.gz: 5caf2c0788cdf075dbaf56d6912e49cb9c77b115e760dcd81e67bd5a9b6e8ea24a5fd4264e2ad6c6d50fb5cee251a271e4cfee1c9eb98e180aa99fa6e5d7bb25
data/config/crawler-user-agents.json CHANGED
@@ -1,289 +1,713 @@
1
1
  [
2
2
  {
3
3
  "pattern": "Googlebot\\/",
4
- "url": "http://www.google.com/bot.html"
4
+ "url": "http://www.google.com/bot.html",
5
+ "instances": [
6
+ "Googlebot/2.1 (+http://www.google.com/bot.html)",
7
+ "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
8
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
9
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
10
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
11
+ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
12
+ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
13
+ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36",
14
+ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
15
+ ]
16
+ },
17
+ {
18
+ "pattern": "Googlebot-Mobile",
19
+ "instances": [
20
+ "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
21
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
22
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
23
+ "Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
24
+ "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"
25
+ ]
5
26
  },
6
27
  {
7
- "pattern": "Googlebot-Mobile"
28
+ "pattern": "Googlebot-Image",
29
+ "instances": [
30
+ "Googlebot-Image/1.0"
31
+ ]
8
32
  },
9
33
  {
10
- "pattern": "Googlebot-Image"
34
+ "pattern": "Googlebot-News",
35
+ "instances": [
36
+ "Googlebot-News"
37
+ ]
11
38
  },
12
39
  {
13
- "pattern": "Googlebot-News"
40
+ "pattern": "Googlebot-Video",
41
+ "instances": [
42
+ "Googlebot-Video/1.0"
43
+ ]
14
44
  },
15
45
  {
16
- "pattern": "Googlebot-Video"
46
+ "pattern": "AdsBot-Google([^-]|$)",
47
+ "url": "https://support.google.com/webmasters/answer/1061943?hl=en",
48
+ "instances": [
49
+ "AdsBot-Google (+http://www.google.com/adsbot.html)"
50
+ ]
17
51
  },
18
52
  {
19
- "pattern": "AdsBot-Google",
20
- "url": "https://support.google.com/webmasters/answer/1061943?hl=en"
53
+ "pattern": "AdsBot-Google-Mobile",
54
+ "addition_date": "2017/08/21",
55
+ "url": "https://support.google.com/adwords/answer/2404197",
56
+ "instances": [
57
+ "AdsBot-Google-Mobile-Apps",
58
+ "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
59
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)"
60
+ ]
21
61
  },
22
62
  {
23
63
  "pattern": "Mediapartners-Google",
24
- "url": "https://support.google.com/webmasters/answer/1061943?hl=en"
64
+ "url": "https://support.google.com/webmasters/answer/1061943?hl=en",
65
+ "instances": [
66
+ "Mediapartners-Google",
67
+ "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)",
68
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)",
69
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"
70
+ ]
71
+ },
72
+ {
73
+ "pattern": "Mediapartners \\(Googlebot\\)",
74
+ "addition_date": "2017/08/08",
75
+ "url": "https://support.google.com/webmasters/answer/1061943?hl=en",
76
+ "instances": []
77
+ },
78
+ {
79
+ "pattern": "APIs-Google",
80
+ "addition_date": "2017/08/08",
81
+ "url": "https://support.google.com/webmasters/answer/1061943?hl=en",
82
+ "instances": [
83
+ "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)"
84
+ ]
25
85
  },
26
86
  {
27
87
  "pattern": "bingbot",
28
- "url": "http://www.bing.com/bingbot.htm"
88
+ "url": "http://www.bing.com/bingbot.htm",
89
+ "instances": [
90
+ "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",
91
+ "Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)",
92
+ "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",
93
+ "Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)",
94
+ "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm",
95
+ "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
96
+ "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe",
97
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)",
98
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",
99
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)",
100
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
101
+ "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"
102
+ ]
29
103
  },
30
104
  {
31
105
  "pattern": "slurp",
32
- "url": "http://help.yahoo.com/help/us/ysearch/slurp"
106
+ "url": "http://help.yahoo.com/help/us/ysearch/slurp",
107
+ "instances": [
108
+ "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)",
109
+ "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
110
+ ]
33
111
  },
34
112
  {
35
- "pattern": "java"
113
+ "pattern": "java",
114
+ "instances": [
115
+ "Apache-HttpClient/4.2.3 (java 1.5)",
116
+ "Apache-HttpClient/4.2.5 (java 1.5)",
117
+ "Apache-HttpClient/4.3.1 (java 1.5)",
118
+ "Apache-HttpClient/4.3.3 (java 1.5)",
119
+ "Apache-HttpClient/4.3.5 (java 1.5)",
120
+ "Apache-HttpClient/UNAVAILABLE (java 1.4)",
121
+ "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
122
+ "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
123
+ "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
124
+ "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html"
125
+ ]
36
126
  },
37
127
  {
38
- "pattern": "wget"
128
+ "pattern": "[wW]get",
129
+ "instances": [
130
+ "WGETbot/1.0 (+http://wget.alanreed.org)",
131
+ "Wget/1.14 (linux-gnu)"
132
+ ]
39
133
  },
40
134
  {
41
- "pattern": "curl"
135
+ "pattern": "curl",
136
+ "instances": [
137
+ "eCairn-Grabber/1.0 (+http://ecairn.com/grabber) curl/7.15"
138
+ ]
42
139
  },
43
140
  {
44
- "pattern": "Commons-HttpClient"
141
+ "pattern": "Commons-HttpClient",
142
+ "instances": [
143
+ "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)",
144
+ "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)"
145
+ ]
45
146
  },
46
147
  {
47
- "pattern": "Python-urllib"
148
+ "pattern": "Python-urllib",
149
+ "instances": [
150
+ "Python-urllib/2.7 (+http://aranea.juls.savba.sk/aranea_about.html)",
151
+ "Python-urllib/2.7 (+http://ella.juls.savba.sk/aranea_about)",
152
+ "Python-urllib/2.7 (+http://sketch.juls.savba.sk/aranea_about)"
153
+ ]
48
154
  },
49
155
  {
50
- "pattern": "libwww"
156
+ "pattern": "libwww",
157
+ "instances": [
158
+ "2Bone_LinkChecker/1.0 libwww-perl/6.03",
159
+ "2Bone_LinkChkr/1.0 libwww-perl/6.03",
160
+ "W3C-checklink/2.90 libwww-perl/5.64",
161
+ "W3C-checklink/3.6.2.3 libwww-perl/5.64",
162
+ "W3C-checklink/4.2 [4.20] libwww-perl/5.803",
163
+ "W3C-checklink/4.2.1 [4.21] libwww-perl/5.803",
164
+ "W3C-checklink/4.3 [4.42] libwww-perl/5.805",
165
+ "W3C-checklink/4.3 [4.42] libwww-perl/5.808",
166
+ "W3C-checklink/4.3 [4.42] libwww-perl/5.820",
167
+ "W3C-checklink/4.5 [4.154] libwww-perl/5.823",
168
+ "W3C-checklink/4.5 [4.160] libwww-perl/5.823",
169
+ "amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831"
170
+ ]
51
171
  },
52
172
  {
53
- "pattern": "httpunit"
173
+ "pattern": "httpunit",
174
+ "instances": [
175
+ "httpunit/1.x"
176
+ ]
54
177
  },
55
178
  {
56
- "pattern": "nutch"
179
+ "pattern": "nutch",
180
+ "instances": [
181
+ "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
182
+ "istellabot-nutch/Nutch-1.10"
183
+ ]
57
184
  },
58
185
  {
59
186
  "pattern": "Go-http-client",
60
187
  "addition_date": "2016/03/26",
61
188
  "url": "https://golang.org/pkg/net/http/",
62
- "instances": ["Go-http-client/1.1"]
189
+ "instances": [
190
+ "Go-http-client/1.1"
191
+ ]
63
192
  },
64
193
  {
65
194
  "pattern": "phpcrawl",
66
195
  "addition_date": "2012-09/17",
67
- "url": "http://phpcrawl.cuab.de/"
196
+ "url": "http://phpcrawl.cuab.de/",
197
+ "instances": [
198
+ "phpcrawl"
199
+ ]
68
200
  },
69
201
  {
70
202
  "pattern": "msnbot",
71
- "url": "http://search.msn.com/msnbot.htm"
203
+ "url": "http://search.msn.com/msnbot.htm",
204
+ "instances": [
205
+ "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)",
206
+ "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)",
207
+ "librabot/1.0 (+http://search.msn.com/msnbot.htm)",
208
+ "librabot/2.0 (+http://search.msn.com/msnbot.htm)",
209
+ "msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)",
210
+ "msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)",
211
+ "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)",
212
+ "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)",
213
+ "msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)",
214
+ "msnbot/1.0 (+http://search.msn.com/msnbot.htm)",
215
+ "msnbot/1.1 (+http://search.msn.com/msnbot.htm)",
216
+ "msnbot/2.0b (+http://search.msn.com/msnbot.htm)",
217
+ "msnbot/2.0b (+http://search.msn.com/msnbot.htm).",
218
+ "msnbot/2.0b (+http://search.msn.com/msnbot.htm)._"
219
+ ]
72
220
  },
73
221
  {
74
- "pattern": "jyxobot"
222
+ "pattern": "jyxobot",
223
+ "instances": []
75
224
  },
76
225
  {
77
- "pattern": "FAST-WebCrawler"
226
+ "pattern": "FAST-WebCrawler",
227
+ "instances": [
228
+ "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",
229
+ "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
230
+ "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",
231
+ "FAST-WebCrawler/3.8"
232
+ ]
78
233
  },
79
234
  {
80
- "pattern": "FAST Enterprise Crawler"
235
+ "pattern": "FAST Enterprise Crawler",
236
+ "instances": [
237
+ "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/",
238
+ "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)"
239
+ ]
81
240
  },
82
241
  {
83
- "pattern": "biglotron"
242
+ "pattern": "BIGLOTRON",
243
+ "instances": [
244
+ "BIGLOTRON (Beta 2;GNU/Linux)"
245
+ ]
84
246
  },
85
247
  {
86
- "pattern": "teoma"
248
+ "pattern": "Teoma",
249
+ "instances": [
250
+ "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)",
251
+ "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)"
252
+ ],
253
+ "url": "http://about.ask.com/en/docs/about/webmasters.shtml"
87
254
  },
88
255
  {
89
- "pattern": "convera"
256
+ "pattern": "convera",
257
+ "instances": [
258
+ "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)"
259
+ ],
260
+ "url": "http://ews.converasearch.com/crawl.htm"
90
261
  },
91
262
  {
92
- "pattern": "seekbot"
263
+ "pattern": "seekbot",
264
+ "instances": [
265
+ "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2"
266
+ ],
267
+ "url": "http://www.seekbot.net/bot.html"
93
268
  },
94
269
  {
95
- "pattern": "gigabot",
96
- "instances": ["Gigabot/1.0", "Gigabot/2.0 (http://www.gigablast.com/spider.html)", "Gigabot/2.0 (http://www.gigablast.com/spider.html)"],
97
- "url": "https://github.com/gigablast/open-source-search-engine"
270
+ "pattern": "Gigabot",
271
+ "instances": [
272
+ "Gigabot/1.0",
273
+ "Gigabot/2.0 (http://www.gigablast.com/spider.html)"
274
+ ],
275
+ "url": "http://www.gigablast.com/spider.html"
98
276
  },
99
277
  {
100
- "pattern": "gigablast",
101
- "instances": ["GigablastOpenSource/1.0"],
278
+ "pattern": "Gigablast",
279
+ "instances": [
280
+ "GigablastOpenSource/1.0"
281
+ ],
102
282
  "url": "https://github.com/gigablast/open-source-search-engine"
103
283
  },
104
284
  {
105
- "pattern": "exabot"
106
- },
107
- {
108
- "pattern": "ngbot"
109
- },
110
- {
111
- "pattern": "ia_archiver"
112
- },
113
- {
114
- "pattern": "GingerCrawler"
285
+ "pattern": "exabot",
286
+ "instances": [
287
+ "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)",
288
+ "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)",
289
+ "Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)",
290
+ "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)",
291
+ "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)"
292
+ ]
115
293
  },
116
294
  {
117
- "pattern": "webmon "
295
+ "pattern": "ia_archiver",
296
+ "instances": [
297
+ "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",
298
+ "ia_archiver-web.archive.org"
299
+ ]
118
300
  },
119
301
  {
120
- "pattern": "httrack"
302
+ "pattern": "GingerCrawler",
303
+ "instances": [
304
+ "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)"
305
+ ]
121
306
  },
122
307
  {
123
- "pattern": "webcrawler"
308
+ "pattern": "webmon ",
309
+ "instances": []
124
310
  },
125
311
  {
126
- "pattern": "grub.org"
312
+ "pattern": "HTTrack",
313
+ "instances": [
314
+ "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"
315
+ ]
127
316
  },
128
317
  {
129
- "pattern": "UsineNouvelleCrawler"
318
+ "pattern": "grub.org",
319
+ "instances": [
320
+ "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)",
321
+ "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)",
322
+ "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)",
323
+ "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)",
324
+ "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)",
325
+ "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)",
326
+ "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)",
327
+ "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)",
328
+ "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)",
329
+ "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)",
330
+ "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)"
331
+ ]
130
332
  },
131
333
  {
132
- "pattern": "antibot"
334
+ "pattern": "UsineNouvelleCrawler",
335
+ "instances": []
133
336
  },
134
337
  {
135
- "pattern": "netresearchserver"
338
+ "pattern": "antibot",
339
+ "instances": []
136
340
  },
137
341
  {
138
- "pattern": "speedy"
342
+ "pattern": "netresearchserver",
343
+ "instances": []
139
344
  },
140
345
  {
141
- "pattern": "fluffy"
346
+ "pattern": "speedy",
347
+ "instances": [
348
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)",
349
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)",
350
+ "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)",
351
+ "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)",
352
+ "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)"
353
+ ]
142
354
  },
143
355
  {
144
- "pattern": "bibnum.bnf"
356
+ "pattern": "fluffy",
357
+ "instances": []
145
358
  },
146
359
  {
147
- "pattern": "findlink"
360
+ "pattern": "bibnum.bnf",
361
+ "instances": [
362
+ "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)"
363
+ ]
148
364
  },
149
365
  {
150
- "pattern": "msrbot"
366
+ "pattern": "findlink",
367
+ "instances": [
368
+ "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)",
369
+ "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)",
370
+ "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)",
371
+ "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)",
372
+ "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)",
373
+ "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)",
374
+ "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)",
375
+ "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)",
376
+ "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)",
377
+ "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)",
378
+ "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)",
379
+ "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)",
380
+ "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)",
381
+ "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)",
382
+ "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)",
383
+ "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",
384
+ "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)",
385
+ "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)",
386
+ "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)",
387
+ "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",
388
+ "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)",
389
+ "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",
390
+ "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)"
391
+ ]
151
392
  },
152
393
  {
153
- "pattern": "panscient"
394
+ "pattern": "msrbot",
395
+ "instances": []
154
396
  },
155
397
  {
156
- "pattern": "yacybot"
398
+ "pattern": "panscient",
399
+ "instances": [
400
+ "panscient.com"
401
+ ]
157
402
  },
158
403
  {
159
- "pattern": "AISearchBot"
404
+ "pattern": "yacybot",
405
+ "instances": [
406
+ "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
407
+ "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
408
+ "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
409
+ "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
410
+ "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html",
411
+ "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
412
+ "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html",
413
+ "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",
414
+ "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html",
415
+ "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
416
+ "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html",
417
+ "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html",
418
+ "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",
419
+ "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",
420
+ "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html",
421
+ "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html",
422
+ "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html",
423
+ "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html",
424
+ "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html",
425
+ "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html",
426
+ "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html",
427
+ "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html",
428
+ "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html",
429
+ "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
430
+ "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html",
431
+ "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
432
+ "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",
433
+ "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html",
434
+ "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html",
435
+ "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html",
436
+ "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html",
437
+ "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html",
438
+ "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html",
439
+ "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",
440
+ "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",
441
+ "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
442
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html",
443
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html",
444
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",
445
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html",
446
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
447
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html",
448
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html"
449
+ ]
160
450
  },
161
451
  {
162
- "pattern": "IOI"
452
+ "pattern": "AISearchBot",
453
+ "instances": []
163
454
  },
164
455
  {
165
- "pattern": "ips-agent"
456
+ "pattern": "IOI",
457
+ "instances": []
166
458
  },
167
459
  {
168
- "pattern": "tagoobot"
460
+ "pattern": "ips-agent",
461
+ "instances": [
462
+ "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent",
463
+ "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7",
464
+ "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3",
465
+ "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24",
466
+ "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1"
467
+ ]
169
468
  },
170
469
  {
171
- "pattern": "MJ12bot"
470
+ "pattern": "tagoobot",
471
+ "instances": []
172
472
  },
173
473
  {
174
- "pattern": "dotbot"
474
+ "pattern": "MJ12bot",
475
+ "instances": [
476
+ "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)",
477
+ "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)",
478
+ "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)",
479
+ "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)",
480
+ "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)",
481
+ "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)",
482
+ "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)",
483
+ "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)",
484
+ "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)",
485
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)",
486
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)",
487
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)",
488
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)",
489
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)",
490
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)",
491
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)",
492
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)",
493
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)"
494
+ ]
175
495
  },
176
496
  {
177
- "pattern": "woriobot"
497
+ "pattern": "woriobot",
498
+ "instances": [
499
+ "Mozilla/5.0 (compatible; woriobot +http://worio.com)",
500
+ "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)"
501
+ ]
178
502
  },
179
503
  {
180
- "pattern": "yanga"
504
+ "pattern": "yanga",
505
+ "instances": [
506
+ "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)"
507
+ ]
181
508
  },
182
509
  {
183
- "pattern": "buzzbot"
510
+ "pattern": "buzzbot",
511
+ "instances": [
512
+ "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)"
513
+ ]
184
514
  },
185
515
  {
186
- "pattern": "mlbot"
516
+ "pattern": "mlbot",
517
+ "instances": [
518
+ "MLBot (www.metadatalabs.com/mlbot)"
519
+ ]
187
520
  },
188
521
  {
189
- "pattern": "yandexbot",
522
+ "pattern": "YandexBot",
190
523
  "url": "http://yandex.com/bots",
191
- "instances": ["Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"],
524
+ "instances": [
525
+ "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"
526
+ ],
192
527
  "addition_date": "2015/04/14"
193
528
  },
529
+ {
530
+ "pattern": "yandex.com\\/bots",
531
+ "url": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml#robot-in-logs",
532
+ "instances": [
533
+ "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)"
534
+ ],
535
+ "addition_date": "2016/12/01"
536
+ },
194
537
  {
195
538
  "pattern": "purebot",
196
- "addition_date": "2010/01/19"
539
+ "addition_date": "2010/01/19",
540
+ "instances": []
197
541
  },
198
542
  {
199
543
  "pattern": "Linguee Bot",
200
544
  "addition_date": "2010/01/26",
201
- "url": "http://www.linguee.com/bot"
202
- },
203
- {
204
- "pattern": "Voyager",
205
- "addition_date": "2010/02/01",
206
- "url": "http://www.kosmix.com/crawler.html"
545
+ "url": "http://www.linguee.com/bot",
546
+ "instances": [
547
+ "Linguee Bot (http://www.linguee.com/bot)",
548
+ "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)"
549
+ ]
207
550
  },
208
551
  {
209
552
  "pattern": "CyberPatrol",
210
553
  "addition_date": "2010/02/11",
211
- "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp"
554
+ "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp",
555
+ "instances": [
556
+ "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)"
557
+ ]
212
558
  },
213
559
  {
214
560
  "pattern": "voilabot",
215
- "addition_date": "2010/05/18"
561
+ "addition_date": "2010/05/18",
562
+ "instances": [
563
+ "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
564
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
565
+ "Mozilla/5.0 (compatible; OrangeBot/2.0; support.voilabot@orange.com)"
566
+ ]
216
567
  },
217
568
  {
218
- "pattern": "baiduspider",
569
+ "pattern": "Baiduspider",
219
570
  "addition_date": "2010/07/15",
220
- "url": "http://www.baidu.jp/spider/"
571
+ "url": "http://www.baidu.jp/spider/",
572
+ "instances": [
573
+ "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
574
+ ]
221
575
  },
222
576
  {
223
577
  "pattern": "citeseerxbot",
224
- "addition_date": "2010/07/17"
578
+ "addition_date": "2010/07/17",
579
+ "instances": []
225
580
  },
226
581
  {
227
582
  "pattern": "spbot",
228
583
  "addition_date": "2010/07/31",
229
- "url": "http://www.seoprofiler.com/bot"
584
+ "url": "http://www.seoprofiler.com/bot",
585
+ "instances": [
586
+ "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )",
587
+ "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )",
588
+ "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )",
589
+ "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )",
590
+ "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )",
591
+ "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )",
592
+ "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )",
593
+ "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )",
594
+ "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )",
595
+ "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )",
596
+ "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )",
597
+ "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )",
598
+ "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )",
599
+ "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )",
600
+ "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )",
601
+ "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )",
602
+ "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )",
603
+ "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )",
604
+ "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )",
605
+ "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )",
606
+ "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )",
607
+ "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )",
608
+ "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )",
609
+ "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )",
610
+ "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )",
611
+ "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )",
612
+ "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )",
613
+ "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )",
614
+ "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )",
615
+ "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )",
616
+ "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )",
617
+ "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )",
618
+ "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )",
619
+ "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )"
620
+ ]
230
621
  },
231
622
  {
232
623
  "pattern": "twengabot",
233
624
  "addition_date": "2010/08/03",
234
- "url": "http://www.twenga.com/bot.html"
625
+ "url": "http://www.twenga.com/bot.html",
626
+ "instances": []
235
627
  },
236
628
  {
237
629
  "pattern": "postrank",
238
630
  "addition_date": "2010/08/03",
239
- "url": "http://www.postrank.com"
631
+ "url": "http://www.postrank.com",
632
+ "instances": [
633
+ "PostRank/2.0 (postrank.com)",
634
+ "PostRank/2.0 (postrank.com; 1 subscribers)"
635
+ ]
240
636
  },
241
637
  {
242
638
  "pattern": "turnitinbot",
243
639
  "addition_date": "2010/09/26",
244
- "url": "http://www.turnitin.com"
640
+ "url": "http://www.turnitin.com",
641
+ "instances": []
245
642
  },
246
643
  {
247
644
  "pattern": "scribdbot",
248
645
  "addition_date": "2010/09/28",
249
- "url": "http://www.scribd.com"
646
+ "url": "http://www.scribd.com",
647
+ "instances": []
250
648
  },
251
649
  {
252
650
  "pattern": "page2rss",
253
651
  "addition_date": "2010/10/07",
254
- "url": "http://www.page2rss.com"
652
+ "url": "http://www.page2rss.com",
653
+ "instances": [
654
+ "Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)"
655
+ ]
255
656
  },
256
657
  {
257
658
  "pattern": "sitebot",
258
659
  "addition_date": "2010/12/15",
259
- "url": "http://www.sitebot.org"
660
+ "url": "http://www.sitebot.org",
661
+ "instances": [
662
+ "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)"
663
+ ]
260
664
  },
261
665
  {
262
666
  "pattern": "linkdex",
263
667
  "addition_date": "2011/01/06",
264
- "url": "http://www.linkdex.com"
668
+ "url": "http://www.linkdex.com",
669
+ "instances": [
670
+ "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)",
671
+ "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)",
672
+ "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)",
673
+ "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)",
674
+ "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)",
675
+ "linkdex.com/v2.0",
676
+ "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)"
677
+ ]
265
678
  },
266
679
  {
267
680
  "pattern": "Adidxbot",
268
- "url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx"
681
+ "url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx",
682
+ "instances": []
269
683
  },
270
684
  {
271
685
  "pattern": "blekkobot",
272
- "url": "http://blekko.com/about/blekkobot"
686
+ "url": "http://blekko.com/about/blekkobot",
687
+ "instances": [
688
+ "Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)"
689
+ ]
273
690
  },
274
691
  {
275
692
  "pattern": "ezooms",
276
693
  "addition_date": "2011/04/27",
277
- "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289"
694
+ "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289",
695
+ "instances": [
696
+ "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)"
697
+ ]
278
698
  },
279
699
  {
280
700
  "pattern": "dotbot",
281
- "addition_date": "2011/04/27"
701
+ "addition_date": "2011/04/27",
702
+ "instances": [
703
+ "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)",
704
+ "dotbot"
705
+ ]
282
706
  },
283
707
  {
284
708
  "pattern": "Mail.RU_Bot",
285
709
  "addition_date": "2011/04/27",
286
- "instances" : [
710
+ "instances": [
287
711
  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/",
288
712
  "Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/"
289
713
  ]
@@ -291,291 +715,502 @@
291
715
  {
292
716
  "pattern": "discobot",
293
717
  "addition_date": "2011/05/03",
294
- "url": "http://discoveryengine.com/discobot.html"
718
+ "url": "http://discoveryengine.com/discobot.html",
719
+ "instances": [
720
+ "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)",
721
+ "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)",
722
+ "mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)"
723
+ ]
295
724
  },
296
725
  {
297
726
  "pattern": "heritrix",
298
727
  "addition_date": "2011/06/21",
299
- "url": "http://crawler.archive.org/"
728
+ "url": "http://crawler.archive.org/",
729
+ "instances": [
730
+ "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)",
731
+ "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)",
732
+ "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)",
733
+ "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)",
734
+ "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)",
735
+ "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)",
736
+ "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)",
737
+ "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)",
738
+ "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)",
739
+ "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)",
740
+ "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)",
741
+ "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)",
742
+ "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)",
743
+ "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)",
744
+ "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)",
745
+ "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)",
746
+ "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)",
747
+ "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)",
748
+ "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)",
749
+ "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)",
750
+ "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)",
751
+ "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)",
752
+ "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)",
753
+ "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)",
754
+ "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)"
755
+ ]
300
756
  },
301
757
  {
302
758
  "pattern": "findthatfile",
303
759
  "addition_date": "2011/06/21",
304
- "url": "http://www.findthatfile.com/"
760
+ "url": "http://www.findthatfile.com/",
761
+ "instances": []
305
762
  },
306
763
  {
307
764
  "pattern": "europarchive.org",
308
765
  "addition_date": "2011/06/21",
309
- "url": ""
766
+ "url": "",
767
+ "instances": [
768
+ "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)"
769
+ ]
310
770
  },
311
771
  {
312
772
  "pattern": "NerdByNature.Bot",
313
773
  "addition_date": "2011/07/12",
314
- "url": "http://www.nerdbynature.net/bot"
774
+ "url": "http://www.nerdbynature.net/bot",
775
+ "instances": [
776
+ "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)"
777
+ ]
315
778
  },
316
779
  {
317
780
  "pattern": "sistrix crawler",
318
- "addition_date": "2011/08/02"
781
+ "addition_date": "2011/08/02",
782
+ "instances": []
319
783
  },
320
784
  {
321
- "pattern": "ahrefsbot",
322
- "addition_date": "2011/08/28"
785
+ "pattern": "AhrefsBot",
786
+ "addition_date": "2011/08/28",
787
+ "instances": [
788
+ "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)"
789
+ ]
323
790
  },
324
791
  {
325
792
  "pattern": "Aboundex",
326
793
  "addition_date": "2011/09/28",
327
- "url": "http://www.aboundex.com/crawler/"
794
+ "url": "http://www.aboundex.com/crawler/",
795
+ "instances": [
796
+ "Aboundex/0.2 (http://www.aboundex.com/crawler/)",
797
+ "Aboundex/0.3 (http://www.aboundex.com/crawler/)"
798
+ ]
328
799
  },
329
800
  {
330
801
  "pattern": "domaincrawler",
331
- "addition_date": "2011/10/21"
802
+ "addition_date": "2011/10/21",
803
+ "instances": [
804
+ "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)"
805
+ ]
332
806
  },
333
807
  {
334
808
  "pattern": "wbsearchbot",
335
809
  "addition_date": "2011/12/21",
336
- "url": "http://www.warebay.com/bot.html"
810
+ "url": "http://www.warebay.com/bot.html",
811
+ "instances": []
337
812
  },
338
813
  {
339
814
  "pattern": "summify",
340
815
  "addition_date": "2012/01/04",
341
- "url": "http://summify.com"
816
+ "url": "http://summify.com",
817
+ "instances": [
818
+ "Summify (Summify/1.0.1; +http://summify.com)"
819
+ ]
342
820
  },
343
821
  {
344
- "pattern": "ccbot",
822
+ "pattern": "CCBot",
345
823
  "addition_date": "2012/02/05",
346
- "url": "http://www.commoncrawl.org/bot.html"
824
+ "url": "http://www.commoncrawl.org/bot.html",
825
+ "instances": [
826
+ "CCBot/2.0 (http://commoncrawl.org/faq/)"
827
+ ]
347
828
  },
348
829
  {
349
830
  "pattern": "edisterbot",
350
- "addition_date": "2012/02/25"
831
+ "addition_date": "2012/02/25",
832
+ "instances": []
351
833
  },
352
834
  {
353
835
  "pattern": "seznambot",
354
- "addition_date": "2012/03/14"
836
+ "addition_date": "2012/03/14",
837
+ "instances": [
838
+ "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)",
839
+ "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)",
840
+ "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)",
841
+ "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)",
842
+ "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)"
843
+ ]
355
844
  },
356
845
  {
357
846
  "pattern": "ec2linkfinder",
358
- "addition_date": "2012/03/22"
847
+ "addition_date": "2012/03/22",
848
+ "instances": [
849
+ "ec2linkfinder"
850
+ ]
359
851
  },
360
852
  {
361
853
  "pattern": "gslfbot",
362
- "addition_date": "2012/04/03"
854
+ "addition_date": "2012/04/03",
855
+ "instances": []
363
856
  },
364
857
  {
365
858
  "pattern": "aihitbot",
366
- "addition_date": "2012/04/16"
859
+ "addition_date": "2012/04/16",
860
+ "instances": []
367
861
  },
368
862
  {
369
863
  "pattern": "intelium_bot",
370
- "addition_date": "2012/05/07"
864
+ "addition_date": "2012/05/07",
865
+ "instances": []
371
866
  },
372
867
  {
373
868
  "pattern": "facebookexternalhit",
374
- "addition_date": "2012/05/07"
869
+ "addition_date": "2012/05/07",
870
+ "instances": [
871
+ "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)",
872
+ "facebookexternalhit/1.1",
873
+ "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"
874
+ ]
375
875
  },
376
876
  {
377
- "pattern": "yeti",
378
- "addition_date": "2012/05/07"
877
+ "pattern": "Yeti",
878
+ "addition_date": "2012/05/07",
879
+ "url": "http://naver.me/bot",
880
+ "instances": [
881
+ "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)"
882
+ ]
379
883
  },
380
884
  {
381
885
  "pattern": "RetrevoPageAnalyzer",
382
- "addition_date": "2012/05/07"
886
+ "addition_date": "2012/05/07",
887
+ "instances": [
888
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)"
889
+ ]
383
890
  },
384
891
  {
385
892
  "pattern": "lb-spider",
386
- "addition_date": "2012/05/07"
893
+ "addition_date": "2012/05/07",
894
+ "instances": []
387
895
  },
388
896
  {
389
- "pattern": "sogou",
897
+ "pattern": "Sogou",
390
898
  "addition_date": "2012/05/13",
391
- "url": "http://www.sogou.com/docs/help/webmasters.htm#07"
899
+ "url": "http://www.sogou.com/docs/help/webmasters.htm#07",
900
+ "instances": [
901
+ "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
902
+ "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
903
+ "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
904
+ ]
392
905
  },
393
906
  {
394
907
  "pattern": "lssbot",
395
- "addition_date": "2012/05/15"
908
+ "addition_date": "2012/05/15",
909
+ "instances": []
396
910
  },
397
911
  {
398
912
  "pattern": "careerbot",
399
913
  "addition_date": "2012/05/23",
400
- "url": "http://www.career-x.de/bot.html"
914
+ "url": "http://www.career-x.de/bot.html",
915
+ "instances": []
401
916
  },
402
917
  {
403
918
  "pattern": "wotbox",
404
919
  "addition_date": "2012/06/12",
405
- "url": "http://www.wotbox.com"
920
+ "url": "http://www.wotbox.com",
921
+ "instances": [
922
+ "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)",
923
+ "Wotbox/2.01 (+http://www.wotbox.com/bot/)"
924
+ ]
406
925
  },
407
926
  {
408
927
  "pattern": "wocbot",
409
928
  "addition_date": "2012/07/25",
410
- "url": "http://www.wocodi.com/crawler"
929
+ "url": "http://www.wocodi.com/crawler",
930
+ "instances": []
411
931
  },
412
932
  {
413
933
  "pattern": "ichiro",
414
934
  "addition_date": "2012/08/28",
415
- "url": "http://help.goo.ne.jp/help/article/1142"
935
+ "url": "http://help.goo.ne.jp/help/article/1142",
936
+ "instances": [
937
+ "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)",
938
+ "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
939
+ "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
940
+ "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)",
941
+ "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)",
942
+ "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)",
943
+ "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
944
+ "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
945
+ "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)",
946
+ "ichiro/2.0 (ichiro@nttr.co.jp)",
947
+ "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)",
948
+ "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)",
949
+ "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
950
+ "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)",
951
+ "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)"
952
+ ]
416
953
  },
417
954
  {
418
955
  "pattern": "DuckDuckBot",
419
956
  "addition_date": "2012/09/19",
420
- "url": "http://duckduckgo.com/duckduckbot.html"
957
+ "url": "http://duckduckgo.com/duckduckbot.html",
958
+ "instances": [
959
+ "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
960
+ "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)"
961
+ ]
421
962
  },
422
963
  {
423
964
  "pattern": "lssrocketcrawler",
424
- "addition_date": "2012/09/24"
965
+ "addition_date": "2012/09/24",
966
+ "instances": []
425
967
  },
426
968
  {
427
969
  "pattern": "drupact",
428
970
  "addition_date": "2012/09/27",
429
- "url": "http://www.arocom.de/drupact"
971
+ "url": "http://www.arocom.de/drupact",
972
+ "instances": [
973
+ "drupact/0.7; http://www.arocom.de/drupact"
974
+ ]
430
975
  },
431
976
  {
432
977
  "pattern": "webcompanycrawler",
433
- "addition_date": "2012/10/03"
978
+ "addition_date": "2012/10/03",
979
+ "instances": []
434
980
  },
435
981
  {
436
982
  "pattern": "acoonbot",
437
983
  "addition_date": "2012/10/07",
438
- "url": "http://www.acoon.de/robot.asp"
984
+ "url": "http://www.acoon.de/robot.asp",
985
+ "instances": []
439
986
  },
440
987
  {
441
988
  "pattern": "openindexspider",
442
989
  "addition_date": "2012/10/26",
443
- "url": "http://www.openindex.io/en/webmasters/spider.html"
990
+ "url": "http://www.openindex.io/en/webmasters/spider.html",
991
+ "instances": []
444
992
  },
445
993
  {
446
994
  "pattern": "gnam gnam spider",
447
- "addition_date": "2012/10/31"
995
+ "addition_date": "2012/10/31",
996
+ "instances": []
448
997
  },
449
998
  {
450
- "pattern": "web-archive-net.com.bot"
999
+ "pattern": "web-archive-net.com.bot",
1000
+ "instances": []
451
1001
  },
452
1002
  {
453
1003
  "pattern": "backlinkcrawler",
454
- "addition_date": "2013/01/04"
1004
+ "addition_date": "2013/01/04",
1005
+ "instances": []
455
1006
  },
456
1007
  {
457
1008
  "pattern": "coccoc",
458
1009
  "addition_date": "2013/01/04",
459
- "url": "http://help.coccoc.vn/"
1010
+ "url": "http://help.coccoc.vn/",
1011
+ "instances": [
1012
+ "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)",
1013
+ "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)",
1014
+ "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)",
1015
+ "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)",
1016
+ "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)",
1017
+ "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)",
1018
+ "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)",
1019
+ "coccoc",
1020
+ "coccoc/1.0 ()",
1021
+ "coccoc/1.0 (http://help.coccoc.com/)",
1022
+ "coccoc/1.0 (http://help.coccoc.vn/)"
1023
+ ]
460
1024
  },
461
1025
  {
462
1026
  "pattern": "integromedb",
463
1027
  "addition_date": "2013/01/10",
464
- "url": "http://www.integromedb.org/Crawler"
1028
+ "url": "http://www.integromedb.org/Crawler",
1029
+ "instances": [
1030
+ "www.integromedb.org/Crawler"
1031
+ ]
465
1032
  },
466
1033
  {
467
1034
  "pattern": "content crawler spider",
468
- "addition_date": "2013/01/11"
1035
+ "addition_date": "2013/01/11",
1036
+ "instances": []
469
1037
  },
470
1038
  {
471
1039
  "pattern": "toplistbot",
472
- "addition_date": "2013/02/05"
1040
+ "addition_date": "2013/02/05",
1041
+ "instances": []
473
1042
  },
474
1043
  {
475
1044
  "pattern": "seokicks-robot",
476
- "addition_date": "2013/02/25"
1045
+ "addition_date": "2013/02/25",
1046
+ "instances": []
477
1047
  },
478
1048
  {
479
1049
  "pattern": "it2media-domain-crawler",
480
- "addition_date": "2013/03/12"
1050
+ "addition_date": "2013/03/12",
1051
+ "instances": [
1052
+ "it2media-domain-crawler/1.0 on crawler-prod.it2media.de",
1053
+ "it2media-domain-crawler/2.0"
1054
+ ]
481
1055
  },
482
1056
  {
483
1057
  "pattern": "ip-web-crawler.com",
484
- "addition_date": "2013/03/22"
1058
+ "addition_date": "2013/03/22",
1059
+ "instances": []
485
1060
  },
486
1061
  {
487
1062
  "pattern": "siteexplorer.info",
488
- "addition_date": "2013/05/01"
1063
+ "addition_date": "2013/05/01",
1064
+ "instances": [
1065
+ "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)",
1066
+ "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)"
1067
+ ]
489
1068
  },
490
1069
  {
491
1070
  "pattern": "elisabot",
492
- "addition_date": "2013/06/27"
1071
+ "addition_date": "2013/06/27",
1072
+ "instances": []
493
1073
  },
494
1074
  {
495
1075
  "pattern": "proximic",
496
1076
  "addition_date": "2013/09/12",
497
- "url": "http://www.proximic.com/info/spider.php"
1077
+ "url": "http://www.proximic.com/info/spider.php",
1078
+ "instances": [
1079
+ "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)",
1080
+ "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)"
1081
+ ]
498
1082
  },
499
1083
  {
500
1084
  "pattern": "changedetection",
501
1085
  "addition_date": "2013/09/13",
502
- "url": "http://www.changedetection.com/bot.html"
1086
+ "url": "http://www.changedetection.com/bot.html",
1087
+ "instances": [
1088
+ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )"
1089
+ ]
503
1090
  },
504
1091
  {
505
1092
  "pattern": "blexbot",
506
1093
  "addition_date": "2013/10/03",
507
- "url": "http://webmeup-crawler.com/"
1094
+ "url": "http://webmeup-crawler.com/",
1095
+ "instances": []
508
1096
  },
509
1097
  {
510
1098
  "pattern": "arabot",
511
- "addition_date": "2013/10/09"
1099
+ "addition_date": "2013/10/09",
1100
+ "instances": []
512
1101
  },
513
1102
  {
514
1103
  "pattern": "WeSEE:Search",
515
- "addition_date": "2013/11/18"
1104
+ "addition_date": "2013/11/18",
1105
+ "instances": [
1106
+ "WeSEE:Search",
1107
+ "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)"
1108
+ ]
516
1109
  },
517
1110
  {
518
1111
  "pattern": "niki-bot",
519
- "addition_date": "2014/01/01"
1112
+ "addition_date": "2014/01/01",
1113
+ "instances": []
520
1114
  },
521
1115
  {
522
1116
  "pattern": "CrystalSemanticsBot",
523
1117
  "addition_date": "2014/02/17",
524
- "url": "http://www.crystalsemantics.com/user-agent/"
1118
+ "url": "http://www.crystalsemantics.com/user-agent/",
1119
+ "instances": []
525
1120
  },
526
1121
  {
527
1122
  "pattern": "rogerbot",
528
1123
  "addition_date": "2014/02/28",
529
- "url": "http://moz.com/help/pro/what-is-rogerbot-"
1124
+ "url": "http://moz.com/help/pro/what-is-rogerbot-",
1125
+ "instances": [
1126
+ "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)",
1127
+ "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)",
1128
+ "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)",
1129
+ "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com",
1130
+ "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)",
1131
+ "rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)",
1132
+ "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)",
1133
+ "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)",
1134
+ "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)",
1135
+ "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)",
1136
+ "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)",
1137
+ "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)",
1138
+ "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)"
1139
+ ]
530
1140
  },
531
1141
  {
532
1142
  "pattern": "360Spider",
533
1143
  "addition_date": "2014/03/14",
534
- "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html"
1144
+ "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html",
1145
+ "instances": [
1146
+ "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider",
1147
+ "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)",
1148
+ "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider",
1149
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider",
1150
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11; 360Spider",
1151
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;",
1152
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider",
1153
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider",
1154
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)"
1155
+ ]
535
1156
  },
536
1157
  {
537
1158
  "pattern": "psbot",
538
1159
  "addition_date": "2014/03/31",
539
- "url": "http://www.picsearch.com/bot.html"
1160
+ "url": "http://www.picsearch.com/bot.html",
1161
+ "instances": [
1162
+ "psbot-image (+http://www.picsearch.com/bot.html)",
1163
+ "psbot-page (+http://www.picsearch.com/bot.html)",
1164
+ "psbot/0.1 (+http://www.picsearch.com/bot.html)"
1165
+ ]
540
1166
  },
541
1167
  {
542
1168
  "pattern": "InterfaxScanBot",
543
1169
  "addition_date": "2014/03/31",
544
- "url": "http://scan-interfax.ru"
545
- },
546
- {
547
- "pattern": "Lipperhey SEO Service",
548
- "addition_date": "2014/04/01",
549
- "url": "http://www.lipperhey.com/"
1170
+ "url": "http://scan-interfax.ru",
1171
+ "instances": []
550
1172
  },
551
1173
  {
552
1174
  "pattern": "CC Metadata Scaper",
553
1175
  "addition_date": "2014/04/01",
554
- "url": "http://wiki.creativecommons.org/Metadata_Scraper"
1176
+ "url": "http://wiki.creativecommons.org/Metadata_Scraper",
1177
+ "instances": [
1178
+ "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper"
1179
+ ]
555
1180
  },
556
1181
  {
557
1182
  "pattern": "g00g1e.net",
558
1183
  "addition_date": "2014/04/01",
559
- "url": "http://www.g00g1e.net/"
1184
+ "url": "http://www.g00g1e.net/",
1185
+ "instances": []
560
1186
  },
561
1187
  {
562
1188
  "pattern": "GrapeshotCrawler",
563
1189
  "addition_date": "2014/04/01",
564
- "url": "http://www.grapeshot.co.uk/crawler.php"
1190
+ "url": "http://www.grapeshot.co.uk/crawler.php",
1191
+ "instances": [
1192
+ "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)"
1193
+ ]
565
1194
  },
566
1195
  {
567
1196
  "pattern": "urlappendbot",
568
1197
  "addition_date": "2014/05/10",
569
- "url": "http://www.profound.net/urlappendbot.html"
1198
+ "url": "http://www.profound.net/urlappendbot.html",
1199
+ "instances": [
1200
+ "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)"
1201
+ ]
570
1202
  },
571
1203
  {
572
1204
  "pattern": "brainobot",
573
- "addition_date": "2014/06/24"
1205
+ "addition_date": "2014/06/24",
1206
+ "instances": []
574
1207
  },
575
1208
  {
576
1209
  "pattern": "fr-crawler",
577
1210
  "addition_date": "2014/07/31",
578
- "instances": ["Mozilla/5.0 (compatible; fr-crawler/1.1)"]
1211
+ "instances": [
1212
+ "Mozilla/5.0 (compatible; fr-crawler/1.1)"
1213
+ ]
579
1214
  },
580
1215
  {
581
1216
  "pattern": "binlar",
@@ -591,349 +1226,1101 @@
591
1226
  {
592
1227
  "pattern": "SimpleCrawler",
593
1228
  "addition_date": "2014/09/12",
594
- "instances": ["SimpleCrawler/0.1" ]
595
- },
596
- {
597
- "pattern": "Livelapbot",
598
- "addition_date": "2014/09/12",
599
- "instances": ["Livelapbot/0.1" ]
1229
+ "instances": [
1230
+ "SimpleCrawler/0.1"
1231
+ ]
600
1232
  },
601
1233
  {
602
1234
  "pattern": "Twitterbot",
603
1235
  "addition_date": "2014/09/12",
604
- "instances": ["Twitterbot/0.1", "Twitterbot/1.0" ]
1236
+ "url": "https://dev.twitter.com/cards/getting-started",
1237
+ "instances": [
1238
+ "Twitterbot/0.1",
1239
+ "Twitterbot/1.0"
1240
+ ]
605
1241
  },
606
1242
  {
607
1243
  "pattern": "cXensebot",
608
1244
  "addition_date": "2014/10/05",
609
- "instances": ["cXensebot/1.1a"],
1245
+ "instances": [
1246
+ "cXensebot/1.1a"
1247
+ ],
610
1248
  "url": "http://www.cxense.com/bot.html"
611
1249
  },
612
1250
  {
613
1251
  "pattern": "smtbot",
614
1252
  "addition_date": "2014/10/04",
615
- "instances": ["Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbo)t", "SMTBot (similartech.com/smtbot)"],
1253
+ "instances": [
1254
+ "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",
1255
+ "SMTBot (similartech.com/smtbot)"
1256
+ ],
616
1257
  "url": "http://www.similartech.com/smtbot"
617
1258
  },
618
1259
  {
619
1260
  "pattern": "bnf.fr_bot",
620
1261
  "addition_date": "2014/11/18",
621
1262
  "url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html",
622
- "instances": ["Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)"]
1263
+ "instances": [
1264
+ "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)"
1265
+ ]
623
1266
  },
624
1267
  {
625
1268
  "pattern": "A6-Indexer",
626
1269
  "addition_date": "2014/12/05",
627
1270
  "url": "http://www.a6corp.com/a6-web-scraping-policy/",
628
- "instances": ["A6-Indexer"]
1271
+ "instances": [
1272
+ "A6-Indexer"
1273
+ ]
629
1274
  },
630
1275
  {
631
1276
  "pattern": "ADmantX",
632
1277
  "addition_date": "2014/12/05",
633
1278
  "url": "http://www.admantx.com",
634
- "instances": ["ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com"]
1279
+ "instances": [
1280
+ "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com"
1281
+ ]
635
1282
  },
636
1283
  {
637
1284
  "pattern": "Facebot",
638
1285
  "url": "https://developers.facebook.com/docs/sharing/best-practices#crawl",
639
- "addition_date": "2014/12/30"
640
- },
641
- {
642
- "pattern": "Twitterbot",
643
- "url": "https://dev.twitter.com/cards/getting-started",
644
- "addition_date": "2014/12/30"
1286
+ "addition_date": "2014/12/30",
1287
+ "instances": [
1288
+ "Facebot/1.0"
1289
+ ]
645
1290
  },
646
1291
  {
647
1292
  "pattern": "OrangeBot",
648
- "instances": ["Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com"],
1293
+ "instances": [
1294
+ "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com"
1295
+ ],
649
1296
  "addition_date": "2015/01/12"
650
1297
  },
651
1298
  {
652
1299
  "pattern": "memorybot",
653
1300
  "url": "http://mignify.com/bot.htm",
654
- "instances": ["Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)"],
1301
+ "instances": [
1302
+ "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)"
1303
+ ],
655
1304
  "addition_date": "2015/02/01"
656
1305
  },
657
1306
  {
658
1307
  "pattern": "AdvBot",
659
1308
  "url": "http://advbot.net/bot.html",
660
- "instances": ["Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)"],
1309
+ "instances": [
1310
+ "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)"
1311
+ ],
661
1312
  "addition_date": "2015/02/01"
662
1313
  },
663
1314
  {
664
1315
  "pattern": "MegaIndex",
665
1316
  "url": "https://www.megaindex.ru/?tab=linkAnalyze",
666
- "instances": ["Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)"],
1317
+ "instances": [
1318
+ "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)"
1319
+ ],
667
1320
  "addition_date": "2015/03/28"
668
1321
  },
669
1322
  {
670
1323
  "pattern": "SemanticScholarBot",
671
1324
  "url": "http://s2.allenai.org/bot.html",
672
- "instances": ["SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)"],
1325
+ "instances": [
1326
+ "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)"
1327
+ ],
673
1328
  "addition_date": "2015/03/28"
674
1329
  },
675
1330
  {
676
1331
  "pattern": "ltx71",
677
1332
  "url": "http://ltx71.com/",
678
- "instances": ["ltx71 - (http://ltx71.com/)"],
1333
+ "instances": [
1334
+ "ltx71 - (http://ltx71.com/)"
1335
+ ],
679
1336
  "addition_date": "2015/04/04"
680
1337
  },
681
1338
  {
682
1339
  "pattern": "nerdybot",
683
1340
  "url": "http://nerdybot.com/",
684
- "instances": ["nerdybot"],
1341
+ "instances": [
1342
+ "nerdybot"
1343
+ ],
685
1344
  "addition_date": "2015/04/05"
686
1345
  },
687
1346
  {
688
1347
  "pattern": "xovibot",
689
1348
  "url": "http://www.xovibot.net/",
690
- "instances": ["Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)"],
1349
+ "instances": [
1350
+ "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)"
1351
+ ],
691
1352
  "addition_date": "2015/04/05"
692
1353
  },
693
1354
  {
694
1355
  "pattern": "BUbiNG",
695
1356
  "url": "http://law.di.unimi.it/BUbiNG.html",
696
- "instances": ["BUbiNG (+http://law.di.unimi.it/BUbiNG.html)"],
1357
+ "instances": [
1358
+ "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)"
1359
+ ],
697
1360
  "addition_date": "2015/04/06"
698
1361
  },
699
1362
  {
700
1363
  "pattern": "Qwantify",
701
1364
  "url": "https://www.qwant.com/",
702
- "instances": ["Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*"],
1365
+ "instances": [
1366
+ "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*"
1367
+ ],
703
1368
  "addition_date": "2015/04/06"
704
1369
  },
705
1370
  {
706
1371
  "pattern": "archive.org_bot",
707
1372
  "url": "http://www.archive.org/details/archive.org_bot",
708
- "instances": ["Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)"],
1373
+ "instances": [
1374
+ "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)"
1375
+ ],
709
1376
  "addition_date": "2015/04/14"
710
1377
  },
711
1378
  {
712
1379
  "pattern": "Applebot",
713
1380
  "url": "http://www.apple.com/go/applebot",
714
- "addition_date": "2015/04/15"
1381
+ "addition_date": "2015/04/15",
1382
+ "instances": [
1383
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)",
1384
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)",
1385
+ "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",
1386
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",
1387
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)"
1388
+ ]
715
1389
  },
716
1390
  {
717
1391
  "pattern": "TweetmemeBot",
718
1392
  "url": "http://datasift.com/bot.html",
719
- "instances": ["Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0"],
1393
+ "instances": [
1394
+ "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0"
1395
+ ],
720
1396
  "addition_date": "2015/04/15"
721
1397
  },
722
1398
  {
723
1399
  "pattern": "crawler4j",
724
1400
  "url": "https://github.com/yasserg/crawler4j",
725
- "instances": ["crawler4j (http://code.google.com/p/crawler4j/)"],
1401
+ "instances": [
1402
+ "crawler4j (http://code.google.com/p/crawler4j/)"
1403
+ ],
726
1404
  "addition_date": "2015/05/07"
727
1405
  },
728
1406
  {
729
1407
  "pattern": "findxbot",
730
1408
  "url": "http://www.findxbot.com",
731
- "instances": ["Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)"],
1409
+ "instances": [
1410
+ "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)"
1411
+ ],
732
1412
  "addition_date": "2015/05/07"
733
1413
  },
734
1414
  {
735
- "pattern": "SemrushBot",
1415
+ "pattern": "S[eE][mM]rushBot",
736
1416
  "url": "http://www.semrush.com/bot.html",
737
- "instances": ["Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)"],
1417
+ "instances": [
1418
+ "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)",
1419
+ "SEMrushBot"
1420
+ ],
738
1421
  "addition_date": "2015/05/26"
739
1422
  },
740
1423
  {
741
1424
  "pattern": "yoozBot",
742
1425
  "url": "http://yooz.ir",
743
- "instances": ["Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)"],
1426
+ "instances": [
1427
+ "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)"
1428
+ ],
744
1429
  "addition_date": "2015/05/26"
745
1430
  },
746
1431
  {
747
1432
  "pattern": "lipperhey",
748
1433
  "url": "http://www.lipperhey.com/",
749
- "instances": ["Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)"],
1434
+ "instances": [
1435
+ "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)",
1436
+ "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)",
1437
+ "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)",
1438
+ "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)"
1439
+ ],
750
1440
  "addition_date": "2015/08/26"
751
1441
  },
752
1442
  {
753
- "pattern": "y!j-asr",
1443
+ "pattern": "Y!J-ASR",
754
1444
  "url": "http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/",
755
- "instances": ["Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)"],
1445
+ "instances": [
1446
+ "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)"
1447
+ ],
756
1448
  "addition_date": "2015/05/26"
757
- },
758
- {
1449
+ },
1450
+ {
759
1451
  "pattern": "Domain Re-Animator Bot",
760
1452
  "url": "http://domainreanimator.com",
761
- "instances": ["Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com"],
1453
+ "instances": [
1454
+ "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com"
1455
+ ],
762
1456
  "addition_date": "2015/04/14"
763
- },
764
- {
1457
+ },
1458
+ {
765
1459
  "pattern": "AddThis",
766
1460
  "url": "https://www.addthis.com",
767
- "instances": ["AddThis.com robot tech.support@clearspring.com"],
1461
+ "instances": [
1462
+ "AddThis.com robot tech.support@clearspring.com"
1463
+ ],
768
1464
  "addition_date": "2015/06/02"
769
- },
770
- {
1465
+ },
1466
+ {
771
1467
  "pattern": "Screaming Frog SEO Spider",
772
1468
  "url": "http://www.screamingfrog.co.uk/seo-spider",
773
- "instances": ["Screaming Frog SEO Spider/5.1"],
1469
+ "instances": [
1470
+ "Screaming Frog SEO Spider/5.1"
1471
+ ],
774
1472
  "addition_date": "2016/01/08"
775
- },
776
- {
1473
+ },
1474
+ {
777
1475
  "pattern": "MetaURI",
778
1476
  "url": "http://www.useragentstring.com/MetaURI_id_17683.php",
779
- "instances": ["MetaURI API/2.0 +metauri.com"],
1477
+ "instances": [
1478
+ "MetaURI API/2.0 +metauri.com"
1479
+ ],
780
1480
  "addition_date": "2016/01/02"
781
- },
782
- {
1481
+ },
1482
+ {
783
1483
  "pattern": "Scrapy",
784
1484
  "url": "http://scrapy.org/",
785
- "instances": ["Scrapy/1.0.3 (+http://scrapy.org)"],
1485
+ "instances": [
1486
+ "Scrapy/1.0.3 (+http://scrapy.org)"
1487
+ ],
786
1488
  "addition_date": "2016/01/02"
787
- },
788
- {
789
- "pattern": "LivelapBot",
1489
+ },
1490
+ {
1491
+ "pattern": "Livelap[bB]ot",
790
1492
  "url": "http://site.livelap.com/crawler",
791
- "instances": ["LivelapBot/0.2 (http://site.livelap.com/crawler)"],
1493
+ "instances": [
1494
+ "LivelapBot/0.2 (http://site.livelap.com/crawler)",
1495
+ "Livelapbot/0.1"
1496
+ ],
792
1497
  "addition_date": "2016/01/02"
793
- },
794
- {
1498
+ },
1499
+ {
795
1500
  "pattern": "OpenHoseBot",
796
1501
  "url": "http://www.openhose.org/bot.html",
797
- "instances": ["Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)"],
1502
+ "instances": [
1503
+ "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)"
1504
+ ],
798
1505
  "addition_date": "2016/01/02"
799
- },
800
- {
1506
+ },
1507
+ {
801
1508
  "pattern": "CapsuleChecker",
802
1509
  "url": "http://www.capsulink.com/about",
803
- "instances": ["CapsuleChecker (http://www.capsulink.com/)"],
1510
+ "instances": [
1511
+ "CapsuleChecker (http://www.capsulink.com/)"
1512
+ ],
804
1513
  "addition_date": "2016/01/02"
805
- },
806
- {
1514
+ },
1515
+ {
807
1516
  "pattern": "collection@infegy.com",
808
1517
  "url": "http://infegy.com/",
809
- "instances": ["Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com"],
1518
+ "instances": [
1519
+ "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com"
1520
+ ],
810
1521
  "addition_date": "2016/01/03"
811
- },
812
- {
1522
+ },
1523
+ {
813
1524
  "pattern": "IstellaBot",
814
1525
  "url": "http://www.tiscali.it/",
815
- "instances": ["Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)"],
1526
+ "instances": [
1527
+ "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)"
1528
+ ],
816
1529
  "addition_date": "2016/01/09"
817
- },
1530
+ },
818
1531
  {
819
1532
  "pattern": "DeuSu\\/",
820
1533
  "addition_date": "2016/01/23",
821
- "url": "https://deusu.de/robot.html"
1534
+ "url": "https://deusu.de/robot.html",
1535
+ "instances": [
1536
+ "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)",
1537
+ "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)"
1538
+ ]
822
1539
  },
823
1540
  {
824
1541
  "pattern": "betaBot",
825
- "addition_date": "2016/01/23"
1542
+ "addition_date": "2016/01/23",
1543
+ "instances": []
826
1544
  },
827
1545
  {
828
1546
  "pattern": "Cliqzbot\\/",
829
1547
  "addition_date": "2016/01/23",
830
- "url": "http://cliqz.com/company/cliqzbot"
1548
+ "url": "http://cliqz.com/company/cliqzbot",
1549
+ "instances": [
1550
+ "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)",
1551
+ "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)",
1552
+ "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)",
1553
+ "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)"
1554
+ ]
831
1555
  },
832
1556
  {
833
1557
  "pattern": "MojeekBot\\/",
834
1558
  "addition_date": "2016/01/23",
835
- "url": "https://www.mojeek.com/bot.html"
1559
+ "url": "https://www.mojeek.com/bot.html",
1560
+ "instances": [
1561
+ "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)",
1562
+ "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)",
1563
+ "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)",
1564
+ "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)",
1565
+ "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)",
1566
+ "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)"
1567
+ ]
836
1568
  },
837
1569
  {
838
1570
  "pattern": "netEstate NE Crawler",
839
1571
  "addition_date": "2016/01/23",
840
- "url": "+http://www.website-datenbank.de/"
1572
+ "url": "+http://www.website-datenbank.de/",
1573
+ "instances": [
1574
+ "netEstate NE Crawler (+http://www.sengine.info/)",
1575
+ "netEstate NE Crawler (+http://www.website-datenbank.de/)"
1576
+ ]
841
1577
  },
842
1578
  {
843
1579
  "pattern": "SafeSearch microdata crawler",
844
1580
  "addition_date": "2016/01/23",
845
- "url": "https://safesearch.avira.com"
1581
+ "url": "https://safesearch.avira.com",
1582
+ "instances": [
1583
+ "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)"
1584
+ ]
846
1585
  },
847
1586
  {
848
1587
  "pattern": "Gluten Free Crawler\\/",
849
1588
  "addition_date": "2016/01/23",
850
- "url": "http://glutenfreepleasure.com/"
1589
+ "url": "http://glutenfreepleasure.com/",
1590
+ "instances": [
1591
+ "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)"
1592
+ ]
851
1593
  },
852
1594
  {
853
1595
  "pattern": "Sonic",
854
1596
  "addition_date": "2016/02/08",
855
- "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html"
1597
+ "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html",
1598
+ "instances": [
1599
+ "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)",
1600
+ "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)",
1601
+ "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)"
1602
+ ]
856
1603
  },
857
1604
  {
858
1605
  "pattern": "Sysomos",
859
1606
  "addition_date": "2016/02/08",
860
- "url": "http://www.sysomos.com"
1607
+ "url": "http://www.sysomos.com",
1608
+ "instances": [
1609
+ "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)"
1610
+ ]
861
1611
  },
862
1612
  {
863
1613
  "pattern": "Trove",
864
1614
  "addition_date": "2016/02/08",
865
- "url": "http://www.trove.com"
1615
+ "url": "http://www.trove.com",
1616
+ "instances": []
866
1617
  },
867
1618
  {
868
1619
  "pattern": "deadlinkchecker",
869
1620
  "addition_date": "2016/02/08",
870
- "url": "http://www.deadlinkchecker.com"
1621
+ "url": "http://www.deadlinkchecker.com",
1622
+ "instances": [
1623
+ "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36",
1624
+ "www.deadlinkchecker.com XMLHTTP/1.0",
1625
+ "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36"
1626
+ ]
871
1627
  },
872
1628
  {
873
1629
  "pattern": "Slack-ImgProxy",
874
1630
  "addition_date": "2016/04/25",
875
- "url": "https://api.slack.com/robots"
1631
+ "url": "https://api.slack.com/robots",
1632
+ "instances": [
1633
+ "Slack-ImgProxy (+https://api.slack.com/robots)",
1634
+ "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)",
1635
+ "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)",
1636
+ "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)",
1637
+ "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)",
1638
+ "Slack-ImgProxy 149 (+https://api.slack.com/robots)"
1639
+ ]
876
1640
  },
877
1641
  {
878
1642
  "pattern": "Embedly",
879
1643
  "addition_date": "2016/04/25",
880
- "url": "http://support.embed.ly"
1644
+ "url": "http://support.embed.ly",
1645
+ "instances": [
1646
+ "Embedly +support@embed.ly",
1647
+ "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)",
1648
+ "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)"
1649
+ ]
881
1650
  },
882
1651
  {
883
1652
  "pattern": "RankActiveLinkBot",
884
1653
  "addition_date": "2016/06/20",
885
- "url": "https://rankactive.com/resources/rankactive-linkbot"
1654
+ "url": "https://rankactive.com/resources/rankactive-linkbot",
1655
+ "instances": [
1656
+ "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)"
1657
+ ]
886
1658
  },
887
1659
  {
888
1660
  "pattern": "iskanie",
889
1661
  "addition_date": "2016/09/02",
890
1662
  "url": "http://www.iskanie.com",
891
- "instances": ["iskanie (+http://www.iskanie.com)"]
1663
+ "instances": [
1664
+ "iskanie (+http://www.iskanie.com)"
1665
+ ]
892
1666
  },
893
1667
  {
894
1668
  "pattern": "SafeDNSBot",
895
1669
  "addition_date": "2016/09/10",
896
1670
  "url": "https://www.safedns.com/searchbot",
897
- "instances": ["SafeDNSBot (https://www.safedns.com/searchbot)"]
1671
+ "instances": [
1672
+ "SafeDNSBot (https://www.safedns.com/searchbot)"
1673
+ ]
898
1674
  },
899
1675
  {
900
1676
  "pattern": "SkypeUriPreview",
901
1677
  "addition_date": "2016/10/10",
902
- "instances": ["Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5"]
1678
+ "instances": [
1679
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5"
1680
+ ]
903
1681
  },
904
1682
  {
905
1683
  "pattern": "Veoozbot",
906
1684
  "addition_date": "2016/11/03",
907
- "url": "http://www.veooz.com/veoozbot.html"
1685
+ "url": "http://www.veooz.com/veoozbot.html",
1686
+ "instances": [
1687
+ "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)"
1688
+ ]
908
1689
  },
909
1690
  {
910
1691
  "pattern": "Slackbot",
911
1692
  "addition_date": "2016/11/03",
912
- "url": "https://api.slack.com/robots"
1693
+ "url": "https://api.slack.com/robots",
1694
+ "instances": [
1695
+ "Slackbot-LinkExpanding (+https://api.slack.com/robots)",
1696
+ "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"
1697
+ ]
913
1698
  },
914
1699
  {
915
1700
  "pattern": "redditbot",
916
1701
  "addition_date": "2016/11/03",
917
- "url": "http://www.reddit.com/feedback"
1702
+ "url": "http://www.reddit.com/feedback",
1703
+ "instances": [
1704
+ "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)"
1705
+ ]
918
1706
  },
919
1707
  {
920
1708
  "pattern": "datagnionbot",
921
1709
  "addition_date": "2016/11/03",
922
- "url": "http://www.datagnion.com/bot.html"
1710
+ "url": "http://www.datagnion.com/bot.html",
1711
+ "instances": [
1712
+ "datagnionbot (+http://www.datagnion.com/bot.html)"
1713
+ ]
923
1714
  },
924
1715
  {
925
- "pattern": "Veoozbot",
1716
+ "pattern": "Google-Adwords-Instant",
926
1717
  "addition_date": "2016/11/03",
927
- "url": "http://www.veooz.com/veoozbot.html"
1718
+ "url": "http://www.google.com/adsbot.html",
1719
+ "instances": [
1720
+ "Google-Adwords-Instant (+http://www.google.com/adsbot.html)"
1721
+ ]
928
1722
  },
929
1723
  {
930
- "pattern": "Slackbot",
931
- "addition_date": "2016/11/03",
932
- "url": "https://api.slack.com/robots"
1724
+ "pattern": "adbeat_bot",
1725
+ "addition_date": "2016/11/04",
1726
+ "instances": [
1727
+ "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)",
1728
+ "adbeat_bot"
1729
+ ]
933
1730
  },
934
1731
  {
935
- "pattern": "Google-Adwords-Instant",
936
- "addition_date": "2016/11/03",
937
- "url": "http://www.google.com/adsbot.html"
1732
+ "pattern": "WhatsApp",
1733
+ "addition_date": "2016/11/15",
1734
+ "url": "https://www.whatsapp.com/",
1735
+ "instances": [
1736
+ "WhatsApp",
1737
+ "WhatsApp/2.12.15/i",
1738
+ "WhatsApp/2.12.16/i",
1739
+ "WhatsApp/2.12.17/i",
1740
+ "WhatsApp/2.12.449 A",
1741
+ "WhatsApp/2.12.453 A",
1742
+ "WhatsApp/2.12.510 A",
1743
+ "WhatsApp/2.12.540 A",
1744
+ "WhatsApp/2.12.548 A",
1745
+ "WhatsApp/2.12.555 A",
1746
+ "WhatsApp/2.12.556 A",
1747
+ "WhatsApp/2.16.1/i",
1748
+ "WhatsApp/2.16.13 A",
1749
+ "WhatsApp/2.16.2/i",
1750
+ "WhatsApp/2.16.42 A",
1751
+ "WhatsApp/2.16.57 A"
1752
+ ]
1753
+ },
1754
+ {
1755
+ "pattern": "contxbot",
1756
+ "addition_date": "2017/02/25",
1757
+ "instances": [
1758
+ "Mozilla/5.0 (compatible;contxbot/1.0)"
1759
+ ]
1760
+ },
1761
+ {
1762
+ "pattern": "pinterest",
1763
+ "addition_date": "2017/03/03",
1764
+ "instances": [
1765
+ "Pinterest/0.2 (+http://www.pinterest.com/bot.html)"
1766
+ ],
1767
+ "url": "http://www.pinterest.com/bot.html"
1768
+ },
1769
+ {
1770
+ "pattern": "electricmonk",
1771
+ "addition_date": "2017/03/04",
1772
+ "instances": [
1773
+ "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)"
1774
+ ],
1775
+ "url": "https://www.duedil.com/our-crawler/"
1776
+ },
1777
+ {
1778
+ "pattern": "GarlikCrawler",
1779
+ "addition_date": "2017/03/18",
1780
+ "instances": [
1781
+ "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)"
1782
+ ],
1783
+ "url": "http://garlik.com/"
1784
+ },
1785
+ {
1786
+ "pattern": "BingPreview\\/",
1787
+ "addition_date": "2017/04/23",
1788
+ "url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0",
1789
+ "instances": [
1790
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b",
1791
+ "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko",
1792
+ "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; WOW64; Trident/6.0; BingPreview/1.0b)",
1793
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)",
1794
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b"
1795
+ ]
1796
+ },
1797
+ {
1798
+ "pattern": "vebidoobot",
1799
+ "addition_date": "2017/05/08",
1800
+ "instances": [
1801
+ "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/"
1802
+ ],
1803
+ "url": "https://blog.vebidoo.de/vebidoobot/"
1804
+ },
1805
+ {
1806
+ "pattern": "FemtosearchBot",
1807
+ "addition_date": "2017/05/16",
1808
+ "instances": [
1809
+ "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)"
1810
+ ],
1811
+ "url": "http://femtosearch.com"
1812
+ },
1813
+ {
1814
+ "pattern": "Yahoo Link Preview",
1815
+ "addition_date": "2017/06/28",
1816
+ "instances": [
1817
+ "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)"
1818
+ ],
1819
+ "url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html"
1820
+ },
1821
+ {
1822
+ "pattern": "MetaJobBot",
1823
+ "addition_date": "2017/08/16",
1824
+ "instances": [
1825
+ "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)"
1826
+ ],
1827
+ "url": "http://www.metajob.de/the/crawler"
1828
+ },
1829
+ {
1830
+ "pattern": "DomainStatsBot",
1831
+ "addition_date": "2017/08/16",
1832
+ "instances": [
1833
+ "DomainStatsBot/1.0 (http://domainstats.io/our-bot)"
1834
+ ],
1835
+ "url": "http://domainstats.io/our-bot"
1836
+ },
1837
+ {
1838
+ "pattern": "mindUpBot",
1839
+ "addition_date": "2017/08/16",
1840
+ "instances": [
1841
+ "mindUpBot (datenbutler.de)"
1842
+ ],
1843
+ "url": "http://www.datenbutler.de/"
1844
+ },
1845
+ {
1846
+ "pattern": "Daum",
1847
+ "addition_date": "2017/08/16",
1848
+ "instances": [
1849
+ "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)"
1850
+ ],
1851
+ "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966"
1852
+ },
1853
+ {
1854
+ "pattern": "Jugendschutzprogramm-Crawler",
1855
+ "addition_date": "2017/08/16",
1856
+ "instances": [
1857
+ "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de"
1858
+ ],
1859
+ "url": "http://www.jugendschutzprogramm.de"
1860
+ },
1861
+ {
1862
+ "pattern": "Xenu Link Sleuth",
1863
+ "addition_date": "2017/08/19",
1864
+ "instances": [
1865
+ "Xenu Link Sleuth/1.3.8"
1866
+ ],
1867
+ "url": "http://home.snafu.de/tilman/xenulink.html"
1868
+ },
1869
+ {
1870
+ "pattern": "Pcore-HTTP",
1871
+ "addition_date": "2017/08/19",
1872
+ "instances": [
1873
+ "Pcore-HTTP/v0.40.3"
1874
+ ],
1875
+ "url": "https://bitbucket.org/softvisio/pcore/overview"
1876
+ },
1877
+ {
1878
+ "pattern": "moatbot",
1879
+ "addition_date": "2017/09/16",
1880
+ "instances": [
1881
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot",
1882
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot"
1883
+ ],
1884
+ "url": "https://moat.com"
1885
+ },
1886
+ {
1887
+ "pattern": "KosmioBot",
1888
+ "addition_date": "2017/09/16",
1889
+ "instances": [
1890
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)"
1891
+ ],
1892
+ "url": "http://kosm.io/bot.html"
1893
+ },
1894
+ {
1895
+ "pattern": "Pingdom",
1896
+ "addition_date": "2017/09/16",
1897
+ "instances": [
1898
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)"
1899
+ ],
1900
+ "url": "http://www.pingdom.com"
1901
+ },
1902
+ {
1903
+ "pattern": "PhantomJS",
1904
+ "addition_date": "2017/09/18",
1905
+ "instances": [
1906
+ "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)"
1907
+ ],
1908
+ "url": "http://phantomjs.org/"
1909
+ },
1910
+ {
1911
+ "pattern": "Gowikibot",
1912
+ "addition_date": "2017/10/26",
1913
+ "instances": [
1914
+ "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)"
1915
+ ],
1916
+ "url": "http://www.gowikibot.com"
1917
+ },
1918
+ {
1919
+ "pattern": "PiplBot",
1920
+ "addition_date": "2017/10/30",
1921
+ "instances": [
1922
+ "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)"
1923
+ ],
1924
+ "url": "http://www.pipl.com/bot/"
1925
+ },
1926
+ {
1927
+ "pattern": "Discordbot",
1928
+ "addition_date": "2017/09/22",
1929
+ "url": "https://discordapp.com",
1930
+ "instances": [
1931
+ "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"
1932
+ ]
1933
+ },
1934
+ {
1935
+ "pattern": "TelegramBot",
1936
+ "addition_date": "2017/10/01",
1937
+ "instances": [
1938
+ "TelegramBot (like TwitterBot)"
1939
+ ]
1940
+ },
1941
+ {
1942
+ "pattern": "InfoPath.2",
1943
+ "addition_date": "2017/10/07",
1944
+ "instances": [
1945
+ "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; InfoPath.2)"
1946
+ ]
1947
+ },
1948
+ {
1949
+ "pattern": "Jetslide",
1950
+ "addition_date": "2017/09/27",
1951
+ "url": "http://jetsli.de/crawler",
1952
+ "instances": [
1953
+ "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)"
1954
+ ]
1955
+ },
1956
+ {
1957
+ "pattern": "newsharecounts",
1958
+ "addition_date": "2017/09/30",
1959
+ "url": "http://newsharecounts.com/crawler",
1960
+ "instances": [
1961
+ "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)"
1962
+ ]
1963
+ },
1964
+ {
1965
+ "pattern": "James BOT",
1966
+ "addition_date": "2017/10/12",
1967
+ "url": "http://cognitiveseo.com/bot.html",
1968
+ "instances": [
1969
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html"
1970
+ ]
1971
+ },
1972
+ {
1973
+ "pattern": "Barkrowler",
1974
+ "addition_date": "2017/10/09",
1975
+ "url": "http://www.exensa.com/crawl",
1976
+ "instances": [
1977
+ "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG"
1978
+ ]
1979
+ },
1980
+ {
1981
+ "pattern": "TinEye-bot",
1982
+ "addition_date": "2017/10/14",
1983
+ "url": "http://www.tineye.com/crawler.html",
1984
+ "instances": [
1985
+ "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)"
1986
+ ]
1987
+ },
1988
+ {
1989
+ "pattern": "SocialRankIOBot",
1990
+ "addition_date": "2017/10/19",
1991
+ "url": "http://socialrank.io/about",
1992
+ "instances": [
1993
+ "SocialRankIOBot; http://socialrank.io/about"
1994
+ ]
1995
+ },
1996
+ {
1997
+ "pattern": "trendictionbot",
1998
+ "addition_date": "2017/10/30",
1999
+ "url": "http://www.trendiction.de/bot",
2000
+ "instances": [
2001
+ "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11"
2002
+ ]
2003
+ },
2004
+ {
2005
+ "pattern": "Ocarinabot",
2006
+ "addition_date": "2017/09/27",
2007
+ "instances": [
2008
+ "Ocarinabot"
2009
+ ]
2010
+ },
2011
+ {
2012
+ "pattern": "epicbot",
2013
+ "addition_date": "2017/10/31",
2014
+ "url": "http://www.epictions.com/epicbot",
2015
+ "instances": [
2016
+ "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)"
2017
+ ]
2018
+ },
2019
+ {
2020
+ "pattern": "Primalbot",
2021
+ "addition_date": "2017/09/27",
2022
+ "url": "https://www.primal.com",
2023
+ "instances": [
2024
+ "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)"
2025
+ ]
2026
+ },
2027
+ {
2028
+ "pattern": "DuckDuckGo-Favicons-Bot",
2029
+ "addition_date": "2017/10/06",
2030
+ "url": "http://duckduckgo.com",
2031
+ "instances": [
2032
+ "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)"
2033
+ ]
2034
+ },
2035
+ {
2036
+ "pattern": "GnowitNewsbot",
2037
+ "addition_date": "2017/10/30",
2038
+ "url": "http://www.gnowit.com",
2039
+ "instances": [
2040
+ "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com"
2041
+ ]
2042
+ },
2043
+ {
2044
+ "pattern": "Leikibot",
2045
+ "addition_date": "2017/09/24",
2046
+ "url": "http://www.leiki.com",
2047
+ "instances": [
2048
+ "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)"
2049
+ ]
2050
+ },
2051
+ {
2052
+ "pattern": "LinkArchiver",
2053
+ "addition_date": "2017/09/24",
2054
+ "instances": [
2055
+ "@LinkArchiver twitter bot"
2056
+ ]
2057
+ },
2058
+ {
2059
+ "pattern": "YaK",
2060
+ "addition_date": "2017/09/25",
2061
+ "url": "http://linkfluence.com",
2062
+ "instances": [
2063
+ "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)"
2064
+ ]
2065
+ },
2066
+ {
2067
+ "pattern": "PaperLiBot",
2068
+ "addition_date": "2017/09/25",
2069
+ "url": "http://support.paper.li/entries/20023257-what-is-paper-li",
2070
+ "instances": [
2071
+ "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)"
2072
+ ]
2073
+ },
2074
+ {
2075
+ "pattern": "Digg Deeper",
2076
+ "addition_date": "2017/09/26",
2077
+ "url": "http://digg.com/about",
2078
+ "instances": [
2079
+ "Digg Deeper/v1 (http://digg.com/about)"
2080
+ ]
2081
+ },
2082
+ {
2083
+ "pattern": "dcrawl",
2084
+ "addition_date": "2017/09/22",
2085
+ "instances": [
2086
+ "dcrawl/1.0"
2087
+ ]
2088
+ },
2089
+ {
2090
+ "pattern": "Snacktory",
2091
+ "addition_date": "2017/09/23",
2092
+ "url": "https://github.com/karussell/snacktory",
2093
+ "instances": [
2094
+ "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)"
2095
+ ]
2096
+ },
2097
+ {
2098
+ "pattern": "AndersPinkBot",
2099
+ "addition_date": "2017/09/24",
2100
+ "url": "http://anderspink.com/bot.html",
2101
+ "instances": [
2102
+ "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)"
2103
+ ]
2104
+ },
2105
+ {
2106
+ "pattern": "Fyrebot",
2107
+ "addition_date": "2017/09/22",
2108
+ "instances": [
2109
+ "Fyrebot/1.0"
2110
+ ]
2111
+ },
2112
+ {
2113
+ "pattern": "EveryoneSocialBot",
2114
+ "addition_date": "2017/09/22",
2115
+ "url": "http://everyonesocial.com",
2116
+ "instances": [
2117
+ "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)"
2118
+ ]
2119
+ },
2120
+ {
2121
+ "pattern": "Mediatoolkitbot",
2122
+ "addition_date": "2017/10/06",
2123
+ "url": "http://mediatoolkit.com",
2124
+ "instances": [
2125
+ "Mediatoolkitbot (complaints@mediatoolkit.com)"
2126
+ ]
2127
+ },
2128
+ {
2129
+ "pattern": "Luminator-robots",
2130
+ "addition_date": "2017/09/22",
2131
+ "instances": [
2132
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0"
2133
+ ]
2134
+ },
2135
+ {
2136
+ "pattern": "ExtLinksBot",
2137
+ "addition_date": "2017/11/02",
2138
+ "url": "https://extlinks.com/Bot.html",
2139
+ "instances": [
2140
+ "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)"
2141
+ ]
2142
+ },
2143
+ {
2144
+ "pattern": "SurveyBot",
2145
+ "addition_date": "2017/11/02",
2146
+ "instances": [
2147
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)"
2148
+ ]
2149
+ },
2150
+ {
2151
+ "pattern": "NING",
2152
+ "addition_date": "2017/11/02",
2153
+ "instances": [
2154
+ "NING/1.0"
2155
+ ]
2156
+ },
2157
+ {
2158
+ "pattern": "okhttp",
2159
+ "addition_date": "2017/11/02",
2160
+ "instances": [
2161
+ "okhttp/2.5.0"
2162
+ ]
2163
+ },
2164
+ {
2165
+ "pattern": "Nuzzel",
2166
+ "addition_date": "2017/11/02",
2167
+ "instances": [
2168
+ "Nuzzel"
2169
+ ]
2170
+ },
2171
+ {
2172
+ "pattern": "omgili",
2173
+ "addition_date": "2017/11/02",
2174
+ "url": "http://omgili.com",
2175
+ "instances": [
2176
+ "omgili/0.5 +http://omgili.com"
2177
+ ]
2178
+ },
2179
+ {
2180
+ "pattern": "PocketParser",
2181
+ "addition_date": "2017/11/02",
2182
+ "url": "https://getpocket.com/pocketparser_ua",
2183
+ "instances": [
2184
+ "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)"
2185
+ ]
2186
+ },
2187
+ {
2188
+ "pattern": "YisouSpider",
2189
+ "addition_date": "2017/11/02",
2190
+ "instances": [
2191
+ "YisouSpider"
2192
+ ]
2193
+ },
2194
+ {
2195
+ "pattern": "um-LN",
2196
+ "addition_date": "2017/11/02",
2197
+ "instances": [
2198
+ "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)"
2199
+ ]
2200
+ },
2201
+ {
2202
+ "pattern": "ToutiaoSpider",
2203
+ "addition_date": "2017/11/02",
2204
+ "url": "http://web.toutiao.com/media_cooperation/",
2205
+ "instances": [
2206
+ "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)"
2207
+ ]
2208
+ },
2209
+ {
2210
+ "pattern": "MuckRack",
2211
+ "addition_date": "2017/11/02",
2212
+ "url": "http://muckrack.com",
2213
+ "instances": [
2214
+ "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)"
2215
+ ]
2216
+ },
2217
+ {
2218
+ "pattern": "Jamie's Spider",
2219
+ "addition_date": "2017/11/02",
2220
+ "url": "http://jamiembrown.com/",
2221
+ "instances": [
2222
+ "Jamie's Spider (http://jamiembrown.com/)"
2223
+ ]
2224
+ },
2225
+ {
2226
+ "pattern": "AHC",
2227
+ "addition_date": "2017/11/02",
2228
+ "instances": [
2229
+ "AHC/2.0"
2230
+ ]
2231
+ },
2232
+ {
2233
+ "pattern": "NetcraftSurveyAgent",
2234
+ "addition_date": "2017/11/02",
2235
+ "instances": [
2236
+ "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)"
2237
+ ]
2238
+ },
2239
+ {
2240
+ "pattern": "Laserlikebot",
2241
+ "addition_date": "2017/11/02",
2242
+ "instances": [
2243
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)"
2244
+ ]
2245
+ },
2246
+ {
2247
+ "pattern": "Apache-HttpClient",
2248
+ "addition_date": "2017/11/02",
2249
+ "instances": [
2250
+ "Apache-HttpClient/4.4.1 (Java/1.8.0_65)"
2251
+ ]
2252
+ },
2253
+ {
2254
+ "pattern": "AppEngine-Google",
2255
+ "addition_date": "2017/11/02",
2256
+ "instances": [
2257
+ "AppEngine-Google; (+http://code.google.com/appengine; appid: example)"
2258
+ ]
2259
+ },
2260
+ {
2261
+ "pattern": "Jetty",
2262
+ "addition_date": "2017/11/02",
2263
+ "instances": [
2264
+ "Jetty/9.3.z-SNAPSHOT"
2265
+ ]
2266
+ },
2267
+ {
2268
+ "pattern": "Upflow",
2269
+ "addition_date": "2017/11/02",
2270
+ "instances": [
2271
+ "Upflow/1.0"
2272
+ ]
2273
+ },
2274
+ {
2275
+ "pattern": "Thinklab",
2276
+ "addition_date": "2017/11/02",
2277
+ "url": "thinklab.com",
2278
+ "instances": [
2279
+ "Thinklab (thinklab.com)"
2280
+ ]
2281
+ },
2282
+ {
2283
+ "pattern": "Traackr.com",
2284
+ "addition_date": "2017/11/02",
2285
+ "url": "Traackr.com",
2286
+ "instances": [
2287
+ "Traackr.com"
2288
+ ]
2289
+ },
2290
+ {
2291
+ "pattern": "Twurly",
2292
+ "addition_date": "2017/11/02",
2293
+ "url": "http://twurly.org",
2294
+ "instances": [
2295
+ "Ruby, Twurly v1.1 (http://twurly.org)"
2296
+ ]
2297
+ },
2298
+ {
2299
+ "pattern": "Mastodon",
2300
+ "addition_date": "2017/11/02",
2301
+ "instances": [
2302
+ "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)"
2303
+ ]
2304
+ },
2305
+ {
2306
+ "pattern": "http_get",
2307
+ "addition_date": "2017/11/02",
2308
+ "instances": [
2309
+ "http_get"
2310
+ ]
2311
+ },
2312
+ {
2313
+ "pattern": "DnyzBot",
2314
+ "addition_date": "2017/11/20",
2315
+ "instances": [
2316
+ "Mozilla/5.0 (compatible; DnyzBot/1.0)"
2317
+ ]
2318
+ },
2319
+ {
2320
+ "pattern": "Botify",
2321
+ "addition_date": "2018/02/01",
2322
+ "instances": [
2323
+ "Mozilla/5.0 (compatible; botify; http://botify.com)"
2324
+ ]
938
2325
  }
939
2326
  ]