voight_kampff 1.1.1 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. checksums.yaml +4 -4
  2. data/config/crawler-user-agents.json +1653 -266
  3. data/lib/voight_kampff/version.rb +1 -1
  4. metadata +3 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d84469d1aed5ee0786d045d4a064d96a9c96c6a1
4
- data.tar.gz: 14e7e52f8fd161842dfd36eefc003dc4acf68c06
3
+ metadata.gz: 51a3721f69a6b32a7c619496d3369fbe3a8121e7
4
+ data.tar.gz: 59596dc12c149da92840f78c8d96d2e891deb056
5
5
  SHA512:
6
- metadata.gz: b20c3443891cc19f27c0fe8f278330f2e7077a023a237f728b2ef8803a423d730378b6b312eced348c61cbf0d5e8f95b18006502c39c9171812984f02baffd43
7
- data.tar.gz: 7fa0cfba7a668b9cdd760aeb04f2073b997908e429236aaf6e2c8bab2563448b14972b413eb5878abb7da132ee7e4cc419fc5dc574947067ed52fe94ddaf5816
6
+ metadata.gz: 4c9dd899fd331df778fd8f7592144532ee15a4f7964ddaffccc1c3fdefd58c1326e8645a782511737b527729f509c31d9edcdbfeb7fc2a22a561a6886834cee5
7
+ data.tar.gz: 5caf2c0788cdf075dbaf56d6912e49cb9c77b115e760dcd81e67bd5a9b6e8ea24a5fd4264e2ad6c6d50fb5cee251a271e4cfee1c9eb98e180aa99fa6e5d7bb25
data/config/crawler-user-agents.json CHANGED
@@ -1,289 +1,713 @@
1
1
  [
2
2
  {
3
3
  "pattern": "Googlebot\\/",
4
- "url": "http://www.google.com/bot.html"
4
+ "url": "http://www.google.com/bot.html",
5
+ "instances": [
6
+ "Googlebot/2.1 (+http://www.google.com/bot.html)",
7
+ "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
8
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
9
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
10
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
11
+ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
12
+ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
13
+ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36",
14
+ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
15
+ ]
16
+ },
17
+ {
18
+ "pattern": "Googlebot-Mobile",
19
+ "instances": [
20
+ "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
21
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
22
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
23
+ "Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
24
+ "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"
25
+ ]
5
26
  },
6
27
  {
7
- "pattern": "Googlebot-Mobile"
28
+ "pattern": "Googlebot-Image",
29
+ "instances": [
30
+ "Googlebot-Image/1.0"
31
+ ]
8
32
  },
9
33
  {
10
- "pattern": "Googlebot-Image"
34
+ "pattern": "Googlebot-News",
35
+ "instances": [
36
+ "Googlebot-News"
37
+ ]
11
38
  },
12
39
  {
13
- "pattern": "Googlebot-News"
40
+ "pattern": "Googlebot-Video",
41
+ "instances": [
42
+ "Googlebot-Video/1.0"
43
+ ]
14
44
  },
15
45
  {
16
- "pattern": "Googlebot-Video"
46
+ "pattern": "AdsBot-Google([^-]|$)",
47
+ "url": "https://support.google.com/webmasters/answer/1061943?hl=en",
48
+ "instances": [
49
+ "AdsBot-Google (+http://www.google.com/adsbot.html)"
50
+ ]
17
51
  },
18
52
  {
19
- "pattern": "AdsBot-Google",
20
- "url": "https://support.google.com/webmasters/answer/1061943?hl=en"
53
+ "pattern": "AdsBot-Google-Mobile",
54
+ "addition_date": "2017/08/21",
55
+ "url": "https://support.google.com/adwords/answer/2404197",
56
+ "instances": [
57
+ "AdsBot-Google-Mobile-Apps",
58
+ "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
59
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)"
60
+ ]
21
61
  },
22
62
  {
23
63
  "pattern": "Mediapartners-Google",
24
- "url": "https://support.google.com/webmasters/answer/1061943?hl=en"
64
+ "url": "https://support.google.com/webmasters/answer/1061943?hl=en",
65
+ "instances": [
66
+ "Mediapartners-Google",
67
+ "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)",
68
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)",
69
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"
70
+ ]
71
+ },
72
+ {
73
+ "pattern": "Mediapartners \\(Googlebot\\)",
74
+ "addition_date": "2017/08/08",
75
+ "url": "https://support.google.com/webmasters/answer/1061943?hl=en",
76
+ "instances": []
77
+ },
78
+ {
79
+ "pattern": "APIs-Google",
80
+ "addition_date": "2017/08/08",
81
+ "url": "https://support.google.com/webmasters/answer/1061943?hl=en",
82
+ "instances": [
83
+ "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)"
84
+ ]
25
85
  },
26
86
  {
27
87
  "pattern": "bingbot",
28
- "url": "http://www.bing.com/bingbot.htm"
88
+ "url": "http://www.bing.com/bingbot.htm",
89
+ "instances": [
90
+ "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",
91
+ "Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)",
92
+ "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",
93
+ "Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)",
94
+ "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm",
95
+ "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
96
+ "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe",
97
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)",
98
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",
99
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)",
100
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
101
+ "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"
102
+ ]
29
103
  },
30
104
  {
31
105
  "pattern": "slurp",
32
- "url": "http://help.yahoo.com/help/us/ysearch/slurp"
106
+ "url": "http://help.yahoo.com/help/us/ysearch/slurp",
107
+ "instances": [
108
+ "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)",
109
+ "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
110
+ ]
33
111
  },
34
112
  {
35
- "pattern": "java"
113
+ "pattern": "java",
114
+ "instances": [
115
+ "Apache-HttpClient/4.2.3 (java 1.5)",
116
+ "Apache-HttpClient/4.2.5 (java 1.5)",
117
+ "Apache-HttpClient/4.3.1 (java 1.5)",
118
+ "Apache-HttpClient/4.3.3 (java 1.5)",
119
+ "Apache-HttpClient/4.3.5 (java 1.5)",
120
+ "Apache-HttpClient/UNAVAILABLE (java 1.4)",
121
+ "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
122
+ "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
123
+ "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
124
+ "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html"
125
+ ]
36
126
  },
37
127
  {
38
- "pattern": "wget"
128
+ "pattern": "[wW]get",
129
+ "instances": [
130
+ "WGETbot/1.0 (+http://wget.alanreed.org)",
131
+ "Wget/1.14 (linux-gnu)"
132
+ ]
39
133
  },
40
134
  {
41
- "pattern": "curl"
135
+ "pattern": "curl",
136
+ "instances": [
137
+ "eCairn-Grabber/1.0 (+http://ecairn.com/grabber) curl/7.15"
138
+ ]
42
139
  },
43
140
  {
44
- "pattern": "Commons-HttpClient"
141
+ "pattern": "Commons-HttpClient",
142
+ "instances": [
143
+ "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)",
144
+ "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)"
145
+ ]
45
146
  },
46
147
  {
47
- "pattern": "Python-urllib"
148
+ "pattern": "Python-urllib",
149
+ "instances": [
150
+ "Python-urllib/2.7 (+http://aranea.juls.savba.sk/aranea_about.html)",
151
+ "Python-urllib/2.7 (+http://ella.juls.savba.sk/aranea_about)",
152
+ "Python-urllib/2.7 (+http://sketch.juls.savba.sk/aranea_about)"
153
+ ]
48
154
  },
49
155
  {
50
- "pattern": "libwww"
156
+ "pattern": "libwww",
157
+ "instances": [
158
+ "2Bone_LinkChecker/1.0 libwww-perl/6.03",
159
+ "2Bone_LinkChkr/1.0 libwww-perl/6.03",
160
+ "W3C-checklink/2.90 libwww-perl/5.64",
161
+ "W3C-checklink/3.6.2.3 libwww-perl/5.64",
162
+ "W3C-checklink/4.2 [4.20] libwww-perl/5.803",
163
+ "W3C-checklink/4.2.1 [4.21] libwww-perl/5.803",
164
+ "W3C-checklink/4.3 [4.42] libwww-perl/5.805",
165
+ "W3C-checklink/4.3 [4.42] libwww-perl/5.808",
166
+ "W3C-checklink/4.3 [4.42] libwww-perl/5.820",
167
+ "W3C-checklink/4.5 [4.154] libwww-perl/5.823",
168
+ "W3C-checklink/4.5 [4.160] libwww-perl/5.823",
169
+ "amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831"
170
+ ]
51
171
  },
52
172
  {
53
- "pattern": "httpunit"
173
+ "pattern": "httpunit",
174
+ "instances": [
175
+ "httpunit/1.x"
176
+ ]
54
177
  },
55
178
  {
56
- "pattern": "nutch"
179
+ "pattern": "nutch",
180
+ "instances": [
181
+ "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
182
+ "istellabot-nutch/Nutch-1.10"
183
+ ]
57
184
  },
58
185
  {
59
186
  "pattern": "Go-http-client",
60
187
  "addition_date": "2016/03/26",
61
188
  "url": "https://golang.org/pkg/net/http/",
62
- "instances": ["Go-http-client/1.1"]
189
+ "instances": [
190
+ "Go-http-client/1.1"
191
+ ]
63
192
  },
64
193
  {
65
194
  "pattern": "phpcrawl",
66
195
  "addition_date": "2012-09/17",
67
- "url": "http://phpcrawl.cuab.de/"
196
+ "url": "http://phpcrawl.cuab.de/",
197
+ "instances": [
198
+ "phpcrawl"
199
+ ]
68
200
  },
69
201
  {
70
202
  "pattern": "msnbot",
71
- "url": "http://search.msn.com/msnbot.htm"
203
+ "url": "http://search.msn.com/msnbot.htm",
204
+ "instances": [
205
+ "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)",
206
+ "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)",
207
+ "librabot/1.0 (+http://search.msn.com/msnbot.htm)",
208
+ "librabot/2.0 (+http://search.msn.com/msnbot.htm)",
209
+ "msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)",
210
+ "msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)",
211
+ "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)",
212
+ "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)",
213
+ "msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)",
214
+ "msnbot/1.0 (+http://search.msn.com/msnbot.htm)",
215
+ "msnbot/1.1 (+http://search.msn.com/msnbot.htm)",
216
+ "msnbot/2.0b (+http://search.msn.com/msnbot.htm)",
217
+ "msnbot/2.0b (+http://search.msn.com/msnbot.htm).",
218
+ "msnbot/2.0b (+http://search.msn.com/msnbot.htm)._"
219
+ ]
72
220
  },
73
221
  {
74
- "pattern": "jyxobot"
222
+ "pattern": "jyxobot",
223
+ "instances": []
75
224
  },
76
225
  {
77
- "pattern": "FAST-WebCrawler"
226
+ "pattern": "FAST-WebCrawler",
227
+ "instances": [
228
+ "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",
229
+ "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
230
+ "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",
231
+ "FAST-WebCrawler/3.8"
232
+ ]
78
233
  },
79
234
  {
80
- "pattern": "FAST Enterprise Crawler"
235
+ "pattern": "FAST Enterprise Crawler",
236
+ "instances": [
237
+ "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/",
238
+ "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)"
239
+ ]
81
240
  },
82
241
  {
83
- "pattern": "biglotron"
242
+ "pattern": "BIGLOTRON",
243
+ "instances": [
244
+ "BIGLOTRON (Beta 2;GNU/Linux)"
245
+ ]
84
246
  },
85
247
  {
86
- "pattern": "teoma"
248
+ "pattern": "Teoma",
249
+ "instances": [
250
+ "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)",
251
+ "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)"
252
+ ],
253
+ "url": "http://about.ask.com/en/docs/about/webmasters.shtml"
87
254
  },
88
255
  {
89
- "pattern": "convera"
256
+ "pattern": "convera",
257
+ "instances": [
258
+ "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)"
259
+ ],
260
+ "url": "http://ews.converasearch.com/crawl.htm"
90
261
  },
91
262
  {
92
- "pattern": "seekbot"
263
+ "pattern": "seekbot",
264
+ "instances": [
265
+ "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2"
266
+ ],
267
+ "url": "http://www.seekbot.net/bot.html"
93
268
  },
94
269
  {
95
- "pattern": "gigabot",
96
- "instances": ["Gigabot/1.0", "Gigabot/2.0 (http://www.gigablast.com/spider.html)", "Gigabot/2.0 (http://www.gigablast.com/spider.html)"],
97
- "url": "https://github.com/gigablast/open-source-search-engine"
270
+ "pattern": "Gigabot",
271
+ "instances": [
272
+ "Gigabot/1.0",
273
+ "Gigabot/2.0 (http://www.gigablast.com/spider.html)"
274
+ ],
275
+ "url": "http://www.gigablast.com/spider.html"
98
276
  },
99
277
  {
100
- "pattern": "gigablast",
101
- "instances": ["GigablastOpenSource/1.0"],
278
+ "pattern": "Gigablast",
279
+ "instances": [
280
+ "GigablastOpenSource/1.0"
281
+ ],
102
282
  "url": "https://github.com/gigablast/open-source-search-engine"
103
283
  },
104
284
  {
105
- "pattern": "exabot"
106
- },
107
- {
108
- "pattern": "ngbot"
109
- },
110
- {
111
- "pattern": "ia_archiver"
112
- },
113
- {
114
- "pattern": "GingerCrawler"
285
+ "pattern": "exabot",
286
+ "instances": [
287
+ "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)",
288
+ "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)",
289
+ "Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)",
290
+ "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)",
291
+ "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)"
292
+ ]
115
293
  },
116
294
  {
117
- "pattern": "webmon "
295
+ "pattern": "ia_archiver",
296
+ "instances": [
297
+ "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",
298
+ "ia_archiver-web.archive.org"
299
+ ]
118
300
  },
119
301
  {
120
- "pattern": "httrack"
302
+ "pattern": "GingerCrawler",
303
+ "instances": [
304
+ "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)"
305
+ ]
121
306
  },
122
307
  {
123
- "pattern": "webcrawler"
308
+ "pattern": "webmon ",
309
+ "instances": []
124
310
  },
125
311
  {
126
- "pattern": "grub.org"
312
+ "pattern": "HTTrack",
313
+ "instances": [
314
+ "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"
315
+ ]
127
316
  },
128
317
  {
129
- "pattern": "UsineNouvelleCrawler"
318
+ "pattern": "grub.org",
319
+ "instances": [
320
+ "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)",
321
+ "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)",
322
+ "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)",
323
+ "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)",
324
+ "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)",
325
+ "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)",
326
+ "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)",
327
+ "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)",
328
+ "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)",
329
+ "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)",
330
+ "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)"
331
+ ]
130
332
  },
131
333
  {
132
- "pattern": "antibot"
334
+ "pattern": "UsineNouvelleCrawler",
335
+ "instances": []
133
336
  },
134
337
  {
135
- "pattern": "netresearchserver"
338
+ "pattern": "antibot",
339
+ "instances": []
136
340
  },
137
341
  {
138
- "pattern": "speedy"
342
+ "pattern": "netresearchserver",
343
+ "instances": []
139
344
  },
140
345
  {
141
- "pattern": "fluffy"
346
+ "pattern": "speedy",
347
+ "instances": [
348
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)",
349
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)",
350
+ "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)",
351
+ "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)",
352
+ "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)"
353
+ ]
142
354
  },
143
355
  {
144
- "pattern": "bibnum.bnf"
356
+ "pattern": "fluffy",
357
+ "instances": []
145
358
  },
146
359
  {
147
- "pattern": "findlink"
360
+ "pattern": "bibnum.bnf",
361
+ "instances": [
362
+ "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)"
363
+ ]
148
364
  },
149
365
  {
150
- "pattern": "msrbot"
366
+ "pattern": "findlink",
367
+ "instances": [
368
+ "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)",
369
+ "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)",
370
+ "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)",
371
+ "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)",
372
+ "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)",
373
+ "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)",
374
+ "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)",
375
+ "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)",
376
+ "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)",
377
+ "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)",
378
+ "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)",
379
+ "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)",
380
+ "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)",
381
+ "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)",
382
+ "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)",
383
+ "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",
384
+ "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)",
385
+ "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)",
386
+ "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)",
387
+ "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",
388
+ "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)",
389
+ "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",
390
+ "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)"
391
+ ]
151
392
  },
152
393
  {
153
- "pattern": "panscient"
394
+ "pattern": "msrbot",
395
+ "instances": []
154
396
  },
155
397
  {
156
- "pattern": "yacybot"
398
+ "pattern": "panscient",
399
+ "instances": [
400
+ "panscient.com"
401
+ ]
157
402
  },
158
403
  {
159
- "pattern": "AISearchBot"
404
+ "pattern": "yacybot",
405
+ "instances": [
406
+ "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
407
+ "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
408
+ "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
409
+ "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
410
+ "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html",
411
+ "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
412
+ "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html",
413
+ "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",
414
+ "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html",
415
+ "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
416
+ "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html",
417
+ "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html",
418
+ "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",
419
+ "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",
420
+ "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html",
421
+ "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html",
422
+ "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html",
423
+ "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html",
424
+ "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html",
425
+ "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html",
426
+ "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html",
427
+ "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html",
428
+ "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html",
429
+ "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
430
+ "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html",
431
+ "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
432
+ "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",
433
+ "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html",
434
+ "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html",
435
+ "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html",
436
+ "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html",
437
+ "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html",
438
+ "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html",
439
+ "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",
440
+ "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",
441
+ "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",
442
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html",
443
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html",
444
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",
445
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html",
446
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",
447
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html",
448
+ "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html"
449
+ ]
160
450
  },
161
451
  {
162
- "pattern": "IOI"
452
+ "pattern": "AISearchBot",
453
+ "instances": []
163
454
  },
164
455
  {
165
- "pattern": "ips-agent"
456
+ "pattern": "IOI",
457
+ "instances": []
166
458
  },
167
459
  {
168
- "pattern": "tagoobot"
460
+ "pattern": "ips-agent",
461
+ "instances": [
462
+ "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent",
463
+ "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7",
464
+ "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3",
465
+ "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24",
466
+ "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1"
467
+ ]
169
468
  },
170
469
  {
171
- "pattern": "MJ12bot"
470
+ "pattern": "tagoobot",
471
+ "instances": []
172
472
  },
173
473
  {
174
- "pattern": "dotbot"
474
+ "pattern": "MJ12bot",
475
+ "instances": [
476
+ "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)",
477
+ "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)",
478
+ "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)",
479
+ "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)",
480
+ "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)",
481
+ "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)",
482
+ "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)",
483
+ "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)",
484
+ "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)",
485
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)",
486
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)",
487
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)",
488
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)",
489
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)",
490
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)",
491
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)",
492
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)",
493
+ "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)"
494
+ ]
175
495
  },
176
496
  {
177
- "pattern": "woriobot"
497
+ "pattern": "woriobot",
498
+ "instances": [
499
+ "Mozilla/5.0 (compatible; woriobot +http://worio.com)",
500
+ "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)"
501
+ ]
178
502
  },
179
503
  {
180
- "pattern": "yanga"
504
+ "pattern": "yanga",
505
+ "instances": [
506
+ "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)"
507
+ ]
181
508
  },
182
509
  {
183
- "pattern": "buzzbot"
510
+ "pattern": "buzzbot",
511
+ "instances": [
512
+ "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)"
513
+ ]
184
514
  },
185
515
  {
186
- "pattern": "mlbot"
516
+ "pattern": "mlbot",
517
+ "instances": [
518
+ "MLBot (www.metadatalabs.com/mlbot)"
519
+ ]
187
520
  },
188
521
  {
189
- "pattern": "yandexbot",
522
+ "pattern": "YandexBot",
190
523
  "url": "http://yandex.com/bots",
191
- "instances": ["Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"],
524
+ "instances": [
525
+ "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"
526
+ ],
192
527
  "addition_date": "2015/04/14"
193
528
  },
529
+ {
530
+ "pattern": "yandex.com\\/bots",
531
+ "url": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml#robot-in-logs",
532
+ "instances": [
533
+ "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)"
534
+ ],
535
+ "addition_date": "2016/12/01"
536
+ },
194
537
  {
195
538
  "pattern": "purebot",
196
- "addition_date": "2010/01/19"
539
+ "addition_date": "2010/01/19",
540
+ "instances": []
197
541
  },
198
542
  {
199
543
  "pattern": "Linguee Bot",
200
544
  "addition_date": "2010/01/26",
201
- "url": "http://www.linguee.com/bot"
202
- },
203
- {
204
- "pattern": "Voyager",
205
- "addition_date": "2010/02/01",
206
- "url": "http://www.kosmix.com/crawler.html"
545
+ "url": "http://www.linguee.com/bot",
546
+ "instances": [
547
+ "Linguee Bot (http://www.linguee.com/bot)",
548
+ "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)"
549
+ ]
207
550
  },
208
551
  {
209
552
  "pattern": "CyberPatrol",
210
553
  "addition_date": "2010/02/11",
211
- "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp"
554
+ "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp",
555
+ "instances": [
556
+ "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)"
557
+ ]
212
558
  },
213
559
  {
214
560
  "pattern": "voilabot",
215
- "addition_date": "2010/05/18"
561
+ "addition_date": "2010/05/18",
562
+ "instances": [
563
+ "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
564
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
565
+ "Mozilla/5.0 (compatible; OrangeBot/2.0; support.voilabot@orange.com)"
566
+ ]
216
567
  },
217
568
  {
218
- "pattern": "baiduspider",
569
+ "pattern": "Baiduspider",
219
570
  "addition_date": "2010/07/15",
220
- "url": "http://www.baidu.jp/spider/"
571
+ "url": "http://www.baidu.jp/spider/",
572
+ "instances": [
573
+ "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
574
+ ]
221
575
  },
222
576
  {
223
577
  "pattern": "citeseerxbot",
224
- "addition_date": "2010/07/17"
578
+ "addition_date": "2010/07/17",
579
+ "instances": []
225
580
  },
226
581
  {
227
582
  "pattern": "spbot",
228
583
  "addition_date": "2010/07/31",
229
- "url": "http://www.seoprofiler.com/bot"
584
+ "url": "http://www.seoprofiler.com/bot",
585
+ "instances": [
586
+ "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )",
587
+ "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )",
588
+ "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )",
589
+ "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )",
590
+ "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )",
591
+ "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )",
592
+ "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )",
593
+ "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )",
594
+ "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )",
595
+ "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )",
596
+ "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )",
597
+ "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )",
598
+ "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )",
599
+ "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )",
600
+ "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )",
601
+ "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )",
602
+ "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )",
603
+ "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )",
604
+ "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )",
605
+ "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )",
606
+ "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )",
607
+ "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )",
608
+ "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )",
609
+ "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )",
610
+ "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )",
611
+ "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )",
612
+ "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )",
613
+ "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )",
614
+ "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )",
615
+ "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )",
616
+ "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )",
617
+ "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )",
618
+ "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )",
619
+ "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )"
620
+ ]
230
621
  },
231
622
  {
232
623
  "pattern": "twengabot",
233
624
  "addition_date": "2010/08/03",
234
- "url": "http://www.twenga.com/bot.html"
625
+ "url": "http://www.twenga.com/bot.html",
626
+ "instances": []
235
627
  },
236
628
  {
237
629
  "pattern": "postrank",
238
630
  "addition_date": "2010/08/03",
239
- "url": "http://www.postrank.com"
631
+ "url": "http://www.postrank.com",
632
+ "instances": [
633
+ "PostRank/2.0 (postrank.com)",
634
+ "PostRank/2.0 (postrank.com; 1 subscribers)"
635
+ ]
240
636
  },
241
637
  {
242
638
  "pattern": "turnitinbot",
243
639
  "addition_date": "2010/09/26",
244
- "url": "http://www.turnitin.com"
640
+ "url": "http://www.turnitin.com",
641
+ "instances": []
245
642
  },
246
643
  {
247
644
  "pattern": "scribdbot",
248
645
  "addition_date": "2010/09/28",
249
- "url": "http://www.scribd.com"
646
+ "url": "http://www.scribd.com",
647
+ "instances": []
250
648
  },
251
649
  {
252
650
  "pattern": "page2rss",
253
651
  "addition_date": "2010/10/07",
254
- "url": "http://www.page2rss.com"
652
+ "url": "http://www.page2rss.com",
653
+ "instances": [
654
+ "Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)"
655
+ ]
255
656
  },
256
657
  {
257
658
  "pattern": "sitebot",
258
659
  "addition_date": "2010/12/15",
259
- "url": "http://www.sitebot.org"
660
+ "url": "http://www.sitebot.org",
661
+ "instances": [
662
+ "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)"
663
+ ]
260
664
  },
261
665
  {
262
666
  "pattern": "linkdex",
263
667
  "addition_date": "2011/01/06",
264
- "url": "http://www.linkdex.com"
668
+ "url": "http://www.linkdex.com",
669
+ "instances": [
670
+ "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)",
671
+ "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)",
672
+ "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)",
673
+ "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)",
674
+ "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)",
675
+ "linkdex.com/v2.0",
676
+ "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)"
677
+ ]
265
678
  },
266
679
  {
267
680
  "pattern": "Adidxbot",
268
- "url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx"
681
+ "url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx",
682
+ "instances": []
269
683
  },
270
684
  {
271
685
  "pattern": "blekkobot",
272
- "url": "http://blekko.com/about/blekkobot"
686
+ "url": "http://blekko.com/about/blekkobot",
687
+ "instances": [
688
+ "Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)"
689
+ ]
273
690
  },
274
691
  {
275
692
  "pattern": "ezooms",
276
693
  "addition_date": "2011/04/27",
277
- "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289"
694
+ "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289",
695
+ "instances": [
696
+ "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)"
697
+ ]
278
698
  },
279
699
  {
280
700
  "pattern": "dotbot",
281
- "addition_date": "2011/04/27"
701
+ "addition_date": "2011/04/27",
702
+ "instances": [
703
+ "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)",
704
+ "dotbot"
705
+ ]
282
706
  },
283
707
  {
284
708
  "pattern": "Mail.RU_Bot",
285
709
  "addition_date": "2011/04/27",
286
- "instances" : [
710
+ "instances": [
287
711
  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/",
288
712
  "Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/"
289
713
  ]
@@ -291,291 +715,502 @@
291
715
  {
292
716
  "pattern": "discobot",
293
717
  "addition_date": "2011/05/03",
294
- "url": "http://discoveryengine.com/discobot.html"
718
+ "url": "http://discoveryengine.com/discobot.html",
719
+ "instances": [
720
+ "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)",
721
+ "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)",
722
+ "mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)"
723
+ ]
295
724
  },
296
725
  {
297
726
  "pattern": "heritrix",
298
727
  "addition_date": "2011/06/21",
299
- "url": "http://crawler.archive.org/"
728
+ "url": "http://crawler.archive.org/",
729
+ "instances": [
730
+ "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)",
731
+ "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)",
732
+ "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)",
733
+ "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)",
734
+ "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)",
735
+ "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)",
736
+ "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)",
737
+ "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)",
738
+ "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)",
739
+ "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)",
740
+ "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)",
741
+ "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)",
742
+ "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)",
743
+ "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)",
744
+ "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)",
745
+ "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)",
746
+ "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)",
747
+ "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)",
748
+ "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)",
749
+ "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)",
750
+ "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)",
751
+ "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)",
752
+ "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)",
753
+ "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)",
754
+ "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)"
755
+ ]
300
756
  },
301
757
  {
302
758
  "pattern": "findthatfile",
303
759
  "addition_date": "2011/06/21",
304
- "url": "http://www.findthatfile.com/"
760
+ "url": "http://www.findthatfile.com/",
761
+ "instances": []
305
762
  },
306
763
  {
307
764
  "pattern": "europarchive.org",
308
765
  "addition_date": "2011/06/21",
309
- "url": ""
766
+ "url": "",
767
+ "instances": [
768
+ "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)"
769
+ ]
310
770
  },
311
771
  {
312
772
  "pattern": "NerdByNature.Bot",
313
773
  "addition_date": "2011/07/12",
314
- "url": "http://www.nerdbynature.net/bot"
774
+ "url": "http://www.nerdbynature.net/bot",
775
+ "instances": [
776
+ "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)"
777
+ ]
315
778
  },
316
779
  {
317
780
  "pattern": "sistrix crawler",
318
- "addition_date": "2011/08/02"
781
+ "addition_date": "2011/08/02",
782
+ "instances": []
319
783
  },
320
784
  {
321
- "pattern": "ahrefsbot",
322
- "addition_date": "2011/08/28"
785
+ "pattern": "AhrefsBot",
786
+ "addition_date": "2011/08/28",
787
+ "instances": [
788
+ "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)"
789
+ ]
323
790
  },
324
791
  {
325
792
  "pattern": "Aboundex",
326
793
  "addition_date": "2011/09/28",
327
- "url": "http://www.aboundex.com/crawler/"
794
+ "url": "http://www.aboundex.com/crawler/",
795
+ "instances": [
796
+ "Aboundex/0.2 (http://www.aboundex.com/crawler/)",
797
+ "Aboundex/0.3 (http://www.aboundex.com/crawler/)"
798
+ ]
328
799
  },
329
800
  {
330
801
  "pattern": "domaincrawler",
331
- "addition_date": "2011/10/21"
802
+ "addition_date": "2011/10/21",
803
+ "instances": [
804
+ "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)"
805
+ ]
332
806
  },
333
807
  {
334
808
  "pattern": "wbsearchbot",
335
809
  "addition_date": "2011/12/21",
336
- "url": "http://www.warebay.com/bot.html"
810
+ "url": "http://www.warebay.com/bot.html",
811
+ "instances": []
337
812
  },
338
813
  {
339
814
  "pattern": "summify",
340
815
  "addition_date": "2012/01/04",
341
- "url": "http://summify.com"
816
+ "url": "http://summify.com",
817
+ "instances": [
818
+ "Summify (Summify/1.0.1; +http://summify.com)"
819
+ ]
342
820
  },
343
821
  {
344
- "pattern": "ccbot",
822
+ "pattern": "CCBot",
345
823
  "addition_date": "2012/02/05",
346
- "url": "http://www.commoncrawl.org/bot.html"
824
+ "url": "http://www.commoncrawl.org/bot.html",
825
+ "instances": [
826
+ "CCBot/2.0 (http://commoncrawl.org/faq/)"
827
+ ]
347
828
  },
348
829
  {
349
830
  "pattern": "edisterbot",
350
- "addition_date": "2012/02/25"
831
+ "addition_date": "2012/02/25",
832
+ "instances": []
351
833
  },
352
834
  {
353
835
  "pattern": "seznambot",
354
- "addition_date": "2012/03/14"
836
+ "addition_date": "2012/03/14",
837
+ "instances": [
838
+ "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)",
839
+ "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)",
840
+ "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)",
841
+ "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)",
842
+ "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)"
843
+ ]
355
844
  },
356
845
  {
357
846
  "pattern": "ec2linkfinder",
358
- "addition_date": "2012/03/22"
847
+ "addition_date": "2012/03/22",
848
+ "instances": [
849
+ "ec2linkfinder"
850
+ ]
359
851
  },
360
852
  {
361
853
  "pattern": "gslfbot",
362
- "addition_date": "2012/04/03"
854
+ "addition_date": "2012/04/03",
855
+ "instances": []
363
856
  },
364
857
  {
365
858
  "pattern": "aihitbot",
366
- "addition_date": "2012/04/16"
859
+ "addition_date": "2012/04/16",
860
+ "instances": []
367
861
  },
368
862
  {
369
863
  "pattern": "intelium_bot",
370
- "addition_date": "2012/05/07"
864
+ "addition_date": "2012/05/07",
865
+ "instances": []
371
866
  },
372
867
  {
373
868
  "pattern": "facebookexternalhit",
374
- "addition_date": "2012/05/07"
869
+ "addition_date": "2012/05/07",
870
+ "instances": [
871
+ "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)",
872
+ "facebookexternalhit/1.1",
873
+ "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"
874
+ ]
375
875
  },
376
876
  {
377
- "pattern": "yeti",
378
- "addition_date": "2012/05/07"
877
+ "pattern": "Yeti",
878
+ "addition_date": "2012/05/07",
879
+ "url": "http://naver.me/bot",
880
+ "instances": [
881
+ "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)"
882
+ ]
379
883
  },
380
884
  {
381
885
  "pattern": "RetrevoPageAnalyzer",
382
- "addition_date": "2012/05/07"
886
+ "addition_date": "2012/05/07",
887
+ "instances": [
888
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)"
889
+ ]
383
890
  },
384
891
  {
385
892
  "pattern": "lb-spider",
386
- "addition_date": "2012/05/07"
893
+ "addition_date": "2012/05/07",
894
+ "instances": []
387
895
  },
388
896
  {
389
- "pattern": "sogou",
897
+ "pattern": "Sogou",
390
898
  "addition_date": "2012/05/13",
391
- "url": "http://www.sogou.com/docs/help/webmasters.htm#07"
899
+ "url": "http://www.sogou.com/docs/help/webmasters.htm#07",
900
+ "instances": [
901
+ "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
902
+ "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
903
+ "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
904
+ ]
392
905
  },
393
906
  {
394
907
  "pattern": "lssbot",
395
- "addition_date": "2012/05/15"
908
+ "addition_date": "2012/05/15",
909
+ "instances": []
396
910
  },
397
911
  {
398
912
  "pattern": "careerbot",
399
913
  "addition_date": "2012/05/23",
400
- "url": "http://www.career-x.de/bot.html"
914
+ "url": "http://www.career-x.de/bot.html",
915
+ "instances": []
401
916
  },
402
917
  {
403
918
  "pattern": "wotbox",
404
919
  "addition_date": "2012/06/12",
405
- "url": "http://www.wotbox.com"
920
+ "url": "http://www.wotbox.com",
921
+ "instances": [
922
+ "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)",
923
+ "Wotbox/2.01 (+http://www.wotbox.com/bot/)"
924
+ ]
406
925
  },
407
926
  {
408
927
  "pattern": "wocbot",
409
928
  "addition_date": "2012/07/25",
410
- "url": "http://www.wocodi.com/crawler"
929
+ "url": "http://www.wocodi.com/crawler",
930
+ "instances": []
411
931
  },
412
932
  {
413
933
  "pattern": "ichiro",
414
934
  "addition_date": "2012/08/28",
415
- "url": "http://help.goo.ne.jp/help/article/1142"
935
+ "url": "http://help.goo.ne.jp/help/article/1142",
936
+ "instances": [
937
+ "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)",
938
+ "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
939
+ "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
940
+ "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)",
941
+ "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)",
942
+ "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)",
943
+ "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
944
+ "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
945
+ "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)",
946
+ "ichiro/2.0 (ichiro@nttr.co.jp)",
947
+ "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)",
948
+ "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)",
949
+ "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
950
+ "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)",
951
+ "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)"
952
+ ]
416
953
  },
417
954
  {
418
955
  "pattern": "DuckDuckBot",
419
956
  "addition_date": "2012/09/19",
420
- "url": "http://duckduckgo.com/duckduckbot.html"
957
+ "url": "http://duckduckgo.com/duckduckbot.html",
958
+ "instances": [
959
+ "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
960
+ "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)"
961
+ ]
421
962
  },
422
963
  {
423
964
  "pattern": "lssrocketcrawler",
424
- "addition_date": "2012/09/24"
965
+ "addition_date": "2012/09/24",
966
+ "instances": []
425
967
  },
426
968
  {
427
969
  "pattern": "drupact",
428
970
  "addition_date": "2012/09/27",
429
- "url": "http://www.arocom.de/drupact"
971
+ "url": "http://www.arocom.de/drupact",
972
+ "instances": [
973
+ "drupact/0.7; http://www.arocom.de/drupact"
974
+ ]
430
975
  },
431
976
  {
432
977
  "pattern": "webcompanycrawler",
433
- "addition_date": "2012/10/03"
978
+ "addition_date": "2012/10/03",
979
+ "instances": []
434
980
  },
435
981
  {
436
982
  "pattern": "acoonbot",
437
983
  "addition_date": "2012/10/07",
438
- "url": "http://www.acoon.de/robot.asp"
984
+ "url": "http://www.acoon.de/robot.asp",
985
+ "instances": []
439
986
  },
440
987
  {
441
988
  "pattern": "openindexspider",
442
989
  "addition_date": "2012/10/26",
443
- "url": "http://www.openindex.io/en/webmasters/spider.html"
990
+ "url": "http://www.openindex.io/en/webmasters/spider.html",
991
+ "instances": []
444
992
  },
445
993
  {
446
994
  "pattern": "gnam gnam spider",
447
- "addition_date": "2012/10/31"
995
+ "addition_date": "2012/10/31",
996
+ "instances": []
448
997
  },
449
998
  {
450
- "pattern": "web-archive-net.com.bot"
999
+ "pattern": "web-archive-net.com.bot",
1000
+ "instances": []
451
1001
  },
452
1002
  {
453
1003
  "pattern": "backlinkcrawler",
454
- "addition_date": "2013/01/04"
1004
+ "addition_date": "2013/01/04",
1005
+ "instances": []
455
1006
  },
456
1007
  {
457
1008
  "pattern": "coccoc",
458
1009
  "addition_date": "2013/01/04",
459
- "url": "http://help.coccoc.vn/"
1010
+ "url": "http://help.coccoc.vn/",
1011
+ "instances": [
1012
+ "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)",
1013
+ "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)",
1014
+ "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)",
1015
+ "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)",
1016
+ "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)",
1017
+ "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)",
1018
+ "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)",
1019
+ "coccoc",
1020
+ "coccoc/1.0 ()",
1021
+ "coccoc/1.0 (http://help.coccoc.com/)",
1022
+ "coccoc/1.0 (http://help.coccoc.vn/)"
1023
+ ]
460
1024
  },
461
1025
  {
462
1026
  "pattern": "integromedb",
463
1027
  "addition_date": "2013/01/10",
464
- "url": "http://www.integromedb.org/Crawler"
1028
+ "url": "http://www.integromedb.org/Crawler",
1029
+ "instances": [
1030
+ "www.integromedb.org/Crawler"
1031
+ ]
465
1032
  },
466
1033
  {
467
1034
  "pattern": "content crawler spider",
468
- "addition_date": "2013/01/11"
1035
+ "addition_date": "2013/01/11",
1036
+ "instances": []
469
1037
  },
470
1038
  {
471
1039
  "pattern": "toplistbot",
472
- "addition_date": "2013/02/05"
1040
+ "addition_date": "2013/02/05",
1041
+ "instances": []
473
1042
  },
474
1043
  {
475
1044
  "pattern": "seokicks-robot",
476
- "addition_date": "2013/02/25"
1045
+ "addition_date": "2013/02/25",
1046
+ "instances": []
477
1047
  },
478
1048
  {
479
1049
  "pattern": "it2media-domain-crawler",
480
- "addition_date": "2013/03/12"
1050
+ "addition_date": "2013/03/12",
1051
+ "instances": [
1052
+ "it2media-domain-crawler/1.0 on crawler-prod.it2media.de",
1053
+ "it2media-domain-crawler/2.0"
1054
+ ]
481
1055
  },
482
1056
  {
483
1057
  "pattern": "ip-web-crawler.com",
484
- "addition_date": "2013/03/22"
1058
+ "addition_date": "2013/03/22",
1059
+ "instances": []
485
1060
  },
486
1061
  {
487
1062
  "pattern": "siteexplorer.info",
488
- "addition_date": "2013/05/01"
1063
+ "addition_date": "2013/05/01",
1064
+ "instances": [
1065
+ "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)",
1066
+ "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)"
1067
+ ]
489
1068
  },
490
1069
  {
491
1070
  "pattern": "elisabot",
492
- "addition_date": "2013/06/27"
1071
+ "addition_date": "2013/06/27",
1072
+ "instances": []
493
1073
  },
494
1074
  {
495
1075
  "pattern": "proximic",
496
1076
  "addition_date": "2013/09/12",
497
- "url": "http://www.proximic.com/info/spider.php"
1077
+ "url": "http://www.proximic.com/info/spider.php",
1078
+ "instances": [
1079
+ "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)",
1080
+ "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)"
1081
+ ]
498
1082
  },
499
1083
  {
500
1084
  "pattern": "changedetection",
501
1085
  "addition_date": "2013/09/13",
502
- "url": "http://www.changedetection.com/bot.html"
1086
+ "url": "http://www.changedetection.com/bot.html",
1087
+ "instances": [
1088
+ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )"
1089
+ ]
503
1090
  },
504
1091
  {
505
1092
  "pattern": "blexbot",
506
1093
  "addition_date": "2013/10/03",
507
- "url": "http://webmeup-crawler.com/"
1094
+ "url": "http://webmeup-crawler.com/",
1095
+ "instances": []
508
1096
  },
509
1097
  {
510
1098
  "pattern": "arabot",
511
- "addition_date": "2013/10/09"
1099
+ "addition_date": "2013/10/09",
1100
+ "instances": []
512
1101
  },
513
1102
  {
514
1103
  "pattern": "WeSEE:Search",
515
- "addition_date": "2013/11/18"
1104
+ "addition_date": "2013/11/18",
1105
+ "instances": [
1106
+ "WeSEE:Search",
1107
+ "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)"
1108
+ ]
516
1109
  },
517
1110
  {
518
1111
  "pattern": "niki-bot",
519
- "addition_date": "2014/01/01"
1112
+ "addition_date": "2014/01/01",
1113
+ "instances": []
520
1114
  },
521
1115
  {
522
1116
  "pattern": "CrystalSemanticsBot",
523
1117
  "addition_date": "2014/02/17",
524
- "url": "http://www.crystalsemantics.com/user-agent/"
1118
+ "url": "http://www.crystalsemantics.com/user-agent/",
1119
+ "instances": []
525
1120
  },
526
1121
  {
527
1122
  "pattern": "rogerbot",
528
1123
  "addition_date": "2014/02/28",
529
- "url": "http://moz.com/help/pro/what-is-rogerbot-"
1124
+ "url": "http://moz.com/help/pro/what-is-rogerbot-",
1125
+ "instances": [
1126
+ "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)",
1127
+ "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)",
1128
+ "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)",
1129
+ "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com",
1130
+ "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)",
1131
+ "rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)",
1132
+ "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)",
1133
+ "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)",
1134
+ "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)",
1135
+ "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)",
1136
+ "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)",
1137
+ "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)",
1138
+ "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)"
1139
+ ]
530
1140
  },
531
1141
  {
532
1142
  "pattern": "360Spider",
533
1143
  "addition_date": "2014/03/14",
534
- "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html"
1144
+ "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html",
1145
+ "instances": [
1146
+ "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider",
1147
+ "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)",
1148
+ "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider",
1149
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider",
1150
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11; 360Spider",
1151
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;",
1152
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider",
1153
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider",
1154
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)"
1155
+ ]
535
1156
  },
536
1157
  {
537
1158
  "pattern": "psbot",
538
1159
  "addition_date": "2014/03/31",
539
- "url": "http://www.picsearch.com/bot.html"
1160
+ "url": "http://www.picsearch.com/bot.html",
1161
+ "instances": [
1162
+ "psbot-image (+http://www.picsearch.com/bot.html)",
1163
+ "psbot-page (+http://www.picsearch.com/bot.html)",
1164
+ "psbot/0.1 (+http://www.picsearch.com/bot.html)"
1165
+ ]
540
1166
  },
541
1167
  {
542
1168
  "pattern": "InterfaxScanBot",
543
1169
  "addition_date": "2014/03/31",
544
- "url": "http://scan-interfax.ru"
545
- },
546
- {
547
- "pattern": "Lipperhey SEO Service",
548
- "addition_date": "2014/04/01",
549
- "url": "http://www.lipperhey.com/"
1170
+ "url": "http://scan-interfax.ru",
1171
+ "instances": []
550
1172
  },
551
1173
  {
552
1174
  "pattern": "CC Metadata Scaper",
553
1175
  "addition_date": "2014/04/01",
554
- "url": "http://wiki.creativecommons.org/Metadata_Scraper"
1176
+ "url": "http://wiki.creativecommons.org/Metadata_Scraper",
1177
+ "instances": [
1178
+ "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper"
1179
+ ]
555
1180
  },
556
1181
  {
557
1182
  "pattern": "g00g1e.net",
558
1183
  "addition_date": "2014/04/01",
559
- "url": "http://www.g00g1e.net/"
1184
+ "url": "http://www.g00g1e.net/",
1185
+ "instances": []
560
1186
  },
561
1187
  {
562
1188
  "pattern": "GrapeshotCrawler",
563
1189
  "addition_date": "2014/04/01",
564
- "url": "http://www.grapeshot.co.uk/crawler.php"
1190
+ "url": "http://www.grapeshot.co.uk/crawler.php",
1191
+ "instances": [
1192
+ "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)"
1193
+ ]
565
1194
  },
566
1195
  {
567
1196
  "pattern": "urlappendbot",
568
1197
  "addition_date": "2014/05/10",
569
- "url": "http://www.profound.net/urlappendbot.html"
1198
+ "url": "http://www.profound.net/urlappendbot.html",
1199
+ "instances": [
1200
+ "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)"
1201
+ ]
570
1202
  },
571
1203
  {
572
1204
  "pattern": "brainobot",
573
- "addition_date": "2014/06/24"
1205
+ "addition_date": "2014/06/24",
1206
+ "instances": []
574
1207
  },
575
1208
  {
576
1209
  "pattern": "fr-crawler",
577
1210
  "addition_date": "2014/07/31",
578
- "instances": ["Mozilla/5.0 (compatible; fr-crawler/1.1)"]
1211
+ "instances": [
1212
+ "Mozilla/5.0 (compatible; fr-crawler/1.1)"
1213
+ ]
579
1214
  },
580
1215
  {
581
1216
  "pattern": "binlar",
@@ -591,349 +1226,1101 @@
591
1226
  {
592
1227
  "pattern": "SimpleCrawler",
593
1228
  "addition_date": "2014/09/12",
594
- "instances": ["SimpleCrawler/0.1" ]
595
- },
596
- {
597
- "pattern": "Livelapbot",
598
- "addition_date": "2014/09/12",
599
- "instances": ["Livelapbot/0.1" ]
1229
+ "instances": [
1230
+ "SimpleCrawler/0.1"
1231
+ ]
600
1232
  },
601
1233
  {
602
1234
  "pattern": "Twitterbot",
603
1235
  "addition_date": "2014/09/12",
604
- "instances": ["Twitterbot/0.1", "Twitterbot/1.0" ]
1236
+ "url": "https://dev.twitter.com/cards/getting-started",
1237
+ "instances": [
1238
+ "Twitterbot/0.1",
1239
+ "Twitterbot/1.0"
1240
+ ]
605
1241
  },
606
1242
  {
607
1243
  "pattern": "cXensebot",
608
1244
  "addition_date": "2014/10/05",
609
- "instances": ["cXensebot/1.1a"],
1245
+ "instances": [
1246
+ "cXensebot/1.1a"
1247
+ ],
610
1248
  "url": "http://www.cxense.com/bot.html"
611
1249
  },
612
1250
  {
613
1251
  "pattern": "smtbot",
614
1252
  "addition_date": "2014/10/04",
615
- "instances": ["Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbo)t", "SMTBot (similartech.com/smtbot)"],
1253
+ "instances": [
1254
+ "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",
1255
+ "SMTBot (similartech.com/smtbot)"
1256
+ ],
616
1257
  "url": "http://www.similartech.com/smtbot"
617
1258
  },
618
1259
  {
619
1260
  "pattern": "bnf.fr_bot",
620
1261
  "addition_date": "2014/11/18",
621
1262
  "url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html",
622
- "instances": ["Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)"]
1263
+ "instances": [
1264
+ "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)"
1265
+ ]
623
1266
  },
624
1267
  {
625
1268
  "pattern": "A6-Indexer",
626
1269
  "addition_date": "2014/12/05",
627
1270
  "url": "http://www.a6corp.com/a6-web-scraping-policy/",
628
- "instances": ["A6-Indexer"]
1271
+ "instances": [
1272
+ "A6-Indexer"
1273
+ ]
629
1274
  },
630
1275
  {
631
1276
  "pattern": "ADmantX",
632
1277
  "addition_date": "2014/12/05",
633
1278
  "url": "http://www.admantx.com",
634
- "instances": ["ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com"]
1279
+ "instances": [
1280
+ "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com"
1281
+ ]
635
1282
  },
636
1283
  {
637
1284
  "pattern": "Facebot",
638
1285
  "url": "https://developers.facebook.com/docs/sharing/best-practices#crawl",
639
- "addition_date": "2014/12/30"
640
- },
641
- {
642
- "pattern": "Twitterbot",
643
- "url": "https://dev.twitter.com/cards/getting-started",
644
- "addition_date": "2014/12/30"
1286
+ "addition_date": "2014/12/30",
1287
+ "instances": [
1288
+ "Facebot/1.0"
1289
+ ]
645
1290
  },
646
1291
  {
647
1292
  "pattern": "OrangeBot",
648
- "instances": ["Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com"],
1293
+ "instances": [
1294
+ "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com"
1295
+ ],
649
1296
  "addition_date": "2015/01/12"
650
1297
  },
651
1298
  {
652
1299
  "pattern": "memorybot",
653
1300
  "url": "http://mignify.com/bot.htm",
654
- "instances": ["Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)"],
1301
+ "instances": [
1302
+ "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)"
1303
+ ],
655
1304
  "addition_date": "2015/02/01"
656
1305
  },
657
1306
  {
658
1307
  "pattern": "AdvBot",
659
1308
  "url": "http://advbot.net/bot.html",
660
- "instances": ["Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)"],
1309
+ "instances": [
1310
+ "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)"
1311
+ ],
661
1312
  "addition_date": "2015/02/01"
662
1313
  },
663
1314
  {
664
1315
  "pattern": "MegaIndex",
665
1316
  "url": "https://www.megaindex.ru/?tab=linkAnalyze",
666
- "instances": ["Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)"],
1317
+ "instances": [
1318
+ "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)"
1319
+ ],
667
1320
  "addition_date": "2015/03/28"
668
1321
  },
669
1322
  {
670
1323
  "pattern": "SemanticScholarBot",
671
1324
  "url": "http://s2.allenai.org/bot.html",
672
- "instances": ["SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)"],
1325
+ "instances": [
1326
+ "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)"
1327
+ ],
673
1328
  "addition_date": "2015/03/28"
674
1329
  },
675
1330
  {
676
1331
  "pattern": "ltx71",
677
1332
  "url": "http://ltx71.com/",
678
- "instances": ["ltx71 - (http://ltx71.com/)"],
1333
+ "instances": [
1334
+ "ltx71 - (http://ltx71.com/)"
1335
+ ],
679
1336
  "addition_date": "2015/04/04"
680
1337
  },
681
1338
  {
682
1339
  "pattern": "nerdybot",
683
1340
  "url": "http://nerdybot.com/",
684
- "instances": ["nerdybot"],
1341
+ "instances": [
1342
+ "nerdybot"
1343
+ ],
685
1344
  "addition_date": "2015/04/05"
686
1345
  },
687
1346
  {
688
1347
  "pattern": "xovibot",
689
1348
  "url": "http://www.xovibot.net/",
690
- "instances": ["Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)"],
1349
+ "instances": [
1350
+ "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)"
1351
+ ],
691
1352
  "addition_date": "2015/04/05"
692
1353
  },
693
1354
  {
694
1355
  "pattern": "BUbiNG",
695
1356
  "url": "http://law.di.unimi.it/BUbiNG.html",
696
- "instances": ["BUbiNG (+http://law.di.unimi.it/BUbiNG.html)"],
1357
+ "instances": [
1358
+ "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)"
1359
+ ],
697
1360
  "addition_date": "2015/04/06"
698
1361
  },
699
1362
  {
700
1363
  "pattern": "Qwantify",
701
1364
  "url": "https://www.qwant.com/",
702
- "instances": ["Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*"],
1365
+ "instances": [
1366
+ "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*"
1367
+ ],
703
1368
  "addition_date": "2015/04/06"
704
1369
  },
705
1370
  {
706
1371
  "pattern": "archive.org_bot",
707
1372
  "url": "http://www.archive.org/details/archive.org_bot",
708
- "instances": ["Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)"],
1373
+ "instances": [
1374
+ "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)"
1375
+ ],
709
1376
  "addition_date": "2015/04/14"
710
1377
  },
711
1378
  {
712
1379
  "pattern": "Applebot",
713
1380
  "url": "http://www.apple.com/go/applebot",
714
- "addition_date": "2015/04/15"
1381
+ "addition_date": "2015/04/15",
1382
+ "instances": [
1383
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)",
1384
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)",
1385
+ "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",
1386
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",
1387
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)"
1388
+ ]
715
1389
  },
716
1390
  {
717
1391
  "pattern": "TweetmemeBot",
718
1392
  "url": "http://datasift.com/bot.html",
719
- "instances": ["Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0"],
1393
+ "instances": [
1394
+ "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0"
1395
+ ],
720
1396
  "addition_date": "2015/04/15"
721
1397
  },
722
1398
  {
723
1399
  "pattern": "crawler4j",
724
1400
  "url": "https://github.com/yasserg/crawler4j",
725
- "instances": ["crawler4j (http://code.google.com/p/crawler4j/)"],
1401
+ "instances": [
1402
+ "crawler4j (http://code.google.com/p/crawler4j/)"
1403
+ ],
726
1404
  "addition_date": "2015/05/07"
727
1405
  },
728
1406
  {
729
1407
  "pattern": "findxbot",
730
1408
  "url": "http://www.findxbot.com",
731
- "instances": ["Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)"],
1409
+ "instances": [
1410
+ "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)"
1411
+ ],
732
1412
  "addition_date": "2015/05/07"
733
1413
  },
734
1414
  {
735
- "pattern": "SemrushBot",
1415
+ "pattern": "S[eE][mM]rushBot",
736
1416
  "url": "http://www.semrush.com/bot.html",
737
- "instances": ["Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)"],
1417
+ "instances": [
1418
+ "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)",
1419
+ "SEMrushBot"
1420
+ ],
738
1421
  "addition_date": "2015/05/26"
739
1422
  },
740
1423
  {
741
1424
  "pattern": "yoozBot",
742
1425
  "url": "http://yooz.ir",
743
- "instances": ["Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)"],
1426
+ "instances": [
1427
+ "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)"
1428
+ ],
744
1429
  "addition_date": "2015/05/26"
745
1430
  },
746
1431
  {
747
1432
  "pattern": "lipperhey",
748
1433
  "url": "http://www.lipperhey.com/",
749
- "instances": ["Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)", "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)"],
1434
+ "instances": [
1435
+ "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)",
1436
+ "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)",
1437
+ "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)",
1438
+ "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)"
1439
+ ],
750
1440
  "addition_date": "2015/08/26"
751
1441
  },
752
1442
  {
753
- "pattern": "y!j-asr",
1443
+ "pattern": "Y!J-ASR",
754
1444
  "url": "http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/",
755
- "instances": ["Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)"],
1445
+ "instances": [
1446
+ "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)"
1447
+ ],
756
1448
  "addition_date": "2015/05/26"
757
- },
758
- {
1449
+ },
1450
+ {
759
1451
  "pattern": "Domain Re-Animator Bot",
760
1452
  "url": "http://domainreanimator.com",
761
- "instances": ["Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com"],
1453
+ "instances": [
1454
+ "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com"
1455
+ ],
762
1456
  "addition_date": "2015/04/14"
763
- },
764
- {
1457
+ },
1458
+ {
765
1459
  "pattern": "AddThis",
766
1460
  "url": "https://www.addthis.com",
767
- "instances": ["AddThis.com robot tech.support@clearspring.com"],
1461
+ "instances": [
1462
+ "AddThis.com robot tech.support@clearspring.com"
1463
+ ],
768
1464
  "addition_date": "2015/06/02"
769
- },
770
- {
1465
+ },
1466
+ {
771
1467
  "pattern": "Screaming Frog SEO Spider",
772
1468
  "url": "http://www.screamingfrog.co.uk/seo-spider",
773
- "instances": ["Screaming Frog SEO Spider/5.1"],
1469
+ "instances": [
1470
+ "Screaming Frog SEO Spider/5.1"
1471
+ ],
774
1472
  "addition_date": "2016/01/08"
775
- },
776
- {
1473
+ },
1474
+ {
777
1475
  "pattern": "MetaURI",
778
1476
  "url": "http://www.useragentstring.com/MetaURI_id_17683.php",
779
- "instances": ["MetaURI API/2.0 +metauri.com"],
1477
+ "instances": [
1478
+ "MetaURI API/2.0 +metauri.com"
1479
+ ],
780
1480
  "addition_date": "2016/01/02"
781
- },
782
- {
1481
+ },
1482
+ {
783
1483
  "pattern": "Scrapy",
784
1484
  "url": "http://scrapy.org/",
785
- "instances": ["Scrapy/1.0.3 (+http://scrapy.org)"],
1485
+ "instances": [
1486
+ "Scrapy/1.0.3 (+http://scrapy.org)"
1487
+ ],
786
1488
  "addition_date": "2016/01/02"
787
- },
788
- {
789
- "pattern": "LivelapBot",
1489
+ },
1490
+ {
1491
+ "pattern": "Livelap[bB]ot",
790
1492
  "url": "http://site.livelap.com/crawler",
791
- "instances": ["LivelapBot/0.2 (http://site.livelap.com/crawler)"],
1493
+ "instances": [
1494
+ "LivelapBot/0.2 (http://site.livelap.com/crawler)",
1495
+ "Livelapbot/0.1"
1496
+ ],
792
1497
  "addition_date": "2016/01/02"
793
- },
794
- {
1498
+ },
1499
+ {
795
1500
  "pattern": "OpenHoseBot",
796
1501
  "url": "http://www.openhose.org/bot.html",
797
- "instances": ["Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)"],
1502
+ "instances": [
1503
+ "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)"
1504
+ ],
798
1505
  "addition_date": "2016/01/02"
799
- },
800
- {
1506
+ },
1507
+ {
801
1508
  "pattern": "CapsuleChecker",
802
1509
  "url": "http://www.capsulink.com/about",
803
- "instances": ["CapsuleChecker (http://www.capsulink.com/)"],
1510
+ "instances": [
1511
+ "CapsuleChecker (http://www.capsulink.com/)"
1512
+ ],
804
1513
  "addition_date": "2016/01/02"
805
- },
806
- {
1514
+ },
1515
+ {
807
1516
  "pattern": "collection@infegy.com",
808
1517
  "url": "http://infegy.com/",
809
- "instances": ["Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com"],
1518
+ "instances": [
1519
+ "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com"
1520
+ ],
810
1521
  "addition_date": "2016/01/03"
811
- },
812
- {
1522
+ },
1523
+ {
813
1524
  "pattern": "IstellaBot",
814
1525
  "url": "http://www.tiscali.it/",
815
- "instances": ["Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)"],
1526
+ "instances": [
1527
+ "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)"
1528
+ ],
816
1529
  "addition_date": "2016/01/09"
817
- },
1530
+ },
818
1531
  {
819
1532
  "pattern": "DeuSu\\/",
820
1533
  "addition_date": "2016/01/23",
821
- "url": "https://deusu.de/robot.html"
1534
+ "url": "https://deusu.de/robot.html",
1535
+ "instances": [
1536
+ "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)",
1537
+ "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)"
1538
+ ]
822
1539
  },
823
1540
  {
824
1541
  "pattern": "betaBot",
825
- "addition_date": "2016/01/23"
1542
+ "addition_date": "2016/01/23",
1543
+ "instances": []
826
1544
  },
827
1545
  {
828
1546
  "pattern": "Cliqzbot\\/",
829
1547
  "addition_date": "2016/01/23",
830
- "url": "http://cliqz.com/company/cliqzbot"
1548
+ "url": "http://cliqz.com/company/cliqzbot",
1549
+ "instances": [
1550
+ "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)",
1551
+ "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)",
1552
+ "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)",
1553
+ "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)"
1554
+ ]
831
1555
  },
832
1556
  {
833
1557
  "pattern": "MojeekBot\\/",
834
1558
  "addition_date": "2016/01/23",
835
- "url": "https://www.mojeek.com/bot.html"
1559
+ "url": "https://www.mojeek.com/bot.html",
1560
+ "instances": [
1561
+ "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)",
1562
+ "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)",
1563
+ "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)",
1564
+ "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)",
1565
+ "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)",
1566
+ "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)"
1567
+ ]
836
1568
  },
837
1569
  {
838
1570
  "pattern": "netEstate NE Crawler",
839
1571
  "addition_date": "2016/01/23",
840
- "url": "+http://www.website-datenbank.de/"
1572
+ "url": "+http://www.website-datenbank.de/",
1573
+ "instances": [
1574
+ "netEstate NE Crawler (+http://www.sengine.info/)",
1575
+ "netEstate NE Crawler (+http://www.website-datenbank.de/)"
1576
+ ]
841
1577
  },
842
1578
  {
843
1579
  "pattern": "SafeSearch microdata crawler",
844
1580
  "addition_date": "2016/01/23",
845
- "url": "https://safesearch.avira.com"
1581
+ "url": "https://safesearch.avira.com",
1582
+ "instances": [
1583
+ "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)"
1584
+ ]
846
1585
  },
847
1586
  {
848
1587
  "pattern": "Gluten Free Crawler\\/",
849
1588
  "addition_date": "2016/01/23",
850
- "url": "http://glutenfreepleasure.com/"
1589
+ "url": "http://glutenfreepleasure.com/",
1590
+ "instances": [
1591
+ "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)"
1592
+ ]
851
1593
  },
852
1594
  {
853
1595
  "pattern": "Sonic",
854
1596
  "addition_date": "2016/02/08",
855
- "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html"
1597
+ "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html",
1598
+ "instances": [
1599
+ "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)",
1600
+ "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)",
1601
+ "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)"
1602
+ ]
856
1603
  },
857
1604
  {
858
1605
  "pattern": "Sysomos",
859
1606
  "addition_date": "2016/02/08",
860
- "url": "http://www.sysomos.com"
1607
+ "url": "http://www.sysomos.com",
1608
+ "instances": [
1609
+ "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)"
1610
+ ]
861
1611
  },
862
1612
  {
863
1613
  "pattern": "Trove",
864
1614
  "addition_date": "2016/02/08",
865
- "url": "http://www.trove.com"
1615
+ "url": "http://www.trove.com",
1616
+ "instances": []
866
1617
  },
867
1618
  {
868
1619
  "pattern": "deadlinkchecker",
869
1620
  "addition_date": "2016/02/08",
870
- "url": "http://www.deadlinkchecker.com"
1621
+ "url": "http://www.deadlinkchecker.com",
1622
+ "instances": [
1623
+ "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36",
1624
+ "www.deadlinkchecker.com XMLHTTP/1.0",
1625
+ "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36"
1626
+ ]
871
1627
  },
872
1628
  {
873
1629
  "pattern": "Slack-ImgProxy",
874
1630
  "addition_date": "2016/04/25",
875
- "url": "https://api.slack.com/robots"
1631
+ "url": "https://api.slack.com/robots",
1632
+ "instances": [
1633
+ "Slack-ImgProxy (+https://api.slack.com/robots)",
1634
+ "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)",
1635
+ "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)",
1636
+ "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)",
1637
+ "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)",
1638
+ "Slack-ImgProxy 149 (+https://api.slack.com/robots)"
1639
+ ]
876
1640
  },
877
1641
  {
878
1642
  "pattern": "Embedly",
879
1643
  "addition_date": "2016/04/25",
880
- "url": "http://support.embed.ly"
1644
+ "url": "http://support.embed.ly",
1645
+ "instances": [
1646
+ "Embedly +support@embed.ly",
1647
+ "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)",
1648
+ "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)"
1649
+ ]
881
1650
  },
882
1651
  {
883
1652
  "pattern": "RankActiveLinkBot",
884
1653
  "addition_date": "2016/06/20",
885
- "url": "https://rankactive.com/resources/rankactive-linkbot"
1654
+ "url": "https://rankactive.com/resources/rankactive-linkbot",
1655
+ "instances": [
1656
+ "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)"
1657
+ ]
886
1658
  },
887
1659
  {
888
1660
  "pattern": "iskanie",
889
1661
  "addition_date": "2016/09/02",
890
1662
  "url": "http://www.iskanie.com",
891
- "instances": ["iskanie (+http://www.iskanie.com)"]
1663
+ "instances": [
1664
+ "iskanie (+http://www.iskanie.com)"
1665
+ ]
892
1666
  },
893
1667
  {
894
1668
  "pattern": "SafeDNSBot",
895
1669
  "addition_date": "2016/09/10",
896
1670
  "url": "https://www.safedns.com/searchbot",
897
- "instances": ["SafeDNSBot (https://www.safedns.com/searchbot)"]
1671
+ "instances": [
1672
+ "SafeDNSBot (https://www.safedns.com/searchbot)"
1673
+ ]
898
1674
  },
899
1675
  {
900
1676
  "pattern": "SkypeUriPreview",
901
1677
  "addition_date": "2016/10/10",
902
- "instances": ["Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5"]
1678
+ "instances": [
1679
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5"
1680
+ ]
903
1681
  },
904
1682
  {
905
1683
  "pattern": "Veoozbot",
906
1684
  "addition_date": "2016/11/03",
907
- "url": "http://www.veooz.com/veoozbot.html"
1685
+ "url": "http://www.veooz.com/veoozbot.html",
1686
+ "instances": [
1687
+ "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)"
1688
+ ]
908
1689
  },
909
1690
  {
910
1691
  "pattern": "Slackbot",
911
1692
  "addition_date": "2016/11/03",
912
- "url": "https://api.slack.com/robots"
1693
+ "url": "https://api.slack.com/robots",
1694
+ "instances": [
1695
+ "Slackbot-LinkExpanding (+https://api.slack.com/robots)",
1696
+ "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"
1697
+ ]
913
1698
  },
914
1699
  {
915
1700
  "pattern": "redditbot",
916
1701
  "addition_date": "2016/11/03",
917
- "url": "http://www.reddit.com/feedback"
1702
+ "url": "http://www.reddit.com/feedback",
1703
+ "instances": [
1704
+ "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)"
1705
+ ]
918
1706
  },
919
1707
  {
920
1708
  "pattern": "datagnionbot",
921
1709
  "addition_date": "2016/11/03",
922
- "url": "http://www.datagnion.com/bot.html"
1710
+ "url": "http://www.datagnion.com/bot.html",
1711
+ "instances": [
1712
+ "datagnionbot (+http://www.datagnion.com/bot.html)"
1713
+ ]
923
1714
  },
924
1715
  {
925
- "pattern": "Veoozbot",
1716
+ "pattern": "Google-Adwords-Instant",
926
1717
  "addition_date": "2016/11/03",
927
- "url": "http://www.veooz.com/veoozbot.html"
1718
+ "url": "http://www.google.com/adsbot.html",
1719
+ "instances": [
1720
+ "Google-Adwords-Instant (+http://www.google.com/adsbot.html)"
1721
+ ]
928
1722
  },
929
1723
  {
930
- "pattern": "Slackbot",
931
- "addition_date": "2016/11/03",
932
- "url": "https://api.slack.com/robots"
1724
+ "pattern": "adbeat_bot",
1725
+ "addition_date": "2016/11/04",
1726
+ "instances": [
1727
+ "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)",
1728
+ "adbeat_bot"
1729
+ ]
933
1730
  },
934
1731
  {
935
- "pattern": "Google-Adwords-Instant",
936
- "addition_date": "2016/11/03",
937
- "url": "http://www.google.com/adsbot.html"
1732
+ "pattern": "WhatsApp",
1733
+ "addition_date": "2016/11/15",
1734
+ "url": "https://www.whatsapp.com/",
1735
+ "instances": [
1736
+ "WhatsApp",
1737
+ "WhatsApp/2.12.15/i",
1738
+ "WhatsApp/2.12.16/i",
1739
+ "WhatsApp/2.12.17/i",
1740
+ "WhatsApp/2.12.449 A",
1741
+ "WhatsApp/2.12.453 A",
1742
+ "WhatsApp/2.12.510 A",
1743
+ "WhatsApp/2.12.540 A",
1744
+ "WhatsApp/2.12.548 A",
1745
+ "WhatsApp/2.12.555 A",
1746
+ "WhatsApp/2.12.556 A",
1747
+ "WhatsApp/2.16.1/i",
1748
+ "WhatsApp/2.16.13 A",
1749
+ "WhatsApp/2.16.2/i",
1750
+ "WhatsApp/2.16.42 A",
1751
+ "WhatsApp/2.16.57 A"
1752
+ ]
1753
+ },
1754
+ {
1755
+ "pattern": "contxbot",
1756
+ "addition_date": "2017/02/25",
1757
+ "instances": [
1758
+ "Mozilla/5.0 (compatible;contxbot/1.0)"
1759
+ ]
1760
+ },
1761
+ {
1762
+ "pattern": "pinterest",
1763
+ "addition_date": "2017/03/03",
1764
+ "instances": [
1765
+ "Pinterest/0.2 (+http://www.pinterest.com/bot.html)"
1766
+ ],
1767
+ "url": "http://www.pinterest.com/bot.html"
1768
+ },
1769
+ {
1770
+ "pattern": "electricmonk",
1771
+ "addition_date": "2017/03/04",
1772
+ "instances": [
1773
+ "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)"
1774
+ ],
1775
+ "url": "https://www.duedil.com/our-crawler/"
1776
+ },
1777
+ {
1778
+ "pattern": "GarlikCrawler",
1779
+ "addition_date": "2017/03/18",
1780
+ "instances": [
1781
+ "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)"
1782
+ ],
1783
+ "url": "http://garlik.com/"
1784
+ },
1785
+ {
1786
+ "pattern": "BingPreview\\/",
1787
+ "addition_date": "2017/04/23",
1788
+ "url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0",
1789
+ "instances": [
1790
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b",
1791
+ "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko",
1792
+ "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; WOW64; Trident/6.0; BingPreview/1.0b)",
1793
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)",
1794
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b"
1795
+ ]
1796
+ },
1797
+ {
1798
+ "pattern": "vebidoobot",
1799
+ "addition_date": "2017/05/08",
1800
+ "instances": [
1801
+ "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/"
1802
+ ],
1803
+ "url": "https://blog.vebidoo.de/vebidoobot/"
1804
+ },
1805
+ {
1806
+ "pattern": "FemtosearchBot",
1807
+ "addition_date": "2017/05/16",
1808
+ "instances": [
1809
+ "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)"
1810
+ ],
1811
+ "url": "http://femtosearch.com"
1812
+ },
1813
+ {
1814
+ "pattern": "Yahoo Link Preview",
1815
+ "addition_date": "2017/06/28",
1816
+ "instances": [
1817
+ "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)"
1818
+ ],
1819
+ "url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html"
1820
+ },
1821
+ {
1822
+ "pattern": "MetaJobBot",
1823
+ "addition_date": "2017/08/16",
1824
+ "instances": [
1825
+ "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)"
1826
+ ],
1827
+ "url": "http://www.metajob.de/the/crawler"
1828
+ },
1829
+ {
1830
+ "pattern": "DomainStatsBot",
1831
+ "addition_date": "2017/08/16",
1832
+ "instances": [
1833
+ "DomainStatsBot/1.0 (http://domainstats.io/our-bot)"
1834
+ ],
1835
+ "url": "http://domainstats.io/our-bot"
1836
+ },
1837
+ {
1838
+ "pattern": "mindUpBot",
1839
+ "addition_date": "2017/08/16",
1840
+ "instances": [
1841
+ "mindUpBot (datenbutler.de)"
1842
+ ],
1843
+ "url": "http://www.datenbutler.de/"
1844
+ },
1845
+ {
1846
+ "pattern": "Daum",
1847
+ "addition_date": "2017/08/16",
1848
+ "instances": [
1849
+ "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)"
1850
+ ],
1851
+ "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966"
1852
+ },
1853
+ {
1854
+ "pattern": "Jugendschutzprogramm-Crawler",
1855
+ "addition_date": "2017/08/16",
1856
+ "instances": [
1857
+ "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de"
1858
+ ],
1859
+ "url": "http://www.jugendschutzprogramm.de"
1860
+ },
1861
+ {
1862
+ "pattern": "Xenu Link Sleuth",
1863
+ "addition_date": "2017/08/19",
1864
+ "instances": [
1865
+ "Xenu Link Sleuth/1.3.8"
1866
+ ],
1867
+ "url": "http://home.snafu.de/tilman/xenulink.html"
1868
+ },
1869
+ {
1870
+ "pattern": "Pcore-HTTP",
1871
+ "addition_date": "2017/08/19",
1872
+ "instances": [
1873
+ "Pcore-HTTP/v0.40.3"
1874
+ ],
1875
+ "url": "https://bitbucket.org/softvisio/pcore/overview"
1876
+ },
1877
+ {
1878
+ "pattern": "moatbot",
1879
+ "addition_date": "2017/09/16",
1880
+ "instances": [
1881
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot",
1882
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot"
1883
+ ],
1884
+ "url": "https://moat.com"
1885
+ },
1886
+ {
1887
+ "pattern": "KosmioBot",
1888
+ "addition_date": "2017/09/16",
1889
+ "instances": [
1890
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)"
1891
+ ],
1892
+ "url": "http://kosm.io/bot.html"
1893
+ },
1894
+ {
1895
+ "pattern": "Pingdom",
1896
+ "addition_date": "2017/09/16",
1897
+ "instances": [
1898
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)"
1899
+ ],
1900
+ "url": "http://www.pingdom.com"
1901
+ },
1902
+ {
1903
+ "pattern": "PhantomJS",
1904
+ "addition_date": "2017/09/18",
1905
+ "instances": [
1906
+ "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)"
1907
+ ],
1908
+ "url": "http://phantomjs.org/"
1909
+ },
1910
+ {
1911
+ "pattern": "Gowikibot",
1912
+ "addition_date": "2017/10/26",
1913
+ "instances": [
1914
+ "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)"
1915
+ ],
1916
+ "url": "http://www.gowikibot.com"
1917
+ },
1918
+ {
1919
+ "pattern": "PiplBot",
1920
+ "addition_date": "2017/10/30",
1921
+ "instances": [
1922
+ "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)"
1923
+ ],
1924
+ "url": "http://www.pipl.com/bot/"
1925
+ },
1926
+ {
1927
+ "pattern": "Discordbot",
1928
+ "addition_date": "2017/09/22",
1929
+ "url": "https://discordapp.com",
1930
+ "instances": [
1931
+ "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"
1932
+ ]
1933
+ },
1934
+ {
1935
+ "pattern": "TelegramBot",
1936
+ "addition_date": "2017/10/01",
1937
+ "instances": [
1938
+ "TelegramBot (like TwitterBot)"
1939
+ ]
1940
+ },
1941
+ {
1942
+ "pattern": "InfoPath.2",
1943
+ "addition_date": "2017/10/07",
1944
+ "instances": [
1945
+ "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; InfoPath.2)"
1946
+ ]
1947
+ },
1948
+ {
1949
+ "pattern": "Jetslide",
1950
+ "addition_date": "2017/09/27",
1951
+ "url": "http://jetsli.de/crawler",
1952
+ "instances": [
1953
+ "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)"
1954
+ ]
1955
+ },
1956
+ {
1957
+ "pattern": "newsharecounts",
1958
+ "addition_date": "2017/09/30",
1959
+ "url": "http://newsharecounts.com/crawler",
1960
+ "instances": [
1961
+ "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)"
1962
+ ]
1963
+ },
1964
+ {
1965
+ "pattern": "James BOT",
1966
+ "addition_date": "2017/10/12",
1967
+ "url": "http://cognitiveseo.com/bot.html",
1968
+ "instances": [
1969
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html"
1970
+ ]
1971
+ },
1972
+ {
1973
+ "pattern": "Barkrowler",
1974
+ "addition_date": "2017/10/09",
1975
+ "url": "http://www.exensa.com/crawl",
1976
+ "instances": [
1977
+ "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG"
1978
+ ]
1979
+ },
1980
+ {
1981
+ "pattern": "TinEye-bot",
1982
+ "addition_date": "2017/10/14",
1983
+ "url": "http://www.tineye.com/crawler.html",
1984
+ "instances": [
1985
+ "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)"
1986
+ ]
1987
+ },
1988
+ {
1989
+ "pattern": "SocialRankIOBot",
1990
+ "addition_date": "2017/10/19",
1991
+ "url": "http://socialrank.io/about",
1992
+ "instances": [
1993
+ "SocialRankIOBot; http://socialrank.io/about"
1994
+ ]
1995
+ },
1996
+ {
1997
+ "pattern": "trendictionbot",
1998
+ "addition_date": "2017/10/30",
1999
+ "url": "http://www.trendiction.de/bot",
2000
+ "instances": [
2001
+ "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11"
2002
+ ]
2003
+ },
2004
+ {
2005
+ "pattern": "Ocarinabot",
2006
+ "addition_date": "2017/09/27",
2007
+ "instances": [
2008
+ "Ocarinabot"
2009
+ ]
2010
+ },
2011
+ {
2012
+ "pattern": "epicbot",
2013
+ "addition_date": "2017/10/31",
2014
+ "url": "http://www.epictions.com/epicbot",
2015
+ "instances": [
2016
+ "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)"
2017
+ ]
2018
+ },
2019
+ {
2020
+ "pattern": "Primalbot",
2021
+ "addition_date": "2017/09/27",
2022
+ "url": "https://www.primal.com",
2023
+ "instances": [
2024
+ "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)"
2025
+ ]
2026
+ },
2027
+ {
2028
+ "pattern": "DuckDuckGo-Favicons-Bot",
2029
+ "addition_date": "2017/10/06",
2030
+ "url": "http://duckduckgo.com",
2031
+ "instances": [
2032
+ "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)"
2033
+ ]
2034
+ },
2035
+ {
2036
+ "pattern": "GnowitNewsbot",
2037
+ "addition_date": "2017/10/30",
2038
+ "url": "http://www.gnowit.com",
2039
+ "instances": [
2040
+ "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com"
2041
+ ]
2042
+ },
2043
+ {
2044
+ "pattern": "Leikibot",
2045
+ "addition_date": "2017/09/24",
2046
+ "url": "http://www.leiki.com",
2047
+ "instances": [
2048
+ "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)"
2049
+ ]
2050
+ },
2051
+ {
2052
+ "pattern": "LinkArchiver",
2053
+ "addition_date": "2017/09/24",
2054
+ "instances": [
2055
+ "@LinkArchiver twitter bot"
2056
+ ]
2057
+ },
2058
+ {
2059
+ "pattern": "YaK",
2060
+ "addition_date": "2017/09/25",
2061
+ "url": "http://linkfluence.com",
2062
+ "instances": [
2063
+ "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)"
2064
+ ]
2065
+ },
2066
+ {
2067
+ "pattern": "PaperLiBot",
2068
+ "addition_date": "2017/09/25",
2069
+ "url": "http://support.paper.li/entries/20023257-what-is-paper-li",
2070
+ "instances": [
2071
+ "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)"
2072
+ ]
2073
+ },
2074
+ {
2075
+ "pattern": "Digg Deeper",
2076
+ "addition_date": "2017/09/26",
2077
+ "url": "http://digg.com/about",
2078
+ "instances": [
2079
+ "Digg Deeper/v1 (http://digg.com/about)"
2080
+ ]
2081
+ },
2082
+ {
2083
+ "pattern": "dcrawl",
2084
+ "addition_date": "2017/09/22",
2085
+ "instances": [
2086
+ "dcrawl/1.0"
2087
+ ]
2088
+ },
2089
+ {
2090
+ "pattern": "Snacktory",
2091
+ "addition_date": "2017/09/23",
2092
+ "url": "https://github.com/karussell/snacktory",
2093
+ "instances": [
2094
+ "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)"
2095
+ ]
2096
+ },
2097
+ {
2098
+ "pattern": "AndersPinkBot",
2099
+ "addition_date": "2017/09/24",
2100
+ "url": "http://anderspink.com/bot.html",
2101
+ "instances": [
2102
+ "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)"
2103
+ ]
2104
+ },
2105
+ {
2106
+ "pattern": "Fyrebot",
2107
+ "addition_date": "2017/09/22",
2108
+ "instances": [
2109
+ "Fyrebot/1.0"
2110
+ ]
2111
+ },
2112
+ {
2113
+ "pattern": "EveryoneSocialBot",
2114
+ "addition_date": "2017/09/22",
2115
+ "url": "http://everyonesocial.com",
2116
+ "instances": [
2117
+ "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)"
2118
+ ]
2119
+ },
2120
+ {
2121
+ "pattern": "Mediatoolkitbot",
2122
+ "addition_date": "2017/10/06",
2123
+ "url": "http://mediatoolkit.com",
2124
+ "instances": [
2125
+ "Mediatoolkitbot (complaints@mediatoolkit.com)"
2126
+ ]
2127
+ },
2128
+ {
2129
+ "pattern": "Luminator-robots",
2130
+ "addition_date": "2017/09/22",
2131
+ "instances": [
2132
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0"
2133
+ ]
2134
+ },
2135
+ {
2136
+ "pattern": "ExtLinksBot",
2137
+ "addition_date": "2017/11/02",
2138
+ "url": "https://extlinks.com/Bot.html",
2139
+ "instances": [
2140
+ "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)"
2141
+ ]
2142
+ },
2143
+ {
2144
+ "pattern": "SurveyBot",
2145
+ "addition_date": "2017/11/02",
2146
+ "instances": [
2147
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)"
2148
+ ]
2149
+ },
2150
+ {
2151
+ "pattern": "NING",
2152
+ "addition_date": "2017/11/02",
2153
+ "instances": [
2154
+ "NING/1.0"
2155
+ ]
2156
+ },
2157
+ {
2158
+ "pattern": "okhttp",
2159
+ "addition_date": "2017/11/02",
2160
+ "instances": [
2161
+ "okhttp/2.5.0"
2162
+ ]
2163
+ },
2164
+ {
2165
+ "pattern": "Nuzzel",
2166
+ "addition_date": "2017/11/02",
2167
+ "instances": [
2168
+ "Nuzzel"
2169
+ ]
2170
+ },
2171
+ {
2172
+ "pattern": "omgili",
2173
+ "addition_date": "2017/11/02",
2174
+ "url": "http://omgili.com",
2175
+ "instances": [
2176
+ "omgili/0.5 +http://omgili.com"
2177
+ ]
2178
+ },
2179
+ {
2180
+ "pattern": "PocketParser",
2181
+ "addition_date": "2017/11/02",
2182
+ "url": "https://getpocket.com/pocketparser_ua",
2183
+ "instances": [
2184
+ "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)"
2185
+ ]
2186
+ },
2187
+ {
2188
+ "pattern": "YisouSpider",
2189
+ "addition_date": "2017/11/02",
2190
+ "instances": [
2191
+ "YisouSpider"
2192
+ ]
2193
+ },
2194
+ {
2195
+ "pattern": "um-LN",
2196
+ "addition_date": "2017/11/02",
2197
+ "instances": [
2198
+ "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)"
2199
+ ]
2200
+ },
2201
+ {
2202
+ "pattern": "ToutiaoSpider",
2203
+ "addition_date": "2017/11/02",
2204
+ "url": "http://web.toutiao.com/media_cooperation/",
2205
+ "instances": [
2206
+ "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)"
2207
+ ]
2208
+ },
2209
+ {
2210
+ "pattern": "MuckRack",
2211
+ "addition_date": "2017/11/02",
2212
+ "url": "http://muckrack.com",
2213
+ "instances": [
2214
+ "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)"
2215
+ ]
2216
+ },
2217
+ {
2218
+ "pattern": "Jamie's Spider",
2219
+ "addition_date": "2017/11/02",
2220
+ "url": "http://jamiembrown.com/",
2221
+ "instances": [
2222
+ "Jamie's Spider (http://jamiembrown.com/)"
2223
+ ]
2224
+ },
2225
+ {
2226
+ "pattern": "AHC",
2227
+ "addition_date": "2017/11/02",
2228
+ "instances": [
2229
+ "AHC/2.0"
2230
+ ]
2231
+ },
2232
+ {
2233
+ "pattern": "NetcraftSurveyAgent",
2234
+ "addition_date": "2017/11/02",
2235
+ "instances": [
2236
+ "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)"
2237
+ ]
2238
+ },
2239
+ {
2240
+ "pattern": "Laserlikebot",
2241
+ "addition_date": "2017/11/02",
2242
+ "instances": [
2243
+ "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)"
2244
+ ]
2245
+ },
2246
+ {
2247
+ "pattern": "Apache-HttpClient",
2248
+ "addition_date": "2017/11/02",
2249
+ "instances": [
2250
+ "Apache-HttpClient/4.4.1 (Java/1.8.0_65)"
2251
+ ]
2252
+ },
2253
+ {
2254
+ "pattern": "AppEngine-Google",
2255
+ "addition_date": "2017/11/02",
2256
+ "instances": [
2257
+ "AppEngine-Google; (+http://code.google.com/appengine; appid: example)"
2258
+ ]
2259
+ },
2260
+ {
2261
+ "pattern": "Jetty",
2262
+ "addition_date": "2017/11/02",
2263
+ "instances": [
2264
+ "Jetty/9.3.z-SNAPSHOT"
2265
+ ]
2266
+ },
2267
+ {
2268
+ "pattern": "Upflow",
2269
+ "addition_date": "2017/11/02",
2270
+ "instances": [
2271
+ "Upflow/1.0"
2272
+ ]
2273
+ },
2274
+ {
2275
+ "pattern": "Thinklab",
2276
+ "addition_date": "2017/11/02",
2277
+ "url": "thinklab.com",
2278
+ "instances": [
2279
+ "Thinklab (thinklab.com)"
2280
+ ]
2281
+ },
2282
+ {
2283
+ "pattern": "Traackr.com",
2284
+ "addition_date": "2017/11/02",
2285
+ "url": "Traackr.com",
2286
+ "instances": [
2287
+ "Traackr.com"
2288
+ ]
2289
+ },
2290
+ {
2291
+ "pattern": "Twurly",
2292
+ "addition_date": "2017/11/02",
2293
+ "url": "http://twurly.org",
2294
+ "instances": [
2295
+ "Ruby, Twurly v1.1 (http://twurly.org)"
2296
+ ]
2297
+ },
2298
+ {
2299
+ "pattern": "Mastodon",
2300
+ "addition_date": "2017/11/02",
2301
+ "instances": [
2302
+ "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)"
2303
+ ]
2304
+ },
2305
+ {
2306
+ "pattern": "http_get",
2307
+ "addition_date": "2017/11/02",
2308
+ "instances": [
2309
+ "http_get"
2310
+ ]
2311
+ },
2312
+ {
2313
+ "pattern": "DnyzBot",
2314
+ "addition_date": "2017/11/20",
2315
+ "instances": [
2316
+ "Mozilla/5.0 (compatible; DnyzBot/1.0)"
2317
+ ]
2318
+ },
2319
+ {
2320
+ "pattern": "Botify",
2321
+ "addition_date": "2018/02/01",
2322
+ "instances": [
2323
+ "Mozilla/5.0 (compatible; botify; http://botify.com)"
2324
+ ]
938
2325
  }
939
2326
  ]